diff --git a/Include/parsetok.h b/Include/parsetok.h index 4b7694f..7b3effd 100644 --- a/Include/parsetok.h +++ b/Include/parsetok.h @@ -9,7 +9,11 @@ extern "C" { typedef struct { int error; +#ifdef PGEN const char *filename; /* decoded from the filesystem encoding */ +#else + PyObject *filename_obj; +#endif int lineno; int offset; char *text; /* UTF-8-encoded string */ @@ -66,8 +70,10 @@ PyAPI_FUNC(node *) PyParser_ParseStringFlagsFilenameEx( perrdetail *err_ret, int *flags); -/* Note that he following function is defined in pythonrun.c not parsetok.c. */ +/* Note that the following functions are defined in pythonrun.c, + not in parsetok.c */ PyAPI_FUNC(void) PyParser_SetError(perrdetail *); +PyAPI_FUNC(void) PyParser_FreeError(perrdetail *); #ifdef __cplusplus } diff --git a/Makefile.pre.in b/Makefile.pre.in index edbb7dd..a91310e 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -235,14 +235,13 @@ POBJS= \ Parser/listnode.o \ Parser/node.o \ Parser/parser.o \ - Parser/parsetok.o \ Parser/bitset.o \ Parser/metagrammar.o \ Parser/firstsets.o \ Parser/grammar.o \ Parser/pgen.o -PARSER_OBJS= $(POBJS) Parser/myreadline.o Parser/tokenizer.o +PARSER_OBJS= $(POBJS) Parser/myreadline.o Parser/parsetok.o Parser/tokenizer.o PGOBJS= \ Objects/obmalloc.o \ @@ -251,10 +250,12 @@ PGOBJS= \ Python/pyctype.o \ Parser/tokenizer_pgen.o \ Parser/printgrammar.o \ + Parser/parsetok_pgen.o \ Parser/pgenmain.o PARSER_HEADERS= \ Parser/parser.h \ + Include/parsetok.h \ Parser/tokenizer.h PGENOBJS= $(PGENMAIN) $(POBJS) $(PGOBJS) @@ -589,6 +590,7 @@ Parser/grammar.o: $(srcdir)/Parser/grammar.c \ Parser/metagrammar.o: $(srcdir)/Parser/metagrammar.c Parser/tokenizer_pgen.o: $(srcdir)/Parser/tokenizer.c +Parser/parsetok_pgen.o: $(srcdir)/Parser/parsetok.c Parser/pgenmain.o: $(srcdir)/Include/parsetok.h @@ -696,7 +698,6 @@ PYTHON_HEADERS= \ Include/objimpl.h \ Include/opcode.h \ Include/osdefs.h \ - Include/parsetok.h \ Include/patchlevel.h \ Include/pgen.h \ Include/pgenheaders.h \ diff --git a/Modules/parsermodule.c b/Modules/parsermodule.c index e959321..5c37f8b 100644 --- a/Modules/parsermodule.c +++ b/Modules/parsermodule.c @@ -584,6 +584,7 @@ parser_do_parse(PyObject *args, PyObject *kw, char *argspec, int type) else PyParser_SetError(&err); } + PyParser_FreeError(&err); return (res); } diff --git a/Parser/parsetok.c b/Parser/parsetok.c index 7636a54..f4d8c68 100644 --- a/Parser/parsetok.c +++ b/Parser/parsetok.c @@ -13,7 +13,7 @@ /* Forward */ static node *parsetok(struct tok_state *, grammar *, int, perrdetail *, int *); -static void initerr(perrdetail *err_ret, const char* filename); +static int initerr(perrdetail *err_ret, const char* filename); /* Parse input coming from a string. Return error code, print some errors. */ node * @@ -48,7 +48,8 @@ PyParser_ParseStringFlagsFilenameEx(const char *s, const char *filename, struct tok_state *tok; int exec_input = start == file_input; - initerr(err_ret, filename); + if (initerr(err_ret, filename) < 0) + return NULL; if (*flags & PyPARSE_IGNORE_COOKIE) tok = PyTokenizer_FromUTF8(s, exec_input); @@ -59,7 +60,12 @@ PyParser_ParseStringFlagsFilenameEx(const char *s, const char *filename, return NULL; } +#ifdef PGEN tok->filename = filename ? filename : ""; +#else + Py_INCREF(err_ret->filename_obj); + tok->filename_obj = err_ret->filename_obj; +#endif return parsetok(tok, g, start, err_ret, flags); } @@ -90,13 +96,19 @@ PyParser_ParseFileFlagsEx(FILE *fp, const char *filename, { struct tok_state *tok; - initerr(err_ret, filename); + if (initerr(err_ret, filename) < 0) + return NULL; if ((tok = PyTokenizer_FromFile(fp, (char *)enc, ps1, ps2)) == NULL) { err_ret->error = E_NOMEM; return NULL; } +#ifdef PGEN tok->filename = filename; +#else + Py_INCREF(err_ret->filename_obj); + tok->filename_obj = err_ret->filename_obj; +#endif return parsetok(tok, g, start, err_ret, flags); } @@ -268,14 +280,26 @@ done: return n; } -static void +static int initerr(perrdetail *err_ret, const char *filename) { err_ret->error = E_OK; - err_ret->filename = filename; err_ret->lineno = 0; err_ret->offset = 0; err_ret->text = NULL; err_ret->token = -1; err_ret->expected = -1; +#ifdef PGEN + err_ret->filename = filename; +#else + if (filename) + err_ret->filename_obj = PyUnicode_DecodeFSDefault(filename); + else + err_ret->filename_obj = PyUnicode_FromString(""); + if (err_ret->filename_obj == NULL) { + err_ret->error = E_ERROR; + return -1; + } +#endif + return 0; } diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 441d05a..03927b1 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -128,7 +128,11 @@ tok_new(void) tok->prompt = tok->nextprompt = NULL; tok->lineno = 0; tok->level = 0; +#ifdef PGEN tok->filename = NULL; +#else + tok->filename_obj = NULL; +#endif tok->altwarning = 1; tok->alterror = 1; tok->alttabsize = 1; @@ -545,7 +549,6 @@ decoding_fgets(char *s, int size, struct tok_state *tok) { char *line = NULL; int badchar = 0; - PyObject *filename; for (;;) { if (tok->decoding_state == STATE_NORMAL) { /* We already have a codec associated with @@ -586,16 +589,12 @@ decoding_fgets(char *s, int size, struct tok_state *tok) if (badchar) { /* Need to add 1 to the line number, since this line has not been counted, yet. */ - filename = PyUnicode_DecodeFSDefault(tok->filename); - if (filename != NULL) { - PyErr_Format(PyExc_SyntaxError, - "Non-UTF-8 code starting with '\\x%.2x' " - "in file %.200U on line %i, " - "but no encoding declared; " - "see http://python.org/dev/peps/pep-0263/ for details", - badchar, filename, tok->lineno + 1); - Py_DECREF(filename); - } + PyErr_Format(PyExc_SyntaxError, + "Non-UTF-8 code starting with '\\x%.2x' " + "in file %.200U on line %i, " + "but no encoding declared; " + "see http://python.org/dev/peps/pep-0263/ for details", + badchar, tok->filename_obj, tok->lineno + 1); return error_ret(tok); } #endif @@ -853,6 +852,7 @@ PyTokenizer_Free(struct tok_state *tok) #ifndef PGEN Py_XDECREF(tok->decoding_readline); Py_XDECREF(tok->decoding_buffer); + Py_XDECREF(tok->filename_obj); #endif if (tok->fp != NULL && tok->buf != NULL) PyMem_FREE(tok->buf); @@ -1240,8 +1240,13 @@ indenterror(struct tok_state *tok) return 1; } if (tok->altwarning) { +#ifdef PGEN PySys_WriteStderr("%s: inconsistent use of tabs and spaces " "in indentation\n", tok->filename); +#else + PySys_FormatStderr("%U: inconsistent use of tabs and spaces " + "in indentation\n", tok->filename_obj); +#endif tok->altwarning = 0; } return 0; diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h index 2be3bf2..89cb7c2 100644 --- a/Parser/tokenizer.h +++ b/Parser/tokenizer.h @@ -40,7 +40,11 @@ struct tok_state { int level; /* () [] {} Parentheses nesting level */ /* Used to allow free continuations inside them */ /* Stuff for checking on different tab sizes */ - const char *filename; /* encoded to the filesystem encoding */ +#ifdef PGEN + const char *filename; /* decoded from the filesystem encoding */ +#else + PyObject *filename_obj; +#endif int altwarning; /* Issue warning if alternate tabs don't match */ int alterror; /* Issue error if alternate tabs don't match */ int alttabsize; /* Alternate tab spacing */ diff --git a/Python/pythonrun.c b/Python/pythonrun.c index f7335a2..22d2ad5 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -62,6 +62,7 @@ static PyObject *run_mod(mod_ty, const char *, PyObject *, PyObject *, static PyObject *run_pyc_file(FILE *, const char *, PyObject *, PyObject *, PyCompilerFlags *); static void err_input(perrdetail *); +static void err_free(perrdetail *); static void initsigs(void); static void call_py_exitfuncs(void); static void wait_for_thread_shutdown(void); @@ -1867,12 +1868,13 @@ PyParser_ASTFromString(const char *s, const char *filename, int start, flags->cf_flags |= iflags & PyCF_MASK; mod = PyAST_FromNode(n, flags, filename, arena); PyNode_Free(n); - return mod; } else { err_input(&err); - return NULL; + mod = NULL; } + err_free(&err); + return mod; } mod_ty @@ -1897,14 +1899,15 @@ PyParser_ASTFromFile(FILE *fp, const char *filename, const char* enc, flags->cf_flags |= iflags & PyCF_MASK; mod = PyAST_FromNode(n, flags, filename, arena); PyNode_Free(n); - return mod; } else { err_input(&err); if (errcode) *errcode = err.error; - return NULL; + mod = NULL; } + err_free(&err); + return mod; } /* Simplified interface to parsefile -- return node or set exception */ @@ -1918,6 +1921,7 @@ PyParser_SimpleParseFileFlags(FILE *fp, const char *filename, int start, int fla start, NULL, NULL, &err, flags); if (n == NULL) err_input(&err); + err_free(&err); return n; } @@ -1932,6 +1936,7 @@ PyParser_SimpleParseStringFlags(const char *str, int start, int flags) start, &err, flags); if (n == NULL) err_input(&err); + err_free(&err); return n; } @@ -1944,6 +1949,7 @@ PyParser_SimpleParseStringFlagsFilename(const char *str, const char *filename, &_PyParser_Grammar, start, &err, flags); if (n == NULL) err_input(&err); + err_free(&err); return n; } @@ -1957,11 +1963,23 @@ PyParser_SimpleParseStringFilename(const char *str, const char *filename, int st even parser modules. */ void +PyParser_FreeError(perrdetail *err) +{ + err_free(err); +} + +void PyParser_SetError(perrdetail *err) { err_input(err); } +static void +err_free(perrdetail *err) +{ + Py_XDECREF(err->filename_obj); +} + /* Set the error appropriate to the given input error code (see errcode.h) */ static void @@ -1969,7 +1987,6 @@ err_input(perrdetail *err) { PyObject *v, *w, *errtype, *errtext; PyObject *msg_obj = NULL; - PyObject *filename; char *msg = NULL; errtype = PyExc_SyntaxError; @@ -2055,17 +2072,8 @@ err_input(perrdetail *err) errtext = PyUnicode_DecodeUTF8(err->text, strlen(err->text), "replace"); } - if (err->filename != NULL) - filename = PyUnicode_DecodeFSDefault(err->filename); - else { - Py_INCREF(Py_None); - filename = Py_None; - } - if (filename != NULL) - v = Py_BuildValue("(NiiN)", filename, - err->lineno, err->offset, errtext); - else - v = NULL; + v = Py_BuildValue("(OiiN)", err->filename_obj, + err->lineno, err->offset, errtext); if (v != NULL) { if (msg_obj) w = Py_BuildValue("(OO)", msg_obj, v); diff --git a/Parser/parsetok_pgen.c b/Parser/parsetok_pgen.c new file mode 100644 index 0000000..97b9288 --- /dev/null +++ b/Parser/parsetok_pgen.c @@ -0,0 +1,2 @@ +#define PGEN +#include "parsetok.c"