Remove the DocString AST type.

python · serhiy-storchaka · May 29, 2018 · May 25, 2018 · May 25, 2018 · May 28, 2018
commit 8ee9227599583c460ff8faf9c1ab1670559a1224
diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst
@@ -151,11 +151,6 @@ and classes for traversing abstract syntax trees:
    .. versionchanged:: 3.5
       :class:`AsyncFunctionDef` is now supported.
 
-   .. versionchanged:: 3.7
-      ``Str`` and ``Constant`` are not treated as docstring anymore.
-      The new AST node named ``DocString`` is added and only it is
-      considered as docstring.
-
 
 .. function:: fix_missing_locations(node)
 

diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst
@@ -2161,11 +2161,6 @@ Changes in Python Behavior
 Changes in the Python API
 -------------------------
 
-* Add new AST node ``DocString``.  Only it is considered as docstring.
-  ``Str`` and ``Const`` are not considered as docstring anymore.
-  (Contributed by INADA Naoki and Eugene Toder in :issue:`29463` and
-  :issue:`32911`.)
-
 * :meth:`socketserver.ThreadingMixIn.server_close` now waits until all
   non-daemon threads complete.  Set the new
   :attr:`socketserver.ThreadingMixIn.block_on_close` class attribute to

diff --git a/Include/Python-ast.h b/Include/Python-ast.h
@@ -70,7 +70,7 @@ enum _stmt_kind {FunctionDef_kind=1, AsyncFunctionDef_kind=2, ClassDef_kind=3,
                   AsyncWith_kind=14, Raise_kind=15, Try_kind=16,
                   Assert_kind=17, Import_kind=18, ImportFrom_kind=19,
                   Global_kind=20, Nonlocal_kind=21, Expr_kind=22, Pass_kind=23,
-                  Break_kind=24, Continue_kind=25, DocString_kind=26};
+                  Break_kind=24, Continue_kind=25};
 struct _stmt {
     enum _stmt_kind kind;
     union {
@@ -199,10 +199,6 @@ struct _stmt {
             expr_ty value;
         } Expr;
 
-        struct {
-            string s;
-        } DocString;
-
     } v;
     int lineno;
     int col_offset;
@@ -525,8 +521,6 @@ stmt_ty _Py_Pass(int lineno, int col_offset, PyArena *arena);
 stmt_ty _Py_Break(int lineno, int col_offset, PyArena *arena);
 #define Continue(a0, a1, a2) _Py_Continue(a0, a1, a2)
 stmt_ty _Py_Continue(int lineno, int col_offset, PyArena *arena);
-#define DocString(a0, a1, a2, a3) _Py_DocString(a0, a1, a2, a3)
-stmt_ty _Py_DocString(string s, int lineno, int col_offset, PyArena *arena);
 #define BoolOp(a0, a1, a2, a3, a4) _Py_BoolOp(a0, a1, a2, a3, a4)
 expr_ty _Py_BoolOp(boolop_ty op, asdl_seq * values, int lineno, int col_offset,
                    PyArena *arena);

diff --git a/Lib/ast.py b/Lib/ast.py
@@ -206,9 +206,15 @@ def get_docstring(node, clean=True):
     """
     if not isinstance(node, (AsyncFunctionDef, FunctionDef, ClassDef, Module)):
         raise TypeError("%r can't have docstrings" % node.__class__.__name__)
-    if not(node.body and isinstance(node.body[0], DocString)):
-        return
-    text = node.body[0].s
+    if not node.body:
+        return None
+    node = node.body[0].value
+    if isinstance(node, Str):
+        text = node.s
+    elif isinstance(node, Constant) and isinstance(node.value, str):
+        text = node.value
+    else:
+        return None
     if clean and text:
         import inspect
         text = inspect.cleandoc(text)

diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py
@@ -330,21 +330,6 @@ def test_module(self):
         x = ast.Module(body)
         self.assertEqual(x.body, body)
 
-    def test_docstring(self):
-        body = [ast.DocString("module docstring")]
-        x = ast.Module(body)
-        self.assertEqual(x.body, body)
-
-        a = ast.arguments()
-        x = ast.FunctionDef("x", a, body, [], None)
-        self.assertEqual(x.body, body)
-
-        x = ast.AsyncFunctionDef("x", a, body, [], None)
-        self.assertEqual(x.body, body)
-
-        x = ast.ClassDef("x", [], [], body, [])
-        self.assertEqual(x.body, body)
-
     def test_nodeclasses(self):
         # Zero arguments constructor explicitly allowed
         x = ast.BinOp()
@@ -1125,16 +1110,16 @@ def main():
 #### EVERYTHING BELOW IS GENERATED #####
 exec_results = [
 ('Module', [('Expr', (1, 0), ('NameConstant', (1, 0), None))]),
-('Module', [('DocString', (1, 0), 'module docstring')]),
+('Module', [('Expr', (1, 0), ('Str', (1, 0), 'module docstring'))]),
 ('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Pass', (1, 9))], [], None)]),
-('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('DocString', (1, 9), 'function docstring')], [], None)]),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Expr', (1, 9), ('Str', (1, 9), 'function docstring'))], [], None)]),
 ('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', (1, 6), 'a', None)], None, [], [], None, []), [('Pass', (1, 10))], [], None)]),
 ('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', (1, 6), 'a', None)], None, [], [], None, [('Num', (1, 8), 0)]), [('Pass', (1, 12))], [], None)]),
 ('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], ('arg', (1, 7), 'args', None), [], [], None, []), [('Pass', (1, 14))], [], None)]),
 ('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], ('arg', (1, 8), 'kwargs', None), []), [('Pass', (1, 17))], [], None)]),
-('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', (1, 6), 'a', None), ('arg', (1, 9), 'b', None), ('arg', (1, 14), 'c', None), ('arg', (1, 22), 'd', None), ('arg', (1, 28), 'e', None)], ('arg', (1, 35), 'args', None), [('arg', (1, 41), 'f', None)], [('Num', (1, 43), 42)], ('arg', (1, 49), 'kwargs', None), [('Num', (1, 11), 1), ('NameConstant', (1, 16), None), ('List', (1, 24), [], ('Load',)), ('Dict', (1, 30), [], [])]), [('DocString', (1, 58), 'doc for f()')], [], None)]),
+('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [('arg', (1, 6), 'a', None), ('arg', (1, 9), 'b', None), ('arg', (1, 14), 'c', None), ('arg', (1, 22), 'd', None), ('arg', (1, 28), 'e', None)], ('arg', (1, 35), 'args', None), [('arg', (1, 41), 'f', None)], [('Num', (1, 43), 42)], ('arg', (1, 49), 'kwargs', None), [('Num', (1, 11), 1), ('NameConstant', (1, 16), None), ('List', (1, 24), [], ('Load',)), ('Dict', (1, 30), [], [])]), [('Expr', (1, 58), ('Str', (1, 58), 'doc for f()'))], [], None)]),
 ('Module', [('ClassDef', (1, 0), 'C', [], [], [('Pass', (1, 8))], [])]),
-('Module', [('ClassDef', (1, 0), 'C', [], [], [('DocString', (1, 9), 'docstring for class C')], [])]),
+('Module', [('ClassDef', (1, 0), 'C', [], [], [('Expr', (1, 9), ('Str', (1, 9), 'docstring for class C'))], [])]),
 ('Module', [('ClassDef', (1, 0), 'C', [('Name', (1, 8), 'object', ('Load',))], [], [('Pass', (1, 17))], [])]),
 ('Module', [('FunctionDef', (1, 0), 'f', ('arguments', [], None, [], [], None, []), [('Return', (1, 8), ('Num', (1, 15), 1))], [], None)]),
 ('Module', [('Delete', (1, 0), [('Name', (1, 4), 'v', ('Del',))])]),
@@ -1165,7 +1150,7 @@ def main():
 ('Module', [('Expr', (1, 0), ('DictComp', (1, 0), ('Name', (1, 1), 'a', ('Load',)), ('Name', (1, 5), 'b', ('Load',)), [('comprehension', ('Tuple', (1, 11), [('Name', (1, 11), 'v', ('Store',)), ('Name', (1, 13), 'w', ('Store',))], ('Store',)), ('Name', (1, 18), 'x', ('Load',)), [], 0)]))]),
 ('Module', [('Expr', (1, 0), ('SetComp', (1, 0), ('Name', (1, 1), 'r', ('Load',)), [('comprehension', ('Name', (1, 7), 'l', ('Store',)), ('Name', (1, 12), 'x', ('Load',)), [('Name', (1, 17), 'g', ('Load',))], 0)]))]),
 ('Module', [('Expr', (1, 0), ('SetComp', (1, 0), ('Name', (1, 1), 'r', ('Load',)), [('comprehension', ('Tuple', (1, 7), [('Name', (1, 7), 'l', ('Store',)), ('Name', (1, 9), 'm', ('Store',))], ('Store',)), ('Name', (1, 14), 'x', ('Load',)), [], 0)]))]),
-('Module', [('AsyncFunctionDef', (1, 6), 'f', ('arguments', [], None, [], [], None, []), [('DocString', (2, 1), 'async function'), ('Expr', (3, 1), ('Await', (3, 1), ('Call', (3, 7), ('Name', (3, 7), 'something', ('Load',)), [], [])))], [], None)]),
+('Module', [('AsyncFunctionDef', (1, 6), 'f', ('arguments', [], None, [], [], None, []), [('Expr', (2, 1), ('Str', (2, 1), 'async function')), ('Expr', (3, 1), ('Await', (3, 1), ('Call', (3, 7), ('Name', (3, 7), 'something', ('Load',)), [], [])))], [], None)]),
 ('Module', [('AsyncFunctionDef', (1, 6), 'f', ('arguments', [], None, [], [], None, []), [('AsyncFor', (2, 7), ('Name', (2, 11), 'e', ('Store',)), ('Name', (2, 16), 'i', ('Load',)), [('Expr', (2, 19), ('Num', (2, 19), 1))], [('Expr', (3, 7), ('Num', (3, 7), 2))])], [], None)]),
 ('Module', [('AsyncFunctionDef', (1, 6), 'f', ('arguments', [], None, [], [], None, []), [('AsyncWith', (2, 7), [('withitem', ('Name', (2, 12), 'a', ('Load',)), ('Name', (2, 17), 'b', ('Store',)))], [('Expr', (2, 20), ('Num', (2, 20), 1))])], [], None)]),
 ('Module', [('Expr', (1, 0), ('Dict', (1, 0), [None, ('Num', (1, 10), 2)], [('Dict', (1, 3), [('Num', (1, 4), 1)], [('Num', (1, 6), 2)]), ('Num', (1, 12), 3)]))]),

diff --git a/Misc/NEWS.d/3.7.0a1.rst b/Misc/NEWS.d/3.7.0a1.rst
@@ -788,7 +788,7 @@ objects.
 Add ``docstring`` field to Module, ClassDef, FunctionDef, and
 AsyncFunctionDef ast nodes.  docstring is not first stmt in their body
 anymore.  It affects ``co_firstlineno`` and ``co_lnotab`` of code object for
-module and class.
+module and class. (Reverted in :issue:`32911`.)
 
 ..
 

diff --git a/Parser/Python.asdl b/Parser/Python.asdl
@@ -50,7 +50,6 @@ module Python
           | Nonlocal(identifier* names)
           | Expr(expr value)
           | Pass | Break | Continue
-          | DocString(string s)
 
           -- XXX Jython will be different
           -- col_offset is the byte offset in the utf8 string the parser uses

diff --git a/Python/Python-ast.c b/Python/Python-ast.c
@@ -185,11 +185,6 @@ static char *Expr_fields[]={
 static PyTypeObject *Pass_type;
 static PyTypeObject *Break_type;
 static PyTypeObject *Continue_type;
-static PyTypeObject *DocString_type;
-_Py_IDENTIFIER(s);
-static char *DocString_fields[]={
-    "s",
-};
 static PyTypeObject *expr_type;
 static char *expr_attributes[] = {
     "lineno",
@@ -295,6 +290,7 @@ static char *Num_fields[]={
     "n",
 };
 static PyTypeObject *Str_type;
+_Py_IDENTIFIER(s);
 static char *Str_fields[]={
     "s",
 };
@@ -910,8 +906,6 @@ static int init_types(void)
     if (!Break_type) return 0;
     Continue_type = make_type("Continue", stmt_type, NULL, 0);
     if (!Continue_type) return 0;
-    DocString_type = make_type("DocString", stmt_type, DocString_fields, 1);
-    if (!DocString_type) return 0;
     expr_type = make_type("expr", &AST_type, NULL, 0);
     if (!expr_type) return 0;
     if (!add_attributes(expr_type, expr_attributes, 2)) return 0;
@@ -1741,25 +1735,6 @@ Continue(int lineno, int col_offset, PyArena *arena)
     return p;
 }
 
-stmt_ty
-DocString(string s, int lineno, int col_offset, PyArena *arena)
-{
-    stmt_ty p;
-    if (!s) {
-        PyErr_SetString(PyExc_ValueError,
-                        "field s is required for DocString");
-        return NULL;
-    }
-    p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
-    if (!p)
-        return NULL;
-    p->kind = DocString_kind;
-    p->v.DocString.s = s;
-    p->lineno = lineno;
-    p->col_offset = col_offset;
-    return p;
-}
-
 expr_ty
 BoolOp(boolop_ty op, asdl_seq * values, int lineno, int col_offset, PyArena
        *arena)
@@ -3049,15 +3024,6 @@ ast2obj_stmt(void* _o)
         result = PyType_GenericNew(Continue_type, NULL, NULL);
         if (!result) goto failed;
         break;
-    case DocString_kind:
-        result = PyType_GenericNew(DocString_type, NULL, NULL);
-        if (!result) goto failed;
-        value = ast2obj_string(o->v.DocString.s);
-        if (!value) goto failed;
-        if (_PyObject_SetAttrId(result, &PyId_s, value) == -1)
-            goto failed;
-        Py_DECREF(value);
-        break;
     }
     value = ast2obj_int(o->lineno);
     if (!value) goto failed;
@@ -5767,30 +5733,6 @@ obj2ast_stmt(PyObject* obj, stmt_ty* out, PyArena* arena)
         if (*out == NULL) goto failed;
         return 0;
     }
-    isinstance = PyObject_IsInstance(obj, (PyObject*)DocString_type);
-    if (isinstance == -1) {
-        return 1;
-    }
-    if (isinstance) {
-        string s;
-
-        if (_PyObject_LookupAttrId(obj, &PyId_s, &tmp) < 0) {
-            return 1;
-        }
-        if (tmp == NULL) {
-            PyErr_SetString(PyExc_TypeError, "required field \"s\" missing from DocString");
-            return 1;
-        }
-        else {
-            int res;
-            res = obj2ast_string(tmp, &s, arena);
-            if (res != 0) goto failed;
-            Py_CLEAR(tmp);
-        }
-        *out = DocString(s, lineno, col_offset, arena);
-        if (*out == NULL) goto failed;
-        return 0;
-    }
 
     PyErr_Format(PyExc_TypeError, "expected some sort of stmt, but got %R", obj);
     failed:
@@ -8271,8 +8213,6 @@ PyInit__ast(void)
         NULL;
     if (PyDict_SetItemString(d, "Continue", (PyObject*)Continue_type) < 0)
         return NULL;
-    if (PyDict_SetItemString(d, "DocString", (PyObject*)DocString_type) < 0)
-        return NULL;
     if (PyDict_SetItemString(d, "expr", (PyObject*)expr_type) < 0) return NULL;
     if (PyDict_SetItemString(d, "BoolOp", (PyObject*)BoolOp_type) < 0) return
         NULL;

diff --git a/Python/ast.c b/Python/ast.c
@@ -507,7 +507,6 @@ validate_stmt(stmt_ty stmt)
     case Pass_kind:
     case Break_kind:
     case Continue_kind:
-    case DocString_kind:
         return 1;
     default:
         PyErr_SetString(PyExc_SystemError, "unexpected statement");
@@ -595,8 +594,7 @@ struct compiling {
 static asdl_seq *seq_for_testlist(struct compiling *, const node *);
 static expr_ty ast_for_expr(struct compiling *, const node *);
 static stmt_ty ast_for_stmt(struct compiling *, const node *);
-static asdl_seq *ast_for_body(struct compiling *c, const node *n);
-static int docstring_from_stmts(asdl_seq *stmts, PyArena *arena);
+static asdl_seq *ast_for_suite(struct compiling *c, const node *n);
 static asdl_seq *ast_for_exprlist(struct compiling *, const node *,
                                   expr_context_ty);
 static expr_ty ast_for_testlist(struct compiling *, const node *);
@@ -814,9 +812,6 @@ PyAST_FromNodeObject(const node *n, PyCompilerFlags *flags,
                     }
                 }
             }
-            if (!docstring_from_stmts(stmts, arena)) {
-                goto out;
-            }
             res = Module(stmts, arena);
             break;
         case eval_input: {
@@ -1600,7 +1595,7 @@ ast_for_funcdef_impl(struct compiling *c, const node *n,
             return NULL;
         name_i += 2;
     }
-    body = ast_for_body(c, CHILD(n, name_i + 3));
+    body = ast_for_suite(c, CHILD(n, name_i + 3));
     if (!body)
         return NULL;
 
@@ -3522,41 +3517,6 @@ ast_for_suite(struct compiling *c, const node *n)
     return seq;
 }
 
-/* When first element of the stmts is string literal,
- * convert it to DocString
- */
-static int
-docstring_from_stmts(asdl_seq *stmts, PyArena *arena)
-{
-    if (stmts && stmts->size) {
-        stmt_ty s = (stmt_ty)asdl_seq_GET(stmts, 0);
-        /* If first statement is a literal string, it's the doc string. */
-        if (s->kind == Expr_kind && s->v.Expr.value->kind == Str_kind) {
-            stmt_ty doc = DocString(s->v.Expr.value->v.Str.s,
-                                    s->v.Expr.value->lineno,
-                                    s->v.Expr.value->col_offset, arena);
-            if (doc == NULL) {
-                return 0;
-            }
-            asdl_seq_SET(stmts, 0, doc);
-        }
-    }
-    return 1;
-}
-
-static asdl_seq *
-ast_for_body(struct compiling *c, const node *n)
-{
-    asdl_seq *stmts = ast_for_suite(c, n);
-    if (stmts == NULL) {
-        return NULL;
-    }
-    if (!docstring_from_stmts(stmts, c->c_arena)) {
-        return NULL;
-    }
-    return stmts;
-}
-
 static stmt_ty
 ast_for_if_stmt(struct compiling *c, const node *n)
 {
@@ -3946,7 +3906,7 @@ ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq)
     REQ(n, classdef);
 
     if (NCH(n) == 4) { /* class NAME ':' suite */
-        s = ast_for_body(c, CHILD(n, 3));
+        s = ast_for_suite(c, CHILD(n, 3));
         if (!s)
             return NULL;
         classname = NEW_IDENTIFIER(CHILD(n, 1));
@@ -3959,7 +3919,7 @@ ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq)
     }
 
     if (TYPE(CHILD(n, 3)) == RPAR) { /* class NAME '(' ')' ':' suite */
-        s = ast_for_body(c, CHILD(n, 5));
+        s = ast_for_suite(c, CHILD(n, 5));
         if (!s)
             return NULL;
         classname = NEW_IDENTIFIER(CHILD(n, 1));
@@ -3984,7 +3944,7 @@ ast_for_classdef(struct compiling *c, const node *n, asdl_seq *decorator_seq)
         if (!call)
             return NULL;
     }
-    s = ast_for_body(c, CHILD(n, 6));
+    s = ast_for_suite(c, CHILD(n, 6));
     if (!s)
         return NULL;
     classname = NEW_IDENTIFIER(CHILD(n, 1));