Provisional implementation of PEP 3104.

jeremyhylton · jeremyhylton · commit 81e9502df693 · 2007-02-27T06:50:52.000Z
Add nonlocal_stmt to Grammar and Nonlocal node to AST.  They both
parallel the definitions for globals.  The symbol table treats
variables declared as nonlocal just like variables that are free
implicitly.

This change is missing the language spec changes, but makes some
decisions about what the spec should say via the unittests.  The PEP
is silent on a number of decisions, so we should review those before
claiming that nonlocal is complete.

Thomas Wouters made the grammer and ast changes.  Jeremy Hylton added
the symbol table changes and the tests.  Pete Shinners and Neal
Norwitz helped review the code.
diff --git a/Grammar/Grammar b/Grammar/Grammar
@@ -39,7 +39,7 @@ vfplist: vfpdef (',' vfpdef)* [',']
 stmt: simple_stmt | compound_stmt
 simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE
 small_stmt: (expr_stmt | del_stmt | pass_stmt | flow_stmt |
-             import_stmt | global_stmt | assert_stmt)
+             import_stmt | global_stmt | nonlocal_stmt | assert_stmt)
 expr_stmt: testlist (augassign (yield_expr|testlist) |
                      ('=' (yield_expr|testlist))*)
 augassign: ('+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' |
@@ -63,6 +63,7 @@ import_as_names: import_as_name (',' import_as_name)* [',']
 dotted_as_names: dotted_as_name (',' dotted_as_name)*
 dotted_name: NAME ('.' NAME)*
 global_stmt: 'global' NAME (',' NAME)*
+nonlocal_stmt: 'nonlocal' NAME (',' NAME)*
 assert_stmt: 'assert' test [',' test]
 
 compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef
diff --git a/Include/Python-ast.h b/Include/Python-ast.h
@@ -66,7 +66,8 @@ enum _stmt_kind {FunctionDef_kind=1, ClassDef_kind=2, Return_kind=3,
                   While_kind=8, If_kind=9, With_kind=10, Raise_kind=11,
                   TryExcept_kind=12, TryFinally_kind=13, Assert_kind=14,
                   Import_kind=15, ImportFrom_kind=16, Global_kind=17,
-                  Expr_kind=18, Pass_kind=19, Break_kind=20, Continue_kind=21};
+                  Nonlocal_kind=18, Expr_kind=19, Pass_kind=20, Break_kind=21,
+                  Continue_kind=22};
 struct _stmt {
         enum _stmt_kind kind;
         union {
@@ -164,6 +165,10 @@ struct _stmt {
                         asdl_seq *names;
                 } Global;
                 
+                struct {
+                        asdl_seq *names;
+                } Nonlocal;
+                
                 struct {
                         expr_ty value;
                 } Expr;
@@ -422,6 +427,9 @@ stmt_ty _Py_ImportFrom(identifier module, asdl_seq * names, int level, int
 #define Global(a0, a1, a2, a3) _Py_Global(a0, a1, a2, a3)
 stmt_ty _Py_Global(asdl_seq * names, int lineno, int col_offset, PyArena
                    *arena);
+#define Nonlocal(a0, a1, a2, a3) _Py_Nonlocal(a0, a1, a2, a3)
+stmt_ty _Py_Nonlocal(asdl_seq * names, int lineno, int col_offset, PyArena
+                     *arena);
 #define Expr(a0, a1, a2, a3) _Py_Expr(a0, a1, a2, a3)
 stmt_ty _Py_Expr(expr_ty value, int lineno, int col_offset, PyArena *arena);
 #define Pass(a0, a1, a2) _Py_Pass(a0, a1, a2)
diff --git a/Include/graminit.h b/Include/graminit.h
@@ -35,53 +35,54 @@
 #define dotted_as_names 290
 #define dotted_name 291
 #define global_stmt 292
-#define assert_stmt 293
-#define compound_stmt 294
-#define if_stmt 295
-#define while_stmt 296
-#define for_stmt 297
-#define try_stmt 298
-#define with_stmt 299
-#define with_var 300
-#define except_clause 301
-#define suite 302
-#define testlist_safe 303
-#define old_test 304
-#define old_lambdef 305
-#define test 306
-#define or_test 307
-#define and_test 308
-#define not_test 309
-#define comparison 310
-#define comp_op 311
-#define expr 312
-#define xor_expr 313
-#define and_expr 314
-#define shift_expr 315
-#define arith_expr 316
-#define term 317
-#define factor 318
-#define power 319
-#define atom 320
-#define listmaker 321
-#define testlist_gexp 322
-#define lambdef 323
-#define trailer 324
-#define subscriptlist 325
-#define subscript 326
-#define sliceop 327
-#define exprlist 328
-#define testlist 329
-#define dictsetmaker 330
-#define classdef 331
-#define arglist 332
-#define argument 333
-#define list_iter 334
-#define list_for 335
-#define list_if 336
-#define gen_iter 337
-#define gen_for 338
-#define gen_if 339
-#define testlist1 340
-#define encoding_decl 341
-#define yield_expr 342
+#define nonlocal_stmt 293
+#define assert_stmt 294
+#define compound_stmt 295
+#define if_stmt 296
+#define while_stmt 297
+#define for_stmt 298
+#define try_stmt 299
+#define with_stmt 300
+#define with_var 301
+#define except_clause 302
+#define suite 303
+#define testlist_safe 304
+#define old_test 305
+#define old_lambdef 306
+#define test 307
+#define or_test 308
+#define and_test 309
+#define not_test 310
+#define comparison 311
+#define comp_op 312
+#define expr 313
+#define xor_expr 314
+#define and_expr 315
+#define shift_expr 316
+#define arith_expr 317
+#define term 318
+#define factor 319
+#define power 320
+#define atom 321
+#define listmaker 322
+#define testlist_gexp 323
+#define lambdef 324
+#define trailer 325
+#define subscriptlist 326
+#define subscript 327
+#define sliceop 328
+#define exprlist 329
+#define testlist 330
+#define dictsetmaker 331
+#define classdef 332
+#define arglist 333
+#define argument 334
+#define list_iter 335
+#define list_for 336
+#define list_if 337
+#define gen_iter 338
+#define gen_for 339
+#define gen_if 340
+#define testlist1 341
+#define encoding_decl 342
+#define yield_expr 343
diff --git a/Include/symtable.h b/Include/symtable.h
@@ -64,23 +64,24 @@ PyAPI_FUNC(void) PySymtable_Free(struct symtable *);
 #define DEF_GLOBAL 1           /* global stmt */
 #define DEF_LOCAL 2            /* assignment in code block */
 #define DEF_PARAM 2<<1         /* formal parameter */
-#define USE 2<<2               /* name is used */
-#define DEF_STAR 2<<3          /* parameter is star arg */
-#define DEF_DOUBLESTAR 2<<4    /* parameter is star-star arg */
-#define DEF_INTUPLE 2<<5       /* name defined in tuple in parameters */
-#define DEF_FREE 2<<6          /* name used but not defined in nested block */
-#define DEF_FREE_GLOBAL 2<<7   /* free variable is actually implicit global */
-#define DEF_FREE_CLASS 2<<8    /* free variable from class's method */
-#define DEF_IMPORT 2<<9        /* assignment occurred via import */
+#define DEF_NONLOCAL 2<<2      /* nonlocal stmt */
+#define USE 2<<3               /* name is used */
+#define DEF_STAR 2<<4          /* parameter is star arg */
+#define DEF_DOUBLESTAR 2<<5    /* parameter is star-star arg */
+#define DEF_INTUPLE 2<<6       /* name defined in tuple in parameters */
+#define DEF_FREE 2<<7          /* name used but not defined in nested block */
+#define DEF_FREE_GLOBAL 2<<8   /* free variable is actually implicit global */
+#define DEF_FREE_CLASS 2<<9    /* free variable from class's method */
+#define DEF_IMPORT 2<<10        /* assignment occurred via import */
 
 #define DEF_BOUND (DEF_LOCAL | DEF_PARAM | DEF_IMPORT)
 
 /* GLOBAL_EXPLICIT and GLOBAL_IMPLICIT are used internally by the symbol
    table.  GLOBAL is returned from PyST_GetScope() for either of them. 
-   It is stored in ste_symbols at bits 12-14.
+   It is stored in ste_symbols at bits 12-15.
 */
 #define SCOPE_OFF 11
-#define SCOPE_MASK 7
+#define SCOPE_MASK (DEF_GLOBAL | DEF_LOCAL | DEF_PARAM | DEF_NONLOCAL)
 
 #define LOCAL 1
 #define GLOBAL_EXPLICIT 2
diff --git a/Lib/test/test_scope.py b/Lib/test/test_scope.py
@@ -555,6 +555,90 @@ def g():
 
         f(4)()
 
+    def testNonLocalFunction(self):
+
+        def f(x):
+            def inc():
+                nonlocal x
+                x += 1
+                return x
+            def dec():
+                nonlocal x
+                x -= 1
+                return x
+            return inc, dec
+
+        inc, dec = f(0)
+        self.assertEqual(inc(), 1)
+        self.assertEqual(inc(), 2)
+        self.assertEqual(dec(), 1)
+        self.assertEqual(dec(), 0)
+
+    def testNonLocalMethod(self):
+
+        def f(x):
+            class c:
+                def inc(self):
+                    nonlocal x
+                    x += 1
+                    return x
+                def dec(self):
+                    nonlocal x
+                    x -= 1
+                    return x
+            return c()
+
+        c = f(0)
+        self.assertEqual(c.inc(), 1)
+        self.assertEqual(c.inc(), 2)
+        self.assertEqual(c.dec(), 1)
+        self.assertEqual(c.dec(), 0)
+
+    def testNonLocalClass(self):
+
+        def f(x):
+            class c:
+                nonlocal x
+                x += 1
+                def get(self):
+                    return x
+            return c()
+
+        c = f(0)
+        self.assertEqual(c.get(), 1)
+        self.assert_("x" not in c.__class__.__dict__)
+        
+
+    def testNonLocalGenerator(self):
+
+        def f(x):
+            def g(y):
+                nonlocal x
+                for i in range(y):
+                    x += 1
+                    yield x
+            return g
+
+        g = f(0)
+        self.assertEqual(list(g(5)), [1, 2, 3, 4, 5])
+
+    def testNestedNonLocal(self):
+
+        def f(x):
+            def g():
+                nonlocal x
+                x -= 2
+                def h():
+                    nonlocal x
+                    x += 4
+                    return x
+                return h
+            return g
+
+        g = f(1)
+        h = g()
+        self.assertEqual(h(), 3)
+
 
 def test_main():
     run_unittest(ScopeTests)
diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py
@@ -367,6 +367,46 @@
      ...
    SystemError: too many statically nested blocks
 
+Misuse of the nonlocal statement can lead to a few unique syntax errors.
+
+   >>> def f(x):
+   ...     nonlocal x
+   Traceback (most recent call last):
+     ...
+   SyntaxError: name 'x' is local and nonlocal
+   
+   >>> def f():
+   ...     global x
+   ...     nonlocal x
+   Traceback (most recent call last):
+     ...
+   SyntaxError: name 'x' is nonlocal and global
+
+   >>> def f():
+   ...     nonlocal x
+   Traceback (most recent call last):
+     ...
+   SyntaxError: no binding for nonlocal 'x' found
+
+TODO(jhylton): Figure out how to test SyntaxWarning with doctest.
+
+##   >>> def f(x):
+##   ...     def f():
+##   ...         print(x)
+##   ...         nonlocal x
+##   Traceback (most recent call last):
+##     ...
+##   SyntaxWarning: name 'x' is assigned to before nonlocal declaration
+   
+##   >>> def f():
+##   ...     x = 1
+##   ...     nonlocal x
+##   Traceback (most recent call last):
+##     ...
+##   SyntaxWarning: name 'x' is assigned to before nonlocal declaration
+
+  
+
 """
 
 import re
diff --git a/Parser/Python.asdl b/Parser/Python.asdl
@@ -34,6 +34,7 @@ module Python version "$Revision$"
 	      | ImportFrom(identifier module, alias* names, int? level)
 
 	      | Global(identifier* names)
+	      | Nonlocal(identifier* names)
 	      | Expr(expr value)
 	      | Pass | Break | Continue
 
diff --git a/Python/Python-ast.c b/Python/Python-ast.c
@@ -131,6 +131,10 @@ static PyTypeObject *Global_type;
 static char *Global_fields[]={
         "names",
 };
+static PyTypeObject *Nonlocal_type;
+static char *Nonlocal_fields[]={
+        "names",
+};
 static PyTypeObject *Expr_type;
 static char *Expr_fields[]={
         "value",
@@ -507,6 +511,8 @@ static int init_types(void)
         if (!ImportFrom_type) return 0;
         Global_type = make_type("Global", stmt_type, Global_fields, 1);
         if (!Global_type) return 0;
+        Nonlocal_type = make_type("Nonlocal", stmt_type, Nonlocal_fields, 1);
+        if (!Nonlocal_type) return 0;
         Expr_type = make_type("Expr", stmt_type, Expr_fields, 1);
         if (!Expr_type) return 0;
         Pass_type = make_type("Pass", stmt_type, NULL, 0);
@@ -1145,6 +1151,20 @@ Global(asdl_seq * names, int lineno, int col_offset, PyArena *arena)
         return p;
 }
 
+stmt_ty
+Nonlocal(asdl_seq * names, int lineno, int col_offset, PyArena *arena)
+{
+        stmt_ty p;
+        p = (stmt_ty)PyArena_Malloc(arena, sizeof(*p));
+        if (!p)
+                return NULL;
+        p->kind = Nonlocal_kind;
+        p->v.Nonlocal.names = names;
+        p->lineno = lineno;
+        p->col_offset = col_offset;
+        return p;
+}
+
 stmt_ty
 Expr(expr_ty value, int lineno, int col_offset, PyArena *arena)
 {
@@ -2197,6 +2217,15 @@ ast2obj_stmt(void* _o)
                         goto failed;
                 Py_DECREF(value);
                 break;
+        case Nonlocal_kind:
+                result = PyType_GenericNew(Nonlocal_type, NULL, NULL);
+                if (!result) goto failed;
+                value = ast2obj_list(o->v.Nonlocal.names, ast2obj_identifier);
+                if (!value) goto failed;
+                if (PyObject_SetAttrString(result, "names", value) == -1)
+                        goto failed;
+                Py_DECREF(value);
+                break;
         case Expr_kind:
                 result = PyType_GenericNew(Expr_type, NULL, NULL);
                 if (!result) goto failed;
@@ -3049,6 +3078,8 @@ init_ast(void)
             0) return;
         if (PyDict_SetItemString(d, "Global", (PyObject*)Global_type) < 0)
             return;
+        if (PyDict_SetItemString(d, "Nonlocal", (PyObject*)Nonlocal_type) < 0)
+            return;
         if (PyDict_SetItemString(d, "Expr", (PyObject*)Expr_type) < 0) return;
         if (PyDict_SetItemString(d, "Pass", (PyObject*)Pass_type) < 0) return;
         if (PyDict_SetItemString(d, "Break", (PyObject*)Break_type) < 0) return;
diff --git a/Python/ast.c b/Python/ast.c
diff --git a/Python/compile.c b/Python/compile.c
diff --git a/Python/graminit.c b/Python/graminit.c
diff --git a/Python/symtable.c b/Python/symtable.c
