From 7da1de05fdb793fb2068e7583cf422b3d697cf65 Mon Sep 17 00:00:00 2001 From: Kyle Ellrott Date: Tue, 22 Mar 2022 01:45:37 -0700 Subject: [PATCH 01/60] py: implement __len__ for StringDict --- py/dict.go | 4 ++++ py/tests/dict.py | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/py/dict.go b/py/dict.go index 4f277c47..1710b7da 100644 --- a/py/dict.go +++ b/py/dict.go @@ -114,6 +114,10 @@ func (a StringDict) M__str__() (Object, error) { return a.M__repr__() } +func (a StringDict) M__len__() (Object, error) { + return Int(len(a)), nil +} + func (a StringDict) M__repr__() (Object, error) { var out bytes.Buffer out.WriteRune('{') diff --git a/py/tests/dict.py b/py/tests/dict.py index 2bbcd27e..cb14dbc2 100644 --- a/py/tests/dict.py +++ b/py/tests/dict.py @@ -54,4 +54,9 @@ assert a.__eq__({'a': 'b'}) == True assert a.__ne__({'a': 'b'}) == False +doc="__len__" +a = {"a": "1", "b": "2"} +assert a.__len__() == 2 +assert len(a) == 2 + doc="finished" From d6a392444ba91e6af604f73bf7b9f95b08910e05 Mon Sep 17 00:00:00 2001 From: Sebastien Binet Date: Wed, 23 Mar 2022 11:27:26 +0100 Subject: [PATCH 02/60] all: drop Go-1.16, add Go-1.18 Signed-off-by: Sebastien Binet --- .github/workflows/ci.yml | 2 +- go.mod | 9 ++++++++- go.sum | 12 ++++++------ 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e761437d..1daf04ba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ jobs: name: Build strategy: matrix: - go-version: [1.17.x, 1.16.x] + go-version: [1.18.x, 1.17.x] platform: [ubuntu-latest, macos-latest, windows-latest] runs-on: ${{ matrix.platform }} steps: diff --git a/go.mod b/go.mod index 163319a8..5fdf609c 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,15 @@ module github.com/go-python/gpython -go 1.16 +go 1.17 require ( github.com/gopherjs/gopherwasm v1.1.0 github.com/peterh/liner v1.2.2 ) + +require ( + github.com/gopherjs/gopherjs v0.0.0-20180825215210-0210a2f0f73c // indirect + github.com/mattn/go-runewidth v0.0.13 // indirect + github.com/rivo/uniseg v0.2.0 // indirect + golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 // indirect +) diff --git a/go.sum b/go.sum index 58d8376b..42793c4f 100644 --- a/go.sum +++ b/go.sum @@ -1,14 +1,14 @@ github.com/gopherjs/gopherjs v0.0.0-20180825215210-0210a2f0f73c h1:16eHWuMGvCjSfgRJKqIzapE78onvvTbdi1rMkU00lZw= github.com/gopherjs/gopherjs v0.0.0-20180825215210-0210a2f0f73c/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gopherjs/gopherwasm v1.0.0 h1:32nge/RlujS1Im4HNCJPp0NbBOAeBXFuT1KonUuLl+Y= -github.com/gopherjs/gopherwasm v1.0.0/go.mod h1:SkZ8z7CWBz5VXbhJel8TxCmAcsQqzgWGR/8nMhyhZSI= github.com/gopherjs/gopherwasm v1.1.0 h1:fA2uLoctU5+T3OhOn2vYP0DVT6pxc7xhTlBB1paATqQ= github.com/gopherjs/gopherwasm v1.1.0/go.mod h1:SkZ8z7CWBz5VXbhJel8TxCmAcsQqzgWGR/8nMhyhZSI= -github.com/mattn/go-runewidth v0.0.3 h1:a+kO+98RDGEfo6asOGMmpodZq4FNtnGP54yps8BzLR4= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/peterh/liner v1.1.0 h1:f+aAedNJA6uk7+6rXsYBnhdo4Xux7ESLe+kcuVUF5os= -github.com/peterh/liner v1.1.0/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= +github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= +github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/peterh/liner v1.2.2 h1:aJ4AOodmL+JxOZZEL2u9iJf8omNRpqHc/EbrK+3mAXw= github.com/peterh/liner v1.2.2/go.mod h1:xFwJyiKIXJZUKItq5dGHZSTBRAuG/CpeNpWLyiNRNwI= -golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1 h1:kwrAHlwJ0DUBZwQ238v+Uod/3eZ8B2K5rYsUHBQvzmI= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 h1:OH54vjqzRWmbJ62fjuhxy7AxFFgoHN0/DPc/UrL8cAs= +golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= From c398aacb5be36b256119acfb12d01c229511ba95 Mon Sep 17 00:00:00 2001 From: Sebastien Binet Date: Wed, 23 Mar 2022 14:39:35 +0100 Subject: [PATCH 03/60] all: remove use of deprecated io/ioutil package --- compile/compile_test.go | 4 ++-- modules/runtime.go | 3 +-- parser/testparser/testparser.go | 4 ++-- py/file.go | 3 +-- pytest/pytest.go | 6 +++--- 5 files changed, 9 insertions(+), 11 deletions(-) diff --git a/compile/compile_test.go b/compile/compile_test.go index aee3caac..fe2c8671 100644 --- a/compile/compile_test.go +++ b/compile/compile_test.go @@ -10,7 +10,7 @@ package compile import ( "fmt" - "io/ioutil" + "io" "os/exec" "testing" @@ -118,7 +118,7 @@ func EqCodeCode(t *testing.T, name string, a, b string) { t.Errorf("%s code want %q, got %q", name, a, b) return } - stdoutData, err := ioutil.ReadAll(stdout) + stdoutData, err := io.ReadAll(stdout) if err != nil { t.Fatalf("Failed to read data: %v", err) } diff --git a/modules/runtime.go b/modules/runtime.go index 91fc7092..0b003022 100644 --- a/modules/runtime.go +++ b/modules/runtime.go @@ -6,7 +6,6 @@ package modules import ( "bytes" - "io/ioutil" "os" "path" "path/filepath" @@ -145,7 +144,7 @@ func (ctx *context) ResolveAndCompile(pathname string, opts py.CompileOpts) (py. switch ext { case ".py": var pySrc []byte - pySrc, err = ioutil.ReadFile(fpath) + pySrc, err = os.ReadFile(fpath) if err != nil { return false, py.ExceptionNewf(py.OSError, "Error reading %q: %v", fpath, err) } diff --git a/parser/testparser/testparser.go b/parser/testparser/testparser.go index 6bf2d8e7..410ca043 100644 --- a/parser/testparser/testparser.go +++ b/parser/testparser/testparser.go @@ -7,7 +7,7 @@ package main import ( "flag" "fmt" - "io/ioutil" + "io" "log" "os" @@ -47,7 +47,7 @@ func main() { _, err = parser.Lex(in, path, "exec") } else if *compileFile { var input []byte - input, err = ioutil.ReadAll(in) + input, err = io.ReadAll(in) if err != nil { log.Fatalf("Failed to read %q: %v", path, err) } diff --git a/py/file.go b/py/file.go index fa270865..c60013ad 100644 --- a/py/file.go +++ b/py/file.go @@ -11,7 +11,6 @@ package py import ( "io" - "io/ioutil" "os" ) @@ -129,7 +128,7 @@ func (o *File) Read(args Tuple, kwargs StringDict) (Object, error) { return nil, ExceptionNewf(TypeError, "read() argument 1 must be int, not %s", arg.Type().Name) } - b, err := ioutil.ReadAll(r) + b, err := io.ReadAll(r) if err != nil { if err == io.EOF { return o.readResult(nil) diff --git a/pytest/pytest.go b/pytest/pytest.go index c7af7737..db556f6e 100644 --- a/pytest/pytest.go +++ b/pytest/pytest.go @@ -5,7 +5,7 @@ package pytest import ( - "io/ioutil" + "io" "os" "path" "strings" @@ -31,7 +31,7 @@ func compileProgram(t testing.TB, prog string) (*py.Module, *py.Code) { } }() - str, err := ioutil.ReadAll(f) + str, err := io.ReadAll(f) if err != nil { t.Fatalf("%s: ReadAll failed: %v", prog, err) } @@ -92,7 +92,7 @@ func run(t testing.TB, module *py.Module, code *py.Code) { // find the python files in the directory passed in func findFiles(t testing.TB, testDir string) (names []string) { - files, err := ioutil.ReadDir(testDir) + files, err := os.ReadDir(testDir) if err != nil { t.Fatalf("ReadDir failed: %v", err) } From 92816224f90a57fdc5d7f9e66a5a04fa241dda9d Mon Sep 17 00:00:00 2001 From: Sebastien Binet Date: Wed, 23 Mar 2022 17:26:52 +0100 Subject: [PATCH 04/60] all: move modules to stdlib --- examples/embedding/main.go | 6 +++--- examples/multi-context/main.go | 4 ++-- main.go | 3 ++- pytest/pytest.go | 4 ++-- repl/repl_test.go | 2 +- repl/web/main.go | 5 ++--- modules/runtime.go => stdlib/stdlib.go | 4 +++- 7 files changed, 15 insertions(+), 13 deletions(-) rename modules/runtime.go => stdlib/stdlib.go (97%) diff --git a/examples/embedding/main.go b/examples/embedding/main.go index 37f34a7f..e4aadfef 100644 --- a/examples/embedding/main.go +++ b/examples/embedding/main.go @@ -10,7 +10,7 @@ import ( // This initializes gpython for runtime execution and is essential. // It defines forward-declared symbols and registers native built-in modules, such as sys and time. - _ "github.com/go-python/gpython/modules" + _ "github.com/go-python/gpython/stdlib" // Commonly consumed gpython "github.com/go-python/gpython/py" @@ -27,8 +27,8 @@ func runWithFile(pyFile string) error { // See type Context interface and related docs ctx := py.NewContext(py.DefaultContextOpts()) - - // This drives modules being able to perform cleanup and release resources + + // This drives modules being able to perform cleanup and release resources defer ctx.Close() var err error diff --git a/examples/multi-context/main.go b/examples/multi-context/main.go index 4bbeda8b..f645e766 100644 --- a/examples/multi-context/main.go +++ b/examples/multi-context/main.go @@ -14,7 +14,7 @@ import ( // This initializes gpython for runtime execution and is critical. // It defines forward-declared symbols and registers native built-in modules, such as sys and time. - _ "github.com/go-python/gpython/modules" + _ "github.com/go-python/gpython/stdlib" // This is the primary import for gpython. // It contains all symbols needed to fully compile and run python. @@ -129,7 +129,7 @@ func RunMultiPi(numWorkers, numTimes int) time.Duration { } workersRunning.Done() - // This drives modules being able to perform cleanup and release resources + // This drives modules being able to perform cleanup and release resources w.ctx.Close() }() } diff --git a/main.go b/main.go index d529646a..bc906aac 100644 --- a/main.go +++ b/main.go @@ -14,10 +14,11 @@ import ( "runtime" "runtime/pprof" - _ "github.com/go-python/gpython/modules" "github.com/go-python/gpython/py" "github.com/go-python/gpython/repl" "github.com/go-python/gpython/repl/cli" + + _ "github.com/go-python/gpython/stdlib" ) // Globals diff --git a/pytest/pytest.go b/pytest/pytest.go index db556f6e..7cadadb1 100644 --- a/pytest/pytest.go +++ b/pytest/pytest.go @@ -11,10 +11,10 @@ import ( "strings" "testing" - _ "github.com/go-python/gpython/modules" - "github.com/go-python/gpython/compile" "github.com/go-python/gpython/py" + + _ "github.com/go-python/gpython/stdlib" ) var gContext = py.NewContext(py.DefaultContextOpts()) diff --git a/repl/repl_test.go b/repl/repl_test.go index 9c64879e..3c13c7c6 100644 --- a/repl/repl_test.go +++ b/repl/repl_test.go @@ -6,7 +6,7 @@ import ( "testing" // import required modules - _ "github.com/go-python/gpython/modules" + _ "github.com/go-python/gpython/stdlib" ) type replTest struct { diff --git a/repl/web/main.go b/repl/web/main.go index 0be6f3cd..f0cebfd8 100644 --- a/repl/web/main.go +++ b/repl/web/main.go @@ -13,12 +13,11 @@ import ( "log" "runtime" + "github.com/go-python/gpython/repl" "github.com/gopherjs/gopherwasm/js" // gopherjs to wasm converter shim // import required modules - _ "github.com/go-python/gpython/modules" - - "github.com/go-python/gpython/repl" + _ "github.com/go-python/gpython/stdlib" ) // Implement the replUI interface diff --git a/modules/runtime.go b/stdlib/stdlib.go similarity index 97% rename from modules/runtime.go rename to stdlib/stdlib.go index 0b003022..5298b7e7 100644 --- a/modules/runtime.go +++ b/stdlib/stdlib.go @@ -2,7 +2,9 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -package modules +// Package stdlib provides the bootstrap code to wire in all the stdlib +// (python) modules into a gpython context and VM. +package stdlib import ( "bytes" From e8acd73a892dba1a0d9a87c9dfd68827d1fa8bdb Mon Sep 17 00:00:00 2001 From: Sebastien Binet Date: Wed, 23 Mar 2022 17:33:47 +0100 Subject: [PATCH 05/60] all: move gpython py-modules under stdlib --- .gitignore | 4 ++-- {builtin => stdlib/builtin}/builtin.go | 0 {builtin => stdlib/builtin}/builtin_test.go | 0 {builtin => stdlib/builtin}/tests/builtin.py | 0 {builtin => stdlib/builtin}/tests/lib.py | 0 {builtin => stdlib/builtin}/tests/libtest.py | 0 {math => stdlib/math}/math.go | 0 {math => stdlib/math}/math_test.go | 0 {math => stdlib/math}/tests/libtest.py | 0 {math => stdlib/math}/tests/libulp.py | 0 {math => stdlib/math}/tests/mathtests.py | 0 {math => stdlib/math}/tests/testcases.py | 0 stdlib/stdlib.go | 8 ++++---- {sys => stdlib/sys}/sys.go | 0 {time => stdlib/time}/time.go | 0 15 files changed, 6 insertions(+), 6 deletions(-) rename {builtin => stdlib/builtin}/builtin.go (100%) rename {builtin => stdlib/builtin}/builtin_test.go (100%) rename {builtin => stdlib/builtin}/tests/builtin.py (100%) rename {builtin => stdlib/builtin}/tests/lib.py (100%) rename {builtin => stdlib/builtin}/tests/libtest.py (100%) rename {math => stdlib/math}/math.go (100%) rename {math => stdlib/math}/math_test.go (100%) rename {math => stdlib/math}/tests/libtest.py (100%) rename {math => stdlib/math}/tests/libulp.py (100%) rename {math => stdlib/math}/tests/mathtests.py (100%) rename {math => stdlib/math}/tests/testcases.py (100%) rename {sys => stdlib/sys}/sys.go (100%) rename {time => stdlib/time}/time.go (100%) diff --git a/.gitignore b/.gitignore index ff8d3cdb..dc9b73bb 100644 --- a/.gitignore +++ b/.gitignore @@ -10,5 +10,5 @@ cover.out /dist # tests -builtin/testfile -examples/embedding/embedding \ No newline at end of file +stdlib/builtin/testfile +examples/embedding/embedding diff --git a/builtin/builtin.go b/stdlib/builtin/builtin.go similarity index 100% rename from builtin/builtin.go rename to stdlib/builtin/builtin.go diff --git a/builtin/builtin_test.go b/stdlib/builtin/builtin_test.go similarity index 100% rename from builtin/builtin_test.go rename to stdlib/builtin/builtin_test.go diff --git a/builtin/tests/builtin.py b/stdlib/builtin/tests/builtin.py similarity index 100% rename from builtin/tests/builtin.py rename to stdlib/builtin/tests/builtin.py diff --git a/builtin/tests/lib.py b/stdlib/builtin/tests/lib.py similarity index 100% rename from builtin/tests/lib.py rename to stdlib/builtin/tests/lib.py diff --git a/builtin/tests/libtest.py b/stdlib/builtin/tests/libtest.py similarity index 100% rename from builtin/tests/libtest.py rename to stdlib/builtin/tests/libtest.py diff --git a/math/math.go b/stdlib/math/math.go similarity index 100% rename from math/math.go rename to stdlib/math/math.go diff --git a/math/math_test.go b/stdlib/math/math_test.go similarity index 100% rename from math/math_test.go rename to stdlib/math/math_test.go diff --git a/math/tests/libtest.py b/stdlib/math/tests/libtest.py similarity index 100% rename from math/tests/libtest.py rename to stdlib/math/tests/libtest.py diff --git a/math/tests/libulp.py b/stdlib/math/tests/libulp.py similarity index 100% rename from math/tests/libulp.py rename to stdlib/math/tests/libulp.py diff --git a/math/tests/mathtests.py b/stdlib/math/tests/mathtests.py similarity index 100% rename from math/tests/mathtests.py rename to stdlib/math/tests/mathtests.py diff --git a/math/tests/testcases.py b/stdlib/math/tests/testcases.py similarity index 100% rename from math/tests/testcases.py rename to stdlib/math/tests/testcases.py diff --git a/stdlib/stdlib.go b/stdlib/stdlib.go index 5298b7e7..7b46d391 100644 --- a/stdlib/stdlib.go +++ b/stdlib/stdlib.go @@ -18,10 +18,10 @@ import ( "github.com/go-python/gpython/py" "github.com/go-python/gpython/vm" - _ "github.com/go-python/gpython/builtin" - _ "github.com/go-python/gpython/math" - _ "github.com/go-python/gpython/sys" - _ "github.com/go-python/gpython/time" + _ "github.com/go-python/gpython/stdlib/builtin" + _ "github.com/go-python/gpython/stdlib/math" + _ "github.com/go-python/gpython/stdlib/sys" + _ "github.com/go-python/gpython/stdlib/time" ) func init() { diff --git a/sys/sys.go b/stdlib/sys/sys.go similarity index 100% rename from sys/sys.go rename to stdlib/sys/sys.go diff --git a/time/time.go b/stdlib/time/time.go similarity index 100% rename from time/time.go rename to stdlib/time/time.go From b8d4a9163133d1ad6d0df4e8c1db0fec09b36bf0 Mon Sep 17 00:00:00 2001 From: Sebastien Binet Date: Wed, 23 Mar 2022 17:30:38 +0100 Subject: [PATCH 06/60] pytest: introduce RunScript --- go.mod | 1 + go.sum | 4 +++ pytest/pytest.go | 57 ++++++++++++++++++++++++++++++++ pytest/pytest_test.go | 33 ++++++++++++++++++ pytest/testdata/hello.py | 7 ++++ pytest/testdata/hello_golden.txt | 3 ++ pytest/testdata/tests/libtest.py | 11 ++++++ pytest/testdata/tests/module.py | 12 +++++++ 8 files changed, 128 insertions(+) create mode 100644 pytest/pytest_test.go create mode 100644 pytest/testdata/hello.py create mode 100644 pytest/testdata/hello_golden.txt create mode 100644 pytest/testdata/tests/libtest.py create mode 100644 pytest/testdata/tests/module.py diff --git a/go.mod b/go.mod index 5fdf609c..c87c414b 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/go-python/gpython go 1.17 require ( + github.com/google/go-cmp v0.5.7 github.com/gopherjs/gopherwasm v1.1.0 github.com/peterh/liner v1.2.2 ) diff --git a/go.sum b/go.sum index 42793c4f..34e6510b 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/gopherjs/gopherjs v0.0.0-20180825215210-0210a2f0f73c h1:16eHWuMGvCjSfgRJKqIzapE78onvvTbdi1rMkU00lZw= github.com/gopherjs/gopherjs v0.0.0-20180825215210-0210a2f0f73c/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherwasm v1.1.0 h1:fA2uLoctU5+T3OhOn2vYP0DVT6pxc7xhTlBB1paATqQ= @@ -12,3 +14,5 @@ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 h1:OH54vjqzRWmbJ62fjuhxy7AxFFgoHN0/DPc/UrL8cAs= golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/pytest/pytest.go b/pytest/pytest.go index 7cadadb1..232f7525 100644 --- a/pytest/pytest.go +++ b/pytest/pytest.go @@ -5,14 +5,17 @@ package pytest import ( + "bytes" "io" "os" "path" + "path/filepath" "strings" "testing" "github.com/go-python/gpython/compile" "github.com/go-python/gpython/py" + "github.com/google/go-cmp/cmp" _ "github.com/go-python/gpython/stdlib" ) @@ -126,3 +129,57 @@ func RunBenchmarks(b *testing.B, testDir string) { }) } } + +// RunScript runs the provided path to a script. +// RunScript captures the stdout and stderr while executing the script +// and compares it to a golden file: +// RunScript("./testdata/foo.py") +// will compare the output with "./testdata/foo_golden.txt". +func RunScript(t *testing.T, fname string) { + opts := py.DefaultContextOpts() + opts.SysArgs = []string{fname} + ctx := py.NewContext(opts) + defer ctx.Close() + + sys := ctx.Store().MustGetModule("sys") + tmp, err := os.MkdirTemp("", "gpython-pytest-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmp) + + out, err := os.Create(filepath.Join(tmp, "combined")) + if err != nil { + t.Fatalf("could not create stdout/stderr: %+v", err) + } + defer out.Close() + + sys.Globals["stdout"] = &py.File{File: out, FileMode: py.FileWrite} + sys.Globals["stderr"] = &py.File{File: out, FileMode: py.FileWrite} + + _, err = py.RunFile(ctx, fname, py.CompileOpts{}, nil) + if err != nil { + t.Fatalf("could not run script %q: %+v", fname, err) + } + + err = out.Close() + if err != nil { + t.Fatalf("could not close stdout/stderr: %+v", err) + } + + got, err := os.ReadFile(out.Name()) + if err != nil { + t.Fatalf("could not read script output: %+v", err) + } + + ref := fname[:len(fname)-len(".py")] + "_golden.txt" + want, err := os.ReadFile(ref) + if err != nil { + t.Fatalf("could not read golden output %q: %+v", ref, err) + } + + diff := cmp.Diff(string(want), string(got)) + if !bytes.Equal(got, want) { + t.Fatalf("output differ: -- (-ref +got)\n%s", diff) + } +} diff --git a/pytest/pytest_test.go b/pytest/pytest_test.go new file mode 100644 index 00000000..15e136bf --- /dev/null +++ b/pytest/pytest_test.go @@ -0,0 +1,33 @@ +// Copyright 2018 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package pytest + +import ( + "testing" +) + +func TestCompileSrc(t *testing.T) { + for _, tc := range []struct { + name string + code string + }{ + { + name: "hello", + code: `print("hello")`, + }, + } { + t.Run(tc.name, func(t *testing.T) { + _, _ = CompileSrc(t, gContext, tc.code, tc.name) + }) + } +} + +func TestRunTests(t *testing.T) { + RunTests(t, "./testdata/tests") +} + +func TestRunScript(t *testing.T) { + RunScript(t, "./testdata/hello.py") +} diff --git a/pytest/testdata/hello.py b/pytest/testdata/hello.py new file mode 100644 index 00000000..fdbccfc1 --- /dev/null +++ b/pytest/testdata/hello.py @@ -0,0 +1,7 @@ +# Copyright 2022 The go-python Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +print("hello") +print("world") +print("bye.") diff --git a/pytest/testdata/hello_golden.txt b/pytest/testdata/hello_golden.txt new file mode 100644 index 00000000..e4226353 --- /dev/null +++ b/pytest/testdata/hello_golden.txt @@ -0,0 +1,3 @@ +hello +world +bye. diff --git a/pytest/testdata/tests/libtest.py b/pytest/testdata/tests/libtest.py new file mode 100644 index 00000000..003eb3db --- /dev/null +++ b/pytest/testdata/tests/libtest.py @@ -0,0 +1,11 @@ +# Copyright 2022 The go-python Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +""" +Simple test harness +""" + +def testFunc(): + return + diff --git a/pytest/testdata/tests/module.py b/pytest/testdata/tests/module.py new file mode 100644 index 00000000..4151c996 --- /dev/null +++ b/pytest/testdata/tests/module.py @@ -0,0 +1,12 @@ +# Copyright 2022 The go-python Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +from libtest import testFunc + +doc="module" +assert True +assert not False +assert testFunc() is None + +doc="finished" From 51a6831b9041ec7105912a574548eb60ea22a1fd Mon Sep 17 00:00:00 2001 From: Sebastien Binet Date: Wed, 23 Mar 2022 18:10:25 +0100 Subject: [PATCH 07/60] stdlib/time: add simple tests --- stdlib/time/testdata/test.py | 49 ++++++++++++++++++++++++++++ stdlib/time/testdata/test_golden.txt | 4 +++ stdlib/time/time.go | 12 +++---- stdlib/time/time_test.go | 15 +++++++++ 4 files changed, 73 insertions(+), 7 deletions(-) create mode 100644 stdlib/time/testdata/test.py create mode 100644 stdlib/time/testdata/test_golden.txt create mode 100644 stdlib/time/time_test.go diff --git a/stdlib/time/testdata/test.py b/stdlib/time/testdata/test.py new file mode 100644 index 00000000..1f8536cd --- /dev/null +++ b/stdlib/time/testdata/test.py @@ -0,0 +1,49 @@ +# Copyright 2022 The go-python Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +import time + +now = time.time() +now = time.time_ns() +now = time.clock() + +def notimplemented(fn, *args, **kwargs): + try: + fn(*args, **kwargs) + print("error for %s(%s, %s)" % (fn,args,kwargs)) + except NotImplementedError: + pass + +notimplemented(time.clock_gettime) +notimplemented(time.clock_settime) + +print("# sleep") +time.sleep(0.1) +try: + time.sleep(-1) + print("no error sleep(-1)") +except ValueError as e: + print("caught error: %s" % (e,)) + pass +try: + time.sleep("1") + print("no error sleep('1')") +except TypeError as e: + print("caught error: %s" % (e,)) + pass + +notimplemented(time.gmtime) +notimplemented(time.localtime) +notimplemented(time.asctime) +notimplemented(time.ctime) +notimplemented(time.mktime, 1) +notimplemented(time.strftime) +notimplemented(time.strptime) +notimplemented(time.tzset) +notimplemented(time.monotonic) +notimplemented(time.process_time) +notimplemented(time.perf_counter) +notimplemented(time.get_clock_info) + +print("OK") diff --git a/stdlib/time/testdata/test_golden.txt b/stdlib/time/testdata/test_golden.txt new file mode 100644 index 00000000..cb12313f --- /dev/null +++ b/stdlib/time/testdata/test_golden.txt @@ -0,0 +1,4 @@ +# sleep +caught error: ValueError: 'sleep length must be non-negative' +caught error: TypeError: 'sleep() argument 1 must be float, not str' +OK diff --git a/stdlib/time/time.go b/stdlib/time/time.go index d783ae8f..81d50271 100644 --- a/stdlib/time/time.go +++ b/stdlib/time/time.go @@ -150,7 +150,7 @@ func time_sleep(self py.Object, args py.Tuple) (py.Object, error) { if secs < 0 { return nil, py.ExceptionNewf(py.ValueError, "sleep length must be non-negative") } - time.Sleep(time.Duration(secs * 1e9)) + time.Sleep(time.Duration(secs * py.Float(time.Second))) return py.None, nil } @@ -1007,17 +1007,15 @@ func init() { py.MustNewMethod("perf_counter", time_perf_counter, 0, perf_counter_doc), py.MustNewMethod("get_clock_info", time_get_clock_info, 0, get_clock_info_doc), } - + py.RegisterModule(&py.ModuleImpl{ Info: py.ModuleInfo{ - Name: "time", - Doc: module_doc, + Name: "time", + Doc: module_doc, }, Methods: methods, - Globals: py.StringDict{ - }, + Globals: py.StringDict{}, }) - } const module_doc = `This module provides various functions to manipulate time values. diff --git a/stdlib/time/time_test.go b/stdlib/time/time_test.go new file mode 100644 index 00000000..0afb30c0 --- /dev/null +++ b/stdlib/time/time_test.go @@ -0,0 +1,15 @@ +// Copyright 2022 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package time_test + +import ( + "testing" + + "github.com/go-python/gpython/pytest" +) + +func TestTime(t *testing.T) { + pytest.RunScript(t, "./testdata/test.py") +} From 3d9f636fe4635a3e2c0d4ea24d438f055b138d49 Mon Sep 17 00:00:00 2001 From: Sebastien Binet Date: Fri, 25 Mar 2022 15:35:09 +0100 Subject: [PATCH 08/60] py: add handling of format 'n' to ParseTupleAndKeywords --- py/args.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py/args.go b/py/args.go index 5209a3d3..e3bfc36b 100644 --- a/py/args.go +++ b/py/args.go @@ -476,7 +476,7 @@ func ParseTupleAndKeywords(args Tuple, kwargs StringDict, format string, kwlist return ExceptionNewf(TypeError, "%s() argument %d must be str, not %s", name, i+1, arg.Type().Name) } *result = arg - case 'i': + case 'i', 'n': if _, ok := arg.(Int); !ok { return ExceptionNewf(TypeError, "%s() argument %d must be int, not %s", name, i+1, arg.Type().Name) } From 8ed73c5e5d8026bfbc58bab863206142b929d88b Mon Sep 17 00:00:00 2001 From: Sebastien Binet Date: Fri, 25 Mar 2022 15:35:25 +0100 Subject: [PATCH 09/60] py: refactor str.split into String.Split --- py/string.go | 84 +++++++++++++++++++++++++++------------------------- 1 file changed, 44 insertions(+), 40 deletions(-) diff --git a/py/string.go b/py/string.go index 0f9ddc28..e3b25dd5 100644 --- a/py/string.go +++ b/py/string.go @@ -122,37 +122,8 @@ func fieldsN(s string, n int) []string { } func init() { - StringType.Dict["split"] = MustNewMethod("split", func(self Object, args Tuple) (Object, error) { - selfStr := self.(String) - var value Object = None - zeroRemove := true - if len(args) > 0 { - if _, ok := args[0].(NoneType); !ok { - value = args[0] - zeroRemove = false - } - } - var maxSplit int = -2 - if len(args) > 1 { - if m, ok := args[1].(Int); ok { - maxSplit = int(m) - } - } - var valArray []string - if valStr, ok := value.(String); ok { - valArray = strings.SplitN(string(selfStr), string(valStr), maxSplit+1) - } else if _, ok := value.(NoneType); ok { - valArray = fieldsN(string(selfStr), maxSplit) - } else { - return nil, ExceptionNewf(TypeError, "Can't convert '%s' object to str implicitly", value.Type()) - } - o := List{} - for _, j := range valArray { - if len(j) > 0 || !zeroRemove { - o.Items = append(o.Items, String(j)) - } - } - return &o, nil + StringType.Dict["split"] = MustNewMethod("split", func(self Object, args Tuple, kwargs StringDict) (Object, error) { + return self.(String).Split(args, kwargs) }, 0, "split(sub) -> split string with sub.") StringType.Dict["startswith"] = MustNewMethod("startswith", func(self Object, args Tuple) (Object, error) { @@ -597,13 +568,46 @@ func (s String) M__contains__(item Object) (Object, error) { return NewBool(strings.Contains(string(s), string(needle))), nil } +func (s String) Split(args Tuple, kwargs StringDict) (Object, error) { + var ( + pyval Object = None + pymax Object = Int(-2) + pyfmt = "|Oi:split" + kwlst = []string{"sep", "maxsplit"} + ) + err := ParseTupleAndKeywords(args, kwargs, pyfmt, kwlst, &pyval, &pymax) + if err != nil { + return nil, err + } + + var ( + max = pymax.(Int) + vs []string + ) + switch v := pyval.(type) { + case String: + vs = strings.SplitN(string(s), string(v), int(max)+1) + case NoneType: + vs = fieldsN(string(s), int(max)) + default: + return nil, ExceptionNewf(TypeError, "Can't convert '%s' object to str implicitly", pyval.Type()) + } + o := List{} + for _, j := range vs { + o.Items = append(o.Items, String(j)) + } + return &o, nil +} + // Check stringerface is satisfied -var _ richComparison = String("") -var _ sequenceArithmetic = String("") -var _ I__mod__ = String("") -var _ I__rmod__ = String("") -var _ I__imod__ = String("") -var _ I__len__ = String("") -var _ I__bool__ = String("") -var _ I__getitem__ = String("") -var _ I__contains__ = String("") +var ( + _ richComparison = String("") + _ sequenceArithmetic = String("") + _ I__mod__ = String("") + _ I__rmod__ = String("") + _ I__imod__ = String("") + _ I__len__ = String("") + _ I__bool__ = String("") + _ I__getitem__ = String("") + _ I__contains__ = String("") +) From 759557527b62840733c6aaeb197a75e8d39a90ec Mon Sep 17 00:00:00 2001 From: Sebastien Binet Date: Wed, 23 Mar 2022 17:35:16 +0100 Subject: [PATCH 10/60] stdlib/string: first import --- stdlib/stdlib.go | 1 + stdlib/string/string.go | 117 +++++++++++++++++++++++++ stdlib/string/string_test.go | 15 ++++ stdlib/string/testdata/test.py | 32 +++++++ stdlib/string/testdata/test_golden.txt | 28 ++++++ 5 files changed, 193 insertions(+) create mode 100644 stdlib/string/string.go create mode 100644 stdlib/string/string_test.go create mode 100644 stdlib/string/testdata/test.py create mode 100644 stdlib/string/testdata/test_golden.txt diff --git a/stdlib/stdlib.go b/stdlib/stdlib.go index 7b46d391..ebe23d06 100644 --- a/stdlib/stdlib.go +++ b/stdlib/stdlib.go @@ -20,6 +20,7 @@ import ( _ "github.com/go-python/gpython/stdlib/builtin" _ "github.com/go-python/gpython/stdlib/math" + _ "github.com/go-python/gpython/stdlib/string" _ "github.com/go-python/gpython/stdlib/sys" _ "github.com/go-python/gpython/stdlib/time" ) diff --git a/stdlib/string/string.go b/stdlib/string/string.go new file mode 100644 index 00000000..314fdd09 --- /dev/null +++ b/stdlib/string/string.go @@ -0,0 +1,117 @@ +// Copyright 2022 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package string provides the implementation of the python's 'string' module. +package string + +import ( + "strings" + + "github.com/go-python/gpython/py" +) + +func init() { + py.RegisterModule(&py.ModuleImpl{ + Info: py.ModuleInfo{ + Name: "string", + Doc: module_doc, + }, + Methods: []*py.Method{ + py.MustNewMethod("capwords", capwords, 0, capwords_doc), + }, + Globals: py.StringDict{ + "whitespace": whitespace, + "ascii_lowercase": ascii_lowercase, + "ascii_uppercase": ascii_uppercase, + "ascii_letters": ascii_letters, + "digits": digits, + "hexdigits": hexdigits, + "octdigits": octdigits, + "punctuation": punctuation, + "printable": printable, + }, + }) +} + +const module_doc = `A collection of string constants. + +Public module variables: + +whitespace -- a string containing all ASCII whitespace +ascii_lowercase -- a string containing all ASCII lowercase letters +ascii_uppercase -- a string containing all ASCII uppercase letters +ascii_letters -- a string containing all ASCII letters +digits -- a string containing all ASCII decimal digits +hexdigits -- a string containing all ASCII hexadecimal digits +octdigits -- a string containing all ASCII octal digits +punctuation -- a string containing all ASCII punctuation characters +printable -- a string containing all ASCII characters considered printable +` + +var ( + whitespace = py.String(" \t\n\r\x0b\x0c") + ascii_lowercase = py.String("abcdefghijklmnopqrstuvwxyz") + ascii_uppercase = py.String("ABCDEFGHIJKLMNOPQRSTUVWXYZ") + ascii_letters = ascii_lowercase + ascii_uppercase + digits = py.String("0123456789") + hexdigits = py.String("0123456789abcdefABCDEF") + octdigits = py.String("01234567") + punctuation = py.String("!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~") + printable = py.String("0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~ \t\n\r\x0b\x0c") +) + +const capwords_doc = `capwords(s [,sep]) -> string + +Split the argument into words using split, capitalize each +word using capitalize, and join the capitalized words using +join. If the optional second argument sep is absent or None, +runs of whitespace characters are replaced by a single space +and leading and trailing whitespace are removed, otherwise +sep is used to split and join the words.` + +func capwords(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Object, error) { + var ( + pystr py.Object + pysep py.Object = py.None + ) + err := py.ParseTupleAndKeywords(args, kwargs, "s|z", []string{"s", "sep"}, &pystr, &pysep) + if err != nil { + return nil, err + } + + pystr = py.String(strings.ToLower(string(pystr.(py.String)))) + pyvs, err := pystr.(py.String).Split(py.Tuple{pysep}, nil) + if err != nil { + return nil, err + } + + var ( + lst = pyvs.(*py.List).Items + vs = make([]string, len(lst)) + sep = "" + title = func(s string) string { + if s == "" { + return s + } + return strings.ToUpper(s[:1]) + s[1:] + } + ) + + switch pysep { + case py.None: + for i := range vs { + v := string(lst[i].(py.String)) + vs[i] = title(strings.Trim(v, string(whitespace))) + } + sep = " " + default: + sep = string(pysep.(py.String)) + for i := range vs { + v := string(lst[i].(py.String)) + vs[i] = title(v) + } + } + + return py.String(strings.Join(vs, sep)), nil +} diff --git a/stdlib/string/string_test.go b/stdlib/string/string_test.go new file mode 100644 index 00000000..0ae79ef5 --- /dev/null +++ b/stdlib/string/string_test.go @@ -0,0 +1,15 @@ +// Copyright 2022 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package string_test + +import ( + "testing" + + "github.com/go-python/gpython/pytest" +) + +func TestString(t *testing.T) { + pytest.RunScript(t, "./testdata/test.py") +} diff --git a/stdlib/string/testdata/test.py b/stdlib/string/testdata/test.py new file mode 100644 index 00000000..2bec0736 --- /dev/null +++ b/stdlib/string/testdata/test.py @@ -0,0 +1,32 @@ +# Copyright 2022 The go-python Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +import string + +print("globals:") +for name in ("whitespace", + "ascii_lowercase", + "ascii_uppercase", + "ascii_letters", + "digits", + "hexdigits", + "octdigits", + "punctuation", + "printable"): + v = getattr(string, name) + print("\nstring.%s:\n%s" % (name,repr(v))) + +def assertEqual(x, y): + assert x == y, "got: %s, want: %s" % (repr(x), repr(y)) + +assertEqual(string.capwords('abc def ghi'), 'Abc Def Ghi') +assertEqual(string.capwords('abc\tdef\nghi'), 'Abc Def Ghi') +assertEqual(string.capwords('abc\t def \nghi'), 'Abc Def Ghi') +assertEqual(string.capwords('ABC DEF GHI'), 'Abc Def Ghi') +assertEqual(string.capwords('ABC-DEF-GHI', '-'), 'Abc-Def-Ghi') +assertEqual(string.capwords('ABC-def DEF-ghi GHI'), 'Abc-def Def-ghi Ghi') +assertEqual(string.capwords(' aBc DeF '), 'Abc Def') +assertEqual(string.capwords('\taBc\tDeF\t'), 'Abc Def') +assertEqual(string.capwords('\taBc\tDeF\t', '\t'), '\tAbc\tDef\t') + diff --git a/stdlib/string/testdata/test_golden.txt b/stdlib/string/testdata/test_golden.txt new file mode 100644 index 00000000..95a3e2ef --- /dev/null +++ b/stdlib/string/testdata/test_golden.txt @@ -0,0 +1,28 @@ +globals: + +string.whitespace: +' \t\n\r\x0b\x0c' + +string.ascii_lowercase: +'abcdefghijklmnopqrstuvwxyz' + +string.ascii_uppercase: +'ABCDEFGHIJKLMNOPQRSTUVWXYZ' + +string.ascii_letters: +'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' + +string.digits: +'0123456789' + +string.hexdigits: +'0123456789abcdefABCDEF' + +string.octdigits: +'01234567' + +string.punctuation: +'!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~' + +string.printable: +'0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~ \t\n\r\x0b\x0c' From 12886a2728c232f1fef7b758a1d0f4ff1934e522 Mon Sep 17 00:00:00 2001 From: Sebastien Binet Date: Fri, 22 Apr 2022 16:11:37 +0200 Subject: [PATCH 11/60] repl: reorder want/got into got/want This CL reverts the display of want/got into got/want in order to follow good Go practices. Also align got/want display so as to ease deciphering the error for normal humans. Signed-off-by: Sebastien Binet --- repl/repl_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/repl/repl_test.go b/repl/repl_test.go index 3c13c7c6..06395234 100644 --- a/repl/repl_test.go +++ b/repl/repl_test.go @@ -26,10 +26,10 @@ func (rt *replTest) Print(out string) { func (rt *replTest) assert(t *testing.T, what, wantPrompt, wantOut string) { if rt.prompt != wantPrompt { - t.Errorf("%s: Prompt wrong, want %q got %q", what, wantPrompt, rt.prompt) + t.Errorf("%s: Prompt wrong:\ngot= %q\nwant=%q", what, rt.prompt, wantPrompt) } if rt.out != wantOut { - t.Errorf("%s: Output wrong, want %q got %q", what, wantOut, rt.out) + t.Errorf("%s: Output wrong:\ngot= %q\nwant=%q", what, rt.out, wantOut) } rt.out = "" } @@ -118,13 +118,13 @@ func TestCompleter(t *testing.T) { t.Run(fmt.Sprintf("line=%q,pos=%d)", test.line, test.pos), func(t *testing.T) { gotHead, gotCompletions, gotTail := r.Completer(test.line, test.pos) if test.wantHead != gotHead { - t.Errorf("head: want %q got %q", test.wantHead, gotHead) + t.Errorf("invalid head:\ngot= %q\nwant=%q", gotHead, test.wantHead) } if !reflect.DeepEqual(test.wantCompletions, gotCompletions) { - t.Errorf("completions: want %#v got %#v", test.wantCompletions, gotCompletions) + t.Errorf("invalid completions:\ngot= %#v\nwant=%#v", gotCompletions, test.wantCompletions) } if test.wantTail != gotTail { - t.Errorf("tail: want %q got %q", test.wantTail, gotTail) + t.Errorf("invalid tail:\ngot= %q\nwant=%q", gotTail, test.wantTail) } }) } From 179a6287890ad115efef566bdc50203d92ecd09f Mon Sep 17 00:00:00 2001 From: Jon Poole Date: Sun, 10 Apr 2022 18:30:40 +0100 Subject: [PATCH 12/60] all: improve error logging when parsing a file --- compile/compile.go | 3 ++- main.go | 2 +- parser/grammar_data_test.go | 8 ++++---- parser/lexer.go | 2 +- py/traceback.go | 4 ++-- repl/repl_test.go | 2 +- 6 files changed, 11 insertions(+), 10 deletions(-) diff --git a/compile/compile.go b/compile/compile.go index 775717a9..6b9d926d 100644 --- a/compile/compile.go +++ b/compile/compile.go @@ -9,6 +9,7 @@ package compile // FIXME kill ast.Identifier and turn into string? import ( + "bytes" "fmt" "log" "strings" @@ -107,7 +108,7 @@ func init() { // in addition to any features explicitly specified. func Compile(src, srcDesc string, mode py.CompileMode, futureFlags int, dont_inherit bool) (*py.Code, error) { // Parse Ast - Ast, err := parser.ParseString(src, mode) + Ast, err := parser.Parse(bytes.NewBufferString(src), srcDesc, mode) if err != nil { return nil, err } diff --git a/main.go b/main.go index bc906aac..4598a62d 100644 --- a/main.go +++ b/main.go @@ -78,7 +78,7 @@ func xmain(args []string) { _, err := py.RunFile(ctx, args[0], py.CompileOpts{}, nil) if err != nil { py.TracebackDump(err) - log.Fatal(err) + os.Exit(1) } } } diff --git a/parser/grammar_data_test.go b/parser/grammar_data_test.go index 4079325a..b0b53eeb 100644 --- a/parser/grammar_data_test.go +++ b/parser/grammar_data_test.go @@ -33,8 +33,8 @@ var grammarTestData = []struct { {"b'abc' b'''123'''", "eval", "Expression(body=Bytes(s=b'abc123'))", nil, ""}, {"1234", "eval", "Expression(body=Num(n=1234))", nil, ""}, {"01234", "eval", "", py.SyntaxError, "illegal decimal with leading zero"}, - {"1234d", "eval", "", py.SyntaxError, "invalid syntax"}, - {"1234d", "exec", "", py.SyntaxError, "invalid syntax"}, + {"1234d", "eval", "", py.SyntaxError, "unexpected EOF while parsing"}, + {"1234d", "exec", "", py.SyntaxError, "unexpected EOF while parsing"}, {"1234d", "single", "", py.SyntaxError, "unexpected EOF while parsing"}, {"0x1234", "eval", "Expression(body=Num(n=4660))", nil, ""}, {"12.34", "eval", "Expression(body=Num(n=12.34))", nil, ""}, @@ -325,10 +325,10 @@ var grammarTestData = []struct { {"pass\n", "single", "Interactive(body=[Pass()])", nil, ""}, {"if True:\n pass\n\n", "single", "Interactive(body=[If(test=NameConstant(value=True), body=[Pass()], orelse=[])])", nil, ""}, {"while True:\n pass\nelse:\n return\n", "single", "Interactive(body=[While(test=NameConstant(value=True), body=[Pass()], orelse=[Return(value=None)])])", nil, ""}, - {"a='potato", "eval", "", py.SyntaxError, "invalid syntax"}, + {"a='potato", "eval", "", py.SyntaxError, "unexpected EOF while parsing"}, {"a='potato", "exec", "", py.SyntaxError, "EOL while scanning string literal"}, {"a='potato", "single", "", py.SyntaxError, "EOL while scanning string literal"}, - {"a='''potato", "eval", "", py.SyntaxError, "invalid syntax"}, + {"a='''potato", "eval", "", py.SyntaxError, "unexpected EOF while parsing"}, {"a='''potato", "exec", "", py.SyntaxError, "EOF while scanning triple-quoted string literal"}, {"a='''potato", "single", "", py.SyntaxError, "EOF while scanning triple-quoted string literal"}, } diff --git a/parser/lexer.go b/parser/lexer.go index 76847893..801215ba 100644 --- a/parser/lexer.go +++ b/parser/lexer.go @@ -916,7 +916,7 @@ func (x *yyLex) SyntaxErrorf(format string, a ...interface{}) { func (x *yyLex) ErrorReturn() error { if x.error { if x.errorString == "" { - if x.eof && x.interactive { + if x.eof { x.errorString = "unexpected EOF while parsing" } else { x.errorString = "invalid syntax" diff --git a/py/traceback.go b/py/traceback.go index bf7ba6db..6dcc9ce6 100644 --- a/py/traceback.go +++ b/py/traceback.go @@ -64,10 +64,10 @@ func TracebackDump(err interface{}) { case *ExceptionInfo: e.TracebackDump(os.Stderr) case *Exception: - fmt.Fprintf(os.Stderr, "Exception %#v\n", e) + fmt.Fprintf(os.Stderr, "Exception %v\n", e) fmt.Fprintf(os.Stderr, "-- No traceback available --\n") default: - fmt.Fprintf(os.Stderr, "Error %#v\n", err) + fmt.Fprintf(os.Stderr, "Error %v\n", err) fmt.Fprintf(os.Stderr, "-- No traceback available --\n") } } diff --git a/repl/repl_test.go b/repl/repl_test.go index 06395234..486f12a9 100644 --- a/repl/repl_test.go +++ b/repl/repl_test.go @@ -63,7 +63,7 @@ func TestREPL(t *testing.T) { rt.assert(t, "multi#5", NormalPrompt, "45") r.Run("if") - rt.assert(t, "compileError", NormalPrompt, "Compile error: \n File \"\", line 1, offset 2\n if\n\n\nSyntaxError: 'invalid syntax'") + rt.assert(t, "compileError", NormalPrompt, "Compile error: \n File \"\", line 1, offset 2\n if\n\n\nSyntaxError: 'invalid syntax'") // test comments in the REPL work properly r.Run("# this is a comment") From 23774ddb0433ad10ff63b8813348f563a3f689b6 Mon Sep 17 00:00:00 2001 From: Sebastien Binet Date: Thu, 5 May 2022 14:25:01 +0200 Subject: [PATCH 13/60] repl: make replTest.assert a T.Helper Signed-off-by: Sebastien Binet --- repl/repl_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/repl/repl_test.go b/repl/repl_test.go index 486f12a9..b154bfec 100644 --- a/repl/repl_test.go +++ b/repl/repl_test.go @@ -25,6 +25,7 @@ func (rt *replTest) Print(out string) { } func (rt *replTest) assert(t *testing.T, what, wantPrompt, wantOut string) { + t.Helper() if rt.prompt != wantPrompt { t.Errorf("%s: Prompt wrong:\ngot= %q\nwant=%q", what, rt.prompt, wantPrompt) } From 029a195cb7910511261f0024cd3cfe558ff3ac7a Mon Sep 17 00:00:00 2001 From: Sebastien Binet Date: Thu, 5 May 2022 15:32:58 +0200 Subject: [PATCH 14/60] pytest: store output of tested script This CL stores the output of a failing script for easy human comparison Signed-off-by: Sebastien Binet --- pytest/pytest.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pytest/pytest.go b/pytest/pytest.go index 232f7525..7b3c7cba 100644 --- a/pytest/pytest.go +++ b/pytest/pytest.go @@ -133,7 +133,9 @@ func RunBenchmarks(b *testing.B, testDir string) { // RunScript runs the provided path to a script. // RunScript captures the stdout and stderr while executing the script // and compares it to a golden file: -// RunScript("./testdata/foo.py") +// +// RunScript("./testdata/foo.py") +// // will compare the output with "./testdata/foo_golden.txt". func RunScript(t *testing.T, fname string) { opts := py.DefaultContextOpts() @@ -180,6 +182,8 @@ func RunScript(t *testing.T, fname string) { diff := cmp.Diff(string(want), string(got)) if !bytes.Equal(got, want) { + out := fname[:len(fname)-len(".py")] + ".txt" + _ = os.WriteFile(out, got, 0644) t.Fatalf("output differ: -- (-ref +got)\n%s", diff) } } From 04963cf29e2c82a0e6d7b8764d139beb46e28b94 Mon Sep 17 00:00:00 2001 From: Sebastien Binet Date: Thu, 5 May 2022 15:56:14 +0200 Subject: [PATCH 15/60] ci: add static-check test Signed-off-by: Sebastien Binet --- .github/workflows/ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1daf04ba..b45f9edf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -90,6 +90,12 @@ jobs: if: matrix.platform == 'macos-latest' run: | go run ./ci/run-tests.go $TAGS -race + - name: static-check + uses: dominikh/staticcheck-action@v1.2.0 + with: + install-go: false + cache-key: ${{ matrix.platform }} + version: "2022.1" - name: Upload-Coverage if: matrix.platform == 'ubuntu-latest' uses: codecov/codecov-action@v1 From 26a38d334bcb915c49ca65f6c206670372b9b3c5 Mon Sep 17 00:00:00 2001 From: Sebastien Binet Date: Thu, 5 May 2022 15:56:42 +0200 Subject: [PATCH 16/60] ci: reduce git-fetch depth to 1 Signed-off-by: Sebastien Binet --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b45f9edf..1adebcbc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,6 +35,8 @@ jobs: - name: Checkout code uses: actions/checkout@v2 + with: + fetch-depth: 1 - name: Cache-Go uses: actions/cache@v1 From 01af0fecf70d154deea89a9238dc4a31767f9721 Mon Sep 17 00:00:00 2001 From: Sebastien Binet Date: Thu, 5 May 2022 16:02:15 +0200 Subject: [PATCH 17/60] compile: silence staticcheck Signed-off-by: Sebastien Binet --- compile/compile.go | 1 + 1 file changed, 1 insertion(+) diff --git a/compile/compile.go b/compile/compile.go index 6b9d926d..303729c4 100644 --- a/compile/compile.go +++ b/compile/compile.go @@ -1207,6 +1207,7 @@ func (c *compiler) Stmt(stmt ast.Stmt) { l := c.loops.Top() if l == nil { c.panicSyntaxErrorf(node, loopError) + panic("impossible") } switch l.Type { case loopLoop: From 38b2e3c4719deb0c4605fc2cc1ad50a421e99049 Mon Sep 17 00:00:00 2001 From: Sebastien Binet Date: Thu, 5 May 2022 16:04:06 +0200 Subject: [PATCH 18/60] all: apply gofmt Signed-off-by: Sebastien Binet --- compile/compile.go | 204 ++++++++++++++++++++------------------- parser/y.go | 1 + py/exception.go | 2 +- py/frame.go | 50 +++++----- py/py.go | 78 +++++++-------- py/sequence.go | 2 +- py/string.go | 1 - py/type.go | 7 +- py/util.go | 2 +- stdlib/math/math.go | 221 +++++++++++++++++++++++-------------------- symtable/symtable.go | 58 ++++++------ vm/eval.go | 4 +- 12 files changed, 335 insertions(+), 295 deletions(-) diff --git a/compile/compile.go b/compile/compile.go index 303729c4..76d46c1a 100644 --- a/compile/compile.go +++ b/compile/compile.go @@ -435,9 +435,11 @@ func (c *compiler) Jump(Op vm.OpCode, Dest *Label) { c.OpCodes.Add(instr) } -/* The test for LOCAL must come before the test for FREE in order to - handle classes where name is both local and free. The local var is - a method and the free var is a free var referenced within a method. +/* +The test for LOCAL must come before the test for FREE in order to + + handle classes where name is both local and free. The local var is + a method and the free var is a free var referenced within a method. */ func (c *compiler) getRefType(name string) symtable.Scope { if c.scopeType == compilerScopeClass && name == "__class__" { @@ -666,27 +668,31 @@ func (c *compiler) class(Ast ast.Ast, class *ast.ClassDef) { } /* - Implements the with statement from PEP 343. - - The semantics outlined in that PEP are as follows: - - with EXPR as VAR: - BLOCK - - It is implemented roughly as: - - context = EXPR - exit = context.__exit__ # not calling it - value = context.__enter__() - try: - VAR = value # if VAR present in the syntax - BLOCK - finally: - if an exception was raised: - exc = copy of (exception, instance, traceback) - else: - exc = (None, None, None) - exit(*exc) +Implements the with statement from PEP 343. + +The semantics outlined in that PEP are as follows: + +with EXPR as VAR: + + BLOCK + +It is implemented roughly as: + +context = EXPR +exit = context.__exit__ # not calling it +value = context.__enter__() +try: + + VAR = value # if VAR present in the syntax + BLOCK + +finally: + + if an exception was raised: + exc = copy of (exception, instance, traceback) + else: + exc = (None, None, None) + exit(*exc) */ func (c *compiler) with(node *ast.With, pos int) { item := node.Items[pos] @@ -728,37 +734,38 @@ func (c *compiler) with(node *ast.With, pos int) { c.Op(vm.END_FINALLY) } -/* Code generated for "try: finally: " is as follows: - - SETUP_FINALLY L - - POP_BLOCK - LOAD_CONST - L: - END_FINALLY - - The special instructions use the block stack. Each block - stack entry contains the instruction that created it (here - SETUP_FINALLY), the level of the value stack at the time the - block stack entry was created, and a label (here L). - - SETUP_FINALLY: - Pushes the current value stack level and the label - onto the block stack. - POP_BLOCK: - Pops en entry from the block stack, and pops the value - stack until its level is the same as indicated on the - block stack. (The label is ignored.) - END_FINALLY: - Pops a variable number of entries from the *value* stack - and re-raises the exception they specify. The number of - entries popped depends on the (pseudo) exception type. - - The block stack is unwound when an exception is raised: - when a SETUP_FINALLY entry is found, the exception is pushed - onto the value stack (and the exception condition is cleared), - and the interpreter jumps to the label gotten from the block - stack. +/* +Code generated for "try: finally: " is as follows: + + SETUP_FINALLY L + + POP_BLOCK + LOAD_CONST + L: + END_FINALLY + + The special instructions use the block stack. Each block + stack entry contains the instruction that created it (here + SETUP_FINALLY), the level of the value stack at the time the + block stack entry was created, and a label (here L). + + SETUP_FINALLY: + Pushes the current value stack level and the label + onto the block stack. + POP_BLOCK: + Pops en entry from the block stack, and pops the value + stack until its level is the same as indicated on the + block stack. (The label is ignored.) + END_FINALLY: + Pops a variable number of entries from the *value* stack + and re-raises the exception they specify. The number of + entries popped depends on the (pseudo) exception type. + + The block stack is unwound when an exception is raised: + when a SETUP_FINALLY entry is found, the exception is pushed + onto the value stack (and the exception condition is cleared), + and the interpreter jumps to the label gotten from the block + stack. */ func (c *compiler) tryFinally(node *ast.Try) { end := new(Label) @@ -780,35 +787,36 @@ func (c *compiler) tryFinally(node *ast.Try) { } /* - Code generated for "try: S except E1 as V1: S1 except E2 as V2: S2 ...": - (The contents of the value stack is shown in [], with the top - at the right; 'tb' is trace-back info, 'val' the exception's - associated value, and 'exc' the exception.) - - Value stack Label Instruction Argument - [] SETUP_EXCEPT L1 - [] - [] POP_BLOCK - [] JUMP_FORWARD L0 - - [tb, val, exc] L1: DUP ) - [tb, val, exc, exc] ) - [tb, val, exc, exc, E1] COMPARE_OP EXC_MATCH ) only if E1 - [tb, val, exc, 1-or-0] POP_JUMP_IF_FALSE L2 ) - [tb, val, exc] POP - [tb, val] (or POP if no V1) - [tb] POP - [] - JUMP_FORWARD L0 - - [tb, val, exc] L2: DUP - .............................etc....................... - - [tb, val, exc] Ln+1: END_FINALLY # re-raise exception - - [] L0: - - Of course, parts are not generated if Vi or Ei is not present. +Code generated for "try: S except E1 as V1: S1 except E2 as V2: S2 ...": +(The contents of the value stack is shown in [], with the top +at the right; 'tb' is trace-back info, 'val' the exception's +associated value, and 'exc' the exception.) + +Value stack Label Instruction Argument +[] SETUP_EXCEPT L1 +[] +[] POP_BLOCK +[] JUMP_FORWARD L0 + +[tb, val, exc] L1: DUP ) +[tb, val, exc, exc] ) +[tb, val, exc, exc, E1] COMPARE_OP EXC_MATCH ) only if E1 +[tb, val, exc, 1-or-0] POP_JUMP_IF_FALSE L2 ) +[tb, val, exc] POP +[tb, val] (or POP if no V1) +[tb] POP +[] + + JUMP_FORWARD L0 + +[tb, val, exc] L2: DUP +.............................etc....................... + +[tb, val, exc] Ln+1: END_FINALLY # re-raise exception + +[] L0: + +Of course, parts are not generated if Vi or Ei is not present. */ func (c *compiler) tryExcept(node *ast.Try) { c.loops.Push(loop{Type: exceptLoop}) @@ -897,11 +905,13 @@ func (c *compiler) try(node *ast.Try) { } } -/* The IMPORT_NAME opcode was already generated. This function - merely needs to bind the result to a name. +/* +The IMPORT_NAME opcode was already generated. This function + + merely needs to bind the result to a name. - If there is a dot in name, we need to split it and emit a - LOAD_ATTR for each name. + If there is a dot in name, we need to split it and emit a + LOAD_ATTR for each name. */ func (c *compiler) importAs(name ast.Identifier, asname ast.Identifier) { attrs := strings.Split(string(name), ".") @@ -913,12 +923,14 @@ func (c *compiler) importAs(name ast.Identifier, asname ast.Identifier) { c.NameOp(string(asname), ast.Store) } -/* The Import node stores a module name like a.b.c as a single - string. This is convenient for all cases except - import a.b.c as d - where we need to parse that string to extract the individual - module names. - XXX Perhaps change the representation to make this case simpler? +/* +The Import node stores a module name like a.b.c as a single + + string. This is convenient for all cases except + import a.b.c as d + where we need to parse that string to extract the individual + module names. + XXX Perhaps change the representation to make this case simpler? */ func (c *compiler) import_(node *ast.Import) { //n = asdl_seq_LEN(s.v.Import.names); @@ -1394,7 +1406,9 @@ func (c *compiler) callHelper(n int, Args []ast.Expr, Keywords []*ast.Keyword, S c.OpArg(op, uint32(args+kwargs<<8)) } -/* List and set comprehensions and generator expressions work by creating a +/* + List and set comprehensions and generator expressions work by creating a + nested function to perform the actual iteration. This means that the iteration variables don't leak into the current scope. The defined function is called immediately following its definition, with the diff --git a/parser/y.go b/parser/y.go index 1f5bba73..39027c98 100644 --- a/parser/y.go +++ b/parser/y.go @@ -3,6 +3,7 @@ // license that can be found in the LICENSE file. // Code generated by goyacc -v y.output grammar.y. DO NOT EDIT. +// //line grammar.y:6 package parser diff --git a/py/exception.go b/py/exception.go index 8dde2e67..a2abbcac 100644 --- a/py/exception.go +++ b/py/exception.go @@ -330,7 +330,7 @@ func ExceptionGivenMatches(err, exc Object) bool { // IsException matches the result of recover to an exception // -// For use to catch a single python exception from go code +// # For use to catch a single python exception from go code // // It can be an instance or the class itself func IsException(exception *Type, r interface{}) bool { diff --git a/py/frame.go b/py/frame.go index ce595d8a..3e5c9f99 100644 --- a/py/frame.go +++ b/py/frame.go @@ -158,17 +158,18 @@ func (f *Frame) PopBlock() { } } -/* Convert between "fast" version of locals and dictionary version. +/* +Convert between "fast" version of locals and dictionary version. - map and values are input arguments. map is a tuple of strings. - values is an array of PyObject*. At index i, map[i] is the name of - the variable with value values[i]. The function copies the first - nmap variable from map/values into dict. If values[i] is NULL, - the variable is deleted from dict. + map and values are input arguments. map is a tuple of strings. + values is an array of PyObject*. At index i, map[i] is the name of + the variable with value values[i]. The function copies the first + nmap variable from map/values into dict. If values[i] is NULL, + the variable is deleted from dict. - If deref is true, then the values being copied are cell variables - and the value is extracted from the cell variable before being put - in dict. + If deref is true, then the values being copied are cell variables + and the value is extracted from the cell variable before being put + in dict. */ func map_to_dict(mapping []string, nmap int, dict StringDict, values []Object, deref bool) { for j := nmap - 1; j >= 0; j-- { @@ -189,25 +190,26 @@ func map_to_dict(mapping []string, nmap int, dict StringDict, values []Object, d } } -/* Copy values from the "locals" dict into the fast locals. +/* +Copy values from the "locals" dict into the fast locals. - dict is an input argument containing string keys representing - variables names and arbitrary PyObject* as values. + dict is an input argument containing string keys representing + variables names and arbitrary PyObject* as values. - mapping and values are input arguments. mapping is a tuple of strings. - values is an array of PyObject*. At index i, mapping[i] is the name of - the variable with value values[i]. The function copies the first - nmap variable from mapping/values into dict. If values[i] is nil, - the variable is deleted from dict. + mapping and values are input arguments. mapping is a tuple of strings. + values is an array of PyObject*. At index i, mapping[i] is the name of + the variable with value values[i]. The function copies the first + nmap variable from mapping/values into dict. If values[i] is nil, + the variable is deleted from dict. - If deref is true, then the values being copied are cell variables - and the value is extracted from the cell variable before being put - in dict. If clear is true, then variables in mapping but not in dict - are set to nil in mapping; if clear is false, variables missing in - dict are ignored. + If deref is true, then the values being copied are cell variables + and the value is extracted from the cell variable before being put + in dict. If clear is true, then variables in mapping but not in dict + are set to nil in mapping; if clear is false, variables missing in + dict are ignored. - Exceptions raised while modifying the dict are silently ignored, - because there is no good way to report them. + Exceptions raised while modifying the dict are silently ignored, + because there is no good way to report them. */ func dict_to_map(mapping []string, nmap int, dict StringDict, values []Object, deref bool, clear bool) { for j := nmap - 1; j >= 0; j-- { diff --git a/py/py.go b/py/py.go index 59d0737a..2ebd5fcd 100644 --- a/py/py.go +++ b/py/py.go @@ -39,7 +39,7 @@ var ( // If __new__() does not return an instance of cls, then the new instance’s __init__() method will not be invoked. // // __new__() is intended mainly to allow subclasses of immutable types (like int, str, or tuple) to customize instance creation. It is also commonly overridden in custom metaclasses in order to customize class creation. -//object.__new__(cls[, ...]) +// object.__new__(cls[, ...]) type I__new__ interface { M__new__(cls, args, kwargs Object) (Object, error) } @@ -52,7 +52,7 @@ type I__new__ interface { // [args...]). As a special constraint on constructors, no value may // be returned; doing so will cause a TypeError to be raised at // runtime. -//object.__init__(self[, ...]) +// object.__init__(self[, ...]) type I__init__ interface { M__init__(self, args, kwargs Object) (Object, error) } @@ -101,7 +101,7 @@ type I__init__ interface { // globals are deleted; if no other references to such globals exist, // this may help in assuring that imported modules are still available // at the time when the __del__() method is called. -//object.__del__(self) +// object.__del__(self) type I__del__ interface { M__del__() (Object, error) } @@ -118,7 +118,7 @@ type I__del__ interface { // // This is typically used for debugging, so it is important that the // representation is information-rich and unambiguous. -//object.__repr__(self) +// object.__repr__(self) type I__repr__ interface { M__repr__() (Object, error) } @@ -134,14 +134,14 @@ type I__repr__ interface { // // The default implementation defined by the built-in type object // calls object.__repr__(). -//object.__str__(self) +// object.__str__(self) type I__str__ interface { M__str__() (Object, error) } // Called by bytes() to compute a byte-string representation of an // object. This should return a bytes object. -//object.__bytes__(self) +// object.__bytes__(self) type I__bytes__ interface { M__bytes__() (Object, error) } @@ -159,7 +159,7 @@ type I__bytes__ interface { // standard formatting syntax. // // The return value must be a string object. -//object.__format__(self, format_spec) +// object.__format__(self, format_spec) type I__format__ interface { M__format__(format_spec Object) (Object, error) } @@ -196,32 +196,32 @@ type I__format__ interface { // // To automatically generate ordering operations from a single root // operation, see functools.total_ordering(). -//object.__lt__(self, other) +// object.__lt__(self, other) type I__lt__ interface { M__lt__(other Object) (Object, error) } -//object.__le__(self, other) +// object.__le__(self, other) type I__le__ interface { M__le__(other Object) (Object, error) } -//object.__eq__(self, other) +// object.__eq__(self, other) type I__eq__ interface { M__eq__(other Object) (Object, error) } -//object.__ne__(self, other) +// object.__ne__(self, other) type I__ne__ interface { M__ne__(other Object) (Object, error) } -//object.__gt__(self, other) +// object.__gt__(self, other) type I__gt__ interface { M__gt__(other Object) (Object, error) } -//object.__ge__(self, other) +// object.__ge__(self, other) type I__ge__ interface { M__ge__(other Object) (Object, error) } @@ -300,8 +300,8 @@ type richComparison interface { // // See also PYTHONHASHSEED. // -//Changed in version 3.3: Hash randomization is enabled by default. -//object.__hash__(self) +// Changed in version 3.3: Hash randomization is enabled by default. +// object.__hash__(self) type I__hash__ interface { M__hash__() (Object, error) } @@ -312,7 +312,7 @@ type I__hash__ interface { // considered true if its result is nonzero. If a class defines // neither __len__() nor __bool__(), all its instances are considered // true. -//object.__bool__(self) +// object.__bool__(self) type I__bool__ interface { M__bool__() (Object, error) } @@ -335,7 +335,7 @@ type I__bool__ interface { // instead inserting them in another object). See the // __getattribute__() method below for a way to actually get total // control over attribute access. -//object.__getattr__(self, name) +// object.__getattr__(self, name) type I__getattr__ interface { M__getattr__(name string) (Object, error) } @@ -350,10 +350,10 @@ type I__getattr__ interface { // same name to access any attributes it needs, for example, // object.__getattribute__(self, name). // -//Note This method may still be bypassed when looking up special -//methods as the result of implicit invocation via language syntax or -//built-in functions. See Special method lookup. -//object.__getattribute__(self, name) +// Note This method may still be bypassed when looking up special +// methods as the result of implicit invocation via language syntax or +// built-in functions. See Special method lookup. +// object.__getattribute__(self, name) type I__getattribute__ interface { M__getattribute__(name string) (Object, error) } @@ -366,7 +366,7 @@ type I__getattribute__ interface { // If __setattr__() wants to assign to an instance attribute, it // should call the base class method with the same name, for example, // object.__setattr__(self, name, value). -//object.__setattr__(self, name, value) +// object.__setattr__(self, name, value) type I__setattr__ interface { M__setattr__(name string, value Object) (Object, error) } @@ -374,7 +374,7 @@ type I__setattr__ interface { // Like __setattr__() but for attribute deletion instead of // assignment. This should only be implemented if del obj.name is // meaningful for the object. -//object.__delattr__(self, name) +// object.__delattr__(self, name) type I__delattr__ interface { M__delattr__(name string) (Object, error) } @@ -382,7 +382,7 @@ type I__delattr__ interface { // Called when dir() is called on the object. A sequence must be // returned. dir() converts the returned sequence to a list and sorts // it. -//object.__dir__(self) +// object.__dir__(self) type I__dir__ interface { M__dir__() (Object, error) } @@ -401,21 +401,21 @@ type I__dir__ interface { // attribute is accessed through the owner. This method should return // the (computed) attribute value or raise an AttributeError // exception. -//object.__get__(self, instance, owner) +// object.__get__(self, instance, owner) type I__get__ interface { M__get__(instance, owner Object) (Object, error) } // Called to set the attribute on an instance of the owner // class to a new value. -//object.__set__(self, instance, value) +// object.__set__(self, instance, value) type I__set__ interface { M__set__(instance, value Object) (Object, error) } // Called to delete the attribute on an instance instance of the owner // class. -//object.__delete__(self, instance) +// object.__delete__(self, instance) type I__delete__ interface { M__delete__(instance Object) (Object, error) } @@ -437,7 +437,7 @@ type I__delete__ interface { // Return true if instance should be considered a (direct or indirect) // instance of class. If defined, called to implement // isinstance(instance, class). -//object.__instancecheck__(self, instance) +// object.__instancecheck__(self, instance) type I__instancecheck__ interface { M__instancecheck__(instance Object) (Object, error) } @@ -445,7 +445,7 @@ type I__instancecheck__ interface { // Return true if subclass should be considered a (direct or indirect) // subclass of class. If defined, called to implement // issubclass(subclass, class). -//object.__subclasscheck__(self, subclass) +// object.__subclasscheck__(self, subclass) type I__subclasscheck__ interface { M__subclasscheck__(subclass Object) (Object, error) } @@ -453,7 +453,7 @@ type I__subclasscheck__ interface { // Called when the instance is “called” as a function; if this method // is defined, x(arg1, arg2, ...) is a shorthand for x.__call__(arg1, // arg2, ...). -//object.__call__(self[, args...]) +// object.__call__(self[, args...]) type I__call__ interface { M__call__(args Tuple, kwargs StringDict) (Object, error) } @@ -491,7 +491,7 @@ type I__call__ interface { // length of the object, an integer >= 0. Also, an object that doesn’t // define a __bool__() method and whose __len__() method returns zero // is considered to be false in a Boolean context. -//object.__len__(self) +// object.__len__(self) type I__len__ interface { M__len__() (Object, error) } @@ -502,7 +502,7 @@ type I__len__ interface { // is purely an optimization and is never required for correctness. // // New in version 3.4. -//object.__length_hint__(self) +// object.__length_hint__(self) type I__length_hint__ interface { M__length_hint__() (Object, error) } @@ -525,7 +525,7 @@ type I__length_hint__ interface { // // Note for loops expect that an IndexError will be raised for illegal // indexes to allow proper detection of the end of the sequence. -//object.__getitem__(self, key) +// object.__getitem__(self, key) type I__getitem__ interface { M__getitem__(key Object) (Object, error) } @@ -546,7 +546,7 @@ type I__setitem__ interface { // objects support removal of keys, or for sequences if elements can // be removed from the sequence. The same exceptions should be raised // for improper key values as for the __getitem__() method. -//object.__delitem__(self, key) +// object.__delitem__(self, key) type I__delitem__ interface { M__delitem__(key Object) (Object, error) } @@ -560,7 +560,7 @@ type I__delitem__ interface { // Iterator objects also need to implement this method; they are // required to return themselves. For more information on iterator // objects, see Iterator Types. -//object.__iter__(self) +// object.__iter__(self) type I__iter__ interface { M__iter__() (Object, error) } @@ -605,7 +605,7 @@ type I_generator interface { // should only provide __reversed__() if they can provide an // implementation that is more efficient than the one provided by // reversed(). -//object.__reversed__(self) +// object.__reversed__(self) type I__reversed__ interface { M__reversed__() (Object, error) } @@ -625,7 +625,7 @@ type I__reversed__ interface { // first tries iteration via __iter__(), then the old sequence // iteration protocol via __getitem__(), see this section in the // language reference. -//object.__contains__(self, item) +// object.__contains__(self, item) type I__contains__ interface { M__contains__(item Object) (Object, error) } @@ -1011,7 +1011,7 @@ type sequenceArithmetic interface { // Enter the runtime context related to this object. The with // statement will bind this method’s return value to the target(s) // specified in the as clause of the statement, if any. -//object.__enter__(self) +// object.__enter__(self) type I__enter__ interface { M__enter__() (Object, error) } @@ -1028,7 +1028,7 @@ type I__enter__ interface { // // Note that __exit__() methods should not reraise the passed-in // exception; this is the caller’s responsibility. -//object.__exit__(self, exc_type, exc_value, traceback) +// object.__exit__(self, exc_type, exc_value, traceback) type I__exit__ interface { M__exit__(exc_type, exc_value, traceback Object) (Object, error) } diff --git a/py/sequence.go b/py/sequence.go index 430471c8..9a2a57f2 100644 --- a/py/sequence.go +++ b/py/sequence.go @@ -65,7 +65,7 @@ func SequenceSet(v Object) (*Set, error) { // Call __next__ for the python object // -// Returns the next object +// # Returns the next object // // err == StopIteration or subclass when finished func Next(self Object) (obj Object, err error) { diff --git a/py/string.go b/py/string.go index e3b25dd5..58ab7ad0 100644 --- a/py/string.go +++ b/py/string.go @@ -334,7 +334,6 @@ func (a String) M__ge__(other Object) (Object, error) { // % operator /* - 4.7.2. printf-style String Formatting Note The formatting operations described here exhibit a variety of diff --git a/py/type.go b/py/type.go index a9f835a4..c824c0b0 100644 --- a/py/type.go +++ b/py/type.go @@ -458,7 +458,7 @@ func (t *Type) Lookup(name string) Object { // // Doesn't call __getattr__ etc // -// Returns nil if not found +// # Returns nil if not found // // Doesn't look in the instance dictionary // @@ -478,7 +478,7 @@ func (t *Type) NativeGetAttrOrNil(name string) Object { // // Doesn't call __getattr__ etc // -// Returns nil if not found +// # Returns nil if not found // // FIXME this isn't totally correct! // as we are ignoring getattribute etc @@ -552,7 +552,8 @@ func TypeCall2(self Object, name string, arg1, arg2 Object) (Object, bool, error // Two variants: // // - lookup_maybe() returns nil without raising an exception -// when the _PyType_Lookup() call fails; +// +// when the _PyType_Lookup() call fails; // // - lookup_method() always raises an exception upon errors. func lookup_maybe(self Object, attr string) Object { diff --git a/py/util.go b/py/util.go index 43028bce..56558cac 100644 --- a/py/util.go +++ b/py/util.go @@ -88,7 +88,7 @@ func LoadIntsFromList(list Object) ([]int64, error) { if N <= 0 { return nil, nil } - + intList := make([]int64, N) for i := Int(0); i < N; i++ { item, err := getter.M__getitem__(i) diff --git a/stdlib/math/math.go b/stdlib/math/math.go index 63ecec00..8f1e4dcb 100644 --- a/stdlib/math/math.go +++ b/stdlib/math/math.go @@ -62,13 +62,13 @@ raised for division by zero and mod by zero. */ /* - In general, on an IEEE-754 platform the aim is to follow the C99 - standard, including Annex 'F', whenever possible. Where the - standard recommends raising the 'divide-by-zero' or 'invalid' - floating-point exceptions, Python should raise a ValueError. Where - the standard recommends raising 'overflow', Python should raise an - OverflowError. In all other circumstances a value should be - returned. +In general, on an IEEE-754 platform the aim is to follow the C99 +standard, including Annex 'F', whenever possible. Where the +standard recommends raising the 'divide-by-zero' or 'invalid' +floating-point exceptions, Python should raise a ValueError. Where +the standard recommends raising 'overflow', Python should raise an +OverflowError. In all other circumstances a value should be +returned. */ var ( EDOM = py.ExceptionNewf(py.ValueError, "math domain error") @@ -88,33 +88,38 @@ func isFinite(x float64) bool { } /* - math_1 is used to wrap a libm function f that takes a float64 - arguments and returns a float64. - - The error reporting follows these rules, which are designed to do - the right thing on C89/C99 platforms and IEEE 754/non IEEE 754 - platforms. - - - a NaN result from non-NaN inputs causes ValueError to be raised - - an infinite result from finite inputs causes OverflowError to be - raised if can_overflow is 1, or raises ValueError if can_overflow - is 0. - - if the result is finite and errno == EDOM then ValueError is - raised - - if the result is finite and nonzero and errno == ERANGE then - OverflowError is raised - - The last rule is used to catch overflow on platforms which follow - C89 but for which HUGE_VAL is not an infinity. - - For the majority of one-argument functions these rules are enough - to ensure that Python's functions behave as specified in 'Annex F' - of the C99 standard, with the 'invalid' and 'divide-by-zero' - floating-point exceptions mapping to Python's ValueError and the - 'overflow' floating-point exception mapping to OverflowError. - math_1 only works for functions that don't have singularities *and* - the possibility of overflow; fortunately, that covers everything we - care about right now. +math_1 is used to wrap a libm function f that takes a float64 +arguments and returns a float64. + +The error reporting follows these rules, which are designed to do +the right thing on C89/C99 platforms and IEEE 754/non IEEE 754 +platforms. + +- a NaN result from non-NaN inputs causes ValueError to be raised +- an infinite result from finite inputs causes OverflowError to be + + raised if can_overflow is 1, or raises ValueError if can_overflow + is 0. + +- if the result is finite and errno == EDOM then ValueError is + + raised + +- if the result is finite and nonzero and errno == ERANGE then + + OverflowError is raised + +The last rule is used to catch overflow on platforms which follow +C89 but for which HUGE_VAL is not an infinity. + +For the majority of one-argument functions these rules are enough +to ensure that Python's functions behave as specified in 'Annex F' +of the C99 standard, with the 'invalid' and 'divide-by-zero' +floating-point exceptions mapping to Python's ValueError and the +'overflow' floating-point exception mapping to OverflowError. +math_1 only works for functions that don't have singularities *and* +the possibility of overflow; fortunately, that covers everything we +care about right now. */ func math_1_to_whatever(arg py.Object, fn func(float64) float64, can_overflow bool) (float64, error) { x, err := py.FloatAsFloat64(arg) @@ -140,9 +145,12 @@ func checkResult(x, r float64, can_overflow bool) (float64, error) { return r, nil } -/* variant of math_1, to be used when the function being wrapped is known to - set errno properly (that is, errno = EDOM for invalid or divide-by-zero, - errno = ERANGE for overflow). */ +/* +variant of math_1, to be used when the function being wrapped is known to + + set errno properly (that is, errno = EDOM for invalid or divide-by-zero, + errno = ERANGE for overflow). +*/ func math_1a(arg py.Object, fn func(float64) float64) (py.Object, error) { x, err := py.FloatAsFloat64(arg) if err != nil { @@ -153,30 +161,35 @@ func math_1a(arg py.Object, fn func(float64) float64) (py.Object, error) { } /* - math_2 is used to wrap a libm function f that takes two float64 - arguments and returns a float64. - - The error reporting follows these rules, which are designed to do - the right thing on C89/C99 platforms and IEEE 754/non IEEE 754 - platforms. - - - a NaN result from non-NaN inputs causes ValueError to be raised - - an infinite result from finite inputs causes OverflowError to be - raised. - - if the result is finite and errno == EDOM then ValueError is - raised - - if the result is finite and nonzero and errno == ERANGE then - OverflowError is raised - - The last rule is used to catch overflow on platforms which follow - C89 but for which HUGE_VAL is not an infinity. - - For most two-argument functions (copysign, fmod, hypot, atan2) - these rules are enough to ensure that Python's functions behave as - specified in 'Annex F' of the C99 standard, with the 'invalid' and - 'divide-by-zero' floating-point exceptions mapping to Python's - ValueError and the 'overflow' floating-point exception mapping to - OverflowError. +math_2 is used to wrap a libm function f that takes two float64 +arguments and returns a float64. + +The error reporting follows these rules, which are designed to do +the right thing on C89/C99 platforms and IEEE 754/non IEEE 754 +platforms. + +- a NaN result from non-NaN inputs causes ValueError to be raised +- an infinite result from finite inputs causes OverflowError to be + + raised. + +- if the result is finite and errno == EDOM then ValueError is + + raised + +- if the result is finite and nonzero and errno == ERANGE then + + OverflowError is raised + +The last rule is used to catch overflow on platforms which follow +C89 but for which HUGE_VAL is not an infinity. + +For most two-argument functions (copysign, fmod, hypot, atan2) +these rules are enough to ensure that Python's functions behave as +specified in 'Annex F' of the C99 standard, with the 'invalid' and +'divide-by-zero' floating-point exceptions mapping to Python's +ValueError and the 'overflow' floating-point exception mapping to +OverflowError. */ func math_1(arg py.Object, fn func(float64) float64, can_overflow bool) (py.Object, error) { f, err := math_1_to_whatever(arg, fn, can_overflow) @@ -446,34 +459,35 @@ const math_tanh_doc = "tanh(x)\n\nReturn the hyperbolic tangent of x." accurate result returned by sum(itertools.chain(seq1, seq2)). */ -/* Full precision summation of a sequence of floats. - - def msum(iterable): - partials = [] # sorted, non-overlapping partial sums - for x in iterable: - i = 0 - for y in partials: - if abs(x) < abs(y): - x, y = y, x - hi = x + y - lo = y - (hi - x) - if lo: - partials[i] = lo - i += 1 - x = hi - partials[i:] = [x] - return sum_exact(partials) - - Rounded x+y stored in hi with the roundoff stored in lo. Together hi+lo - are exactly equal to x+y. The inner loop applies hi/lo summation to each - partial so that the list of partial sums remains exact. - - Sum_exact() adds the partial sums exactly and correctly rounds the final - result (using the round-half-to-even rule). The items in partials remain - non-zero, non-special, non-overlapping and strictly increasing in - magnitude, but possibly not all having the same sign. - - Depends on IEEE 754 arithmetic guarantees and half-even rounding. +/* +Full precision summation of a sequence of floats. + + def msum(iterable): + partials = [] # sorted, non-overlapping partial sums + for x in iterable: + i = 0 + for y in partials: + if abs(x) < abs(y): + x, y = y, x + hi = x + y + lo = y - (hi - x) + if lo: + partials[i] = lo + i += 1 + x = hi + partials[i:] = [x] + return sum_exact(partials) + + Rounded x+y stored in hi with the roundoff stored in lo. Together hi+lo + are exactly equal to x+y. The inner loop applies hi/lo summation to each + partial so that the list of partial sums remains exact. + + Sum_exact() adds the partial sums exactly and correctly rounds the final + result (using the round-half-to-even rule). The items in partials remain + non-zero, non-special, non-overlapping and strictly increasing in + magnitude, but possibly not all having the same sign. + + Depends on IEEE 754 arithmetic guarantees and half-even rounding. */ func math_fsum(self py.Object, seq py.Object) (py.Object, error) { const NUM_PARTIALS = 32 /* initial partials array size, on stack */ @@ -939,14 +953,17 @@ const math_modf_doc = `modf(x) Return the fractional and integer parts of x. Both results carry the sign of x and are floats.` -/* A decent logarithm is easy to compute even for huge ints, but libm can't - do that by itself -- loghelper can. func is log or log10, and name is - "log" or "log10". Note that overflow of the result isn't possible: an int - can contain no more than INT_MAX * SHIFT bits, so has value certainly less - than 2**(2**64 * 2**16) == 2**2**80, and log2 of that is 2**80, which is - small enough to fit in an IEEE single. log and log10 are even smaller. - However, intermediate overflow is possible for an int if the number of bits - in that int is larger than PY_SSIZE_T_MAX. */ +/* +A decent logarithm is easy to compute even for huge ints, but libm can't + + do that by itself -- loghelper can. func is log or log10, and name is + "log" or "log10". Note that overflow of the result isn't possible: an int + can contain no more than INT_MAX * SHIFT bits, so has value certainly less + than 2**(2**64 * 2**16) == 2**2**80, and log2 of that is 2**80, which is + small enough to fit in an IEEE single. log and log10 are even smaller. + However, intermediate overflow is possible for an int if the number of bits + in that int is larger than PY_SSIZE_T_MAX. +*/ func loghelper(arg py.Object, fn func(float64) float64, fnname string) (py.Object, error) { /* If it is int, do it ourselves. */ if xBig, err := py.BigIntCheck(arg); err == nil { @@ -1087,10 +1104,12 @@ func math_hypot(self py.Object, args py.Tuple) (py.Object, error) { const math_hypot_doc = `hypot(x, y) Return the Euclidean distance, sqrt(x*x + y*y).` -/* pow can't use math_2, but needs its own wrapper: the problem is - that an infinite result can arise either as a result of overflow - (in which case OverflowError should be raised) or as a result of - e.g. 0.**-5. (for which ValueError needs to be raised.) +/* +pow can't use math_2, but needs its own wrapper: the problem is + + that an infinite result can arise either as a result of overflow + (in which case OverflowError should be raised) or as a result of + e.g. 0.**-5. (for which ValueError needs to be raised.) */ func math_pow(self py.Object, args py.Tuple) (py.Object, error) { var ox, oy py.Object diff --git a/symtable/symtable.go b/symtable/symtable.go index 1513f698..b7d36b4d 100644 --- a/symtable/symtable.go +++ b/symtable/symtable.go @@ -551,11 +551,12 @@ func (s StringSet) Contains(k string) bool { global: set of all symbol names explicitly declared as global */ -/* Decide on scope of name, given flags. +/* +Decide on scope of name, given flags. - The namespace dictionaries may be modified to record information - about the new name. For example, a new global will add an entry to - global. A name that was global can be changed to local. + The namespace dictionaries may be modified to record information + about the new name. For example, a new global will add an entry to + global. A name that was global can be changed to local. */ func (st *SymTable) AnalyzeName(scopes Scopes, name string, symbol Symbol, bound, local, free, global StringSet) { flags := symbol.Flags @@ -618,12 +619,14 @@ func (st *SymTable) AnalyzeName(scopes Scopes, name string, symbol Symbol, bound scopes[name] = ScopeGlobalImplicit } -/* If a name is defined in free and also in locals, then this block - provides the binding for the free variable. The name should be - marked CELL in this block and removed from the free list. +/* +If a name is defined in free and also in locals, then this block - Note that the current block's free variables are included in free. - That's safe because no name can be free and local in the same scope. + provides the binding for the free variable. The name should be + marked CELL in this block and removed from the free list. + + Note that the current block's free variables are included in free. + That's safe because no name can be free and local in the same scope. */ func AnalyzeCells(scopes Scopes, free StringSet) { for name, scope := range scopes { @@ -691,24 +694,25 @@ func (symbols Symbols) Update(scopes Scopes, bound, free StringSet, classflag bo } } -/* Make final symbol table decisions for block of ste. - - Arguments: - st -- current symtable entry (input/output) - bound -- set of variables bound in enclosing scopes (input). bound - is nil for module blocks. - free -- set of free variables in enclosed scopes (output) - globals -- set of declared global variables in enclosing scopes (input) - - The implementation uses two mutually recursive functions, - analyze_block() and analyze_child_block(). analyze_block() is - responsible for analyzing the individual names defined in a block. - analyze_child_block() prepares temporary namespace dictionaries - used to evaluated nested blocks. - - The two functions exist because a child block should see the name - bindings of its enclosing blocks, but those bindings should not - propagate back to a parent block. +/* +Make final symbol table decisions for block of ste. + + Arguments: + st -- current symtable entry (input/output) + bound -- set of variables bound in enclosing scopes (input). bound + is nil for module blocks. + free -- set of free variables in enclosed scopes (output) + globals -- set of declared global variables in enclosing scopes (input) + + The implementation uses two mutually recursive functions, + analyze_block() and analyze_child_block(). analyze_block() is + responsible for analyzing the individual names defined in a block. + analyze_child_block() prepares temporary namespace dictionaries + used to evaluated nested blocks. + + The two functions exist because a child block should see the name + bindings of its enclosing blocks, but those bindings should not + propagate back to a parent block. */ func (st *SymTable) AnalyzeBlock(bound, free, global StringSet) { local := make(StringSet) // collect new names bound in block diff --git a/vm/eval.go b/vm/eval.go index 94bf6d1e..d32cf734 100644 --- a/vm/eval.go +++ b/vm/eval.go @@ -1609,7 +1609,7 @@ func callInternal(fn py.Object, args py.Tuple, kwargs py.StringDict, f *py.Frame // Implements a function call - see CALL_FUNCTION for a description of // how the arguments are arranged. // -// Optionally pass in args and kwargs +// # Optionally pass in args and kwargs // // The result is put on the stack func (vm *Vm) Call(argc int32, starArgs py.Object, starKwargs py.Object) error { @@ -2036,7 +2036,7 @@ func tooManyPositional(co *py.Code, given, defcount int, fastlocals []py.Object) // EvalCode runs a new virtual machine on a Code object. // -// Any parameters are expected to have been decoded into locals +// # Any parameters are expected to have been decoded into locals // // Returns an Object and an error. The error will be a py.ExceptionInfo // From ec3b7dc988701238ae793fb1fb668489b0a914b4 Mon Sep 17 00:00:00 2001 From: Sebastien Binet Date: Thu, 5 May 2022 16:04:45 +0200 Subject: [PATCH 19/60] all: use go:build directives Signed-off-by: Sebastien Binet --- py/gen.go | 1 + py/range_repr110.go | 2 ++ py/range_repr19.go | 2 ++ repl/web/serve.go | 3 ++- 4 files changed, 7 insertions(+), 1 deletion(-) diff --git a/py/gen.go b/py/gen.go index 4b32a883..16db6ad8 100644 --- a/py/gen.go +++ b/py/gen.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build ignore // +build ignore package main diff --git a/py/range_repr110.go b/py/range_repr110.go index dfe4ee8c..2092ff30 100644 --- a/py/range_repr110.go +++ b/py/range_repr110.go @@ -2,7 +2,9 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build go1.10 // +build go1.10 + // Range object package py diff --git a/py/range_repr19.go b/py/range_repr19.go index 0fd8b791..38c5d6d2 100644 --- a/py/range_repr19.go +++ b/py/range_repr19.go @@ -2,7 +2,9 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build !go1.10 // +build !go1.10 + // Range object package py diff --git a/repl/web/serve.go b/repl/web/serve.go index ee9a6b78..30e38e44 100644 --- a/repl/web/serve.go +++ b/repl/web/serve.go @@ -1,4 +1,5 @@ -//+build none +//go:build none +// +build none package main From 6db213773fcb2e30d6be98fc290892bcf93ae642 Mon Sep 17 00:00:00 2001 From: Sebastien Binet Date: Thu, 5 May 2022 16:39:18 +0200 Subject: [PATCH 20/60] all: apply staticcheck fixes Signed-off-by: Sebastien Binet --- main.go | 3 --- parser/lexer_test.go | 19 ++------------- py/complex.go | 7 ------ py/import.go | 2 +- py/type.go | 56 ++++++++++++++++++++++---------------------- py/zip.go | 8 +++---- repl/cli/cli.go | 2 -- stdlib/math/math.go | 7 ------ stdlib/sys/sys.go | 20 ++++++++-------- 9 files changed, 45 insertions(+), 79 deletions(-) diff --git a/main.go b/main.go index 4598a62d..8be7ea2e 100644 --- a/main.go +++ b/main.go @@ -21,10 +21,7 @@ import ( _ "github.com/go-python/gpython/stdlib" ) -// Globals var ( - // Flags - debug = flag.Bool("d", false, "Print lots of debugging") cpuprofile = flag.String("cpuprofile", "", "Write cpu profile to file") ) diff --git a/parser/lexer_test.go b/parser/lexer_test.go index ab252088..9971c2d7 100644 --- a/parser/lexer_test.go +++ b/parser/lexer_test.go @@ -7,8 +7,6 @@ package parser import ( "bytes" "fmt" - "log" - "math" "testing" "github.com/go-python/gpython/ast" @@ -569,19 +567,6 @@ func TestLexerReadOperator(t *testing.T) { } } -// Whether two floats are more or less the same -func approxEq(a, b float64) bool { - log.Printf("ApproxEq(a = %#v, b = %#v)", a, b) - diff := a - b - log.Printf("ApproxEq(diff = %e)", diff) - if math.Abs(diff) > 1e-10 { - log.Printf("ApproxEq(false)") - return false - } - log.Printf("ApproxEq(true)") - return true -} - func TestLexerReadNumber(t *testing.T) { x := yyLex{} for _, test := range []struct { @@ -710,10 +695,10 @@ func TestLexerReadString(t *testing.T) { if testValueBytes, ok := test.value.(py.Bytes); !ok { t.Error("Expecting py.Bytes") } else { - equal = (bytes.Compare(valueBytes, testValueBytes) == 0) + equal = bytes.Equal(valueBytes, testValueBytes) } } else { - equal = (value == test.value) + equal = value == test.value } if token != test.token || !equal || x.line != test.out { diff --git a/py/complex.go b/py/complex.go index 8f42a480..e39f2f64 100644 --- a/py/complex.go +++ b/py/complex.go @@ -152,13 +152,6 @@ func complexFloor(a Complex) Complex { return Complex(complex(math.Floor(real(a)), math.Floor(imag(a)))) } -// Floor divide two complex numbers -func complexFloorDiv(a, b Complex) Complex { - q := complexFloor(a / b) - r := a - q*b - return Complex(r) -} - func (a Complex) M__floordiv__(other Object) (Object, error) { if b, ok := convertToComplex(other); ok { return complexFloor(a / b), nil diff --git a/py/import.go b/py/import.go index 0eaae921..709a4468 100644 --- a/py/import.go +++ b/py/import.go @@ -272,7 +272,7 @@ func XImportModuleLevelObject(ctx Context, nameObj, given_globals, locals, given if err != nil { return nil, err } - } else { + // } else { // not initializing // FIXME locking // if _PyImport_ReleaseLock() < 0 { // return nil, ExceptionNewf(RuntimeError, "not holding the import lock") diff --git a/py/type.go b/py/type.go index c824c0b0..be6d19af 100644 --- a/py/type.go +++ b/py/type.go @@ -570,14 +570,14 @@ func lookup_maybe(self Object, attr string) Object { return res } -func lookup_method(self Object, attr string) Object { - res := lookup_maybe(self, attr) - if res == nil { - // FIXME PyErr_SetObject(PyExc_AttributeError, attrid->object); - return ExceptionNewf(AttributeError, "'%s' object has no attribute '%s'", self.Type().Name, attr) - } - return res -} +// func lookup_method(self Object, attr string) Object { +// res := lookup_maybe(self, attr) +// if res == nil { +// // FIXME PyErr_SetObject(PyExc_AttributeError, attrid->object); +// return ExceptionNewf(AttributeError, "'%s' object has no attribute '%s'", self.Type().Name, attr) +// } +// return res +// } // Method resolution order algorithm C3 described in // "A Monotonic Superclass Linearization for Dylan", @@ -955,26 +955,26 @@ func add_subclass(base, t *Type) { // return result; } -func remove_subclass(base, t *Type) { - // Py_ssize_t i; - // PyObject *list, *ref; - - // list = base->tp_subclasses; - // if (list == nil) { - // return; - // } - // assert(PyList_Check(list)); - // i = PyList_GET_SIZE(list); - // while (--i >= 0) { - // ref = PyList_GET_ITEM(list, i); - // assert(PyWeakref_CheckRef(ref)); - // if (PyWeakref_GET_OBJECT(ref) == (PyObject*)type) { - // /* this can't fail, right? */ - // PySequence_DelItem(list, i); - // return; - // } - // } -} +// func remove_subclass(base, t *Type) { +// // Py_ssize_t i; +// // PyObject *list, *ref; +// +// // list = base->tp_subclasses; +// // if (list == nil) { +// // return; +// // } +// // assert(PyList_Check(list)); +// // i = PyList_GET_SIZE(list); +// // while (--i >= 0) { +// // ref = PyList_GET_ITEM(list, i); +// // assert(PyWeakref_CheckRef(ref)); +// // if (PyWeakref_GET_OBJECT(ref) == (PyObject*)type) { +// // /* this can't fail, right? */ +// // PySequence_DelItem(list, i); +// // return; +// // } +// // } +// } // Ready the type for use // diff --git a/py/zip.go b/py/zip.go index 2626ec01..eee9dc27 100644 --- a/py/zip.go +++ b/py/zip.go @@ -10,10 +10,10 @@ type Zip struct { size int } -// A python ZipIterator iterator -type ZipIterator struct { - zip Zip -} +// // A python ZipIterator iterator +// type ZipIterator struct { +// zip Zip +// } var ZipType = NewTypeX("zip", `zip(iter1 [,iter2 [...]]) --> zip object diff --git a/repl/cli/cli.go b/repl/cli/cli.go index 90f1463b..6648094a 100644 --- a/repl/cli/cli.go +++ b/repl/cli/cli.go @@ -12,7 +12,6 @@ import ( "os/user" "path/filepath" - "github.com/go-python/gpython/py" "github.com/go-python/gpython/repl" "github.com/peterh/liner" ) @@ -36,7 +35,6 @@ type readline struct { *liner.State repl *repl.REPL historyFile string - module *py.Module prompt string } diff --git a/stdlib/math/math.go b/stdlib/math/math.go index 8f1e4dcb..6c6c32d9 100644 --- a/stdlib/math/math.go +++ b/stdlib/math/math.go @@ -75,13 +75,6 @@ var ( ERANGE = py.ExceptionNewf(py.OverflowError, "math range error") ) -// panic if ok is false -func assert(ok bool) { - if !ok { - panic("assertion failed") - } -} - // isFinite is true if x is not Nan or +/-Inf func isFinite(x float64) bool { return !(math.IsInf(x, 0) || math.IsNaN(x)) diff --git a/stdlib/sys/sys.go b/stdlib/sys/sys.go index b8aa3dc2..52d733b6 100644 --- a/stdlib/sys/sys.go +++ b/stdlib/sys/sys.go @@ -347,10 +347,10 @@ func sys_setrecursionlimit(self py.Object, args py.Tuple) (py.Object, error) { return nil, py.NotImplementedError } -const hash_info_doc = `hash_info - -A struct sequence providing parameters used for computing -numeric hashes. The attributes are read only.` +// const hash_info_doc = `hash_info +// +// A struct sequence providing parameters used for computing +// numeric hashes. The attributes are read only.` // PyStructSequence_Field hash_info_fields[] = { // {"width", "width of the type used for hashing, in bits"}, @@ -539,9 +539,9 @@ func sys_clear_type_cache(self py.Object, args py.Tuple) (py.Object, error) { return nil, py.NotImplementedError } -const flags__doc__ = `sys.flags - -Flags provided through command line arguments or environment vars.` +// const flags__doc__ = `sys.flags +// +// Flags provided through command line arguments or environment vars.` // PyTypeObject FlagsType; @@ -603,9 +603,9 @@ Flags provided through command line arguments or environment vars.` // return seq; // } -const version_info__doc__ = `sys.version_info - -Version information as a named tuple.` +//const version_info__doc__ = `sys.version_info +// +//Version information as a named tuple.` // PyStructSequence_Field version_info_fields[] = { // {"major", "Major release number"}, From 739246d708ac810f3890c7e072c0686b06f151cb Mon Sep 17 00:00:00 2001 From: Sebastien Binet Date: Fri, 6 May 2022 19:26:49 +0200 Subject: [PATCH 21/60] all: move marshal to stdlib/marshal Signed-off-by: Sebastien Binet --- compile/legacy.go | 2 +- {marshal => stdlib/marshal}/marshal.go | 0 stdlib/stdlib.go | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename {marshal => stdlib/marshal}/marshal.go (100%) diff --git a/compile/legacy.go b/compile/legacy.go index 6578b18f..c5f2571e 100644 --- a/compile/legacy.go +++ b/compile/legacy.go @@ -11,8 +11,8 @@ import ( "os/exec" "strings" - "github.com/go-python/gpython/marshal" "github.com/go-python/gpython/py" + "github.com/go-python/gpython/stdlib/marshal" ) // Compile with python3.4 - not used any more but keep for the moment! diff --git a/marshal/marshal.go b/stdlib/marshal/marshal.go similarity index 100% rename from marshal/marshal.go rename to stdlib/marshal/marshal.go diff --git a/stdlib/stdlib.go b/stdlib/stdlib.go index ebe23d06..38483df1 100644 --- a/stdlib/stdlib.go +++ b/stdlib/stdlib.go @@ -14,8 +14,8 @@ import ( "strings" "sync" - "github.com/go-python/gpython/marshal" "github.com/go-python/gpython/py" + "github.com/go-python/gpython/stdlib/marshal" "github.com/go-python/gpython/vm" _ "github.com/go-python/gpython/stdlib/builtin" From af8341ee071138455dc92592a5368243c873bcba Mon Sep 17 00:00:00 2001 From: Sebastien Binet Date: Fri, 6 May 2022 17:15:45 +0200 Subject: [PATCH 22/60] py: improve ParseTuple{,AndKeywords} to handle 's{,*,#}' Signed-off-by: Sebastien Binet --- py/args.go | 55 ++++++++-- py/args_test.go | 277 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 326 insertions(+), 6 deletions(-) create mode 100644 py/args_test.go diff --git a/py/args.go b/py/args.go index e3bfc36b..d7e5a35b 100644 --- a/py/args.go +++ b/py/args.go @@ -465,17 +465,60 @@ func ParseTupleAndKeywords(args Tuple, kwargs StringDict, format string, kwlist switch op.code { case 'O': *result = arg - case 'Z', 'z': - if _, ok := arg.(NoneType); ok { - *result = arg - break + case 'Z': + switch op.modifier { + default: + return ExceptionNewf(TypeError, "%s() argument %d must be str or None, not %s", name, i+1, arg.Type().Name) + case '#', 0: + switch arg := arg.(type) { + case String, NoneType: + default: + return ExceptionNewf(TypeError, "%s() argument %d must be str or None, not %s", name, i+1, arg.Type().Name) + } + } + *result = arg + case 'z': + switch op.modifier { + default: + switch arg := arg.(type) { + case String, NoneType: + // ok + default: + return ExceptionNewf(TypeError, "%s() argument %d must be str or None, not %s", name, i+1, arg.Type().Name) + } + case '#': + fallthrough // FIXME(sbinet): check for read-only? + case '*': + switch arg := arg.(type) { + case String, Bytes, NoneType: + // ok. + default: + return ExceptionNewf(TypeError, "%s() argument %d must be str, bytes-like or None, not %s", name, i+1, arg.Type().Name) + } } - fallthrough - case 'U', 's': + *result = arg + case 'U': if _, ok := arg.(String); !ok { return ExceptionNewf(TypeError, "%s() argument %d must be str, not %s", name, i+1, arg.Type().Name) } *result = arg + case 's': + switch op.modifier { + default: + if _, ok := arg.(String); !ok { + return ExceptionNewf(TypeError, "%s() argument %d must be str, not %s", name, i+1, arg.Type().Name) + } + case '#': + fallthrough // FIXME(sbinet): check for read-only? + case '*': + switch arg := arg.(type) { + case String, Bytes: + // ok. + default: + return ExceptionNewf(TypeError, "%s() argument %d must be str or bytes-like, not %s", name, i+1, arg.Type().Name) + } + } + *result = arg case 'i', 'n': if _, ok := arg.(Int); !ok { return ExceptionNewf(TypeError, "%s() argument %d must be int, not %s", name, i+1, arg.Type().Name) diff --git a/py/args_test.go b/py/args_test.go new file mode 100644 index 00000000..2f291827 --- /dev/null +++ b/py/args_test.go @@ -0,0 +1,277 @@ +// Copyright 2022 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package py + +import ( + "fmt" + "testing" +) + +func TestParseTupleAndKeywords(t *testing.T) { + for _, tc := range []struct { + args Tuple + kwargs StringDict + format string + kwlist []string + results []Object + err error + }{ + { + args: Tuple{String("a")}, + format: "O:func", + results: []Object{String("a")}, + }, + { + args: Tuple{None}, + format: "Z:func", + results: []Object{None}, + }, + { + args: Tuple{String("a")}, + format: "Z:func", + results: []Object{String("a")}, + }, + { + args: Tuple{Int(42)}, + format: "Z:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be str or None, not int'"), + }, + { + args: Tuple{None}, + format: "Z*:func", // FIXME(sbinet): invalid format. + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be str or None, not NoneType'"), + }, + { + args: Tuple{None}, + format: "Z#:func", + results: []Object{None}, + }, + { + args: Tuple{String("a")}, + format: "Z#:func", + results: []Object{String("a")}, + }, + { + args: Tuple{Int(42)}, + format: "Z#:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be str or None, not int'"), + }, + { + args: Tuple{None}, + format: "z:func", + results: []Object{None}, + }, + { + args: Tuple{String("a")}, + format: "z:func", + results: []Object{String("a")}, + }, + { + args: Tuple{Int(42)}, + format: "z:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be str or None, not int'"), + }, + { + args: Tuple{Bytes("a")}, + format: "z:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be str or None, not bytes'"), + }, + { + args: Tuple{None}, + format: "z*:func", + results: []Object{None}, + }, + { + args: Tuple{Bytes("a")}, + format: "z*:func", + results: []Object{Bytes("a")}, + }, + { + args: Tuple{String("a")}, + format: "z*:func", + results: []Object{String("a")}, + }, + { + args: Tuple{Int(42)}, + format: "z*:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be str, bytes-like or None, not int'"), + }, + { + args: Tuple{None}, + format: "z#:func", + results: []Object{None}, + }, + { + args: Tuple{Bytes("a")}, + format: "z#:func", + results: []Object{Bytes("a")}, + }, + { + args: Tuple{String("a")}, + format: "z#:func", + results: []Object{String("a")}, + }, + { + args: Tuple{Int(42)}, + format: "z#:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be str, bytes-like or None, not int'"), + }, + { + args: Tuple{String("a")}, + format: "s:func", + results: []Object{String("a")}, + }, + { + args: Tuple{None}, + format: "s:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be str, not NoneType'"), + }, + { + args: Tuple{Bytes("a")}, + format: "s:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be str, not bytes'"), + }, + { + args: Tuple{String("a")}, + format: "s#:func", + results: []Object{String("a")}, + }, + { + args: Tuple{Bytes("a")}, + format: "s#:func", + results: []Object{Bytes("a")}, + }, + { + args: Tuple{None}, + format: "s#:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be str or bytes-like, not NoneType'"), + }, + { + args: Tuple{String("a")}, + format: "s*:func", + results: []Object{String("a")}, + }, + { + args: Tuple{Bytes("a")}, + format: "s*:func", + results: []Object{Bytes("a")}, + }, + { + args: Tuple{None}, + format: "s*:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be str or bytes-like, not NoneType'"), + }, + { + args: Tuple{String("a")}, + format: "U:func", + results: []Object{String("a")}, + }, + { + args: Tuple{None}, + format: "U:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be str, not NoneType'"), + }, + { + args: Tuple{Bytes("a")}, + format: "U:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be str, not bytes'"), + }, + { + args: Tuple{Bytes("a")}, + format: "U*:func", // FIXME(sbinet): invalid format + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be str, not bytes'"), + }, + { + args: Tuple{Bytes("a")}, + format: "U#:func", // FIXME(sbinet): invalid format + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be str, not bytes'"), + }, + { + args: Tuple{Int(42)}, + format: "i:func", + results: []Object{Int(42)}, + }, + { + args: Tuple{None}, + format: "i:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be int, not NoneType'"), + }, + { + args: Tuple{Int(42)}, + format: "n:func", + results: []Object{Int(42)}, + }, + { + args: Tuple{None}, + format: "n:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be int, not NoneType'"), + }, + { + args: Tuple{Bool(true)}, + format: "p:func", + results: []Object{Bool(true)}, + }, + { + args: Tuple{None}, + format: "p:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be bool, not NoneType'"), + }, + { + args: Tuple{Float(42)}, + format: "d:func", + results: []Object{Float(42)}, + }, + { + args: Tuple{Int(42)}, + format: "d:func", + results: []Object{Float(42)}, + }, + { + args: Tuple{None}, + format: "p:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be bool, not NoneType'"), + }, + } { + t.Run(tc.format, func(t *testing.T) { + results := make([]*Object, len(tc.results)) + for i := range tc.results { + results[i] = &tc.results[i] + } + err := ParseTupleAndKeywords(tc.args, tc.kwargs, tc.format, tc.kwlist, results...) + switch { + case err != nil && tc.err != nil: + if got, want := err.Error(), tc.err.Error(); got != want { + t.Fatalf("invalid error:\ngot= %s\nwant=%s", got, want) + } + case err != nil && tc.err == nil: + t.Fatalf("could not parse tuple+kwargs: %+v", err) + case err == nil && tc.err != nil: + t.Fatalf("expected an error (got=nil): %+v", tc.err) + case err == nil && tc.err == nil: + // ok. + } + // FIXME(sbinet): check results + }) + } +} From 8ca157b84c9fdfe4726ff015fc1d6626eb95c991 Mon Sep 17 00:00:00 2001 From: Sebastien Binet Date: Fri, 6 May 2022 18:08:59 +0200 Subject: [PATCH 23/60] py: improve ParseTuple{,AndKeywords} to handle 'y{,*,#}' Signed-off-by: Sebastien Binet --- py/args.go | 17 +++++++++++++++++ py/args_test.go | 51 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) diff --git a/py/args.go b/py/args.go index d7e5a35b..04734ef3 100644 --- a/py/args.go +++ b/py/args.go @@ -519,6 +519,23 @@ func ParseTupleAndKeywords(args Tuple, kwargs StringDict, format string, kwlist } } *result = arg + case 'y': + switch op.modifier { + default: + if _, ok := arg.(Bytes); !ok { + return ExceptionNewf(TypeError, "%s() argument %d must be bytes-like, not %s", name, i+1, arg.Type().Name) + } + case '#': + fallthrough // FIXME(sbinet): check for read-only? + case '*': + switch arg := arg.(type) { + case Bytes: + // ok. + default: + return ExceptionNewf(TypeError, "%s() argument %d must be bytes-like, not %s", name, i+1, arg.Type().Name) + } + } + *result = arg case 'i', 'n': if _, ok := arg.(Int); !ok { return ExceptionNewf(TypeError, "%s() argument %d must be int, not %s", name, i+1, arg.Type().Name) diff --git a/py/args_test.go b/py/args_test.go index 2f291827..408ad342 100644 --- a/py/args_test.go +++ b/py/args_test.go @@ -174,6 +174,57 @@ func TestParseTupleAndKeywords(t *testing.T) { results: []Object{nil}, err: fmt.Errorf("TypeError: 'func() argument 1 must be str or bytes-like, not NoneType'"), }, + { + args: Tuple{Bytes("a")}, + format: "y:func", + results: []Object{Bytes("a")}, + }, + { + args: Tuple{None}, + format: "y:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be bytes-like, not NoneType'"), + }, + { + args: Tuple{String("a")}, + format: "y:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be bytes-like, not str'"), + }, + { + args: Tuple{Bytes("a")}, + format: "y#:func", + results: []Object{Bytes("a")}, + }, + { + args: Tuple{String("a")}, + format: "y#:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be bytes-like, not str'"), + }, + { + args: Tuple{None}, + format: "y#:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be bytes-like, not NoneType'"), + }, + { + args: Tuple{Bytes("a")}, + format: "y*:func", + results: []Object{Bytes("a")}, + }, + { + args: Tuple{String("a")}, + format: "y*:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be bytes-like, not str'"), + }, + { + args: Tuple{None}, + format: "y*:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be bytes-like, not NoneType'"), + }, { args: Tuple{String("a")}, format: "U:func", From 6b3e35b14353e9cff3eb89701187515b02cbe664 Mon Sep 17 00:00:00 2001 From: Sebastien Binet Date: Mon, 9 May 2022 15:31:40 +0200 Subject: [PATCH 24/60] py: make Exception implement __{str,repr}__ Signed-off-by: Sebastien Binet --- py/exception.go | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/py/exception.go b/py/exception.go index a2abbcac..2e8f91a2 100644 --- a/py/exception.go +++ b/py/exception.go @@ -360,6 +360,22 @@ func (e *Exception) M__getattr__(name string) (Object, error) { return e.Args, nil // FIXME All attributes are args! } +func (e *Exception) M__str__() (Object, error) { + msg := e.Args.(Tuple)[0] + return msg, nil +} + +func (e *Exception) M__repr__() (Object, error) { + msg := e.Args.(Tuple)[0].(String) + typ := e.Base.Name + return String(fmt.Sprintf("%s(%q)", typ, string(msg))), nil +} + // Check Interfaces -var _ error = (*Exception)(nil) -var _ error = (*ExceptionInfo)(nil) +var ( + _ error = (*ExceptionInfo)(nil) + + _ error = (*Exception)(nil) + _ I__str__ = (*Exception)(nil) + _ I__repr__ = (*Exception)(nil) +) From 189965b62e710677c39f9b84953ce1b941a3fdb1 Mon Sep 17 00:00:00 2001 From: Sebastien Binet Date: Mon, 9 May 2022 11:39:43 +0200 Subject: [PATCH 25/60] stdlib/binascii: first import Signed-off-by: Sebastien Binet --- stdlib/binascii/binascii.go | 158 +++++++++++++++++++++++ stdlib/binascii/binascii_test.go | 15 +++ stdlib/binascii/testdata/test.py | 54 ++++++++ stdlib/binascii/testdata/test_golden.txt | 11 ++ stdlib/stdlib.go | 1 + 5 files changed, 239 insertions(+) create mode 100644 stdlib/binascii/binascii.go create mode 100644 stdlib/binascii/binascii_test.go create mode 100644 stdlib/binascii/testdata/test.py create mode 100644 stdlib/binascii/testdata/test_golden.txt diff --git a/stdlib/binascii/binascii.go b/stdlib/binascii/binascii.go new file mode 100644 index 00000000..4a87d3ee --- /dev/null +++ b/stdlib/binascii/binascii.go @@ -0,0 +1,158 @@ +// Copyright 2022 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package binascii provides the implementation of the python's 'binascii' module. +package binascii + +import ( + "encoding/base64" + "encoding/hex" + "errors" + "hash/crc32" + + "github.com/go-python/gpython/py" +) + +var ( + Incomplete = py.ExceptionType.NewType("binascii.Incomplete", "", nil, nil) + Error = py.ValueError.NewType("binascii.Error", "", nil, nil) +) + +func init() { + py.RegisterModule(&py.ModuleImpl{ + Info: py.ModuleInfo{ + Name: "binascii", + Doc: "Conversion between binary data and ASCII", + }, + Methods: []*py.Method{ + py.MustNewMethod("a2b_base64", a2b_base64, 0, "Decode a line of base64 data."), + py.MustNewMethod("b2a_base64", b2a_base64, 0, "Base64-code line of data."), + py.MustNewMethod("a2b_hex", a2b_hex, 0, a2b_hex_doc), + py.MustNewMethod("b2a_hex", b2a_hex, 0, b2a_hex_doc), + py.MustNewMethod("crc32", crc32_, 0, "Compute CRC-32 incrementally."), + py.MustNewMethod("unhexlify", a2b_hex, 0, unhexlify_doc), + py.MustNewMethod("hexlify", b2a_hex, 0, hexlify_doc), + }, + Globals: py.StringDict{ + "Incomplete": Incomplete, + "Error": Error, + }, + }) +} + +func b2a_base64(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Object, error) { + var ( + pydata py.Object + pynewl py.Object = py.True + ) + err := py.ParseTupleAndKeywords(args, kwargs, "y*|p:binascii.b2a_base64", []string{"data", "newline"}, &pydata, &pynewl) + if err != nil { + return nil, err + } + + var ( + buf = []byte(pydata.(py.Bytes)) + newline = bool(pynewl.(py.Bool)) + ) + + out := base64.StdEncoding.EncodeToString(buf) + if newline { + out += "\n" + } + return py.Bytes(out), nil +} + +func a2b_base64(self py.Object, args py.Tuple) (py.Object, error) { + var pydata py.Object + err := py.ParseTuple(args, "s:binascii.a2b_base64", &pydata) + if err != nil { + return nil, err + } + + out, err := base64.StdEncoding.DecodeString(string(pydata.(py.String))) + if err != nil { + return nil, py.ExceptionNewf(Error, "could not decode base64 data: %+v", err) + } + + return py.Bytes(out), nil +} + +func crc32_(self py.Object, args py.Tuple) (py.Object, error) { + var ( + pydata py.Object + pycrc py.Object = py.Int(0) + ) + + err := py.ParseTuple(args, "y*|i:binascii.crc32", &pydata, &pycrc) + if err != nil { + return nil, err + } + + crc := crc32.Update(uint32(pycrc.(py.Int)), crc32.IEEETable, []byte(pydata.(py.Bytes))) + return py.Int(crc), nil + +} + +const a2b_hex_doc = `Binary data of hexadecimal representation. + +hexstr must contain an even number of hex digits (upper or lower case). +This function is also available as "unhexlify()".` + +func a2b_hex(self py.Object, args py.Tuple) (py.Object, error) { + var ( + hexErr hex.InvalidByteError + pydata py.Object + src string + ) + err := py.ParseTuple(args, "s*:binascii.a2b_hex", &pydata) + if err != nil { + return nil, err + } + + switch v := pydata.(type) { + case py.String: + src = string(v) + case py.Bytes: + src = string(v) + } + + o, err := hex.DecodeString(src) + if err != nil { + switch { + case errors.Is(err, hex.ErrLength): + return nil, py.ExceptionNewf(Error, "Odd-length string") + case errors.As(err, &hexErr): + return nil, py.ExceptionNewf(Error, "Non-hexadecimal digit found") + default: + return nil, py.ExceptionNewf(Error, "could not decode hex data: %+v", err) + } + } + + return py.Bytes(o), nil +} + +const b2a_hex_doc = `Hexadecimal representation of binary data. + +The return value is a bytes object. This function is also +available as "hexlify()".` + +func b2a_hex(self py.Object, args py.Tuple) (py.Object, error) { + var pydata py.Object + err := py.ParseTuple(args, "y*:binascii.b2a_hex", &pydata) + if err != nil { + return nil, err + } + + o := hex.EncodeToString([]byte(pydata.(py.Bytes))) + return py.Bytes(o), nil +} + +const unhexlify_doc = `Binary data of hexadecimal representation. + +hexstr must contain an even number of hex digits (upper or lower case).` + +const hexlify_doc = `Hexadecimal representation of binary data. + +The return value is a bytes object. This function is also +available as "b2a_hex()".` diff --git a/stdlib/binascii/binascii_test.go b/stdlib/binascii/binascii_test.go new file mode 100644 index 00000000..bd40c782 --- /dev/null +++ b/stdlib/binascii/binascii_test.go @@ -0,0 +1,15 @@ +// Copyright 2022 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package binascii_test + +import ( + "testing" + + "github.com/go-python/gpython/pytest" +) + +func TestBinascii(t *testing.T) { + pytest.RunScript(t, "./testdata/test.py") +} diff --git a/stdlib/binascii/testdata/test.py b/stdlib/binascii/testdata/test.py new file mode 100644 index 00000000..f6227aff --- /dev/null +++ b/stdlib/binascii/testdata/test.py @@ -0,0 +1,54 @@ +# Copyright 2022 The go-python Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +import binascii + +print("globals:") +for name in ("Error", "Incomplete"): + v = getattr(binascii, name) + print("\nbinascii.%s:\n%s" % (name,repr(v))) + +def assertEqual(x, y): + assert x == y, "got: %s, want: %s" % (repr(x), repr(y)) + +## base64 +assertEqual(binascii.b2a_base64(b'hello world!'), b'aGVsbG8gd29ybGQh\n') +assertEqual(binascii.b2a_base64(b'hello\nworld!'), b'aGVsbG8Kd29ybGQh\n') +assertEqual(binascii.b2a_base64(b'hello world!', newline=False), b'aGVsbG8gd29ybGQh') +assertEqual(binascii.b2a_base64(b'hello\nworld!', newline=False), b'aGVsbG8Kd29ybGQh') +assertEqual(binascii.a2b_base64("aGVsbG8gd29ybGQh\n"), b'hello world!') + +try: + binascii.b2a_base64("string") + print("expected an exception") +except TypeError as e: + print("expected an exception:", e) + pass + +## crc32 +assertEqual(binascii.crc32(b'hello world!'), 62177901) +assertEqual(binascii.crc32(b'hello world!', 0), 62177901) +assertEqual(binascii.crc32(b'hello world!', 42), 4055036404) + +## hex +assertEqual(binascii.b2a_hex(b'hello world!'), b'68656c6c6f20776f726c6421') +assertEqual(binascii.a2b_hex(b'68656c6c6f20776f726c6421'), b'hello world!') +assertEqual(binascii.hexlify(b'hello world!'), b'68656c6c6f20776f726c6421') +assertEqual(binascii.unhexlify(b'68656c6c6f20776f726c6421'), b'hello world!') + +try: + binascii.a2b_hex(b'123') + print("expected an exception") +except binascii.Error as e: + print("expected an exception:",e) + pass + +try: + binascii.a2b_hex(b'hell') + print("expected an exception") +except binascii.Error as e: + print("expected an exception:",e) + pass + +print("done.") diff --git a/stdlib/binascii/testdata/test_golden.txt b/stdlib/binascii/testdata/test_golden.txt new file mode 100644 index 00000000..29f03223 --- /dev/null +++ b/stdlib/binascii/testdata/test_golden.txt @@ -0,0 +1,11 @@ +globals: + +binascii.Error: + + +binascii.Incomplete: + +expected an exception: binascii.b2a_base64() argument 1 must be bytes-like, not str +expected an exception: Odd-length string +expected an exception: Non-hexadecimal digit found +done. diff --git a/stdlib/stdlib.go b/stdlib/stdlib.go index 38483df1..8182d462 100644 --- a/stdlib/stdlib.go +++ b/stdlib/stdlib.go @@ -18,6 +18,7 @@ import ( "github.com/go-python/gpython/stdlib/marshal" "github.com/go-python/gpython/vm" + _ "github.com/go-python/gpython/stdlib/binascii" _ "github.com/go-python/gpython/stdlib/builtin" _ "github.com/go-python/gpython/stdlib/math" _ "github.com/go-python/gpython/stdlib/string" From 6d34733eef7c2fc0cecf06de6f0ccc07292eccce Mon Sep 17 00:00:00 2001 From: Sebastien Binet Date: Mon, 9 May 2022 17:26:36 +0200 Subject: [PATCH 26/60] py: make bytes implement __{i,}add__ Signed-off-by: Sebastien Binet --- py/bytes.go | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/py/bytes.go b/py/bytes.go index 2c653455..7adf050e 100644 --- a/py/bytes.go +++ b/py/bytes.go @@ -240,5 +240,27 @@ func (a Bytes) M__ge__(other Object) (Object, error) { return NotImplemented, nil } +func (a Bytes) M__add__(other Object) (Object, error) { + if b, ok := convertToBytes(other); ok { + o := make([]byte, len(a)+len(b)) + copy(o[:len(a)], a) + copy(o[len(a):], b) + return Bytes(o), nil + } + return NotImplemented, nil +} + +func (a Bytes) M__iadd__(other Object) (Object, error) { + if b, ok := convertToBytes(other); ok { + a = append(a, b...) + return a, nil + } + return NotImplemented, nil +} + // Check interface is satisfied -var _ richComparison = (Bytes)(nil) +var ( + _ richComparison = (Bytes)(nil) + _ I__add__ = (Bytes)(nil) + _ I__iadd__ = (Bytes)(nil) +) From 7e10deb9bbea0f9e115afd0a93bc3152d643f3dd Mon Sep 17 00:00:00 2001 From: Sebastien Binet Date: Mon, 9 May 2022 17:03:48 +0200 Subject: [PATCH 27/60] stdlib/binascii: implement {b2a,a2b}_qp Signed-off-by: Sebastien Binet --- stdlib/binascii/binascii.go | 77 ++++++++++++++++++++++++++++++++ stdlib/binascii/testdata/test.py | 18 ++++++++ 2 files changed, 95 insertions(+) diff --git a/stdlib/binascii/binascii.go b/stdlib/binascii/binascii.go index 4a87d3ee..f39af440 100644 --- a/stdlib/binascii/binascii.go +++ b/stdlib/binascii/binascii.go @@ -6,10 +6,13 @@ package binascii import ( + "bytes" "encoding/base64" "encoding/hex" "errors" "hash/crc32" + "io" + "mime/quotedprintable" "github.com/go-python/gpython/py" ) @@ -33,6 +36,8 @@ func init() { py.MustNewMethod("crc32", crc32_, 0, "Compute CRC-32 incrementally."), py.MustNewMethod("unhexlify", a2b_hex, 0, unhexlify_doc), py.MustNewMethod("hexlify", b2a_hex, 0, hexlify_doc), + py.MustNewMethod("a2b_qp", a2b_qp, 0, a2b_qp_doc), + py.MustNewMethod("b2a_qp", b2a_qp, 0, b2a_qp_doc), }, Globals: py.StringDict{ "Incomplete": Incomplete, @@ -156,3 +161,75 @@ const hexlify_doc = `Hexadecimal representation of binary data. The return value is a bytes object. This function is also available as "b2a_hex()".` + +const a2b_qp_doc = `Decode a string of qp-encoded data.` + +func a2b_qp(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Object, error) { + var ( + pydata py.Object + pyhdr py.Object = py.Bool(false) + ) + err := py.ParseTupleAndKeywords(args, kwargs, "y*|p:binascii.a2b_qp", []string{"data", "header"}, &pydata, &pyhdr) + if err != nil { + return nil, err + } + + // TODO(sbinet) + if pyhdr.(py.Bool) { + return nil, py.NotImplementedError + } + + o := new(bytes.Buffer) + r := quotedprintable.NewReader(bytes.NewReader([]byte(pydata.(py.Bytes)))) + _, err = io.Copy(o, r) + if err != nil { + // FIXME(sbinet): decorate the error somehow? + return nil, err + } + return py.Bytes(o.Bytes()), nil +} + +const b2a_qp_doc = `Encode a string using quoted-printable encoding. + +On encoding, when istext is set, newlines are not encoded, and white +space at end of lines is. When istext is not set, \r and \n (CR/LF) +are both encoded. When quotetabs is set, space and tabs are encoded.` + +func b2a_qp(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Object, error) { + var ( + pydata py.Object + pyqtabs py.Object = py.Bool(false) + pyistxt py.Object = py.Bool(true) + pyhdr py.Object = py.Bool(false) + ) + err := py.ParseTupleAndKeywords(args, kwargs, "y*|ppp:binascii.b2a_qp", []string{"data", "quotetabs", "istext", "header"}, &pydata, &pyqtabs, &pyistxt, &pyhdr) + if err != nil { + return nil, err + } + + if pyqtabs.(py.Bool) { + return nil, py.NotImplementedError + } + + if !pyistxt.(py.Bool) { + return nil, py.NotImplementedError + } + + if pyhdr.(py.Bool) { + return nil, py.NotImplementedError + } + + o := new(bytes.Buffer) + w := quotedprintable.NewWriter(o) + _, err = w.Write([]byte(pydata.(py.Bytes))) + if err != nil { + // FIXME(sbinet): decorate the error somehow? + return nil, err + } + err = w.Close() + if err != nil { + // FIXME(sbinet): decorate the error somehow? + return nil, err + } + return py.Bytes(o.Bytes()), nil +} diff --git a/stdlib/binascii/testdata/test.py b/stdlib/binascii/testdata/test.py index f6227aff..840c8672 100644 --- a/stdlib/binascii/testdata/test.py +++ b/stdlib/binascii/testdata/test.py @@ -12,6 +12,10 @@ def assertEqual(x, y): assert x == y, "got: %s, want: %s" % (repr(x), repr(y)) +rawdata = b"The quick brown fox jumps over the lazy dog.\r\n" +rawdata += bytes(range(256)) +rawdata += b"\r\nHello world.\n" + ## base64 assertEqual(binascii.b2a_base64(b'hello world!'), b'aGVsbG8gd29ybGQh\n') assertEqual(binascii.b2a_base64(b'hello\nworld!'), b'aGVsbG8Kd29ybGQh\n') @@ -19,6 +23,8 @@ def assertEqual(x, y): assertEqual(binascii.b2a_base64(b'hello\nworld!', newline=False), b'aGVsbG8Kd29ybGQh') assertEqual(binascii.a2b_base64("aGVsbG8gd29ybGQh\n"), b'hello world!') +assertEqual(binascii.b2a_base64(rawdata), b"VGhlIHF1aWNrIGJyb3duIGZveCBqdW1wcyBvdmVyIHRoZSBsYXp5IGRvZy4NCgABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fICEiIyQlJicoKSorLC0uLzAxMjM0NTY3ODk6Ozw9Pj9AQUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVpbXF1eX2BhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ent8fX5/gIGCg4SFhoeIiYqLjI2Oj5CRkpOUlZaXmJmam5ydnp+goaKjpKWmp6ipqqusra6vsLGys7S1tre4ubq7vL2+v8DBwsPExcbHyMnKy8zNzs/Q0dLT1NXW19jZ2tvc3d7f4OHi4+Tl5ufo6err7O3u7/Dx8vP09fb3+Pn6+/z9/v8NCkhlbGxvIHdvcmxkLgo=\n") + try: binascii.b2a_base64("string") print("expected an exception") @@ -37,6 +43,8 @@ def assertEqual(x, y): assertEqual(binascii.hexlify(b'hello world!'), b'68656c6c6f20776f726c6421') assertEqual(binascii.unhexlify(b'68656c6c6f20776f726c6421'), b'hello world!') +assertEqual(binascii.b2a_hex(rawdata), b'54686520717569636b2062726f776e20666f78206a756d7073206f76657220746865206c617a7920646f672e0d0a000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f808182838485868788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9fa0a1a2a3a4a5a6a7a8a9aaabacadaeafb0b1b2b3b4b5b6b7b8b9babbbcbdbebfc0c1c2c3c4c5c6c7c8c9cacbcccdcecfd0d1d2d3d4d5d6d7d8d9dadbdcdddedfe0e1e2e3e4e5e6e7e8e9eaebecedeeeff0f1f2f3f4f5f6f7f8f9fafbfcfdfeff0d0a48656c6c6f20776f726c642e0a') + try: binascii.a2b_hex(b'123') print("expected an exception") @@ -51,4 +59,14 @@ def assertEqual(x, y): print("expected an exception:",e) pass +## quotedprintable +assertEqual(binascii.b2a_qp(b'hello world! = \t'), b'hello world! =3D =09') +assertEqual(binascii.a2b_qp(b'hello world! =3D =09'), b'hello world! = \t') +## ## TODO +## #assertEqual(binascii.a2b_qp(b'hello world!', header=True), b'hello world!') +## assertEqual(binascii.a2b_qp(rawdata, header=False), b'The quick brown fox jumps over the lazy dog.\r\n\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff\r\nHello world.\n') +## assertEqual(binascii.b2a_qp(binascii.a2b_qp(rawdata)), b'The quick brown fox jumps over the lazy dog.\r\n=00=01=02=03=04=05=06=07=08=09\r\n=0B=0C\r=0E=0F=10=11=12=13=14=15=16=17=18=19=1A=1B=1C=1D=1E=1F !"#$%&\'()*+,-=\r\n./0123456789:;<=3D>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuv=\r\nwxyz{|}~=7F=80=81=82=83=84=85=86=87=88=89=8A=8B=8C=8D=8E=8F=90=91=92=93=94=\r\n=95=96=97=98=99=9A=9B=9C=9D=9E=9F=A0=A1=A2=A3=A4=A5=A6=A7=A8=A9=AA=AB=AC=AD=\r\n=AE=AF=B0=B1=B2=B3=B4=B5=B6=B7=B8=B9=BA=BB=BC=BD=BE=BF=C0=C1=C2=C3=C4=C5=C6=\r\n=C7=C8=C9=CA=CB=CC=CD=CE=CF=D0=D1=D2=D3=D4=D5=D6=D7=D8=D9=DA=DB=DC=DD=DE=DF=\r\n=E0=E1=E2=E3=E4=E5=E6=E7=E8=E9=EA=EB=EC=ED=EE=EF=F0=F1=F2=F3=F4=F5=F6=F7=F8=\r\n=F9=FA=FB=FC=FD=FE=FF\r\nHello world.\r\n') +## ## TODO +## #assertEqual(binascii.a2b_qp(rawdata, header=True), b'The quick brown fox jumps over the lazy dog.\r\n\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^ `abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff\r\nHello world.\n') + print("done.") From 433aea80b1dce34aaaa6af09ec21d1b3eb535e53 Mon Sep 17 00:00:00 2001 From: Sebastien Binet Date: Thu, 5 May 2022 15:50:45 +0200 Subject: [PATCH 28/60] py: add convenience function Println Signed-off-by: Sebastien Binet --- py/util.go | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/py/util.go b/py/util.go index 56558cac..e2904187 100644 --- a/py/util.go +++ b/py/util.go @@ -7,6 +7,7 @@ package py import ( "errors" "strconv" + "strings" ) var ( @@ -204,3 +205,32 @@ func loadValue(src Object, data interface{}) error { } return nil } + +// Println prints the provided strings to gpython's stdout. +func Println(self Object, args ...string) bool { + sysModule, err := self.(*Module).Context.GetModule("sys") + if err != nil { + return false + } + stdout := sysModule.Globals["stdout"] + write, err := GetAttrString(stdout, "write") + if err != nil { + return false + } + call, ok := write.(I__call__) + if !ok { + return false + } + for _, v := range args { + if !strings.Contains(v, "\n") { + v += " " + } + _, err := call.M__call__(Tuple{String(v)}, nil) + if err != nil { + return false + } + + } + _, err = call.M__call__(Tuple{String("\n")}, nil) // newline + return err == nil +} From 2c8ecee852b877a5c82facdda5e362096c03028a Mon Sep 17 00:00:00 2001 From: glaukiol1 Date: Wed, 16 Mar 2022 01:35:36 +0100 Subject: [PATCH 29/60] stdlib/os: first import author glaukiol1 committer Sebastien Binet --- stdlib/os/os.go | 228 +++++++++++++++++++++++++++++ stdlib/os/os_test.go | 15 ++ stdlib/os/testdata/test.py | 121 +++++++++++++++ stdlib/os/testdata/test_golden.txt | 27 ++++ stdlib/stdlib.go | 1 + 5 files changed, 392 insertions(+) create mode 100644 stdlib/os/os.go create mode 100644 stdlib/os/os_test.go create mode 100644 stdlib/os/testdata/test.py create mode 100644 stdlib/os/testdata/test_golden.txt diff --git a/stdlib/os/os.go b/stdlib/os/os.go new file mode 100644 index 00000000..d863575f --- /dev/null +++ b/stdlib/os/os.go @@ -0,0 +1,228 @@ +// Copyright 2022 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package os implements the Python os module. +package os + +import ( + "os" + "os/exec" + "runtime" + "strings" + + "github.com/go-python/gpython/py" +) + +var ( + osSep = py.String("/") + osName = py.String("posix") + osPathsep = py.String(":") + osLinesep = py.String("\n") + osDefpath = py.String(":/bin:/usr/bin") + osDevnull = py.String("/dev/null") + + osAltsep py.Object = py.None +) + +func initGlobals() { + switch runtime.GOOS { + case "android": + osName = py.String("java") + case "windows": + osSep = py.String(`\`) + osName = py.String("nt") + osPathsep = py.String(";") + osLinesep = py.String("\r\n") + osDefpath = py.String(`C:\bin`) + osDevnull = py.String("nul") + osAltsep = py.String("/") + } +} + +func init() { + initGlobals() + + methods := []*py.Method{ + py.MustNewMethod("getcwd", getCwd, 0, "Get the current working directory"), + py.MustNewMethod("getcwdb", getCwdb, 0, "Get the current working directory in a byte slice"), + py.MustNewMethod("chdir", chdir, 0, "Change the current working directory"), + py.MustNewMethod("getenv", getenv, 0, "Return the value of the environment variable key if it exists, or default if it doesn’t. key, default and the result are str."), + py.MustNewMethod("getpid", getpid, 0, "Return the current process id."), + py.MustNewMethod("putenv", putenv, 0, "Set the environment variable named key to the string value."), + py.MustNewMethod("unsetenv", unsetenv, 0, "Unset (delete) the environment variable named key."), + py.MustNewMethod("_exit", _exit, 0, "Immediate program termination."), + py.MustNewMethod("system", system, 0, "Run shell commands, prints stdout directly to deault"), + } + globals := py.StringDict{ + "error": py.OSError, + "environ": getEnvVariables(), + "sep": osSep, + "name": osName, + "curdir": py.String("."), + "pardir": py.String(".."), + "extsep": py.String("."), + "altsep": osAltsep, + "pathsep": osPathsep, + "linesep": osLinesep, + "defpath": osDefpath, + "devnull": osDevnull, + } + + py.RegisterModule(&py.ModuleImpl{ + Info: py.ModuleInfo{ + Name: "os", + Doc: "Miscellaneous operating system interfaces", + }, + Methods: methods, + Globals: globals, + }) +} + +// getEnvVariables returns the dictionary of environment variables. +func getEnvVariables() py.StringDict { + vs := os.Environ() + dict := py.NewStringDictSized(len(vs)) + for _, evar := range vs { + key_value := strings.SplitN(evar, "=", 2) // returns a []string containing [key,value] + dict.M__setitem__(py.String(key_value[0]), py.String(key_value[1])) + } + + return dict +} + +// getCwd returns the current working directory. +func getCwd(self py.Object, args py.Tuple) (py.Object, error) { + dir, err := os.Getwd() + if err != nil { + return nil, py.ExceptionNewf(py.OSError, "Unable to get current working directory.") + } + return py.String(dir), nil +} + +// getCwdb returns the current working directory as a byte list. +func getCwdb(self py.Object, args py.Tuple) (py.Object, error) { + dir, err := os.Getwd() + if err != nil { + return nil, py.ExceptionNewf(py.OSError, "Unable to get current working directory.") + } + return py.Bytes(dir), nil +} + +// chdir changes the current working directory to the provided path. +func chdir(self py.Object, args py.Tuple) (py.Object, error) { + if len(args) == 0 { + return nil, py.ExceptionNewf(py.TypeError, "Missing required argument 'path' (pos 1)") + } + dir, ok := args[0].(py.String) + if !ok { + return nil, py.ExceptionNewf(py.TypeError, "str expected, not "+args[0].Type().Name) + } + err := os.Chdir(string(dir)) + if err != nil { + return nil, py.ExceptionNewf(py.NotADirectoryError, "Couldn't change cwd; "+err.Error()) + } + return py.None, nil +} + +// getenv returns the value of the environment variable key. +// If no such environment variable exists and a default value was provided, that value is returned. +func getenv(self py.Object, args py.Tuple) (py.Object, error) { + if len(args) < 1 { + return nil, py.ExceptionNewf(py.TypeError, "missing one required argument: 'name:str'") + } + k, ok := args[0].(py.String) + if !ok { + return nil, py.ExceptionNewf(py.TypeError, "str expected (pos 1), not "+args[0].Type().Name) + } + v, ok := os.LookupEnv(string(k)) + if ok { + return py.String(v), nil + } + if len(args) == 2 { + return args[1], nil + } + return py.None, nil +} + +// getpid returns the current process id. +func getpid(self py.Object, args py.Tuple) (py.Object, error) { + return py.Int(os.Getpid()), nil +} + +// putenv sets the value of an environment variable named by the key. +func putenv(self py.Object, args py.Tuple) (py.Object, error) { + if len(args) != 2 { + return nil, py.ExceptionNewf(py.TypeError, "missing required arguments: 'key:str' and 'value:str'") + } + k, ok := args[0].(py.String) + if !ok { + return nil, py.ExceptionNewf(py.TypeError, "str expected (pos 1), not "+args[0].Type().Name) + } + v, ok := args[1].(py.String) + if !ok { + return nil, py.ExceptionNewf(py.TypeError, "str expected (pos 2), not "+args[1].Type().Name) + } + err := os.Setenv(string(k), string(v)) + if err != nil { + return nil, py.ExceptionNewf(py.OSError, "Unable to set enviroment variable") + } + return py.None, nil +} + +// Unset (delete) the environment variable named key. +func unsetenv(self py.Object, args py.Tuple) (py.Object, error) { + if len(args) != 1 { + return nil, py.ExceptionNewf(py.TypeError, "missing one required argument: 'key:str'") + } + k, ok := args[0].(py.String) + if !ok { + return nil, py.ExceptionNewf(py.TypeError, "str expected (pos 1), not "+args[0].Type().Name) + } + err := os.Unsetenv(string(k)) + if err != nil { + return nil, py.ExceptionNewf(py.OSError, "Unable to unset enviroment variable") + } + return py.None, nil +} + +// os._exit() immediate program termination; unlike sys.exit(), which raises a SystemExit, this function will termninate the program immediately. +func _exit(self py.Object, args py.Tuple) (py.Object, error) { // can never return + if len(args) == 0 { + os.Exit(0) + } + arg, ok := args[0].(py.Int) + if !ok { + return nil, py.ExceptionNewf(py.TypeError, "expected int (pos 1), not "+args[0].Type().Name) + } + os.Exit(int(arg)) + return nil, nil +} + +// os.system(command string) this function runs a shell command and directs the output to standard output. +func system(self py.Object, args py.Tuple) (py.Object, error) { + if len(args) != 1 { + return nil, py.ExceptionNewf(py.TypeError, "missing one required argument: 'command:str'") + } + arg, ok := args[0].(py.String) + if !ok { + return nil, py.ExceptionNewf(py.TypeError, "str expected (pos 1), not "+args[0].Type().Name) + } + + var command *exec.Cmd + if runtime.GOOS != "windows" { + command = exec.Command("/bin/sh", "-c", string(arg)) + } else { + command = exec.Command("cmd.exe", string(arg)) + } + outb, err := command.CombinedOutput() // - commbinedoutput to get both stderr and stdout - + if err != nil { + return nil, py.ExceptionNewf(py.OSError, err.Error()) + } + ok = py.Println(self, string(outb)) + if !ok { + return py.Int(1), nil + } + + return py.Int(0), nil +} diff --git a/stdlib/os/os_test.go b/stdlib/os/os_test.go new file mode 100644 index 00000000..8ae63d09 --- /dev/null +++ b/stdlib/os/os_test.go @@ -0,0 +1,15 @@ +// Copyright 2022 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package os_test + +import ( + "testing" + + "github.com/go-python/gpython/pytest" +) + +func TestOs(t *testing.T) { + pytest.RunScript(t, "./testdata/test.py") +} diff --git a/stdlib/os/testdata/test.py b/stdlib/os/testdata/test.py new file mode 100644 index 00000000..100f48df --- /dev/null +++ b/stdlib/os/testdata/test.py @@ -0,0 +1,121 @@ +# Copyright 2022 The go-python Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +import os + +print("test os") +print("os.error: ", os.error) +print("os.getenv($GPYTHON_TEST_HOME)=", os.getenv("GPYTHON_TEST_HOME")) +os.putenv("GPYTHON_TEST_HOME", "/home/go") +print("os.environ($GPYTHON_TEST_HOME)=", os.environ.get("GPYTHON_TEST_HOME")) +print("os.getenv($GPYTHON_TEST_HOME)=", os.getenv("GPYTHON_TEST_HOME")) +os.unsetenv("GPYTHON_TEST_HOME") +print("os.unsetenv($GPYTHON_TEST_HOME)=", os.getenv("GPYTHON_TEST_HOME")) + +if not os.error is OSError: + print("os.error is not OSError!") +else: + print("os.error is OSError [OK]") + +## FIXME(sbinet): check returned value with a known one +## (ie: when os.mkdir is implemented) +if os.getcwd() == None: + print("os.getcwd() == None !") +else: + print("os.getcwd() != None [OK]") + +## FIXME(sbinet): check returned value with a known one +## (ie: when os.mkdir is implemented) +if os.getcwdb() == None: + print("os.getcwdb() == None !") +else: + print("os.getcwdb() != None [OK]") + +print("os.system('echo hello')...") +if os.name != "nt": + os.system('echo hello') +else: ## FIXME(sbinet): find a way to test this nicely + print("hello\n") + +if os.getpid() > 1: + print("os.getpid is greater than 1 [OK]") +else: + print("invalid os.getpid: ", os.getpid()) + +orig = os.getcwd() +testdir = "/" +if os.name == "nt": + testdir = "C:\\" +os.chdir(testdir) +if os.getcwd() != testdir: + print("invalid getcwd() after os.chdir:",os.getcwd()) +else: + print("os.chdir(testdir) [OK]") +os.chdir(orig) + +try: + os.chdir(1) + print("expected an error with os.chdir(1)") +except TypeError: + print("os.chdir(1) failed [OK]") + +try: + os.environ.get(15) + print("expected an error with os.environ.get(15)") +except KeyError: + print("os.environ.get(15) failed [OK]") + +try: + os.putenv() + print("expected an error with os.putenv()") +except TypeError: + print("os.putenv() failed [OK]") + +try: + os.unsetenv() + print("expected an error with os.unsetenv()") +except TypeError: + print("os.unsetenv() failed [OK]") + +try: + os.getenv() + print("expected an error with os.getenv()") +except TypeError: + print("os.getenv() failed [OK]") + +try: + os.unsetenv("FOO", "BAR") + print("expected an error with os.unsetenv(\"FOO\", \"BAR\")") +except TypeError: + print("os.unsetenv(\"FOO\", \"BAR\") failed [OK]") + +if bytes(os.getcwd(), "utf-8") == os.getcwdb(): + print('bytes(os.getcwd(), "utf-8") == os.getcwdb() [OK]') +else: + print('expected: bytes(os.getcwd(), "utf-8") == os.getcwdb()') + +golden = { + "posix": { + "sep": "/", + "pathsep": ":", + "linesep": "\n", + "devnull": "/dev/null", + "altsep": None + }, + "nt": { + "sep": "\\", + "pathsep": ";", + "linesep": "\r\n", + "devnull": "nul", + "altsep": "/" + }, +}[os.name] + +for k in ("sep", "pathsep", "linesep", "devnull", "altsep"): + if getattr(os, k) != golden[k]: + print("invalid os."+k+": got=",getattr(os,k),", want=", golden[k]) + else: + print("os."+k+": [OK]") + +print("OK") diff --git a/stdlib/os/testdata/test_golden.txt b/stdlib/os/testdata/test_golden.txt new file mode 100644 index 00000000..8aae577d --- /dev/null +++ b/stdlib/os/testdata/test_golden.txt @@ -0,0 +1,27 @@ +test os +os.error: +os.getenv($GPYTHON_TEST_HOME)= None +os.environ($GPYTHON_TEST_HOME)= None +os.getenv($GPYTHON_TEST_HOME)= /home/go +os.unsetenv($GPYTHON_TEST_HOME)= None +os.error is OSError [OK] +os.getcwd() != None [OK] +os.getcwdb() != None [OK] +os.system('echo hello')... +hello + +os.getpid is greater than 1 [OK] +os.chdir(testdir) [OK] +os.chdir(1) failed [OK] +os.environ.get(15) failed [OK] +os.putenv() failed [OK] +os.unsetenv() failed [OK] +os.getenv() failed [OK] +os.unsetenv("FOO", "BAR") failed [OK] +bytes(os.getcwd(), "utf-8") == os.getcwdb() [OK] +os.sep: [OK] +os.pathsep: [OK] +os.linesep: [OK] +os.devnull: [OK] +os.altsep: [OK] +OK diff --git a/stdlib/stdlib.go b/stdlib/stdlib.go index 8182d462..d69268d3 100644 --- a/stdlib/stdlib.go +++ b/stdlib/stdlib.go @@ -21,6 +21,7 @@ import ( _ "github.com/go-python/gpython/stdlib/binascii" _ "github.com/go-python/gpython/stdlib/builtin" _ "github.com/go-python/gpython/stdlib/math" + _ "github.com/go-python/gpython/stdlib/os" _ "github.com/go-python/gpython/stdlib/string" _ "github.com/go-python/gpython/stdlib/sys" _ "github.com/go-python/gpython/stdlib/time" From 90acd73860d6374f2e71d63396c2402fae33ab2e Mon Sep 17 00:00:00 2001 From: Sebastien Binet Date: Tue, 24 May 2022 16:11:47 +0200 Subject: [PATCH 30/60] all: rename master into main Signed-off-by: Sebastien Binet --- .github/workflows/ci.yml | 4 ++-- CONTRIBUTE.md | 6 +++--- README.md | 14 +++++++------- examples/embedding/README.md | 4 ++-- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1adebcbc..8b6f2a2f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,9 +2,9 @@ name: CI on: push: - branches: [ master ] + branches: [ main ] pull_request: - branches: [ master ] + branches: [ main ] schedule: - cron: '0 2 * * 1-5' diff --git a/CONTRIBUTE.md b/CONTRIBUTE.md index 567247e4..76c93fd3 100644 --- a/CONTRIBUTE.md +++ b/CONTRIBUTE.md @@ -40,9 +40,9 @@ If it is large, such as suggesting a new repository, sub-repository, or interfac ### Your First Code Contribution If you are a new contributor, *thank you!* -Before your first merge, you will need to be added to the [CONTRIBUTORS](https://github.com/go-python/license/blob/master/CONTRIBUTORS) and [AUTHORS](https://github.com/go-python/license/blob/master/AUTHORS) files. +Before your first merge, you will need to be added to the [CONTRIBUTORS](https://github.com/go-python/license/blob/main/CONTRIBUTORS) and [AUTHORS](https://github.com/go-python/license/blob/main/AUTHORS) files. Open a pull request adding yourself to these files. -All `go-python` code follows the BSD license in the [license document](https://github.com/go-python/license/blob/master/LICENSE). +All `go-python` code follows the BSD license in the [license document](https://github.com/go-python/license/blob/main/LICENSE). We prefer that code contributions do not come with additional licensing. For exceptions, added code must also follow a BSD license. @@ -88,7 +88,7 @@ Please always format your code with [goimports](https://godoc.org/golang.org/x/t Best is to have it invoked as a hook when you save your `.go` files. Files in the `go-python` repository don't list author names, both to avoid clutter and to avoid having to keep the lists up to date. -Instead, your name will appear in the change log and in the [CONTRIBUTORS](https://github.com/go-python/license/blob/master/CONTRIBUTORS) and [AUTHORS](https://github.com/go-python/license/blob/master/AUTHORS) files. +Instead, your name will appear in the change log and in the [CONTRIBUTORS](https://github.com/go-python/license/blob/main/CONTRIBUTORS) and [AUTHORS](https://github.com/go-python/license/blob/main/AUTHORS) files. New files that you contribute should use the standard copyright header: diff --git a/README.md b/README.md index 802619bc..883ae510 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,9 @@ # gpython [![Build Status](https://github.com/go-python/gpython/workflows/CI/badge.svg)](https://github.com/go-python/gpython/actions) -[![codecov](https://codecov.io/gh/go-python/gpython/branch/master/graph/badge.svg)](https://codecov.io/gh/go-python/gpython) +[![codecov](https://codecov.io/gh/go-python/gpython/branch/main/graph/badge.svg)](https://codecov.io/gh/go-python/gpython) [![GoDoc](https://godoc.org/github.com/go-python/gpython?status.svg)](https://godoc.org/github.com/go-python/gpython) -[![License](https://img.shields.io/badge/License-BSD--3-blue.svg)](https://github.com/go-python/gpython/blob/master/LICENSE) +[![License](https://img.shields.io/badge/License-BSD--3-blue.svg)](https://github.com/go-python/gpython/blob/main/LICENSE) gpython is a part re-implementation, part port of the Python 3.4 interpreter in Go. Although there are many areas of improvement, @@ -54,7 +54,7 @@ gpython currently: - Supports concurrent multi-interpreter ("multi-context") execution Speed hasn't been a goal of the conversions however it runs pystone at -about 20% of the speed of CPython. A [π computation test](https://github.com/go-python/gpython/tree/master/examples/pi_chudnovsky_bs.py) runs quicker under +about 20% of the speed of CPython. A [π computation test](https://github.com/go-python/gpython/tree/main/examples/pi_chudnovsky_bs.py) runs quicker under gpython as the Go long integer primitives are likely faster than the Python ones. @@ -63,15 +63,15 @@ you know would be interested to take it futher, it would be much appreciated. ## Getting Started -The [embedding example](https://github.com/go-python/gpython/tree/master/examples/embedding) demonstrates how to +The [embedding example](https://github.com/go-python/gpython/tree/main/examples/embedding) demonstrates how to easily embed and invoke gpython from any Go application. Of interest, gpython is able to run multiple interpreter instances simultaneously, allowing you to embed gpython naturally into your Go application. This makes it possible to use gpython in a server situation where complete interpreter -independence is paramount. See this in action in the [multi-context example](https://github.com/go-python/gpython/tree/master/examples/multi-context). +independence is paramount. See this in action in the [multi-context example](https://github.com/go-python/gpython/tree/main/examples/multi-context). -If you are looking to get involved, a light and easy place to start is adding more convenience functions to [py/util.go](https://github.com/go-python/gpython/tree/master/py/util.go). See [notes.txt](https://github.com/go-python/gpython/blob/master/notes.txt) for bigger ideas. +If you are looking to get involved, a light and easy place to start is adding more convenience functions to [py/util.go](https://github.com/go-python/gpython/tree/main/py/util.go). See [notes.txt](https://github.com/go-python/gpython/blob/main/notes.txt) for bigger ideas. ## Other Projects of Interest @@ -88,4 +88,4 @@ or on the [Gophers Slack](https://gophers.slack.com/) in the `#go-python` channe This is licensed under the MIT licence, however it contains code which was ported fairly directly directly from the CPython source code under -the [PSF LICENSE](https://github.com/python/cpython/blob/master/LICENSE). +the [PSF LICENSE](https://github.com/python/cpython/blob/main/LICENSE). diff --git a/examples/embedding/README.md b/examples/embedding/README.md index 9c13d748..93a300aa 100644 --- a/examples/embedding/README.md +++ b/examples/embedding/README.md @@ -98,5 +98,5 @@ Spring Break itinerary: - `main.go` demonstrates high-level convenience functions such as `py.RunFile()`. - Embedding a Go `struct` as a Python object only requires that it implement `py.Object`, which is a single function: `Type() *py.Type` - - See [py/run.go](https://github.com/go-python/gpython/tree/master/py/run.go) for more about interpreter instances and `py.Context` - - Helper functions are available in [py/util.go](https://github.com/go-python/gpython/tree/master/py/util.go) and your contributions are welcome! + - See [py/run.go](https://github.com/go-python/gpython/tree/main/py/run.go) for more about interpreter instances and `py.Context` + - Helper functions are available in [py/util.go](https://github.com/go-python/gpython/tree/main/py/util.go) and your contributions are welcome! From e2204efc1bfe1fcc762007fe2d12819c13682571 Mon Sep 17 00:00:00 2001 From: Sebastien Binet Date: Fri, 6 May 2022 17:55:56 +0200 Subject: [PATCH 31/60] py: add str.replace Signed-off-by: Sebastien Binet --- py/string.go | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/py/string.go b/py/string.go index 58ab7ad0..33507970 100644 --- a/py/string.go +++ b/py/string.go @@ -122,6 +122,17 @@ func fieldsN(s string, n int) []string { } func init() { + StringType.Dict["replace"] = MustNewMethod("replace", func(self Object, args Tuple) (Object, error) { + return self.(String).Replace(args) + }, 0, `replace(self, old, new, count=-1) -> return a copy with all occurrences of substring old replaced by new. + + count + Maximum number of occurrences to replace. + -1 (the default value) means replace all occurrences. + +If the optional argument count is given, only the first count occurrences are +replaced.`) + StringType.Dict["split"] = MustNewMethod("split", func(self Object, args Tuple, kwargs StringDict) (Object, error) { return self.(String).Split(args, kwargs) }, 0, "split(sub) -> split string with sub.") @@ -598,6 +609,26 @@ func (s String) Split(args Tuple, kwargs StringDict) (Object, error) { return &o, nil } +func (s String) Replace(args Tuple) (Object, error) { + var ( + pyold Object = None + pynew Object = None + pycnt Object = Int(-1) + ) + err := ParseTuple(args, "ss|i:replace", &pyold, &pynew, &pycnt) + if err != nil { + return nil, err + } + + var ( + old = string(pyold.(String)) + new = string(pynew.(String)) + cnt = int(pycnt.(Int)) + ) + + return String(strings.Replace(string(s), old, new, cnt)), nil +} + // Check stringerface is satisfied var ( _ richComparison = String("") From 9d705975c303b6b8fb9afae542bfeab14f7f09ca Mon Sep 17 00:00:00 2001 From: Sebastien Binet Date: Fri, 6 May 2022 18:09:17 +0200 Subject: [PATCH 32/60] py: add bytes.replace Signed-off-by: Sebastien Binet --- py/bytes.go | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/py/bytes.go b/py/bytes.go index 7adf050e..55a69681 100644 --- a/py/bytes.go +++ b/py/bytes.go @@ -258,9 +258,43 @@ func (a Bytes) M__iadd__(other Object) (Object, error) { return NotImplemented, nil } +func (a Bytes) Replace(args Tuple) (Object, error) { + var ( + pyold Object = None + pynew Object = None + pycnt Object = Int(-1) + ) + err := ParseTuple(args, "yy|i:replace", &pyold, &pynew, &pycnt) + if err != nil { + return nil, err + } + + var ( + old = []byte(pyold.(Bytes)) + new = []byte(pynew.(Bytes)) + cnt = int(pycnt.(Int)) + ) + + return Bytes(bytes.Replace([]byte(a), old, new, cnt)), nil +} + // Check interface is satisfied var ( _ richComparison = (Bytes)(nil) _ I__add__ = (Bytes)(nil) _ I__iadd__ = (Bytes)(nil) ) + +func init() { + BytesType.Dict["replace"] = MustNewMethod("replace", func(self Object, args Tuple) (Object, error) { + return self.(Bytes).Replace(args) + }, 0, `replace(self, old, new, count=-1) -> return a copy with all occurrences of substring old replaced by new. + + count + Maximum number of occurrences to replace. + -1 (the default value) means replace all occurrences. + +If the optional argument count is given, only the first count occurrences are +replaced.`) + +} From 6c9f9e58775387175c21017d3285a37d48317053 Mon Sep 17 00:00:00 2001 From: Sebastien Binet Date: Fri, 6 May 2022 17:13:43 +0200 Subject: [PATCH 33/60] stdlib/glob: first import Signed-off-by: Sebastien Binet --- stdlib/glob/glob.go | 64 ++++++++++++++++++++++++++++ stdlib/glob/glob_test.go | 15 +++++++ stdlib/glob/testdata/test.py | 58 +++++++++++++++++++++++++ stdlib/glob/testdata/test_golden.txt | 0 stdlib/stdlib.go | 1 + 5 files changed, 138 insertions(+) create mode 100644 stdlib/glob/glob.go create mode 100644 stdlib/glob/glob_test.go create mode 100644 stdlib/glob/testdata/test.py create mode 100644 stdlib/glob/testdata/test_golden.txt diff --git a/stdlib/glob/glob.go b/stdlib/glob/glob.go new file mode 100644 index 00000000..173a2493 --- /dev/null +++ b/stdlib/glob/glob.go @@ -0,0 +1,64 @@ +// Copyright 2022 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package glob provides the implementation of the python's 'glob' module. +package glob + +import ( + "path/filepath" + + "github.com/go-python/gpython/py" +) + +func init() { + py.RegisterModule(&py.ModuleImpl{ + Info: py.ModuleInfo{ + Name: "glob", + Doc: "Filename globbing utility.", + }, + Methods: []*py.Method{ + py.MustNewMethod("glob", glob, 0, glob_doc), + }, + }) +} + +const glob_doc = `Return a list of paths matching a pathname pattern. +The pattern may contain simple shell-style wildcards a la +fnmatch. However, unlike fnmatch, filenames starting with a +dot are special cases that are not matched by '*' and '?' +patterns.` + +func glob(self py.Object, args py.Tuple) (py.Object, error) { + var ( + pypathname py.Object + ) + err := py.ParseTuple(args, "s*:glob", &pypathname) + if err != nil { + return nil, err + } + + var ( + pathname string + cnv func(v string) py.Object + ) + switch n := pypathname.(type) { + case py.String: + pathname = string(n) + cnv = func(v string) py.Object { return py.String(v) } + case py.Bytes: + pathname = string(n) + cnv = func(v string) py.Object { return py.Bytes(v) } + } + matches, err := filepath.Glob(pathname) + if err != nil { + return nil, err + } + + lst := py.List{Items: make([]py.Object, len(matches))} + for i, v := range matches { + lst.Items[i] = cnv(v) + } + + return &lst, nil +} diff --git a/stdlib/glob/glob_test.go b/stdlib/glob/glob_test.go new file mode 100644 index 00000000..c90cb091 --- /dev/null +++ b/stdlib/glob/glob_test.go @@ -0,0 +1,15 @@ +// Copyright 2022 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package glob_test + +import ( + "testing" + + "github.com/go-python/gpython/pytest" +) + +func TestGlob(t *testing.T) { + pytest.RunScript(t, "./testdata/test.py") +} diff --git a/stdlib/glob/testdata/test.py b/stdlib/glob/testdata/test.py new file mode 100644 index 00000000..042b6b3b --- /dev/null +++ b/stdlib/glob/testdata/test.py @@ -0,0 +1,58 @@ +# Copyright 2022 The go-python Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +import glob + +def norm(vs): + if len(vs) == 0: + return vs + if type(vs[0]) == type(""): + return normStr(vs) + return normBytes(vs) + +def normStr(vs): + from os import sep + x = [] + for v in vs: + x.append(v.replace('/', sep)) + return x + +def normBytes(vs): + from os import sep + x = [] + for v in vs: + x.append(v.replace(b'/', bytes(sep, encoding="utf-8"))) + return x + +def assertEqual(x, y): + xx = norm(x) + yy = norm(y) + assert xx == yy, "got: %s, want: %s" % (repr(x), repr(y)) + + +## test strings +assertEqual(glob.glob('*'), ["glob.go", "glob_test.go", "testdata"]) +assertEqual(glob.glob('*test*'), ["glob_test.go", "testdata"]) +assertEqual(glob.glob('*/test*'), ["testdata/test.py", "testdata/test_golden.txt"]) +assertEqual(glob.glob('*/test*_*'), ["testdata/test_golden.txt"]) +assertEqual(glob.glob('*/t??t*_*'), ["testdata/test_golden.txt"]) +assertEqual(glob.glob('*/t[e]?t*_*'), ["testdata/test_golden.txt"]) +assertEqual(glob.glob('*/t[oe]?t*_*'), ["testdata/test_golden.txt"]) +assertEqual(glob.glob('*/t[o]?t*_*'), []) + +## FIXME(sbinet) +## assertEqual(glob.glob('*/t[!o]?t*_*'), ["testdata/test_golden.txt"]) + +## test bytes +assertEqual(glob.glob(b'*'), [b"glob.go", b"glob_test.go", b"testdata"]) +assertEqual(glob.glob(b'*test*'), [b"glob_test.go", b"testdata"]) +assertEqual(glob.glob(b'*/test*'), [b"testdata/test.py", b"testdata/test_golden.txt"]) +assertEqual(glob.glob(b'*/test*_*'), [b"testdata/test_golden.txt"]) +assertEqual(glob.glob(b'*/t??t*_*'), [b"testdata/test_golden.txt"]) +assertEqual(glob.glob(b'*/t[e]?t*_*'), [b"testdata/test_golden.txt"]) +assertEqual(glob.glob(b'*/t[oe]?t*_*'), [b"testdata/test_golden.txt"]) +assertEqual(glob.glob(b'*/t[o]?t*_*'), []) + +## FIXME(sbinet) +## assertEqual(glob.glob(b'*/t[!o]?t*_*'), [b"testdata/test_golden.txt"]) diff --git a/stdlib/glob/testdata/test_golden.txt b/stdlib/glob/testdata/test_golden.txt new file mode 100644 index 00000000..e69de29b diff --git a/stdlib/stdlib.go b/stdlib/stdlib.go index d69268d3..c78c2209 100644 --- a/stdlib/stdlib.go +++ b/stdlib/stdlib.go @@ -20,6 +20,7 @@ import ( _ "github.com/go-python/gpython/stdlib/binascii" _ "github.com/go-python/gpython/stdlib/builtin" + _ "github.com/go-python/gpython/stdlib/glob" _ "github.com/go-python/gpython/stdlib/math" _ "github.com/go-python/gpython/stdlib/os" _ "github.com/go-python/gpython/stdlib/string" From ab6c445f6337f4b8a423e00fa11a8fb45fe6fdfd Mon Sep 17 00:00:00 2001 From: Sebastien Binet Date: Tue, 24 May 2022 19:05:17 +0200 Subject: [PATCH 34/60] py: add String.find Updates #139. Signed-off-by: Sebastien Binet --- py/string.go | 89 +++++++++++++++++++++++++++++++++-------------- py/string_test.go | 79 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 142 insertions(+), 26 deletions(-) create mode 100644 py/string_test.go diff --git a/py/string.go b/py/string.go index 33507970..9e50d2dc 100644 --- a/py/string.go +++ b/py/string.go @@ -122,6 +122,43 @@ func fieldsN(s string, n int) []string { } func init() { + StringType.Dict["endswith"] = MustNewMethod("endswith", func(self Object, args Tuple) (Object, error) { + selfStr := string(self.(String)) + suffix := []string{} + if len(args) > 0 { + if s, ok := args[0].(String); ok { + suffix = append(suffix, string(s)) + } else if s, ok := args[0].(Tuple); ok { + for _, t := range s { + if v, ok := t.(String); ok { + suffix = append(suffix, string(v)) + } + } + } else { + return nil, ExceptionNewf(TypeError, "endswith first arg must be str, unicode, or tuple, not %s", args[0].Type()) + } + } else { + return nil, ExceptionNewf(TypeError, "endswith() takes at least 1 argument (0 given)") + } + for _, s := range suffix { + if strings.HasSuffix(selfStr, s) { + return Bool(true), nil + } + } + return Bool(false), nil + }, 0, "endswith(suffix[, start[, end]]) -> bool") + + StringType.Dict["find"] = MustNewMethod("find", func(self Object, args Tuple) (Object, error) { + return self.(String).find(args) + }, 0, `find(...) +S.find(sub[, start[, end]]) -> int + +Return the lowest index in S where substring sub is found, +such that sub is contained within S[start:end]. Optional +arguments start and end are interpreted as in slice notation. + +Return -1 on failure.`) + StringType.Dict["replace"] = MustNewMethod("replace", func(self Object, args Tuple) (Object, error) { return self.(String).Replace(args) }, 0, `replace(self, old, new, count=-1) -> return a copy with all occurrences of substring old replaced by new. @@ -169,32 +206,6 @@ replaced.`) return Bool(false), nil }, 0, "startswith(prefix[, start[, end]]) -> bool") - StringType.Dict["endswith"] = MustNewMethod("endswith", func(self Object, args Tuple) (Object, error) { - selfStr := string(self.(String)) - suffix := []string{} - if len(args) > 0 { - if s, ok := args[0].(String); ok { - suffix = append(suffix, string(s)) - } else if s, ok := args[0].(Tuple); ok { - for _, t := range s { - if v, ok := t.(String); ok { - suffix = append(suffix, string(v)) - } - } - } else { - return nil, ExceptionNewf(TypeError, "endswith first arg must be str, unicode, or tuple, not %s", args[0].Type()) - } - } else { - return nil, ExceptionNewf(TypeError, "endswith() takes at least 1 argument (0 given)") - } - for _, s := range suffix { - if strings.HasSuffix(selfStr, s) { - return Bool(true), nil - } - } - return Bool(false), nil - }, 0, "endswith(suffix[, start[, end]]) -> bool") - } // Type of this object @@ -578,6 +589,32 @@ func (s String) M__contains__(item Object) (Object, error) { return NewBool(strings.Contains(string(s), string(needle))), nil } +func (s String) find(args Tuple) (Object, error) { + var ( + pysub Object + pybeg Object = Int(0) + pyend Object = Int(len(s)) + pyfmt = "s|ii:find" + ) + err := ParseTuple(args, pyfmt, &pysub, &pybeg, &pyend) + if err != nil { + return nil, err + } + + var ( + beg = int(pybeg.(Int)) + end = int(pyend.(Int)) + off = s.slice(0, beg, s.len()).len() + str = string(s.slice(beg, end, s.len())) + sub = string(pysub.(String)) + idx = strings.Index(str, sub) + ) + if idx < 0 { + return Int(idx), nil + } + return Int(off + String(str[:idx]).len()), nil +} + func (s String) Split(args Tuple, kwargs StringDict) (Object, error) { var ( pyval Object = None diff --git a/py/string_test.go b/py/string_test.go new file mode 100644 index 00000000..2eec9ba5 --- /dev/null +++ b/py/string_test.go @@ -0,0 +1,79 @@ +// Copyright 2022 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package py + +import "testing" + +func TestStringFind(t *testing.T) { + for _, tc := range []struct { + str string + sub string + beg int + end int + idx int + }{ + { + str: "hello world", + sub: "world", + idx: 6, + }, + { + str: "hello world", + sub: "o", + idx: 4, + }, + { + str: "hello world", + sub: "o", + beg: 5, + idx: 7, + }, + { + str: "hello world", + sub: "bye", + idx: -1, + }, + { + str: "Hello, 世界", + sub: "界", + idx: 8, + }, + { + str: "01234 6789", + sub: " ", + beg: 6, + idx: -1, + }, + { + str: "0123456789", + sub: "6", + beg: 1, + end: 6, + idx: -1, + }, + { + str: "0123456789", + sub: "6", + beg: 1, + end: 7, + idx: 6, + }, + } { + t.Run(tc.str+":"+tc.sub, func(t *testing.T) { + beg := tc.beg + end := tc.end + if end == 0 { + end = len(tc.str) + } + idx, err := String(tc.str).find(Tuple{String(tc.sub), Int(beg), Int(end)}) + if err != nil { + t.Fatalf("invalid: %+v", err) + } + if got, want := int(idx.(Int)), tc.idx; got != want { + t.Fatalf("got=%d, want=%d", got, want) + } + }) + } +} From e4b6e829e68eb0ce2cde6a25edada9e34bc467bb Mon Sep 17 00:00:00 2001 From: Sebastien Binet Date: Tue, 24 May 2022 19:19:58 +0200 Subject: [PATCH 35/60] py: handle end=-1 in str.find Signed-off-by: Sebastien Binet --- py/string.go | 8 +++++++- py/string_test.go | 7 +++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/py/string.go b/py/string.go index 9e50d2dc..df49253e 100644 --- a/py/string.go +++ b/py/string.go @@ -593,7 +593,7 @@ func (s String) find(args Tuple) (Object, error) { var ( pysub Object pybeg Object = Int(0) - pyend Object = Int(len(s)) + pyend Object = Int(s.len()) pyfmt = "s|ii:find" ) err := ParseTuple(args, pyfmt, &pysub, &pybeg, &pyend) @@ -604,6 +604,12 @@ func (s String) find(args Tuple) (Object, error) { var ( beg = int(pybeg.(Int)) end = int(pyend.(Int)) + ) + if end < 0 { + end = s.len() + } + + var ( off = s.slice(0, beg, s.len()).len() str = string(s.slice(beg, end, s.len())) sub = string(pysub.(String)) diff --git a/py/string_test.go b/py/string_test.go index 2eec9ba5..371e1324 100644 --- a/py/string_test.go +++ b/py/string_test.go @@ -60,6 +60,13 @@ func TestStringFind(t *testing.T) { end: 7, idx: 6, }, + { + str: "0123456789", + sub: "6", + beg: 1, + end: -1, + idx: 6, + }, } { t.Run(tc.str+":"+tc.sub, func(t *testing.T) { beg := tc.beg From 4b114ff8e742115cffa80d334524d704a3b34727 Mon Sep 17 00:00:00 2001 From: Sebastien Binet Date: Tue, 24 May 2022 19:23:34 +0200 Subject: [PATCH 36/60] py: handle corner cases for str.find Signed-off-by: Sebastien Binet --- py/string.go | 13 ++++++++++--- py/string_test.go | 14 ++++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/py/string.go b/py/string.go index df49253e..2c9854bc 100644 --- a/py/string.go +++ b/py/string.go @@ -602,11 +602,18 @@ func (s String) find(args Tuple) (Object, error) { } var ( - beg = int(pybeg.(Int)) - end = int(pyend.(Int)) + beg = int(pybeg.(Int)) + end = int(pyend.(Int)) + size = s.len() ) + if beg > size { + beg = size + } if end < 0 { - end = s.len() + end = size + } + if end > size { + end = size } var ( diff --git a/py/string_test.go b/py/string_test.go index 371e1324..7f6e0c34 100644 --- a/py/string_test.go +++ b/py/string_test.go @@ -67,6 +67,20 @@ func TestStringFind(t *testing.T) { end: -1, idx: 6, }, + { + str: "0123456789", + sub: "6", + beg: 100, + end: -1, + idx: -1, + }, + { + str: "0123456789", + sub: "6", + beg: 2, + end: 1, + idx: -1, + }, } { t.Run(tc.str+":"+tc.sub, func(t *testing.T) { beg := tc.beg From 795df15d5e0e3cf40c46fa3a57b6820a0deb7a8a Mon Sep 17 00:00:00 2001 From: Sebastien Binet Date: Wed, 25 May 2022 16:19:53 +0200 Subject: [PATCH 37/60] stdlib/os: add mkdir, makedirs, remove, removedirs and rmdir Signed-off-by: Sebastien Binet --- stdlib/os/os.go | 216 ++++++++++++++++++++++++++++- stdlib/os/testdata/test.py | 49 +++++++ stdlib/os/testdata/test_golden.txt | 4 + 3 files changed, 267 insertions(+), 2 deletions(-) diff --git a/stdlib/os/os.go b/stdlib/os/os.go index d863575f..4520537e 100644 --- a/stdlib/os/os.go +++ b/stdlib/os/os.go @@ -44,15 +44,20 @@ func init() { initGlobals() methods := []*py.Method{ + py.MustNewMethod("_exit", _exit, 0, "Immediate program termination."), py.MustNewMethod("getcwd", getCwd, 0, "Get the current working directory"), py.MustNewMethod("getcwdb", getCwdb, 0, "Get the current working directory in a byte slice"), py.MustNewMethod("chdir", chdir, 0, "Change the current working directory"), py.MustNewMethod("getenv", getenv, 0, "Return the value of the environment variable key if it exists, or default if it doesn’t. key, default and the result are str."), py.MustNewMethod("getpid", getpid, 0, "Return the current process id."), + py.MustNewMethod("makedirs", makedirs, 0, makedirs_doc), + py.MustNewMethod("mkdir", mkdir, 0, mkdir_doc), py.MustNewMethod("putenv", putenv, 0, "Set the environment variable named key to the string value."), + py.MustNewMethod("remove", remove, 0, remove_doc), + py.MustNewMethod("removedirs", removedirs, 0, removedirs_doc), + py.MustNewMethod("rmdir", rmdir, 0, rmdir_doc), + py.MustNewMethod("system", system, 0, "Run shell commands, prints stdout directly to default"), py.MustNewMethod("unsetenv", unsetenv, 0, "Unset (delete) the environment variable named key."), - py.MustNewMethod("_exit", _exit, 0, "Immediate program termination."), - py.MustNewMethod("system", system, 0, "Run shell commands, prints stdout directly to deault"), } globals := py.StringDict{ "error": py.OSError, @@ -150,6 +155,105 @@ func getpid(self py.Object, args py.Tuple) (py.Object, error) { return py.Int(os.Getpid()), nil } +const makedirs_doc = `makedirs(name [, mode=0o777][, exist_ok=False]) + +Super-mkdir; create a leaf directory and all intermediate ones. Works like +mkdir, except that any intermediate path segment (not just the rightmost) +will be created if it does not exist. If the target directory already +exists, raise an OSError if exist_ok is False. Otherwise no exception is +raised. This is recursive.` + +func makedirs(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Object, error) { + var ( + pypath py.Object + pymode py.Object = py.Int(0o777) + pyok py.Object = py.False + ) + err := py.ParseTupleAndKeywords( + args, kwargs, + "s#|ip:makedirs", []string{"path", "mode", "exist_ok"}, + &pypath, &pymode, &pyok, + ) + if err != nil { + return nil, err + } + + var ( + path = "" + mode = os.FileMode(pymode.(py.Int)) + ) + switch v := pypath.(type) { + case py.String: + path = string(v) + case py.Bytes: + path = string(v) + } + + if pyok.(py.Bool) == py.False { + // check if leaf exists. + _, err := os.Stat(path) + // FIXME(sbinet): handle other errors. + if err == nil { + return nil, py.ExceptionNewf(py.FileExistsError, "File exists: '%s'", path) + } + } + + err = os.MkdirAll(path, mode) + if err != nil { + return nil, err + } + + return py.None, nil +} + +const mkdir_doc = `Create a directory. + +If dir_fd is not None, it should be a file descriptor open to a directory, + and path should be relative; path will then be relative to that directory. +dir_fd may not be implemented on your platform. + If it is unavailable, using it will raise a NotImplementedError. + +The mode argument is ignored on Windows.` + +func mkdir(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Object, error) { + var ( + pypath py.Object + pymode py.Object = py.Int(511) + pydirfd py.Object = py.None + ) + err := py.ParseTupleAndKeywords( + args, kwargs, + "s#|ii:mkdir", []string{"path", "mode", "dir_fd"}, + &pypath, &pymode, &pydirfd, + ) + if err != nil { + return nil, err + } + + var ( + path = "" + mode = os.FileMode(pymode.(py.Int)) + ) + switch v := pypath.(type) { + case py.String: + path = string(v) + case py.Bytes: + path = string(v) + } + + if pydirfd != py.None { + // FIXME(sbinet) + return nil, py.ExceptionNewf(py.NotImplementedError, "mkdir(dir_fd=XXX) not implemented") + } + + err = os.Mkdir(path, mode) + if err != nil { + return nil, err + } + + return py.None, nil +} + // putenv sets the value of an environment variable named by the key. func putenv(self py.Object, args py.Tuple) (py.Object, error) { if len(args) != 2 { @@ -199,6 +303,114 @@ func _exit(self py.Object, args py.Tuple) (py.Object, error) { // can never retu return nil, nil } +const remove_doc = `Remove a file (same as unlink()). + +If dir_fd is not None, it should be a file descriptor open to a directory, + and path should be relative; path will then be relative to that directory. +dir_fd may not be implemented on your platform. + If it is unavailable, using it will raise a NotImplementedError.` + +func remove(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Object, error) { + var ( + pypath py.Object + pydir py.Object = py.None + ) + err := py.ParseTupleAndKeywords(args, kwargs, "s#|i:remove", []string{"path", "dir_fd"}, &pypath, &pydir) + if err != nil { + return nil, err + } + + if pydir != py.None { + // FIXME(sbinet) ? + return nil, py.ExceptionNewf(py.NotImplementedError, "remove(dir_fd=XXX) not implemented") + } + + var name string + switch v := pypath.(type) { + case py.String: + name = string(v) + case py.Bytes: + name = string(v) + } + + err = os.Remove(name) + if err != nil { + return nil, err + } + + return py.None, nil +} + +const removedirs_doc = `removedirs(name) + +Super-rmdir; remove a leaf directory and all empty intermediate +ones. Works like rmdir except that, if the leaf directory is +successfully removed, directories corresponding to rightmost path +segments will be pruned away until either the whole path is +consumed or an error occurs. Errors during this latter phase are +ignored -- they generally mean that a directory was not empty.` + +func removedirs(self py.Object, args py.Tuple) (py.Object, error) { + var pypath py.Object + err := py.ParseTuple(args, "s#:rmdir", &pypath) + if err != nil { + return nil, err + } + + var name string + switch v := pypath.(type) { + case py.String: + name = string(v) + case py.Bytes: + name = string(v) + } + + err = os.RemoveAll(name) + if err != nil { + return nil, err + } + + return py.None, nil +} + +const rmdir_doc = `Remove a directory. + +If dir_fd is not None, it should be a file descriptor open to a directory, + and path should be relative; path will then be relative to that directory. +dir_fd may not be implemented on your platform. + If it is unavailable, using it will raise a NotImplementedError.` + +func rmdir(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Object, error) { + var ( + pypath py.Object + pydir py.Object = py.None + ) + err := py.ParseTupleAndKeywords(args, kwargs, "s#|i:rmdir", []string{"path", "dir_fd"}, &pypath, &pydir) + if err != nil { + return nil, err + } + + if pydir != py.None { + // FIXME(sbinet) ? + return nil, py.ExceptionNewf(py.NotImplementedError, "rmdir(dir_fd=XXX) not implemented") + } + + var name string + switch v := pypath.(type) { + case py.String: + name = string(v) + case py.Bytes: + name = string(v) + } + + err = os.Remove(name) + if err != nil { + return nil, err + } + + return py.None, nil +} + // os.system(command string) this function runs a shell command and directs the output to standard output. func system(self py.Object, args py.Tuple) (py.Object, error) { if len(args) != 1 { diff --git a/stdlib/os/testdata/test.py b/stdlib/os/testdata/test.py index 100f48df..8eb2ec1a 100644 --- a/stdlib/os/testdata/test.py +++ b/stdlib/os/testdata/test.py @@ -118,4 +118,53 @@ else: print("os."+k+": [OK]") +## mkdir,rmdir,remove,removedirs +import tempfile +try: + top = tempfile.mkdtemp(prefix="gpython-os-test-") + dir1 = top + os.sep + "dir1" + dir2 = top + os.sep + "dir2" + dir11 = top + os.sep + "dir1" + os.sep + "dir11" + fname = dir2 + os.sep + "foo.txt" + os.mkdir(dir1) + os.rmdir(dir1) + os.mkdir(dir1) + os.mkdir(dir2) + os.mkdir(dir11) + os.removedirs(dir1) + try: + os.mkdir(dir11) + print("creating nested dirs with os.mkdir should have failed") + except SystemError as e: + print("caught: SystemError - no such file or directory [OK]") + except Exception as e: + print("caught: %s" % e) + + os.makedirs(dir11) + try: + os.makedirs(dir11) + print("creating already existing dirs should have failed") + except FileExistsError as e: + print("caught: FileExistsError [OK]") + except Exception as e: + print("INVALID error caught: %s" % e) + os.makedirs(dir11, exist_ok=True) + + with open(fname, "w+") as f: + pass + try: + os.rmdir(dir2) + print("removing a non-empty directory should have failed") + except SystemError as e: + print("caught: SystemError - directory not empty [OK]") + except Exception as e: + print("INVALID error caught: %s" % e) + os.remove(fname) + os.rmdir(dir2) +except Exception as e: + print("could not create/remove directories: %s" % e) +finally: + os.removedirs(top) + print("os.{mkdir,rmdir,remove,removedirs} worked as expected") + print("OK") diff --git a/stdlib/os/testdata/test_golden.txt b/stdlib/os/testdata/test_golden.txt index 8aae577d..43c83011 100644 --- a/stdlib/os/testdata/test_golden.txt +++ b/stdlib/os/testdata/test_golden.txt @@ -24,4 +24,8 @@ os.pathsep: [OK] os.linesep: [OK] os.devnull: [OK] os.altsep: [OK] +caught: SystemError - no such file or directory [OK] +caught: FileExistsError [OK] +caught: SystemError - directory not empty [OK] +os.{mkdir,rmdir,remove,removedirs} worked as expected OK From 88633c070c748c4e65733bf4866a4a5b73ef8ecc Mon Sep 17 00:00:00 2001 From: Sebastien Binet Date: Wed, 25 May 2022 17:51:08 +0200 Subject: [PATCH 38/60] py: introduce FileModeFrom Signed-off-by: Sebastien Binet --- py/file.go | 131 +++++++++++++++++++++++++++-------------------------- 1 file changed, 67 insertions(+), 64 deletions(-) diff --git a/py/file.go b/py/file.go index c60013ad..3d9f0185 100644 --- a/py/file.go +++ b/py/file.go @@ -166,71 +166,10 @@ func (o *File) M__exit__(exc_type, exc_value, traceback Object) (Object, error) } func OpenFile(filename, mode string, buffering int) (Object, error) { - var fileMode FileMode - var truncate bool - var exclusive bool - - for _, m := range mode { - switch m { - case 'r': - if fileMode&FileReadWrite != 0 { - return nil, ExceptionNewf(ValueError, "must have exactly one of create/read/write/append mode") - } - fileMode |= FileRead - - case 'w': - if fileMode&FileReadWrite != 0 { - return nil, ExceptionNewf(ValueError, "must have exactly one of create/read/write/append mode") - } - fileMode |= FileWrite - truncate = true - - case 'x': - if fileMode&FileReadWrite != 0 { - return nil, ExceptionNewf(ValueError, "must have exactly one of create/read/write/append mode") - } - fileMode |= FileWrite - exclusive = true - - case 'a': - if fileMode&FileReadWrite != 0 { - return nil, ExceptionNewf(ValueError, "must have exactly one of create/read/write/append mode") - } - fileMode |= FileWrite - truncate = false - - case '+': - if fileMode&FileReadWrite == 0 { - return nil, ExceptionNewf(ValueError, "Must have exactly one of create/read/write/append mode and at most one plus") - } - - truncate = (fileMode & FileWrite) != 0 - fileMode |= FileReadWrite - - case 'b': - if fileMode&FileReadWrite == 0 { - return nil, ExceptionNewf(ValueError, "Must have exactly one of create/read/write/append mode and at most one plus") - } - - if fileMode&FileText != 0 { - return nil, ExceptionNewf(ValueError, "can't have text and binary mode at once") - } - - fileMode |= FileBinary - - case 't': - if fileMode&FileReadWrite == 0 { - return nil, ExceptionNewf(ValueError, "Must have exactly one of create/read/write/append mode and at most one plus") - } - - if fileMode&FileBinary != 0 { - return nil, ExceptionNewf(ValueError, "can't have text and binary mode at once") - } - - fileMode |= FileText - } + fileMode, truncate, exclusive, err := FileModeFrom(mode) + if err != nil { + return nil, err } - var fmode int switch fileMode & FileReadWrite { @@ -280,3 +219,67 @@ func OpenFile(filename, mode string, buffering int) (Object, error) { // Check interface is satisfied var _ I__enter__ = (*File)(nil) var _ I__exit__ = (*File)(nil) + +func FileModeFrom(mode string) (perm FileMode, trunc, excl bool, err error) { + for _, m := range mode { + switch m { + case 'r': + if perm&FileReadWrite != 0 { + return 0, false, false, ExceptionNewf(ValueError, "must have exactly one of create/read/write/append mode") + } + perm |= FileRead + + case 'w': + if perm&FileReadWrite != 0 { + return 0, false, false, ExceptionNewf(ValueError, "must have exactly one of create/read/write/append mode") + } + perm |= FileWrite + trunc = true + + case 'x': + if perm&FileReadWrite != 0 { + return 0, false, false, ExceptionNewf(ValueError, "must have exactly one of create/read/write/append mode") + } + perm |= FileWrite + excl = true + + case 'a': + if perm&FileReadWrite != 0 { + return 0, false, false, ExceptionNewf(ValueError, "must have exactly one of create/read/write/append mode") + } + perm |= FileWrite + trunc = false + + case '+': + if perm&FileReadWrite == 0 { + return 0, false, false, ExceptionNewf(ValueError, "Must have exactly one of create/read/write/append mode and at most one plus") + } + + trunc = (perm & FileWrite) != 0 + perm |= FileReadWrite + + case 'b': + if perm&FileReadWrite == 0 { + return 0, false, false, ExceptionNewf(ValueError, "Must have exactly one of create/read/write/append mode and at most one plus") + } + + if perm&FileText != 0 { + return 0, false, false, ExceptionNewf(ValueError, "can't have text and binary mode at once") + } + + perm |= FileBinary + + case 't': + if perm&FileReadWrite == 0 { + return 0, false, false, ExceptionNewf(ValueError, "Must have exactly one of create/read/write/append mode and at most one plus") + } + + if perm&FileBinary != 0 { + return 0, false, false, ExceptionNewf(ValueError, "can't have text and binary mode at once") + } + + perm |= FileText + } + } + return perm, trunc, excl, nil +} From fda1751ef78e0b3a20ffff55bb670720ce6d4989 Mon Sep 17 00:00:00 2001 From: Sebastien Binet Date: Wed, 25 May 2022 17:52:42 +0200 Subject: [PATCH 39/60] stdlib/os: implement fdopen Signed-off-by: Sebastien Binet --- stdlib/os/os.go | 49 ++++++++++++++++++++++++++++++++++++++ stdlib/os/testdata/test.py | 9 +++++++ 2 files changed, 58 insertions(+) diff --git a/stdlib/os/os.go b/stdlib/os/os.go index 4520537e..e8b3ec7a 100644 --- a/stdlib/os/os.go +++ b/stdlib/os/os.go @@ -9,6 +9,7 @@ import ( "os" "os/exec" "runtime" + "strconv" "strings" "github.com/go-python/gpython/py" @@ -45,6 +46,7 @@ func init() { methods := []*py.Method{ py.MustNewMethod("_exit", _exit, 0, "Immediate program termination."), + py.MustNewMethod("fdopen", fdopen, 0, fdopen_doc), py.MustNewMethod("getcwd", getCwd, 0, "Get the current working directory"), py.MustNewMethod("getcwdb", getCwdb, 0, "Get the current working directory in a byte slice"), py.MustNewMethod("chdir", chdir, 0, "Change the current working directory"), @@ -96,6 +98,53 @@ func getEnvVariables() py.StringDict { return dict } +const fdopen_doc = `# Supply os.fdopen()` + +func fdopen(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Object, error) { + var ( + pyfd py.Object + pymode py.Object = py.String("r") + pybuffering py.Object = py.Int(-1) + pyencoding py.Object = py.None + ) + err := py.ParseTupleAndKeywords( + args, kwargs, + "i|s#is#", []string{"fd", "mode", "buffering", "encoding"}, + &pyfd, &pymode, &pybuffering, &pyencoding, + ) + if err != nil { + return nil, err + } + + // FIXME(sbinet): handle buffering + // FIXME(sbinet): handle encoding + + var ( + fd = uintptr(pyfd.(py.Int)) + name = strconv.Itoa(int(fd)) + mode string + ) + + switch v := pymode.(type) { + case py.String: + mode = string(v) + case py.Bytes: + mode = string(v) + } + + perm, _, _, err := py.FileModeFrom(mode) + if err != nil { + return nil, err + } + + f := os.NewFile(fd, name) + if f == nil { + return nil, py.ExceptionNewf(py.OSError, "Bad file descriptor") + } + + return &py.File{f, perm}, nil +} + // getCwd returns the current working directory. func getCwd(self py.Object, args py.Tuple) (py.Object, error) { dir, err := os.Getwd() diff --git a/stdlib/os/testdata/test.py b/stdlib/os/testdata/test.py index 8eb2ec1a..bdbf606e 100644 --- a/stdlib/os/testdata/test.py +++ b/stdlib/os/testdata/test.py @@ -118,6 +118,15 @@ else: print("os."+k+": [OK]") +## fdopen +import tempfile +fd, tmp = tempfile.mkstemp() +f = os.fdopen(fd, "w+") +## if f.name != str(fd): +## print("invalid fd-name:", f.name) +f.close() +os.remove(tmp) + ## mkdir,rmdir,remove,removedirs import tempfile try: From 3e1daa8eeb7ee1983931b4cc5a2567de4f2bd3a7 Mon Sep 17 00:00:00 2001 From: Sebastien Binet Date: Wed, 25 May 2022 16:19:39 +0200 Subject: [PATCH 40/60] stdlib/tempfile: first import Signed-off-by: Sebastien Binet --- stdlib/stdlib.go | 1 + stdlib/tempfile/tempfile.go | 283 +++++++++++++++++++++++ stdlib/tempfile/tempfile_test.go | 15 ++ stdlib/tempfile/testdata/test.py | 110 +++++++++ stdlib/tempfile/testdata/test_golden.txt | 13 ++ 5 files changed, 422 insertions(+) create mode 100644 stdlib/tempfile/tempfile.go create mode 100644 stdlib/tempfile/tempfile_test.go create mode 100644 stdlib/tempfile/testdata/test.py create mode 100644 stdlib/tempfile/testdata/test_golden.txt diff --git a/stdlib/stdlib.go b/stdlib/stdlib.go index c78c2209..7061f486 100644 --- a/stdlib/stdlib.go +++ b/stdlib/stdlib.go @@ -25,6 +25,7 @@ import ( _ "github.com/go-python/gpython/stdlib/os" _ "github.com/go-python/gpython/stdlib/string" _ "github.com/go-python/gpython/stdlib/sys" + _ "github.com/go-python/gpython/stdlib/tempfile" _ "github.com/go-python/gpython/stdlib/time" ) diff --git a/stdlib/tempfile/tempfile.go b/stdlib/tempfile/tempfile.go new file mode 100644 index 00000000..c31612da --- /dev/null +++ b/stdlib/tempfile/tempfile.go @@ -0,0 +1,283 @@ +// Copyright 2022 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package tempfile provides the implementation of the python's 'tempfile' module. +package tempfile + +import ( + "fmt" + "os" + + "github.com/go-python/gpython/py" +) + +var ( + gblTempDir py.Object = py.None +) + +const tempfile_doc = `Temporary files. + +This module provides generic, low- and high-level interfaces for +creating temporary files and directories. All of the interfaces +provided by this module can be used without fear of race conditions +except for 'mktemp'. 'mktemp' is subject to race conditions and +should not be used; it is provided for backward compatibility only. + +The default path names are returned as str. If you supply bytes as +input, all return values will be in bytes. Ex: + + >>> tempfile.mkstemp() + (4, '/tmp/tmptpu9nin8') + >>> tempfile.mkdtemp(suffix=b'') + b'/tmp/tmppbi8f0hy' + +This module also provides some data items to the user: + + TMP_MAX - maximum number of names that will be tried before + giving up. + tempdir - If this is set to a string before the first use of + any routine from this module, it will be considered as + another candidate location to store temporary files.` + +func init() { + py.RegisterModule(&py.ModuleImpl{ + Info: py.ModuleInfo{ + Name: "tempfile", + Doc: tempfile_doc, + }, + Methods: []*py.Method{ + py.MustNewMethod("gettempdir", gettempdir, 0, gettempdir_doc), + py.MustNewMethod("gettempdirb", gettempdirb, 0, gettempdirb_doc), + py.MustNewMethod("mkdtemp", mkdtemp, 0, mkdtemp_doc), + py.MustNewMethod("mkstemp", mkstemp, 0, mkstemp_doc), + }, + Globals: py.StringDict{ + "tempdir": gblTempDir, + }, + }) +} + +const gettempdir_doc = `Returns tempfile.tempdir as str.` + +func gettempdir(self py.Object) (py.Object, error) { + // FIXME(sbinet): lock access to glbTempDir? + if gblTempDir != py.None { + switch dir := gblTempDir.(type) { + case py.String: + return dir, nil + case py.Bytes: + return py.String(dir), nil + default: + return nil, py.ExceptionNewf(py.TypeError, "expected str, bytes or os.PathLike object, not %s", dir.Type().Name) + } + } + return py.String(os.TempDir()), nil +} + +const gettempdirb_doc = `Returns tempfile.tempdir as bytes.` + +func gettempdirb(self py.Object) (py.Object, error) { + // FIXME(sbinet): lock access to glbTempDir? + if gblTempDir != py.None { + switch dir := gblTempDir.(type) { + case py.String: + return py.Bytes(dir), nil + case py.Bytes: + return dir, nil + default: + return nil, py.ExceptionNewf(py.TypeError, "expected str, bytes or os.PathLike object, not %s", dir.Type().Name) + } + } + return py.Bytes(os.TempDir()), nil +} + +const mkdtemp_doc = `mkdtemp(suffix=None, prefix=None, dir=None) + User-callable function to create and return a unique temporary + directory. The return value is the pathname of the directory. + + Arguments are as for mkstemp, except that the 'text' argument is + not accepted. + + The directory is readable, writable, and searchable only by the + creating user. + + Caller is responsible for deleting the directory when done with it.` + +func mkdtemp(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Object, error) { + var ( + pysuffix py.Object = py.None + pyprefix py.Object = py.None + pydir py.Object = py.None + ) + err := py.ParseTupleAndKeywords(args, kwargs, + "|z#z#z#:mkdtemp", + []string{"suffix", "prefix", "dir"}, + &pysuffix, &pyprefix, &pydir, + ) + if err != nil { + return nil, err + } + + str := func(v py.Object, typ *uint8) string { + switch v := v.(type) { + case py.Bytes: + *typ = 2 + return string(v) + case py.String: + *typ = 1 + return string(v) + case py.NoneType: + *typ = 0 + return "" + default: + panic(fmt.Errorf("tempfile: invalid type %T (v=%+v)", v, v)) + } + } + + var ( + t1, t2, t3 uint8 + + suffix = str(pysuffix, &t1) + prefix = str(pyprefix, &t2) + dir = str(pydir, &t3) + pattern = prefix + "*" + suffix + ) + + cmp := func(t1, t2 uint8) bool { + if t1 > 0 && t2 > 0 { + return t1 == t2 + } + return true + } + + if !cmp(t1, t2) || !cmp(t1, t3) || !cmp(t2, t3) { + return nil, py.ExceptionNewf(py.TypeError, "Can't mix bytes and non-bytes in path components") + } + + tmp, err := os.MkdirTemp(dir, pattern) + if err != nil { + return nil, err + } + + typ := t1 + if typ == 0 { + typ = t2 + } + if typ == 0 { + typ = t3 + } + + switch typ { + case 2: + return py.Bytes(tmp), nil + default: + return py.String(tmp), nil + } +} + +const mkstemp_doc = `mkstemp(suffix=None, prefix=None, dir=None, text=False) + +User-callable function to create and return a unique temporary +file. The return value is a pair (fd, name) where fd is the +file descriptor returned by os.open, and name is the filename. + +If 'suffix' is not None, the file name will end with that suffix, +otherwise there will be no suffix. + +If 'prefix' is not None, the file name will begin with that prefix, +otherwise a default prefix is used. + +If 'dir' is not None, the file will be created in that directory, +otherwise a default directory is used. + +If 'text' is specified and true, the file is opened in text +mode. Else (the default) the file is opened in binary mode. + +If any of 'suffix', 'prefix' and 'dir' are not None, they must be the +same type. If they are bytes, the returned name will be bytes; str +otherwise. + +The file is readable and writable only by the creating user ID. +If the operating system uses permission bits to indicate whether a +file is executable, the file is executable by no one. The file +descriptor is not inherited by children of this process. + +Caller is responsible for deleting the file when done with it.` + +func mkstemp(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Object, error) { + var ( + pysuffix py.Object = py.None + pyprefix py.Object = py.None + pydir py.Object = py.None + pytext py.Object = py.False // FIXME(sbinet): can we do something with that? + ) + + err := py.ParseTupleAndKeywords(args, kwargs, + "|z#z#z#p:mkstemp", + []string{"suffix", "prefix", "dir", "text"}, + &pysuffix, &pyprefix, &pydir, &pytext, + ) + if err != nil { + return nil, err + } + + str := func(v py.Object, typ *uint8) string { + switch v := v.(type) { + case py.Bytes: + *typ = 2 + return string(v) + case py.String: + *typ = 1 + return string(v) + case py.NoneType: + *typ = 0 + return "" + default: + panic(fmt.Errorf("tempfile: invalid type %T (v=%+v)", v, v)) + } + } + + var ( + t1, t2, t3 uint8 + + suffix = str(pysuffix, &t1) + prefix = str(pyprefix, &t2) + dir = str(pydir, &t3) + pattern = prefix + "*" + suffix + ) + + cmp := func(t1, t2 uint8) bool { + if t1 > 0 && t2 > 0 { + return t1 == t2 + } + return true + } + + if !cmp(t1, t2) || !cmp(t1, t3) || !cmp(t2, t3) { + return nil, py.ExceptionNewf(py.TypeError, "Can't mix bytes and non-bytes in path components") + } + + f, err := os.CreateTemp(dir, pattern) + if err != nil { + return nil, err + } + + typ := t1 + if typ == 0 { + typ = t2 + } + if typ == 0 { + typ = t3 + } + + tuple := py.Tuple{py.Int(f.Fd())} + switch typ { + case 2: + tuple = append(tuple, py.Bytes(f.Name())) + default: + tuple = append(tuple, py.String(f.Name())) + } + + return tuple, nil +} diff --git a/stdlib/tempfile/tempfile_test.go b/stdlib/tempfile/tempfile_test.go new file mode 100644 index 00000000..33a75e98 --- /dev/null +++ b/stdlib/tempfile/tempfile_test.go @@ -0,0 +1,15 @@ +// Copyright 2022 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tempfile_test + +import ( + "testing" + + "github.com/go-python/gpython/pytest" +) + +func TestTempfile(t *testing.T) { + pytest.RunScript(t, "./testdata/test.py") +} diff --git a/stdlib/tempfile/testdata/test.py b/stdlib/tempfile/testdata/test.py new file mode 100644 index 00000000..19cef5bc --- /dev/null +++ b/stdlib/tempfile/testdata/test.py @@ -0,0 +1,110 @@ +# Copyright 2022 The go-python Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +import tempfile +import os + +print("test tempfile") + +if not tempfile.tempdir is None: + print("tempfile.tempdir is not None: %s" % (tempfile.tempdir,)) +else: + print("tempfile.tempdir is None [OK]") + +v = tempfile.gettempdir() +if type(v) != type(""): + print("tempfile.gettempdir() returned %s (type=%s)" % (v, type(v))) + +v = tempfile.gettempdirb() +if type(v) != type(b""): + print("tempfile.gettempdirb() returned %s (type=%s)" % (v, type(v))) + +## mkdtemp +try: + tmp = tempfile.mkdtemp() + os.rmdir(tmp) + print("mkdtemp() [OK]") +except Exception as e: + print("could not create tmp dir w/ mkdtemp(): %s" % e) + +try: + tmp = tempfile.mkdtemp(prefix="prefix-", suffix="-suffix") + os.rmdir(tmp) + print("mkdtemp(prefix='prefix-', suffix='-suffix') [OK]") +except Exception as e: + print("could not create tmp dir w/ mkdtemp(prefix='prefix-', suffix='-suffix'): %s" % e) + +try: + top = tempfile.mkdtemp(prefix="prefix-", suffix="-suffix") + tmp = tempfile.mkdtemp(prefix="prefix-", suffix="-suffix", dir=top) + os.rmdir(tmp) + os.rmdir(top) + print("mkdtemp(prefix='prefix-', suffix='-suffix', dir=top) [OK]") +except Exception as e: + print("could not create tmp dir w/ mkdtemp(prefix='prefix-', suffix='-suffix', dir=top): %s" % e) + +try: + top = tempfile.mkdtemp(prefix="prefix-", suffix="-suffix") + tmp = tempfile.mkdtemp(prefix="prefix-", suffix="-suffix", dir=top) + os.removedirs(top) + print("mkdtemp(prefix='prefix-', suffix='-suffix', dir=top) [OK]") +except Exception as e: + print("could not create tmp dir w/ mkdtemp(prefix='prefix-', suffix='-suffix', dir=top): %s" % e) + +try: + tmp = tempfile.mkdtemp(prefix=b"prefix-", suffix="-suffix") + print("missing exception!") + os.rmdir(tmp) +except TypeError as e: + print("caught: %s [OK]" % e) +except Exception as e: + print("INVALID error caught: %s" % e) + +def remove(fd, name): + os.fdopen(fd).close() + os.remove(name) + +## mkstemp +try: + fd, tmp = tempfile.mkstemp() + remove(fd, tmp) + print("mkstemp() [OK]") +except Exception as e: + print("could not create tmp dir w/ mkstemp(): %s" % e) + +try: + fd, tmp = tempfile.mkstemp(prefix="prefix-", suffix="-suffix") + remove(fd, tmp) + print("mkstemp(prefix='prefix-', suffix='-suffix') [OK]") +except Exception as e: + print("could not create tmp dir w/ mkstemp(prefix='prefix-', suffix='-suffix'): %s" % e) + +try: + top = tempfile.mkdtemp(prefix="prefix-", suffix="-suffix") + fd, tmp = tempfile.mkstemp(prefix="prefix-", suffix="-suffix", dir=top) + remove(fd, tmp) + os.remove(top) + print("mkstemp(prefix='prefix-', suffix='-suffix', dir=top) [OK]") +except Exception as e: + print("could not create tmp dir w/ mkstemp(prefix='prefix-', suffix='-suffix', dir=top): %s" % e) + +try: + top = tempfile.mkdtemp(prefix="prefix-", suffix="-suffix") + fd, tmp = tempfile.mkstemp(prefix="prefix-", suffix="-suffix", dir=top) + os.fdopen(fd).close() ## needed on Windows. + os.removedirs(top) + print("mkstemp(prefix='prefix-', suffix='-suffix', dir=top) [OK]") +except Exception as e: + print("could not create tmp dir w/ mkstemp(prefix='prefix-', suffix='-suffix', dir=top): %s" % e) + +try: + fd, tmp = tempfile.mkstemp(prefix=b"prefix-", suffix="-suffix") + print("missing exception!") + remove(fd, tmp) +except TypeError as e: + print("caught: %s [OK]" % e) +except Exception as e: + print("INVALID error caught: %s" % e) + +print("OK") diff --git a/stdlib/tempfile/testdata/test_golden.txt b/stdlib/tempfile/testdata/test_golden.txt new file mode 100644 index 00000000..ff7814de --- /dev/null +++ b/stdlib/tempfile/testdata/test_golden.txt @@ -0,0 +1,13 @@ +test tempfile +tempfile.tempdir is None [OK] +mkdtemp() [OK] +mkdtemp(prefix='prefix-', suffix='-suffix') [OK] +mkdtemp(prefix='prefix-', suffix='-suffix', dir=top) [OK] +mkdtemp(prefix='prefix-', suffix='-suffix', dir=top) [OK] +caught: TypeError: "Can't mix bytes and non-bytes in path components" [OK] +mkstemp() [OK] +mkstemp(prefix='prefix-', suffix='-suffix') [OK] +mkstemp(prefix='prefix-', suffix='-suffix', dir=top) [OK] +mkstemp(prefix='prefix-', suffix='-suffix', dir=top) [OK] +caught: TypeError: "Can't mix bytes and non-bytes in path components" [OK] +OK From e563b2e760ba9aed7af9e24f81fb55eba1a04e21 Mon Sep 17 00:00:00 2001 From: Sebastien Binet Date: Wed, 25 May 2022 19:20:25 +0200 Subject: [PATCH 41/60] stdlib/os: add close Signed-off-by: Sebastien Binet --- stdlib/os/os.go | 30 ++++++++++++++++++++++++++++++ stdlib/os/testdata/test.py | 11 +++++++++++ stdlib/os/testdata/test_golden.txt | 1 + 3 files changed, 42 insertions(+) diff --git a/stdlib/os/os.go b/stdlib/os/os.go index e8b3ec7a..f63586ce 100644 --- a/stdlib/os/os.go +++ b/stdlib/os/os.go @@ -46,6 +46,7 @@ func init() { methods := []*py.Method{ py.MustNewMethod("_exit", _exit, 0, "Immediate program termination."), + py.MustNewMethod("close", closefd, 0, closefd_doc), py.MustNewMethod("fdopen", fdopen, 0, fdopen_doc), py.MustNewMethod("getcwd", getCwd, 0, "Get the current working directory"), py.MustNewMethod("getcwdb", getCwdb, 0, "Get the current working directory in a byte slice"), @@ -98,6 +99,35 @@ func getEnvVariables() py.StringDict { return dict } +const closefd_doc = `Close a file descriptor` + +func closefd(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Object, error) { + var ( + pyfd py.Object + ) + err := py.ParseTupleAndKeywords(args, kwargs, "i", []string{"fd"}, &pyfd) + if err != nil { + return nil, err + } + + var ( + fd = uintptr(pyfd.(py.Int)) + name = strconv.Itoa(int(fd)) + ) + + f := os.NewFile(fd, name) + if f == nil { + return nil, py.ExceptionNewf(py.OSError, "Bad file descriptor") + } + + err = f.Close() + if err != nil { + return nil, err + } + + return py.None, nil +} + const fdopen_doc = `# Supply os.fdopen()` func fdopen(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Object, error) { diff --git a/stdlib/os/testdata/test.py b/stdlib/os/testdata/test.py index bdbf606e..bbdf8b69 100644 --- a/stdlib/os/testdata/test.py +++ b/stdlib/os/testdata/test.py @@ -118,6 +118,17 @@ else: print("os."+k+": [OK]") +## close +import tempfile +fd, tmp = tempfile.mkstemp() +os.close(fd=fd) +os.remove(tmp) +try: + os.close(-1) + print("closing a bad file descriptor should have failed") +except Exception as e: + print("caught: %s [OK]" % e) + ## fdopen import tempfile fd, tmp = tempfile.mkstemp() diff --git a/stdlib/os/testdata/test_golden.txt b/stdlib/os/testdata/test_golden.txt index 43c83011..4a0f640a 100644 --- a/stdlib/os/testdata/test_golden.txt +++ b/stdlib/os/testdata/test_golden.txt @@ -24,6 +24,7 @@ os.pathsep: [OK] os.linesep: [OK] os.devnull: [OK] os.altsep: [OK] +caught: OSError: 'Bad file descriptor' [OK] caught: SystemError - no such file or directory [OK] caught: FileExistsError [OK] caught: SystemError - directory not empty [OK] From 80944be95fc263ed7845f6a7eafb6a4b1831c1a6 Mon Sep 17 00:00:00 2001 From: Sebastien Binet Date: Wed, 25 May 2022 19:20:43 +0200 Subject: [PATCH 42/60] stdlib/tempfile: use os.close Signed-off-by: Sebastien Binet --- stdlib/tempfile/testdata/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/tempfile/testdata/test.py b/stdlib/tempfile/testdata/test.py index 19cef5bc..0fdecc9d 100644 --- a/stdlib/tempfile/testdata/test.py +++ b/stdlib/tempfile/testdata/test.py @@ -62,7 +62,7 @@ print("INVALID error caught: %s" % e) def remove(fd, name): - os.fdopen(fd).close() + os.close(fd) os.remove(name) ## mkstemp From 89a6ebc65a0a7ae844f2a202b5d30d65faf9756d Mon Sep 17 00:00:00 2001 From: Sebastien Binet Date: Wed, 17 Aug 2022 16:24:22 +0200 Subject: [PATCH 43/60] pytest: refactor testing infrastructure This CL also exposes a -regen flag to easily regenerate golden files. Co-authored-by: Drew O'Meara Signed-off-by: Sebastien Binet --- pytest/pytest.go | 123 +++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 109 insertions(+), 14 deletions(-) diff --git a/pytest/pytest.go b/pytest/pytest.go index 7b3c7cba..7c331d26 100644 --- a/pytest/pytest.go +++ b/pytest/pytest.go @@ -6,11 +6,14 @@ package pytest import ( "bytes" + "flag" + "fmt" "io" "os" "path" "path/filepath" "strings" + "sync/atomic" "testing" "github.com/go-python/gpython/compile" @@ -20,6 +23,8 @@ import ( _ "github.com/go-python/gpython/stdlib" ) +var RegenTestData = flag.Bool("regen", false, "Regenerate golden files from current testdata.") + var gContext = py.NewContext(py.DefaultContextOpts()) // Compile the program in the file prog to code in the module that is returned @@ -132,58 +137,148 @@ func RunBenchmarks(b *testing.B, testDir string) { // RunScript runs the provided path to a script. // RunScript captures the stdout and stderr while executing the script -// and compares it to a golden file: +// and compares it to a golden file, blocking until completion. // // RunScript("./testdata/foo.py") // // will compare the output with "./testdata/foo_golden.txt". func RunScript(t *testing.T, fname string) { + + RunTestTasks(t, []*Task{ + { + PyFile: fname, + }, + }) +} + +// RunTestTasks runs each given task in a newly created py.Context concurrently. +// If a fatal error is encountered, the given testing.T is signaled. +func RunTestTasks(t *testing.T, tasks []*Task) { + onCompleted := make(chan *Task) + + numTasks := len(tasks) + for ti := 0; ti < numTasks; ti++ { + task := tasks[ti] + go func() { + err := task.run() + task.Err = err + onCompleted <- task + }() + } + + tasks = tasks[:0] + for ti := 0; ti < numTasks; ti++ { + task := <-onCompleted + if task.Err != nil { + t.Error(task.Err) + } + tasks = append(tasks, task) + } +} + +var ( + taskCounter int32 +) + +type Task struct { + num int32 // Assigned when this task is run + ID string // unique key identifying this task. If empty, autogenerated from the basename of PyFile + PyFile string // If set, this file pathname is executed in a newly created ctx + PyTask func(ctx py.Context) error // If set, a new created ctx is created and this blocks until completion + GoldFile string // Filename containing the "gold standard" stdout+stderr. If empty, autogenerated from PyFile or ID + Err error // Non-nil if a fatal error is encountered with this task +} + +func (task *Task) run() error { + fileBase := "" + opts := py.DefaultContextOpts() - opts.SysArgs = []string{fname} + if task.PyFile != "" { + opts.SysArgs = []string{task.PyFile} + if task.ID == "" { + ext := filepath.Ext(task.PyFile) + fileBase = task.PyFile[0 : len(task.PyFile)-len(ext)] + } + } + + task.num = atomic.AddInt32(&taskCounter, 1) + if task.ID == "" { + if fileBase == "" { + task.ID = fmt.Sprintf("task-%04d", atomic.AddInt32(&taskCounter, 1)) + } else { + task.ID = strings.TrimPrefix(fileBase, "./") + } + } + + if task.GoldFile == "" { + task.GoldFile = fileBase + "_golden.txt" + } + ctx := py.NewContext(opts) defer ctx.Close() sys := ctx.Store().MustGetModule("sys") tmp, err := os.MkdirTemp("", "gpython-pytest-") if err != nil { - t.Fatal(err) + return err } defer os.RemoveAll(tmp) out, err := os.Create(filepath.Join(tmp, "combined")) if err != nil { - t.Fatalf("could not create stdout/stderr: %+v", err) + return fmt.Errorf("could not create stdout+stderr output file: %w", err) } defer out.Close() sys.Globals["stdout"] = &py.File{File: out, FileMode: py.FileWrite} sys.Globals["stderr"] = &py.File{File: out, FileMode: py.FileWrite} - _, err = py.RunFile(ctx, fname, py.CompileOpts{}, nil) - if err != nil { - t.Fatalf("could not run script %q: %+v", fname, err) + if task.PyFile != "" { + _, err := py.RunFile(ctx, task.PyFile, py.CompileOpts{}, nil) + if err != nil { + return fmt.Errorf("could not run target script %q: %w", task.PyFile, err) + } } + if task.PyTask != nil { + err := task.PyTask(ctx) + if err != nil { + return fmt.Errorf("PyTask %q failed: %w", task.ID, err) + } + } + + // Close the ctx explicitly as it may legitimately generate output + ctx.Close() + <-ctx.Done() + err = out.Close() if err != nil { - t.Fatalf("could not close stdout/stderr: %+v", err) + return fmt.Errorf("could not close output file: %w", err) } got, err := os.ReadFile(out.Name()) if err != nil { - t.Fatalf("could not read script output: %+v", err) + return fmt.Errorf("could not read script output file: %w", err) } - ref := fname[:len(fname)-len(".py")] + "_golden.txt" - want, err := os.ReadFile(ref) + if *RegenTestData { + err := os.WriteFile(task.GoldFile, got, 0644) + if err != nil { + return fmt.Errorf("could not write golden output %q: %w", task.GoldFile, err) + } + } + + want, err := os.ReadFile(task.GoldFile) if err != nil { - t.Fatalf("could not read golden output %q: %+v", ref, err) + return fmt.Errorf("could not read golden output %q: %w", task.GoldFile, err) } diff := cmp.Diff(string(want), string(got)) if !bytes.Equal(got, want) { - out := fname[:len(fname)-len(".py")] + ".txt" + out := fileBase + ".txt" _ = os.WriteFile(out, got, 0644) - t.Fatalf("output differ: -- (-ref +got)\n%s", diff) + return fmt.Errorf("output differ: -- (-ref +got)\n%s", diff) } + + return nil } From cf9017e39106858eab9f14ccedd0d927834253df Mon Sep 17 00:00:00 2001 From: Sebastien Binet Date: Wed, 17 Aug 2022 16:28:25 +0200 Subject: [PATCH 44/60] examples/embedding: use new pytest testing scaffolding Co-authored-by: Drew O'Meara Signed-off-by: Sebastien Binet --- examples/embedding/README.md | 16 +++--- examples/embedding/main_test.go | 53 ++----------------- examples/embedding/mylib.module.go | 2 +- .../embedding/{ => testdata}/mylib-demo.py | 0 ...g_out_golden.txt => mylib-demo_golden.txt} | 0 5 files changed, 12 insertions(+), 59 deletions(-) rename examples/embedding/{ => testdata}/mylib-demo.py (100%) rename examples/embedding/testdata/{embedding_out_golden.txt => mylib-demo_golden.txt} (100%) diff --git a/examples/embedding/README.md b/examples/embedding/README.md index 93a300aa..1dc5ae69 100644 --- a/examples/embedding/README.md +++ b/examples/embedding/README.md @@ -35,13 +35,13 @@ modules that are only available in CPython. ### Packing List -| | | -|---------------------- | ------------------------------------------------------------------| -| `main.go` | if no args, runs in REPL mode, otherwise runs the given file | -| `lib/mylib.py` | models a library that your application would expose for users | -| `lib/REPL-startup.py` | invoked by `main.go` when starting REPL mode | -| `mylib-demo.py` | models a user-authored script that consumes `mylib` | -| `mylib.module.go` | Go implementation of `mylib_go` consumed by `mylib` | +| | | +|------------------------- | ------------------------------------------------------------------| +| `main.go` | if no args, runs in REPL mode, otherwise runs the given file | +| `lib/mylib.py` | models a library that your application would expose for users | +| `lib/REPL-startup.py` | invoked by `main.go` when starting REPL mode | +| `testdata/mylib-demo.py` | models a user-authored script that consumes `mylib` | +| `mylib.module.go` | Go implementation of `mylib_go` consumed by `mylib` | ### Invoking a Python Script @@ -49,7 +49,7 @@ modules that are only available in CPython. ```bash $ cd examples/embedding/ $ go build . -$ ./embedding mylib-demo.py +$ ./embedding ./testdata/mylib-demo.py ``` ``` Welcome to a gpython embedded example, diff --git a/examples/embedding/main_test.go b/examples/embedding/main_test.go index 642867ec..651d2237 100644 --- a/examples/embedding/main_test.go +++ b/examples/embedding/main_test.go @@ -5,58 +5,11 @@ package main import ( - "bytes" - "flag" - "os" - "os/exec" - "path/filepath" "testing" -) -var regen = flag.Bool("regen", false, "regenerate golden files") + "github.com/go-python/gpython/pytest" +) func TestEmbeddedExample(t *testing.T) { - - tmp, err := os.MkdirTemp("", "go-python-embedding-") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(tmp) - - exe := filepath.Join(tmp, "out.exe") - cmd := exec.Command("go", "build", "-o", exe, ".") - err = cmd.Run() - if err != nil { - t.Fatalf("failed to compile embedding example: %+v", err) - } - - out := new(bytes.Buffer) - cmd = exec.Command(exe, "mylib-demo.py") - cmd.Stdout = out - cmd.Stderr = out - - err = cmd.Run() - if err != nil { - t.Fatalf("failed to run embedding binary: %+v", err) - } - - const fname = "testdata/embedding_out_golden.txt" - - got := out.Bytes() - - flag.Parse() - if *regen { - err = os.WriteFile(fname, got, 0644) - if err != nil { - t.Fatalf("could not write golden file: %+v", err) - } - } - - want, err := os.ReadFile(fname) - if err != nil { - t.Fatalf("could not read golden file: %+v", err) - } - if !bytes.Equal(got, want) { - t.Fatalf("stdout differ:\ngot:\n%s\nwant:\n%s\n", got, want) - } + pytest.RunScript(t, "./testdata/mylib-demo.py") } diff --git a/examples/embedding/mylib.module.go b/examples/embedding/mylib.module.go index 4f2842b2..1efb0fe8 100644 --- a/examples/embedding/mylib.module.go +++ b/examples/embedding/mylib.module.go @@ -48,7 +48,7 @@ func init() { "MYLIB_VERS": py.String("Vacation 1.0 by Fletch F. Fletcher"), }, OnContextClosed: func(instance *py.Module) { - fmt.Print("<<< host py.Context of py.Module instance closing >>>\n+++\n") + py.Println(instance, "<<< host py.Context of py.Module instance closing >>>\n+++") }, }) } diff --git a/examples/embedding/mylib-demo.py b/examples/embedding/testdata/mylib-demo.py similarity index 100% rename from examples/embedding/mylib-demo.py rename to examples/embedding/testdata/mylib-demo.py diff --git a/examples/embedding/testdata/embedding_out_golden.txt b/examples/embedding/testdata/mylib-demo_golden.txt similarity index 100% rename from examples/embedding/testdata/embedding_out_golden.txt rename to examples/embedding/testdata/mylib-demo_golden.txt From 8ac92a9a67dccdc17e5701e5d7d6cebd8b185e3a Mon Sep 17 00:00:00 2001 From: Sebastien Binet Date: Wed, 17 Aug 2022 16:33:28 +0200 Subject: [PATCH 45/60] all: bump to Go-1.19 Signed-off-by: Sebastien Binet --- .github/workflows/ci.yml | 2 +- go.mod | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8b6f2a2f..49657da8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ jobs: name: Build strategy: matrix: - go-version: [1.18.x, 1.17.x] + go-version: [1.19.x, 1.18.x] platform: [ubuntu-latest, macos-latest, windows-latest] runs-on: ${{ matrix.platform }} steps: diff --git a/go.mod b/go.mod index c87c414b..bc6cc9bc 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/go-python/gpython -go 1.17 +go 1.18 require ( github.com/google/go-cmp v0.5.7 From 18159950c925fb1caf56a3a5bc5195121e1950bd Mon Sep 17 00:00:00 2001 From: Sebastien Binet Date: Wed, 17 Aug 2022 16:35:44 +0200 Subject: [PATCH 46/60] all: bump go-cmp@v0.5.8 Signed-off-by: Sebastien Binet --- go.mod | 6 +++--- go.sum | 13 ++++++------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index bc6cc9bc..c27f93bf 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/go-python/gpython go 1.18 require ( - github.com/google/go-cmp v0.5.7 + github.com/google/go-cmp v0.5.8 github.com/gopherjs/gopherwasm v1.1.0 github.com/peterh/liner v1.2.2 ) @@ -11,6 +11,6 @@ require ( require ( github.com/gopherjs/gopherjs v0.0.0-20180825215210-0210a2f0f73c // indirect github.com/mattn/go-runewidth v0.0.13 // indirect - github.com/rivo/uniseg v0.2.0 // indirect - golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 // indirect + github.com/rivo/uniseg v0.3.4 // indirect + golang.org/x/sys v0.0.0-20220817070843-5a390386f1f2 // indirect ) diff --git a/go.sum b/go.sum index 34e6510b..0d0cbc2f 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,5 @@ -github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= -github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/gopherjs/gopherjs v0.0.0-20180825215210-0210a2f0f73c h1:16eHWuMGvCjSfgRJKqIzapE78onvvTbdi1rMkU00lZw= github.com/gopherjs/gopherjs v0.0.0-20180825215210-0210a2f0f73c/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherwasm v1.1.0 h1:fA2uLoctU5+T3OhOn2vYP0DVT6pxc7xhTlBB1paATqQ= @@ -9,10 +9,9 @@ github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4 github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/peterh/liner v1.2.2 h1:aJ4AOodmL+JxOZZEL2u9iJf8omNRpqHc/EbrK+3mAXw= github.com/peterh/liner v1.2.2/go.mod h1:xFwJyiKIXJZUKItq5dGHZSTBRAuG/CpeNpWLyiNRNwI= -github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.3.4 h1:3Z3Eu6FGHZWSfNKJTOUiPatWwfc7DzJRU04jFUqJODw= +github.com/rivo/uniseg v0.3.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 h1:OH54vjqzRWmbJ62fjuhxy7AxFFgoHN0/DPc/UrL8cAs= -golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/sys v0.0.0-20220817070843-5a390386f1f2 h1:fqTvyMIIj+HRzMmnzr9NtpHP6uVpvB5fkHcgPDC4nu8= +golang.org/x/sys v0.0.0-20220817070843-5a390386f1f2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= From 6f8e06a4660709ab44398d8b1a18738aa407b1c3 Mon Sep 17 00:00:00 2001 From: Drew O'Meara <35852900+drew-512@users.noreply.github.com> Date: Wed, 5 Oct 2022 02:25:39 -0500 Subject: [PATCH 47/60] all: minor comment and typos cleanups Co-authored-by: Drew O'Meara --- py/bigint.go | 4 ++-- py/bool.go | 2 +- py/bytes.go | 2 +- py/complex.go | 2 +- py/float.go | 2 +- py/int.go | 2 +- py/none.go | 2 +- py/run.go | 3 +++ py/string.go | 2 +- stdlib/stdlib.go | 10 +++++++--- 10 files changed, 19 insertions(+), 12 deletions(-) diff --git a/py/bigint.go b/py/bigint.go index 0b72804c..bb195888 100644 --- a/py/bigint.go +++ b/py/bigint.go @@ -55,7 +55,7 @@ func BigIntCheckExact(obj Object) (*BigInt, error) { return bigInt, nil } -// Checks that obj is exactly a bigInd and returns an error if not +// Checks that obj is exactly a BigInt and returns an error if not func BigIntCheck(obj Object) (*BigInt, error) { // FIXME should be checking subclasses return BigIntCheckExact(obj) @@ -65,7 +65,7 @@ func BigIntCheck(obj Object) (*BigInt, error) { // Convert an Object to an BigInt // -// Retrurns ok as to whether the conversion worked or not +// Returns ok as to whether the conversion worked or not func ConvertToBigInt(other Object) (*BigInt, bool) { switch b := other.(type) { case Int: diff --git a/py/bool.go b/py/bool.go index 82547754..413b25d4 100644 --- a/py/bool.go +++ b/py/bool.go @@ -52,7 +52,7 @@ func (a Bool) M__repr__() (Object, error) { // Convert an Object to an Bool // -// Retrurns ok as to whether the conversion worked or not +// Returns ok as to whether the conversion worked or not func convertToBool(other Object) (Bool, bool) { switch b := other.(type) { case Bool: diff --git a/py/bytes.go b/py/bytes.go index 55a69681..787807ec 100644 --- a/py/bytes.go +++ b/py/bytes.go @@ -187,7 +187,7 @@ func (a Bytes) M__repr__() (Object, error) { // Convert an Object to an Bytes // -// Retrurns ok as to whether the conversion worked or not +// Returns ok as to whether the conversion worked or not func convertToBytes(other Object) (Bytes, bool) { switch b := other.(type) { case Bytes: diff --git a/py/complex.go b/py/complex.go index e39f2f64..33c09fb3 100644 --- a/py/complex.go +++ b/py/complex.go @@ -42,7 +42,7 @@ func ComplexNew(metatype *Type, args Tuple, kwargs StringDict) (Object, error) { // Convert an Object to an Complex // -// Retrurns ok as to whether the conversion worked or not +// Returns ok as to whether the conversion worked or not func convertToComplex(other Object) (Complex, bool) { switch b := other.(type) { case Complex: diff --git a/py/float.go b/py/float.go index 4f47759b..c4bb96fd 100644 --- a/py/float.go +++ b/py/float.go @@ -118,7 +118,7 @@ var floatDivisionByZero = ExceptionNewf(ZeroDivisionError, "float division by ze // Convert an Object to an Float // -// Retrurns ok as to whether the conversion worked or not +// Returns ok as to whether the conversion worked or not func convertToFloat(other Object) (Float, bool) { switch b := other.(type) { case Float: diff --git a/py/int.go b/py/int.go index d3c84ab8..dde6993e 100644 --- a/py/int.go +++ b/py/int.go @@ -216,7 +216,7 @@ func cantConvert(a Object, to string) (Object, error) { // Convert an Object to an Int // -// Retrurns ok as to whether the conversion worked or not +// Returns ok as to whether the conversion worked or not func convertToInt(other Object) (Int, bool) { switch b := other.(type) { case Int: diff --git a/py/none.go b/py/none.go index 63a9952b..6c453b6c 100644 --- a/py/none.go +++ b/py/none.go @@ -33,7 +33,7 @@ func (a NoneType) M__repr__() (Object, error) { // Convert an Object to an NoneType // -// Retrurns ok as to whether the conversion worked or not +// Returns ok as to whether the conversion worked or not func convertToNoneType(other Object) (NoneType, bool) { switch b := other.(type) { case NoneType: diff --git a/py/run.go b/py/run.go index 5659232b..427cdbe6 100644 --- a/py/run.go +++ b/py/run.go @@ -135,8 +135,11 @@ func RunSrc(ctx Context, pySrc string, pySrcDesc string, inModule interface{}) ( } // RunCode executes the given code object within the given module and returns the Module to indicate success. +// // If inModule is a *Module, then the code is run in that module. +// // If inModule is nil, the code is run in a new __main__ module (and the new Module is returned). +// // If inModule is a string, the code is run in a new module with the given name (and the new Module is returned). func RunCode(ctx Context, code *Code, codeDesc string, inModule interface{}) (*Module, error) { var ( diff --git a/py/string.go b/py/string.go index 2c9854bc..d1c0c6f1 100644 --- a/py/string.go +++ b/py/string.go @@ -300,7 +300,7 @@ func (a String) M__imul__(other Object) (Object, error) { // Convert an Object to an String // -// Retrurns ok as to whether the conversion worked or not +// Returns ok as to whether the conversion worked or not func convertToString(other Object) (String, bool) { switch b := other.(type) { case String: diff --git a/stdlib/stdlib.go b/stdlib/stdlib.go index 7061f486..d945c382 100644 --- a/stdlib/stdlib.go +++ b/stdlib/stdlib.go @@ -47,7 +47,7 @@ type context struct { // NewContext creates a new gpython interpreter instance context. // -// See type Context interface for info. +// See interface py.Context defined in py/run.go func NewContext(opts py.ContextOpts) py.Context { ctx := &context{ opts: opts, @@ -109,6 +109,7 @@ func (ctx *context) ModuleInit(impl *py.ModuleImpl) (*py.Module, error) { return module, nil } +// See interface py.Context defined in py/run.go func (ctx *context) ResolveAndCompile(pathname string, opts py.CompileOpts) (py.CompileOut, error) { err := ctx.pushBusy() defer ctx.popBusy() @@ -202,7 +203,7 @@ func (ctx *context) popBusy() { ctx.running.Done() } -// Close -- see type py.Context +// See interface py.Context defined in py/run.go func (ctx *context) Close() error { ctx.closeOnce.Do(func() { ctx.closing = true @@ -216,7 +217,7 @@ func (ctx *context) Close() error { return nil } -// Done -- see type py.Context +// See interface py.Context defined in py/run.go func (ctx *context) Done() <-chan struct{} { return ctx.done } @@ -274,6 +275,7 @@ func resolveRunPath(runPath string, opts py.CompileOpts, pathObjs []py.Object, t return err } +// See interface py.Context defined in py/run.go func (ctx *context) RunCode(code *py.Code, globals, locals py.StringDict, closure py.Tuple) (py.Object, error) { err := ctx.pushBusy() defer ctx.popBusy() @@ -284,10 +286,12 @@ func (ctx *context) RunCode(code *py.Code, globals, locals py.StringDict, closur return vm.EvalCode(ctx, code, globals, locals, nil, nil, nil, nil, closure) } +// See interface py.Context defined in py/run.go func (ctx *context) GetModule(moduleName string) (*py.Module, error) { return ctx.store.GetModule(moduleName) } +// See interface py.Context defined in py/run.go func (ctx *context) Store() *py.ModuleStore { return ctx.store } From 8e99b322adb3190d8d75309efe92e88f86d279fd Mon Sep 17 00:00:00 2001 From: Kyle Ellrott Date: Mon, 9 Jan 2023 03:21:24 -0800 Subject: [PATCH 48/60] py: add the 'add' method to the set class --- py/set.go | 11 +++++++++++ py/tests/set.py | 14 ++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/py/set.go b/py/set.go index b1b74e26..718e286c 100644 --- a/py/set.go +++ b/py/set.go @@ -46,6 +46,17 @@ func NewSetFromItems(items []Object) *Set { return s } +func init() { + SetType.Dict["add"] = MustNewMethod("add", func(self Object, args Tuple) (Object, error) { + setSelf := self.(*Set) + if len(args) != 1 { + return nil, ExceptionNewf(TypeError, "append() takes exactly one argument (%d given)", len(args)) + } + setSelf.Add(args[0]) + return NoneType{}, nil + }, 0, "add(value)") +} + // Add an item to the set func (s *Set) Add(item Object) { s.items[item] = SetValue{} diff --git a/py/tests/set.py b/py/tests/set.py index 834e457b..3eeaf1d3 100644 --- a/py/tests/set.py +++ b/py/tests/set.py @@ -2,6 +2,8 @@ # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. +from libtest import assertRaises + doc="__and__" a = {1, 2, 3} b = {2, 3, 4, 5} @@ -81,6 +83,18 @@ assert 4 in c assert 5 in c +doc="add" +a = set() +a.add(1) +a.add(2) +a.add(3) +assert len(a) == 3 +assert 1 in a +assert 2 in a +assert 3 in a +assert 4 not in a +assertRaises(TypeError, lambda: a.add()) + doc="__eq__, __ne__" a = set([1,2,3]) assert a.__eq__(3) != True From 7512ac2e41ecd4e912d1aa991171ced2eceac259 Mon Sep 17 00:00:00 2001 From: Kyle Ellrott Date: Mon, 9 Jan 2023 03:23:16 -0800 Subject: [PATCH 49/60] py: add 'keys' and 'values' methods to dict object --- py/dict.go | 26 ++++++++++++++++++++++++++ py/tests/dict.py | 8 ++++++++ 2 files changed, 34 insertions(+) diff --git a/py/dict.go b/py/dict.go index 1710b7da..10c2d33f 100644 --- a/py/dict.go +++ b/py/dict.go @@ -41,6 +41,32 @@ func init() { return NewIterator(o), nil }, 0, "items() -> list of D's (key, value) pairs, as 2-tuples") + StringDictType.Dict["keys"] = MustNewMethod("keys", func(self Object, args Tuple) (Object, error) { + err := UnpackTuple(args, nil, "keys", 0, 0) + if err != nil { + return nil, err + } + sMap := self.(StringDict) + o := make([]Object, 0, len(sMap)) + for k := range sMap { + o = append(o, String(k)) + } + return NewIterator(o), nil + }, 0, "keys() -> list of D's keys, as a list") + + StringDictType.Dict["values"] = MustNewMethod("values", func(self Object, args Tuple) (Object, error) { + err := UnpackTuple(args, nil, "values", 0, 0) + if err != nil { + return nil, err + } + sMap := self.(StringDict) + o := make([]Object, 0, len(sMap)) + for _, v := range sMap { + o = append(o, v) + } + return NewIterator(o), nil + }, 0, "values() -> list of D's values, as a list") + StringDictType.Dict["get"] = MustNewMethod("get", func(self Object, args Tuple) (Object, error) { var length = len(args) switch { diff --git a/py/tests/dict.py b/py/tests/dict.py index cb14dbc2..d6140e47 100644 --- a/py/tests/dict.py +++ b/py/tests/dict.py @@ -28,6 +28,14 @@ assert a.get('b',1) == 1 assert a.get('b',True) == True +doc="check keys" +a = {"a":1} +assert list(a.keys()) == ["a"] + +doc="check values" +a = {"a":1} +assert list(a.values()) == [1] + doc="check items" a = {"a":"b","c":5.5} for k, v in a.items(): From c60d4254cd915422ee376d847c962ca90e5ebe80 Mon Sep 17 00:00:00 2001 From: Kyle Ellrott Date: Wed, 11 Jan 2023 06:30:26 -0800 Subject: [PATCH 50/60] py: add strip, rstrip and lstrip methods to string class --- py/string.go | 60 ++++++++++++++++++++++++++++++++++++++++++++++ py/tests/string.py | 11 +++++++++ 2 files changed, 71 insertions(+) diff --git a/py/string.go b/py/string.go index d1c0c6f1..a28e6e74 100644 --- a/py/string.go +++ b/py/string.go @@ -206,6 +206,18 @@ replaced.`) return Bool(false), nil }, 0, "startswith(prefix[, start[, end]]) -> bool") + StringType.Dict["strip"] = MustNewMethod("strip", func(self Object, args Tuple, kwargs StringDict) (Object, error) { + return self.(String).Strip(args) + }, 0, "strip(chars) -> replace chars from begining and end of string") + + StringType.Dict["rstrip"] = MustNewMethod("rstrip", func(self Object, args Tuple, kwargs StringDict) (Object, error) { + return self.(String).RStrip(args) + }, 0, "rstrip(chars) -> replace chars from end of string") + + StringType.Dict["lstrip"] = MustNewMethod("lstrip", func(self Object, args Tuple, kwargs StringDict) (Object, error) { + return self.(String).LStrip(args) + }, 0, "lstrip(chars) -> replace chars from begining of string") + } // Type of this object @@ -679,6 +691,54 @@ func (s String) Replace(args Tuple) (Object, error) { return String(strings.Replace(string(s), old, new, cnt)), nil } +func stripFunc(args Tuple) (func(rune) bool, error) { + var ( + pyval Object = None + ) + err := ParseTuple(args, "|s", &pyval) + if err != nil { + return nil, err + } + f := unicode.IsSpace + switch v := pyval.(type) { + case String: + chars := []rune(string(v)) + f = func(s rune) bool { + for _, i := range chars { + if s == i { + return true + } + } + return false + } + } + return f, nil +} + +func (s String) Strip(args Tuple) (Object, error) { + f, err := stripFunc(args) + if err != nil { + return nil, err + } + return String(strings.TrimFunc(string(s), f)), nil +} + +func (s String) LStrip(args Tuple) (Object, error) { + f, err := stripFunc(args) + if err != nil { + return nil, err + } + return String(strings.TrimLeftFunc(string(s), f)), nil +} + +func (s String) RStrip(args Tuple) (Object, error) { + f, err := stripFunc(args) + if err != nil { + return nil, err + } + return String(strings.TrimRightFunc(string(s), f)), nil +} + // Check stringerface is satisfied var ( _ richComparison = String("") diff --git a/py/tests/string.py b/py/tests/string.py index 43487328..8af36ca6 100644 --- a/py/tests/string.py +++ b/py/tests/string.py @@ -886,6 +886,17 @@ def index(s, i): assert uni[7:7:2] == '' assert uni[7:7:3] == '' +doc="string strip methods" +a = " adfasd " +assert a.rstrip() == " adfasd" +assert a.lstrip() == "adfasd " +assert a.strip() == "adfasd" + +a = " a bada a" +assert a.rstrip("a ") == " a bad" +assert a.lstrip("a ") == "bada a" +assert a.strip("a ") == "bad" + class Index: def __index__(self): return 1 From 0cc403274b12ec2d73ce8158610c26cba9a68c31 Mon Sep 17 00:00:00 2001 From: Kyle Ellrott Date: Sat, 14 Jan 2023 00:54:39 -0800 Subject: [PATCH 51/60] py: add __delitem__ to dict --- py/dict.go | 13 +++++++++++++ py/tests/dict.py | 10 ++++++++++ 2 files changed, 23 insertions(+) diff --git a/py/dict.go b/py/dict.go index 10c2d33f..f0aa938c 100644 --- a/py/dict.go +++ b/py/dict.go @@ -189,6 +189,19 @@ func (d StringDict) M__getitem__(key Object) (Object, error) { return nil, ExceptionNewf(KeyError, "%v", key) } +func (d StringDict) M__delitem__(key Object) (Object, error) { + str, ok := key.(String) + if !ok { + return nil, ExceptionNewf(KeyError, "%v", key) + } + _, ok = d[string(str)] + if !ok { + return nil, ExceptionNewf(KeyError, "%v", key) + } + delete(d, string(str)) + return None, nil +} + func (d StringDict) M__setitem__(key, value Object) (Object, error) { str, ok := key.(String) if !ok { diff --git a/py/tests/dict.py b/py/tests/dict.py index d6140e47..8fc9619e 100644 --- a/py/tests/dict.py +++ b/py/tests/dict.py @@ -46,6 +46,16 @@ assert v == 5.5 assertRaises(TypeError, a.items, 'a') +doc="del" +a = {'hello': 'world', 'hi': 'there'} +del a["hello"] +def doDel(d, key): + del d[key] +assertRaises(KeyError, lambda: doDel(a, "bob")) +assertRaises(KeyError, lambda: doDel(a, 123)) +assert not a.__contains__('hello') +assert a.__contains__('hi') + doc="__contain__" a = {'hello': 'world'} assert a.__contains__('hello') From 5ef03848c068efc07bee3442b63c0771db18874d Mon Sep 17 00:00:00 2001 From: Yang Yu Date: Tue, 17 Jan 2023 21:57:36 +0800 Subject: [PATCH 52/60] stdlib/os: add listdir --- stdlib/os/os.go | 57 ++++++++++++++++++++++++++++++ stdlib/os/testdata/test.py | 6 ++++ stdlib/os/testdata/test_golden.txt | 3 ++ 3 files changed, 66 insertions(+) diff --git a/stdlib/os/os.go b/stdlib/os/os.go index f63586ce..c37fce19 100644 --- a/stdlib/os/os.go +++ b/stdlib/os/os.go @@ -53,6 +53,7 @@ func init() { py.MustNewMethod("chdir", chdir, 0, "Change the current working directory"), py.MustNewMethod("getenv", getenv, 0, "Return the value of the environment variable key if it exists, or default if it doesn’t. key, default and the result are str."), py.MustNewMethod("getpid", getpid, 0, "Return the current process id."), + py.MustNewMethod("listdir", listDir, 0, listDir_doc), py.MustNewMethod("makedirs", makedirs, 0, makedirs_doc), py.MustNewMethod("mkdir", mkdir, 0, mkdir_doc), py.MustNewMethod("putenv", putenv, 0, "Set the environment variable named key to the string value."), @@ -234,6 +235,62 @@ func getpid(self py.Object, args py.Tuple) (py.Object, error) { return py.Int(os.Getpid()), nil } +const listDir_doc = ` +Return a list containing the names of the files in the directory. + +path can be specified as either str, bytes. If path is bytes, the filenames + returned will also be bytes; in all other circumstances + the filenames returned will be str. +If path is None, uses the path='.'. + +The list is in arbitrary order. It does not include the special +entries '.' and '..' even if they are present in the directory. +` + +func listDir(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Object, error) { + var ( + path py.Object = py.None + ) + err := py.ParseTupleAndKeywords(args, kwargs, "|z*:listdir", []string{"path"}, &path) + if err != nil { + return nil, err + } + + if path == py.None { + cwd, err := os.Getwd() + if err != nil { + return nil, py.ExceptionNewf(py.OSError, "cannot get cwd, error %s", err.Error()) + } + path = py.String(cwd) + } + + dirName := "" + returnsBytes := false + switch v := path.(type) { + case py.String: + dirName = string(v) + case py.Bytes: + dirName = string(v) + returnsBytes = true + default: + return nil, py.ExceptionNewf(py.TypeError, "str or bytes expected, not %T", path) + } + + dirEntries, err := os.ReadDir(dirName) + if err != nil { + return nil, py.ExceptionNewf(py.OSError, "cannot read directory %s, error %s", dirName, err.Error()) + } + result := py.NewListSized(len(dirEntries)) + for i, dirEntry := range dirEntries { + if returnsBytes { + result.Items[i] = py.Bytes(dirEntry.Name()) + } else { + result.Items[i] = py.String(dirEntry.Name()) + } + } + return result, nil +} + const makedirs_doc = `makedirs(name [, mode=0o777][, exist_ok=False]) Super-mkdir; create a leaf directory and all intermediate ones. Works like diff --git a/stdlib/os/testdata/test.py b/stdlib/os/testdata/test.py index bbdf8b69..7b5b6815 100644 --- a/stdlib/os/testdata/test.py +++ b/stdlib/os/testdata/test.py @@ -151,6 +151,11 @@ os.mkdir(dir1) os.mkdir(dir2) os.mkdir(dir11) + print(os.listdir(bytes(top, "utf-8"))) + orig = os.getcwd() + os.chdir(top) + print(os.listdir()) + os.chdir(orig) os.removedirs(dir1) try: os.mkdir(dir11) @@ -181,6 +186,7 @@ print("INVALID error caught: %s" % e) os.remove(fname) os.rmdir(dir2) + print(os.listdir(top)) except Exception as e: print("could not create/remove directories: %s" % e) finally: diff --git a/stdlib/os/testdata/test_golden.txt b/stdlib/os/testdata/test_golden.txt index 4a0f640a..fd5fc39b 100644 --- a/stdlib/os/testdata/test_golden.txt +++ b/stdlib/os/testdata/test_golden.txt @@ -25,8 +25,11 @@ os.linesep: [OK] os.devnull: [OK] os.altsep: [OK] caught: OSError: 'Bad file descriptor' [OK] +[b'dir1', b'dir2'] +['dir1', 'dir2'] caught: SystemError - no such file or directory [OK] caught: FileExistsError [OK] caught: SystemError - directory not empty [OK] +['dir1'] os.{mkdir,rmdir,remove,removedirs} worked as expected OK From 337df2ad1ec2f2a13ef65dc8f56ce1ce8c8cec77 Mon Sep 17 00:00:00 2001 From: Kyle Ellrott Date: Wed, 18 Jan 2023 11:33:50 -0800 Subject: [PATCH 53/60] py: improve dict intialization method --- py/dict.go | 37 +++++++++++++++++++++++++++++++++++-- py/tests/dict.py | 15 +++++++++++++++ 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/py/dict.go b/py/dict.go index f0aa938c..9aac7631 100644 --- a/py/dict.go +++ b/py/dict.go @@ -9,7 +9,9 @@ package py -import "bytes" +import ( + "bytes" +) const dictDoc = `dict() -> new empty dictionary dict(mapping) -> new dictionary initialized from a mapping object's @@ -22,7 +24,7 @@ dict(**kwargs) -> new dictionary initialized with the name=value pairs in the keyword argument list. For example: dict(one=1, two=2)` var ( - StringDictType = NewType("dict", dictDoc) + StringDictType = NewTypeX("dict", dictDoc, DictNew, nil) DictType = NewType("dict", dictDoc) expectingDict = ExceptionNewf(TypeError, "a dict is required") ) @@ -97,6 +99,37 @@ func init() { // Used for variables etc where the keys can only be strings type StringDict map[string]Object +// DictNew +func DictNew(metatype *Type, args Tuple, kwargs StringDict) (Object, error) { + if len(args) > 1 { + return nil, ExceptionNewf(TypeError, "dict expects at most one argument") + } + out := NewStringDict() + if len(args) == 1 { + arg := args[0] + seq, err := SequenceList(arg) + if err != nil { + return nil, err + } + for _, i := range seq.Items { + switch z := i.(type) { + case Tuple: + if zStr, ok := z[0].(String); ok { + out[string(zStr)] = z[1] + } + default: + return nil, ExceptionNewf(TypeError, "non-tuple sequence") + } + } + } + if len(kwargs) > 0 { + for k, v := range kwargs { + out[k] = v + } + } + return out, nil +} + // Type of this StringDict object func (o StringDict) Type() *Type { return StringDictType diff --git a/py/tests/dict.py b/py/tests/dict.py index 8fc9619e..274f6c69 100644 --- a/py/tests/dict.py +++ b/py/tests/dict.py @@ -56,6 +56,21 @@ def doDel(d, key): assert not a.__contains__('hello') assert a.__contains__('hi') +doc="init" +a = dict( zip( "a,b,c".split(","), "1,2,3".split(",") ) ) +assert a["a"] == "1" +assert a["b"] == "2" +assert a["c"] == "3" + +a = dict(a="1", b="2", c="3") +assert a["a"] == "1" +assert a["b"] == "2" +assert a["c"] == "3" + +assertRaises(TypeError, dict, "a") +assertRaises(TypeError, dict, 1) +assertRaises(TypeError, dict, {"a":1}, {"b":2}) + doc="__contain__" a = {'hello': 'world'} assert a.__contains__('hello') From 652daef83367dce372c40e30d2f2100233fa6555 Mon Sep 17 00:00:00 2001 From: wetor Date: Sun, 12 Mar 2023 14:50:51 +0800 Subject: [PATCH 54/60] parser: fix CRLF(\r\n) file parsing error, SyntaxError: 'invalid syntax' --- parser/lexer.go | 3 +++ parser/lexer_test.go | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/parser/lexer.go b/parser/lexer.go index 801215ba..119a741b 100644 --- a/parser/lexer.go +++ b/parser/lexer.go @@ -110,6 +110,9 @@ func (x *yyLex) dequeue() int { func (x *yyLex) refill() { var err error x.line, err = x.reader.ReadString('\n') + if strings.HasSuffix(x.line, "\r\n") { + x.line = x.line[:len(x.line)-2] + "\n" + } if yyDebug >= 2 { fmt.Printf("line = %q, err = %v\n", x.line, err) } diff --git a/parser/lexer_test.go b/parser/lexer_test.go index 9971c2d7..5e044d39 100644 --- a/parser/lexer_test.go +++ b/parser/lexer_test.go @@ -262,7 +262,7 @@ func TestLex(t *testing.T) { {"01", "illegal decimal with leading zero 1:0", "exec", LexTokens{ {FILE_INPUT, nil, ast.Pos{0, 0}}, }}, - {"1\n 2\n 3\n4\n", "", "exec", LexTokens{ + {"1\n 2\r\n 3\r\n4\n", "", "exec", LexTokens{ {FILE_INPUT, nil, ast.Pos{0, 0}}, {NUMBER, py.Int(1), ast.Pos{1, 0}}, {NEWLINE, nil, ast.Pos{1, 1}}, From acd458b80be6badfefd1922d4a189f696f7be65e Mon Sep 17 00:00:00 2001 From: wetor Date: Sat, 18 Mar 2023 14:19:46 +0800 Subject: [PATCH 55/60] py: int() default to decimal --- py/int.go | 2 +- py/tests/int.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/py/int.go b/py/int.go index dde6993e..919c28aa 100644 --- a/py/int.go +++ b/py/int.go @@ -55,7 +55,7 @@ func (o Int) Type() *Type { func IntNew(metatype *Type, args Tuple, kwargs StringDict) (Object, error) { var xObj Object = Int(0) var baseObj Object - base := 0 + base := 10 err := ParseTupleAndKeywords(args, kwargs, "|OO:int", []string{"x", "base"}, &xObj, &baseObj) if err != nil { return nil, err diff --git a/py/tests/int.py b/py/tests/int.py index 758829ab..79356b3c 100644 --- a/py/tests/int.py +++ b/py/tests/int.py @@ -103,6 +103,7 @@ doc="sigils" assert int("7") == 7 +assert int("07") == 7 assert int("07", 10) == 7 assert int("F", 16) == 15 From d5be40c49f6e16b4d3b40cbee8ec8d338fea2398 Mon Sep 17 00:00:00 2001 From: wetor Date: Wed, 7 Jun 2023 19:24:55 +0800 Subject: [PATCH 56/60] py: fix iterable object --- py/arithmetic.go | 18 ------------- py/dict.go | 8 +++--- py/exception.go | 2 ++ py/gen.go | 1 - py/internal.go | 17 +++++++++++++ py/iterator.go | 36 ++++++++++++++++++-------- py/list.go | 8 ++++-- py/object.go | 53 ++++++++++++++++++++++++++++++--------- py/tests/iter.py | 12 +++++++++ stdlib/builtin/builtin.go | 12 +++++++-- 10 files changed, 118 insertions(+), 49 deletions(-) diff --git a/py/arithmetic.go b/py/arithmetic.go index 199fceee..768764b8 100644 --- a/py/arithmetic.go +++ b/py/arithmetic.go @@ -147,24 +147,6 @@ func MakeFloat(a Object) (Object, error) { return nil, ExceptionNewf(TypeError, "unsupported operand type(s) for float: '%s'", a.Type().Name) } -// Iter the python Object returning an Object -// -// Will raise TypeError if Iter can't be run on this object -func Iter(a Object) (Object, error) { - - if A, ok := a.(I__iter__); ok { - res, err := A.M__iter__() - if err != nil { - return nil, err - } - if res != NotImplemented { - return res, nil - } - } - - return nil, ExceptionNewf(TypeError, "unsupported operand type(s) for iter: '%s'", a.Type().Name) -} - // Add two python objects together returning an Object // // Will raise TypeError if can't be add can't be run on these objects diff --git a/py/dict.go b/py/dict.go index 9aac7631..497da3a7 100644 --- a/py/dict.go +++ b/py/dict.go @@ -36,7 +36,7 @@ func init() { return nil, err } sMap := self.(StringDict) - o := make([]Object, 0, len(sMap)) + o := make(Tuple, 0, len(sMap)) for k, v := range sMap { o = append(o, Tuple{String(k), v}) } @@ -49,7 +49,7 @@ func init() { return nil, err } sMap := self.(StringDict) - o := make([]Object, 0, len(sMap)) + o := make(Tuple, 0, len(sMap)) for k := range sMap { o = append(o, String(k)) } @@ -62,7 +62,7 @@ func init() { return nil, err } sMap := self.(StringDict) - o := make([]Object, 0, len(sMap)) + o := make(Tuple, 0, len(sMap)) for _, v := range sMap { o = append(o, v) } @@ -204,7 +204,7 @@ func (a StringDict) M__repr__() (Object, error) { // Returns a list of keys from the dict func (d StringDict) M__iter__() (Object, error) { - o := make([]Object, 0, len(d)) + o := make(Tuple, 0, len(d)) for k := range d { o = append(o, String(k)) } diff --git a/py/exception.go b/py/exception.go index 2e8f91a2..73f92747 100644 --- a/py/exception.go +++ b/py/exception.go @@ -336,6 +336,8 @@ func ExceptionGivenMatches(err, exc Object) bool { func IsException(exception *Type, r interface{}) bool { var t *Type switch ex := r.(type) { + case ExceptionInfo: + t = ex.Type case *Exception: t = ex.Type() case *Type: diff --git a/py/gen.go b/py/gen.go index 16db6ad8..671b0831 100644 --- a/py/gen.go +++ b/py/gen.go @@ -45,7 +45,6 @@ var data = Data{ {Name: "complex", Title: "MakeComplex", Operator: "complex", Unary: true, Conversion: "Complex"}, {Name: "int", Title: "MakeInt", Operator: "int", Unary: true, Conversion: "Int"}, {Name: "float", Title: "MakeFloat", Operator: "float", Unary: true, Conversion: "Float"}, - {Name: "iter", Title: "Iter", Operator: "iter", Unary: true}, }, BinaryOps: Ops{ {Name: "add", Title: "Add", Operator: "+", Binary: true}, diff --git a/py/internal.go b/py/internal.go index df0e285c..e649b299 100644 --- a/py/internal.go +++ b/py/internal.go @@ -430,3 +430,20 @@ func ReprAsString(self Object) (string, error) { } return string(str), nil } + +// Returns an iterator object +// +// Call __Iter__ Returns an iterator object +// +// If object is sequence object, create an iterator +func Iter(self Object) (res Object, err error) { + if I, ok := self.(I__iter__); ok { + return I.M__iter__() + } else if res, ok, err = TypeCall0(self, "__iter__"); ok { + return res, err + } + if ObjectIsSequence(self) { + return NewIterator(self), nil + } + return nil, ExceptionNewf(TypeError, "'%s' object is not iterable", self.Type().Name) +} diff --git a/py/iterator.go b/py/iterator.go index a2368da8..350700a9 100644 --- a/py/iterator.go +++ b/py/iterator.go @@ -8,8 +8,8 @@ package py // A python Iterator object type Iterator struct { - Pos int - Objs []Object + Pos int + Seq Object } var IteratorType = NewType("iterator", "iterator type") @@ -20,10 +20,10 @@ func (o *Iterator) Type() *Type { } // Define a new iterator -func NewIterator(Objs []Object) *Iterator { +func NewIterator(Seq Object) *Iterator { m := &Iterator{ - Pos: 0, - Objs: Objs, + Pos: 0, + Seq: Seq, } return m } @@ -33,13 +33,29 @@ func (it *Iterator) M__iter__() (Object, error) { } // Get next one from the iteration -func (it *Iterator) M__next__() (Object, error) { - if it.Pos >= len(it.Objs) { - return nil, StopIteration +func (it *Iterator) M__next__() (res Object, err error) { + if tuple, ok := it.Seq.(Tuple); ok { + if it.Pos >= len(tuple) { + return nil, StopIteration + } + res = tuple[it.Pos] + it.Pos++ + return res, nil + } + index := Int(it.Pos) + if I, ok := it.Seq.(I__getitem__); ok { + res, err = I.M__getitem__(index) + } else if res, ok, err = TypeCall1(it.Seq, "__getitem__", index); !ok { + return nil, ExceptionNewf(TypeError, "'%s' object is not iterable", it.Type().Name) + } + if err != nil { + if IsException(IndexError, err) { + return nil, StopIteration + } + return nil, err } - r := it.Objs[it.Pos] it.Pos++ - return r, nil + return res, nil } // Check interface is satisfied diff --git a/py/list.go b/py/list.go index 28a118a1..9f6f62b0 100644 --- a/py/list.go +++ b/py/list.go @@ -186,7 +186,7 @@ func (l *List) M__bool__() (Object, error) { } func (l *List) M__iter__() (Object, error) { - return NewIterator(l.Items), nil + return NewIterator(Tuple(l.Items)), nil } func (l *List) M__getitem__(key Object) (Object, error) { @@ -496,7 +496,11 @@ func SortInPlace(l *List, kwargs StringDict, funcName string) error { reverse = False } // FIXME: requires the same bool-check like CPython (or better "|$Op" that doesn't panic on nil). - s := ptrSortable{&sortable{l, keyFunc, ObjectIsTrue(reverse), nil}} + ok, err := ObjectIsTrue(reverse) + if err != nil { + return err + } + s := ptrSortable{&sortable{l, keyFunc, ok, nil}} sort.Stable(s) return s.s.firstErr } diff --git a/py/object.go b/py/object.go index 55141cec..540ea0b6 100644 --- a/py/object.go +++ b/py/object.go @@ -23,24 +23,53 @@ func ObjectRepr(o Object) Object { } // Return whether the object is True or not -func ObjectIsTrue(o Object) bool { - if o == True { - return true +func ObjectIsTrue(o Object) (cmp bool, err error) { + switch o { + case True: + return true, nil + case False: + return false, nil + case None: + return false, nil } - if o == False { - return false + + var res Object + switch t := o.(type) { + case I__bool__: + res, err = t.M__bool__() + case I__len__: + res, err = t.M__len__() + case *Type: + var ok bool + if res, ok, err = TypeCall0(o, "__bool__"); ok { + break + } + if res, ok, err = TypeCall0(o, "__len__"); ok { + break + } + _ = ok // pass static-check + } + if err != nil { + return false, err } - if o == None { - return false + switch t := res.(type) { + case Bool: + return t == True, nil + case Int: + return t > 0, nil } + return true, nil +} - if I, ok := o.(I__bool__); ok { - cmp, err := I.M__bool__() - if err == nil && cmp == True { +// Return whether the object is a sequence +func ObjectIsSequence(o Object) bool { + switch t := o.(type) { + case I__getitem__: + return true + case *Type: + if t.GetAttrOrNil("__getitem__") != nil { return true - } else if err == nil && cmp == False { - return false } } return false diff --git a/py/tests/iter.py b/py/tests/iter.py index 53422b79..4eda19c9 100644 --- a/py/tests/iter.py +++ b/py/tests/iter.py @@ -18,4 +18,16 @@ def f(): words2 = list(iter(words1)) for w1, w2 in zip(words1, words2): assert w1 == w2 + +class SequenceClass: + def __init__(self, n): + self.n = n + def __getitem__(self, i): + if 0 <= i < self.n: + return i + else: + raise IndexError + +assert list(iter(SequenceClass(5))) == [0, 1, 2, 3, 4] + doc="finished" \ No newline at end of file diff --git a/stdlib/builtin/builtin.go b/stdlib/builtin/builtin.go index 83502849..134d8511 100644 --- a/stdlib/builtin/builtin.go +++ b/stdlib/builtin/builtin.go @@ -292,7 +292,11 @@ func builtin_all(self, seq py.Object) (py.Object, error) { } return nil, err } - if !py.ObjectIsTrue(item) { + ok, err := py.ObjectIsTrue(item) + if err != nil { + return nil, err + } + if !ok { return py.False, nil } } @@ -317,7 +321,11 @@ func builtin_any(self, seq py.Object) (py.Object, error) { } return nil, err } - if py.ObjectIsTrue(item) { + ok, err := py.ObjectIsTrue(item) + if err != nil { + return nil, err + } + if ok { return py.True, nil } } From 17dddcd8f647cc11142842523c6846540743dab1 Mon Sep 17 00:00:00 2001 From: wetor Date: Wed, 7 Jun 2023 19:28:05 +0800 Subject: [PATCH 57/60] py: implement `filter` --- py/filter.go | 72 +++++++++++++++++++++++++++++++++++++++ py/tests/filter.py | 65 +++++++++++++++++++++++++++++++++++ stdlib/builtin/builtin.go | 6 ++-- 3 files changed, 140 insertions(+), 3 deletions(-) create mode 100644 py/filter.go create mode 100644 py/tests/filter.py diff --git a/py/filter.go b/py/filter.go new file mode 100644 index 00000000..447c4829 --- /dev/null +++ b/py/filter.go @@ -0,0 +1,72 @@ +// Copyright 2023 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package py + +// A python Filter object +type Filter struct { + it Object + fun Object +} + +var FilterType = NewTypeX("filter", `filter(function or None, iterable) --> filter object + +Return an iterator yielding those items of iterable for which function(item) +is true. If function is None, return the items that are true.`, + FilterTypeNew, nil) + +// Type of this object +func (f *Filter) Type() *Type { + return FilterType +} + +// FilterTypeNew +func FilterTypeNew(metatype *Type, args Tuple, kwargs StringDict) (res Object, err error) { + var fun, seq Object + var it Object + err = UnpackTuple(args, kwargs, "filter", 2, 2, &fun, &seq) + if err != nil { + return nil, err + } + it, err = Iter(seq) + if err != nil { + return nil, err + } + return &Filter{it: it, fun: fun}, nil +} + +func (f *Filter) M__iter__() (Object, error) { + return f, nil +} + +func (f *Filter) M__next__() (Object, error) { + var ok bool + for { + item, err := Next(f.it) + if err != nil { + return nil, err + } + // if (lz->func == Py_None || lz->func == (PyObject *)&PyBool_Type) + if _, _ok := f.fun.(Bool); _ok || f.fun == None { + ok, err = ObjectIsTrue(item) + } else { + var good Object + good, err = Call(f.fun, Tuple{item}, nil) + if err != nil { + return nil, err + } + ok, err = ObjectIsTrue(good) + } + if ok { + return item, nil + } + if err != nil { + return nil, err + } + } +} + +// Check interface is satisfied +var _ I__iter__ = (*Filter)(nil) +var _ I__next__ = (*Filter)(nil) diff --git a/py/tests/filter.py b/py/tests/filter.py new file mode 100644 index 00000000..c42b2b63 --- /dev/null +++ b/py/tests/filter.py @@ -0,0 +1,65 @@ +# test_builtin.py:BuiltinTest.test_filter() +from libtest import assertRaises + +doc="filter" +class T0: + def __bool__(self): + return True +class T1: + def __len__(self): + return 1 +class T2: + def __bool__(self): + return False +class T3: + pass +t0, t1, t2, t3 = T0(), T1(), T2(), T3() +assert list(filter(None, [t0, t1, t2, t3])) == [t0, t1, t3] +assert list(filter(None, [1, [], 2, ''])) == [1, 2] + +class T3: + def __len__(self): + raise ValueError +t3 = T3() +assertRaises(ValueError, list, filter(None, [t3])) + +class Squares: + def __init__(self, max): + self.max = max + self.sofar = [] + + def __len__(self): return len(self.sofar) + + def __getitem__(self, i): + if not 0 <= i < self.max: raise IndexError + n = len(self.sofar) + while n <= i: + self.sofar.append(n*n) + n += 1 + return self.sofar[i] + +assert list(filter(lambda c: 'a' <= c <= 'z', 'Hello World')) == list('elloorld') +assert list(filter(None, [1, 'hello', [], [3], '', None, 9, 0])) == [1, 'hello', [3], 9] +assert list(filter(lambda x: x > 0, [1, -3, 9, 0, 2])) == [1, 9, 2] +assert list(filter(None, Squares(10))) == [1, 4, 9, 16, 25, 36, 49, 64, 81] +assert list(filter(lambda x: x%2, Squares(10))) == [1, 9, 25, 49, 81] +def identity(item): + return 1 +filter(identity, Squares(5)) +assertRaises(TypeError, filter) +class BadSeq(object): + def __getitem__(self, index): + if index<4: + return 42 + raise ValueError +assertRaises(ValueError, list, filter(lambda x: x, BadSeq())) +def badfunc(): + pass +assertRaises(TypeError, list, filter(badfunc, range(5))) + +# test bltinmodule.c::filtertuple() +assert list(filter(None, (1, 2))) == [1, 2] +assert list(filter(lambda x: x>=3, (1, 2, 3, 4))) == [3, 4] +assertRaises(TypeError, list, filter(42, (1, 2))) + +doc="finished" diff --git a/stdlib/builtin/builtin.go b/stdlib/builtin/builtin.go index 134d8511..88d6cffe 100644 --- a/stdlib/builtin/builtin.go +++ b/stdlib/builtin/builtin.go @@ -77,9 +77,9 @@ func init() { "complex": py.ComplexType, "dict": py.StringDictType, // FIXME "enumerate": py.EnumerateType, - // "filter": py.FilterType, - "float": py.FloatType, - "frozenset": py.FrozenSetType, + "filter": py.FilterType, + "float": py.FloatType, + "frozenset": py.FrozenSetType, // "property": py.PropertyType, "int": py.IntType, // FIXME LongType? "list": py.ListType, From 32f9086bff2e8b2fb368759162b9012848fa54cd Mon Sep 17 00:00:00 2001 From: wetor Date: Wed, 7 Jun 2023 19:28:25 +0800 Subject: [PATCH 58/60] py: implement `map` --- py/map.go | 60 +++++++++++++++++++++++++++++++++++++++ py/tests/map.py | 54 +++++++++++++++++++++++++++++++++++ stdlib/builtin/builtin.go | 6 ++-- 3 files changed, 117 insertions(+), 3 deletions(-) create mode 100644 py/map.go create mode 100644 py/tests/map.py diff --git a/py/map.go b/py/map.go new file mode 100644 index 00000000..1c343538 --- /dev/null +++ b/py/map.go @@ -0,0 +1,60 @@ +// Copyright 2023 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package py + +// A python Map object +type Map struct { + iters Tuple + fun Object +} + +var MapType = NewTypeX("filter", `map(func, *iterables) --> map object + +Make an iterator that computes the function using arguments from +each of the iterables. Stops when the shortest iterable is exhausted.`, + MapTypeNew, nil) + +// Type of this object +func (m *Map) Type() *Type { + return FilterType +} + +// MapType +func MapTypeNew(metatype *Type, args Tuple, kwargs StringDict) (res Object, err error) { + numargs := len(args) + if numargs < 2 { + return nil, ExceptionNewf(TypeError, "map() must have at least two arguments.") + } + iters := make(Tuple, numargs-1) + for i := 1; i < numargs; i++ { + iters[i-1], err = Iter(args[i]) + if err != nil { + return nil, err + } + } + return &Map{iters: iters, fun: args[0]}, nil +} + +func (m *Map) M__iter__() (Object, error) { + return m, nil +} + +func (m *Map) M__next__() (Object, error) { + numargs := len(m.iters) + argtuple := make(Tuple, numargs) + + for i := 0; i < numargs; i++ { + val, err := Next(m.iters[i]) + if err != nil { + return nil, err + } + argtuple[i] = val + } + return Call(m.fun, argtuple, nil) +} + +// Check interface is satisfied +var _ I__iter__ = (*Map)(nil) +var _ I__next__ = (*Map)(nil) diff --git a/py/tests/map.py b/py/tests/map.py new file mode 100644 index 00000000..3e5a4e1e --- /dev/null +++ b/py/tests/map.py @@ -0,0 +1,54 @@ +# test_builtin.py:BuiltinTest.test_map() +from libtest import assertRaises + +doc="map" +class Squares: + def __init__(self, max): + self.max = max + self.sofar = [] + + def __len__(self): return len(self.sofar) + + def __getitem__(self, i): + if not 0 <= i < self.max: raise IndexError + n = len(self.sofar) + while n <= i: + self.sofar.append(n*n) + n += 1 + return self.sofar[i] + +assert list(map(lambda x: x*x, range(1,4))) == [1, 4, 9] +try: + from math import sqrt +except ImportError: + def sqrt(x): + return pow(x, 0.5) +assert list(map(lambda x: list(map(sqrt, x)), [[16, 4], [81, 9]])) == [[4.0, 2.0], [9.0, 3.0]] +assert list(map(lambda x, y: x+y, [1,3,2], [9,1,4])) == [10, 4, 6] + +def plus(*v): + accu = 0 + for i in v: accu = accu + i + return accu +assert list(map(plus, [1, 3, 7])) == [1, 3, 7] +assert list(map(plus, [1, 3, 7], [4, 9, 2])) == [1+4, 3+9, 7+2] +assert list(map(plus, [1, 3, 7], [4, 9, 2], [1, 1, 0])) == [1+4+1, 3+9+1, 7+2+0] +assert list(map(int, Squares(10))) == [0, 1, 4, 9, 16, 25, 36, 49, 64, 81] +def Max(a, b): + if a is None: + return b + if b is None: + return a + return max(a, b) +assert list(map(Max, Squares(3), Squares(2))) == [0, 1] +assertRaises(TypeError, map) +assertRaises(TypeError, map, lambda x: x, 42) +class BadSeq: + def __iter__(self): + raise ValueError + yield None +assertRaises(ValueError, list, map(lambda x: x, BadSeq())) +def badfunc(x): + raise RuntimeError +assertRaises(RuntimeError, list, map(badfunc, range(5))) +doc="finished" diff --git a/stdlib/builtin/builtin.go b/stdlib/builtin/builtin.go index 88d6cffe..243f26a3 100644 --- a/stdlib/builtin/builtin.go +++ b/stdlib/builtin/builtin.go @@ -81,9 +81,9 @@ func init() { "float": py.FloatType, "frozenset": py.FrozenSetType, // "property": py.PropertyType, - "int": py.IntType, // FIXME LongType? - "list": py.ListType, - // "map": py.MapType, + "int": py.IntType, // FIXME LongType? + "list": py.ListType, + "map": py.MapType, "object": py.ObjectType, "range": py.RangeType, // "reversed": py.ReversedType, From bd13450143541a222b7428febecda13a717aa0c7 Mon Sep 17 00:00:00 2001 From: wetor Date: Wed, 7 Jun 2023 19:29:15 +0800 Subject: [PATCH 59/60] builtin: implement `oct` and optimise `hex` --- stdlib/builtin/builtin.go | 71 +++++++++++++++++++++++++++++---- stdlib/builtin/tests/builtin.py | 6 +++ 2 files changed, 70 insertions(+), 7 deletions(-) diff --git a/stdlib/builtin/builtin.go b/stdlib/builtin/builtin.go index 243f26a3..290cb939 100644 --- a/stdlib/builtin/builtin.go +++ b/stdlib/builtin/builtin.go @@ -8,6 +8,7 @@ package builtin import ( "fmt" "math/big" + "strconv" "unicode/utf8" "github.com/go-python/gpython/compile" @@ -53,7 +54,7 @@ func init() { py.MustNewMethod("min", builtin_min, 0, min_doc), py.MustNewMethod("next", builtin_next, 0, next_doc), py.MustNewMethod("open", builtin_open, 0, open_doc), - // py.MustNewMethod("oct", builtin_oct, 0, oct_doc), + py.MustNewMethod("oct", builtin_oct, 0, oct_doc), py.MustNewMethod("ord", builtin_ord, 0, ord_doc), py.MustNewMethod("pow", builtin_pow, 0, pow_doc), py.MustNewMethod("print", builtin_print, 0, print_doc), @@ -592,6 +593,56 @@ func builtin_open(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Objec int(buffering.(py.Int))) } +const oct_doc = `oct(number) -> string + +Return the octal representation of an integer. + + >>> oct(342391) + '0o1234567' +` + +func builtin_oct(self, v py.Object) (py.Object, error) { + var ( + i int64 + err error + ) + switch v := v.(type) { + case *py.BigInt: + vv := (*big.Int)(v) + neg := false + if vv.Cmp(big.NewInt(0)) == -1 { + neg = true + } + str := vv.Text(8) + if neg { + str = "-0o" + str[1:] + } else { + str = "0o" + str + } + return py.String(str), nil + case py.IGoInt64: + i, err = v.GoInt64() + case py.IGoInt: + var vv int + vv, err = v.GoInt() + i = int64(vv) + default: + return nil, py.ExceptionNewf(py.TypeError, "'%s' object cannot be interpreted as an integer", v.Type().Name) + } + + if err != nil { + return nil, err + } + + str := strconv.FormatInt(i, 8) + if i < 0 { + str = "-0o" + str[1:] + } else { + str = "0o" + str + } + return py.String(str), nil +} + const ord_doc = `ord(c) -> integer Return the integer ordinal of a one-character string.` @@ -865,11 +916,16 @@ func builtin_hex(self, v py.Object) (py.Object, error) { // test bigint first to make sure we correctly handle the case // where int64 isn't large enough. vv := (*big.Int)(v) - format := "%#x" + neg := false if vv.Cmp(big.NewInt(0)) == -1 { - format = "%+#x" + neg = true + } + str := vv.Text(16) + if neg { + str = "-0x" + str[1:] + } else { + str = "0x" + str } - str := fmt.Sprintf(format, vv) return py.String(str), nil case py.IGoInt64: i, err = v.GoInt64() @@ -885,11 +941,12 @@ func builtin_hex(self, v py.Object) (py.Object, error) { return nil, err } - format := "%#x" + str := strconv.FormatInt(i, 16) if i < 0 { - format = "%+#x" + str = "-0x" + str[1:] + } else { + str = "0x" + str } - str := fmt.Sprintf(format, i) return py.String(str), nil } diff --git a/stdlib/builtin/tests/builtin.py b/stdlib/builtin/tests/builtin.py index 07f1704a..ae4e8a5f 100644 --- a/stdlib/builtin/tests/builtin.py +++ b/stdlib/builtin/tests/builtin.py @@ -275,6 +275,12 @@ def gen2(): ok = True assert ok, "ValueError not raised" +doc="oct" +assert oct(0) == '0o0' +assert oct(100) == '0o144' +assert oct(-100) == '-0o144' +assertRaises(TypeError, oct, ()) + doc="ord" assert 65 == ord("A") assert 163 == ord("£") From 7102b79c9edeff6c73ca3f57553e0959496d08ca Mon Sep 17 00:00:00 2001 From: wetor Date: Thu, 8 Jun 2023 16:46:47 +0800 Subject: [PATCH 60/60] py: fix basic type not run `Ready()` --- py/type.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py/type.go b/py/type.go index be6d19af..509a7628 100644 --- a/py/type.go +++ b/py/type.go @@ -306,7 +306,7 @@ func (t *Type) NewTypeFlags(Name string, Doc string, New NewFunc, Init InitFunc, Dict: StringDict{}, Bases: Tuple{t}, } - TypeDelayReady(t) + TypeDelayReady(tt) return tt }