diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..86a5f633 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,103 @@ +name: CI + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + schedule: + - cron: '0 2 * * 1-5' + +env: + GOPROXY: "https://proxy.golang.org" + TAGS: "-tags=ci" + COVERAGE: "-coverpkg=github.com/go-python/gpython/..." + +jobs: + + build: + name: Build + strategy: + matrix: + go-version: [1.19.x, 1.18.x] + platform: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.platform }} + steps: + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go-version }} + + - name: Setup Git for Windows + run: | + git config --global core.autocrlf false + git config --global core.eol lf + + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 1 + + - name: Cache-Go + uses: actions/cache@v4 + with: + # In order: + # * Module download cache + # * Build cache (Linux) + # * Build cache (Mac) + # * Build cache (Windows) + path: | + ~/go/pkg/mod + ~/.cache/go-build + ~/Library/Caches/go-build + '%LocalAppData%\go-build' + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + + - name: Install Linux packages + if: matrix.platform == 'ubuntu-latest' + run: | + sudo apt-get update + sudo apt-get install -qq pkg-config python3 + + - name: Build-Linux-32b + if: matrix.platform == 'ubuntu-latest' + run: | + GOARCH=386 go install -v $TAGS ./... + - name: Build-Linux-64b + if: matrix.platform == 'ubuntu-latest' + run: | + GOARCH=amd64 go install -v $TAGS ./... + - name: Build-Windows + if: matrix.platform == 'windows-latest' + run: | + go install -v $TAGS ./... + - name: Build-Darwin + if: matrix.platform == 'macos-latest' + run: | + go install -v $TAGS ./... + - name: Test Linux + if: matrix.platform == 'ubuntu-latest' + run: | + GOARCH=386 go test $TAGS ./... + GOARCH=amd64 go run ./ci/run-tests.go $TAGS -race $COVERAGE + ## FIXME(sbinet): bring back python3.4 or upgrade gpython to python3.x + ## python3 py3test.py + - name: Test Windows + if: matrix.platform == 'windows-latest' + run: | + go run ./ci/run-tests.go $TAGS -race + - name: Test Darwin + if: matrix.platform == 'macos-latest' + run: | + go run ./ci/run-tests.go $TAGS -race + - name: static-check + uses: dominikh/staticcheck-action@v1 + with: + install-go: false + cache-key: ${{ matrix.platform }} + version: "2022.1" + - name: Upload-Coverage + if: matrix.platform == 'ubuntu-latest' + uses: codecov/codecov-action@v3 diff --git a/.gitignore b/.gitignore index a3eaead1..dc9b73bb 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,7 @@ __pycache__ cover.out /junk /dist + +# tests +stdlib/builtin/testfile +examples/embedding/embedding diff --git a/.gometalinter.json b/.gometalinter.json new file mode 100644 index 00000000..1959207c --- /dev/null +++ b/.gometalinter.json @@ -0,0 +1,13 @@ +{ + "Enable": [ + "deadcode", + "errcheck", + "goimports", + "ineffassign", + "structcheck", + "varcheck", + "vet" + ], + "EnableGC": true, + "Vendor": true +} diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index cebbdcd2..00000000 --- a/.travis.yml +++ /dev/null @@ -1,45 +0,0 @@ -language: go -sudo: false -dist: trusty - -os: - - linux - -env: - - TAGS="-tags travis" - -matrix: - fast_finish: true - allow_failures: - - go: master - include: - - go: 1.9.x - env: - - TAGS="-tags travis" - - go: 1.10.x - env: - - TAGS="-tags travis" - - go: 1.11.x - env: - - TAGS="-tags travis" - - COVERAGE="-cover" - - go: master - env: - - TAGS="-tags travis" - - GO111MODULE=on - -cache: - directories: - - $HOME/bin/python3.4 - -before_install: - - ./bin/install-python.sh $HOME/bin/python3.4 - -script: - - go install -v $TAGS ./... - - GOARCH=386 go test $TAGS ./... - - GOARCH=amd64 go run ./ci/run-tests.go -race $TAGS $COVERAGE - - python3 py3test.py - -after_success: - - bash <(curl -s https://codecov.io/bash) diff --git a/CONTRIBUTE.md b/CONTRIBUTE.md index 567247e4..76c93fd3 100644 --- a/CONTRIBUTE.md +++ b/CONTRIBUTE.md @@ -40,9 +40,9 @@ If it is large, such as suggesting a new repository, sub-repository, or interfac ### Your First Code Contribution If you are a new contributor, *thank you!* -Before your first merge, you will need to be added to the [CONTRIBUTORS](https://github.com/go-python/license/blob/master/CONTRIBUTORS) and [AUTHORS](https://github.com/go-python/license/blob/master/AUTHORS) files. +Before your first merge, you will need to be added to the [CONTRIBUTORS](https://github.com/go-python/license/blob/main/CONTRIBUTORS) and [AUTHORS](https://github.com/go-python/license/blob/main/AUTHORS) files. Open a pull request adding yourself to these files. -All `go-python` code follows the BSD license in the [license document](https://github.com/go-python/license/blob/master/LICENSE). +All `go-python` code follows the BSD license in the [license document](https://github.com/go-python/license/blob/main/LICENSE). We prefer that code contributions do not come with additional licensing. For exceptions, added code must also follow a BSD license. @@ -88,7 +88,7 @@ Please always format your code with [goimports](https://godoc.org/golang.org/x/t Best is to have it invoked as a hook when you save your `.go` files. Files in the `go-python` repository don't list author names, both to avoid clutter and to avoid having to keep the lists up to date. -Instead, your name will appear in the change log and in the [CONTRIBUTORS](https://github.com/go-python/license/blob/master/CONTRIBUTORS) and [AUTHORS](https://github.com/go-python/license/blob/master/AUTHORS) files. +Instead, your name will appear in the change log and in the [CONTRIBUTORS](https://github.com/go-python/license/blob/main/CONTRIBUTORS) and [AUTHORS](https://github.com/go-python/license/blob/main/AUTHORS) files. New files that you contribute should use the standard copyright header: diff --git a/README.md b/README.md index b0144853..883ae510 100644 --- a/README.md +++ b/README.md @@ -1,22 +1,24 @@ # gpython -[![Build Status](https://travis-ci.org/go-python/gpython.svg?branch=master)](https://travis-ci.org/go-python/gpython) -[![codecov](https://codecov.io/gh/go-python/gpython/branch/master/graph/badge.svg)](https://codecov.io/gh/go-python/gpython) +[![Build Status](https://github.com/go-python/gpython/workflows/CI/badge.svg)](https://github.com/go-python/gpython/actions) +[![codecov](https://codecov.io/gh/go-python/gpython/branch/main/graph/badge.svg)](https://codecov.io/gh/go-python/gpython) [![GoDoc](https://godoc.org/github.com/go-python/gpython?status.svg)](https://godoc.org/github.com/go-python/gpython) -[![License](https://img.shields.io/badge/License-BSD--3-blue.svg)](https://github.com/go-python/gpython/blob/master/LICENSE) - -gpython is a part re-implementation / part port of the Python 3.4 -interpreter to the Go language, "batteries not included". - -It includes: - - * runtime - using compatible byte code to python3.4 - * lexer - * parser - * compiler +[![License](https://img.shields.io/badge/License-BSD--3-blue.svg)](https://github.com/go-python/gpython/blob/main/LICENSE) + +gpython is a part re-implementation, part port of the Python 3.4 +interpreter in Go. Although there are many areas of improvement, +it stands as an noteworthy achievement in capability and potential. + +gpython includes: + + * lexer, parser, and compiler + * runtime and high-level convenience functions + * multi-context interpreter instancing + * easy embedding into your Go application * interactive mode (REPL) ([try online!](https://gpython.org)) -It does not include very many python modules as many of the core + +gpython does not include many python modules as many of the core modules are written in C not python. The converted modules are: * builtins @@ -27,53 +29,52 @@ modules are written in C not python. The converted modules are: ## Install -Gpython is a Go program and comes as a single binary file. - -Download the relevant binary from here: https://github.com/go-python/gpython/releases +Download directly from the [releases page](https://github.com/go-python/gpython/releases) -Or alternatively if you have Go installed use +Or if you have Go installed: - go get github.com/go-python/gpython - -and this will build the binary in `$GOPATH/bin`. You can then modify -the source and submit patches. + go install github.com/go-python/gpython ## Objectives -Gpython was written as a learning experiment to investigate how hard +gpython started as an experiment to investigate how hard porting Python to Go might be. It turns out that all those C modules -are a significant barrier to making a fully functional port. +are a significant barrier to making gpython a complete replacement +to CPython. -## Status +However, to those who want to embed a highly popular and known language +into their Go application, gpython could be a great choice over less +capable (or lesser known) alternatives. -The project works well enough to parse all the code in the python 3.4 -distribution and to compile and run python 3 programs which don't -depend on a module gpython doesn't support. +## Status -See the examples directory for some python programs which run with -gpython. +gpython currently: + - Parses all the code in the Python 3.4 distribution + - Runs Python 3 for the modules that are currently supported + - Supports concurrent multi-interpreter ("multi-context") execution Speed hasn't been a goal of the conversions however it runs pystone at -about 20% of the speed of cpython. The pi test runs quicker under -gpython as I think the Go long integer primitives are faster than the +about 20% of the speed of CPython. A [π computation test](https://github.com/go-python/gpython/tree/main/examples/pi_chudnovsky_bs.py) runs quicker under +gpython as the Go long integer primitives are likely faster than the Python ones. -There are many directions this project could go in. I think the most -profitable would be to re-use the -[grumpy](https://github.com/grumpyhome/grumpy) runtime (which would mean -changing the object model). This would give access to the C modules -that need to be ported and would give grumpy access to a compiler and -interpreter (gpython does support `eval` for instance). +@ncw started gpython in 2013 and work on is sporadic. If you or someone +you know would be interested to take it futher, it would be much appreciated. + +## Getting Started -I (@ncw) haven't had much time to work on gpython (I started it in -2013 and have worked on it very sporadically) so someone who wants to -take it in the next direction would be much appreciated. +The [embedding example](https://github.com/go-python/gpython/tree/main/examples/embedding) demonstrates how to +easily embed and invoke gpython from any Go application. -## Limitations and Bugs +Of interest, gpython is able to run multiple interpreter instances simultaneously, +allowing you to embed gpython naturally into your Go application. This makes it +possible to use gpython in a server situation where complete interpreter +independence is paramount. See this in action in the [multi-context example](https://github.com/go-python/gpython/tree/main/examples/multi-context). + +If you are looking to get involved, a light and easy place to start is adding more convenience functions to [py/util.go](https://github.com/go-python/gpython/tree/main/py/util.go). See [notes.txt](https://github.com/go-python/gpython/blob/main/notes.txt) for bigger ideas. -Lots! -## Similar projects +## Other Projects of Interest * [grumpy](https://github.com/grumpyhome/grumpy) - a python to go transpiler @@ -86,5 +87,5 @@ or on the [Gophers Slack](https://gophers.slack.com/) in the `#go-python` channe ## License This is licensed under the MIT licence, however it contains code which -was ported fairly directly directly from the cpython source code under -the [PSF LICENSE](https://github.com/python/cpython/blob/master/LICENSE). +was ported fairly directly directly from the CPython source code under +the [PSF LICENSE](https://github.com/python/cpython/blob/main/LICENSE). diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index dbaa278f..00000000 --- a/appveyor.yml +++ /dev/null @@ -1,21 +0,0 @@ -build: off - -clone_folder: c:\gopath\src\github.com\go-python\gpython - -branches: - only: - - master - -environment: - GOPATH: c:\gopath - PATH: '%GOPATH%\bin;%PATH%;C:\msys64\mingw64\bin' - matrix: - - TARGET: x86_64-pc-windows-gnu - -stack: go 1.11 - -build_script: - - go get -v -t -race ./... - -test_script: - - go test -race ./... diff --git a/builtin/tests/builtin.py b/builtin/tests/builtin.py deleted file mode 100644 index 623e4187..00000000 --- a/builtin/tests/builtin.py +++ /dev/null @@ -1,236 +0,0 @@ -# Copyright 2018 The go-python Authors. All rights reserved. -# Use of this source code is governed by a BSD-style -# license that can be found in the LICENSE file. - -doc="abs" -assert abs(0) == 0 -assert abs(10) == 10 -assert abs(-10) == 10 - -doc="all" -assert all((0,0,0)) == False -assert all((1,1,0)) == False -assert all(["hello", "world"]) == True -assert all([]) == True - -doc="any" -assert any((0,0,0)) == False -assert any((1,1,0)) == True -assert any(["hello", "world"]) == True -assert any([]) == False - -doc="chr" -assert chr(65) == "A" -assert chr(163) == "£" -assert chr(0x263A) == "☺" - -doc="compile" -code = compile("pass", "", "exec") -assert code is not None -# FIXME - -doc="divmod" -assert divmod(34,7) == (4, 6) - -doc="eval" -# smoke test only - see vm/tests/builtin.py for more tests -assert eval("1+2") == 3 - -doc="exec" -# smoke test only - see vm/tests/builtin.py for more tests -glob = {"a":100} -assert exec("b = a+100", glob) == None -assert glob["b"] == 200 - -doc="getattr" -class C: - def __init__(self): - self.potato = 42 -c = C() -assert getattr(c, "potato") == 42 -assert getattr(c, "potato", 43) == 42 -assert getattr(c, "sausage", 43) == 43 - -doc="globals" -a = 1 -assert globals()["a"] == 1 - -doc="hasattr" -assert hasattr(c, "potato") -assert not hasattr(c, "sausage") - -doc="len" -assert len(()) == 0 -assert len((1,2,3)) == 3 -assert len("hello") == 5 -assert len("£☺") == 2 - -doc="locals" -def fn(x): - assert locals()["x"] == 1 -fn(1) - -doc="next no default" -def gen(): - yield 1 - yield 2 -g = gen() -assert next(g) == 1 -assert next(g) == 2 -ok = False -try: - next(g) -except StopIteration: - ok = True -assert ok, "StopIteration not raised" - -doc="next with default" -g = gen() -assert next(g, 42) == 1 -assert next(g, 42) == 2 -assert next(g, 42) == 42 -assert next(g, 42) == 42 - -doc="next no default with exception" -def gen2(): - yield 1 - raise ValueError("potato") -g = gen2() -assert next(g) == 1 -ok = False -try: - next(g) -except ValueError: - ok = True -assert ok, "ValueError not raised" - -doc="next with default and exception" -g = gen2() -assert next(g, 42) == 1 -ok = False -try: - next(g) -except ValueError: - ok = True -assert ok, "ValueError not raised" - -doc="ord" -assert 65 == ord("A") -assert 163 == ord("£") -assert 0x263A == ord("☺") -assert 65 == ord(b"A") -ok = False -try: - ord("AA") -except TypeError as e: - if e.args[0] != "ord() expected a character, but string of length 2 found": - raise - ok = True -assert ok, "TypeError not raised" -try: - ord(None) -except TypeError as e: - if e.args[0] != "ord() expected string of length 1, but NoneType found": - raise - ok = True -assert ok, "TypeError not raised" - -doc="open" -assert open(__file__) is not None - -doc="pow" -assert pow(2, 10) == 1024 -assert pow(2, 10, 17) == 4 - -doc="repr" -assert repr(5) == "5" -assert repr("hello") == "'hello'" - -doc="print" -ok = False -try: - print("hello", sep=1) -except TypeError as e: - #if e.args[0] != "sep must be None or a string, not int": - # raise - ok = True -assert ok, "TypeError not raised" - -try: - print("hello", sep=" ", end=1) -except TypeError as e: - #if e.args[0] != "end must be None or a string, not int": - # raise - ok = True -assert ok, "TypeError not raised" - -try: - print("hello", sep=" ", end="\n", file=1) -except AttributeError as e: - #if e.args[0] != "'int' object has no attribute 'write'": - # raise - ok = True -assert ok, "AttributeError not raised" - -with open("testfile", "w") as f: - print("hello", "world", sep=" ", end="\n", file=f) - -with open("testfile", "r") as f: - assert f.read() == "hello world\n" - -with open("testfile", "w") as f: - print(1,2,3,sep=",",end=",\n", file=f) - -with open("testfile", "r") as f: - assert f.read() == "1,2,3,\n" - -doc="round" -assert round(1.1) == 1.0 - -doc="setattr" -class C: pass -c = C() -assert not hasattr(c, "potato") -setattr(c, "potato", "spud") -assert getattr(c, "potato") == "spud" -assert c.potato == "spud" - -doc="sum" -assert sum([1,2,3]) == 6 -assert sum([1,2,3], 3) == 9 -assert sum((1,2,3)) == 6 -assert sum((1,2,3), 3) == 9 -assert sum((1, 2.5, 3)) == 6.5 -assert sum((1, 2.5, 3), 3) == 9.5 - -try: - sum([1,2,3], 'hi') -except TypeError as e: - if e.args[0] != "sum() can't sum strings [use ''.join(seq) instead]": - raise - ok = True -assert ok, "TypeError not raised" - -try: - sum([1,2,3], b'hi') -except TypeError as e: - if e.args[0] != "sum() can't sum bytes [use b''.join(seq) instead]": - raise - ok = True -assert ok, "TypeError not raised" - -try: - sum(['h', 'i']) -except TypeError as e: - if e.args[0] != "unsupported operand type(s) for +: 'int' and 'str'": - raise - ok = True -assert ok, "TypeError not raised" - -doc="__import__" -lib = __import__("lib") -assert lib.libfn() == 42 -assert lib.libvar == 43 -assert lib.libclass().method() == 44 - -doc="finished" diff --git a/ci/run-tests.go b/ci/run-tests.go index 031a08ab..42558564 100644 --- a/ci/run-tests.go +++ b/ci/run-tests.go @@ -1,7 +1,8 @@ -// Copyright 2018 The go-python Authors. All rights reserved. +// Copyright ©2018 The go-python Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build ignore // +build ignore package main @@ -10,32 +11,33 @@ import ( "bufio" "bytes" "flag" - "io/ioutil" + "fmt" "log" "os" "os/exec" "strings" + "time" ) func main() { log.SetPrefix("ci: ") log.SetFlags(0) + start := time.Now() + defer func() { + log.Printf("elapsed time: %v\n", time.Since(start)) + }() + var ( - race = flag.Bool("race", false, "enable race detector") - cover = flag.Bool("cover", false, "enable code coverage") - tags = flag.String("tags", "", "build tags") + race = flag.Bool("race", false, "enable race detector") + cover = flag.String("coverpkg", "", "apply coverage analysis in each test to packages matching the patterns.") + tags = flag.String("tags", "", "build tags") + verbose = flag.Bool("v", false, "enable verbose output") ) flag.Parse() - out := new(bytes.Buffer) - cmd := exec.Command("go", "list", "./...") - cmd.Stdout = out - cmd.Stderr = os.Stderr - cmd.Stdin = os.Stdin - - err := cmd.Run() + pkgs, err := pkgList() if err != nil { log.Fatal(err) } @@ -48,23 +50,24 @@ func main() { args := []string{"test"} - if *cover { - args = append(args, "-coverprofile=profile.out", "-covermode=atomic") + if *verbose { + args = append(args, "-v") + } + if *cover != "" { + args = append(args, "-coverprofile=profile.out", "-covermode=atomic", "-coverpkg="+*cover) } if *tags != "" { args = append(args, "-tags="+*tags) } - if *race { - args = append(args, "-race") + switch { + case *race: + args = append(args, "-race", "-timeout=20m") + default: + args = append(args, "-timeout=10m") } args = append(args, "") - scan := bufio.NewScanner(out) - for scan.Scan() { - pkg := scan.Text() - if strings.Contains(pkg, "vendor") { - continue - } + for _, pkg := range pkgs { args[len(args)-1] = pkg cmd := exec.Command("go", args...) cmd.Stdin = os.Stdin @@ -74,8 +77,8 @@ func main() { if err != nil { log.Fatal(err) } - if *cover { - profile, err := ioutil.ReadFile("profile.out") + if *cover != "" { + profile, err := os.ReadFile("profile.out") if err != nil { log.Fatal(err) } @@ -92,3 +95,28 @@ func main() { log.Fatal(err) } } + +func pkgList() ([]string, error) { + out := new(bytes.Buffer) + cmd := exec.Command("go", "list", "./...") + cmd.Stdout = out + cmd.Stderr = os.Stderr + cmd.Stdin = os.Stdin + + err := cmd.Run() + if err != nil { + return nil, fmt.Errorf("could not get package list: %w", err) + } + + var pkgs []string + scan := bufio.NewScanner(out) + for scan.Scan() { + pkg := scan.Text() + if strings.Contains(pkg, "vendor") { + continue + } + pkgs = append(pkgs, pkg) + } + + return pkgs, nil +} diff --git a/compile/compile.go b/compile/compile.go index dc8851cf..fd8d7a2b 100644 --- a/compile/compile.go +++ b/compile/compile.go @@ -9,6 +9,7 @@ package compile // FIXME kill ast.Identifier and turn into string? import ( + "bytes" "fmt" "log" "strings" @@ -89,33 +90,36 @@ func init() { py.Compile = Compile } -// Compile(source, filename, mode, flags, dont_inherit) -> code object +// Compile(src, srcDesc, compileMode, flags, dont_inherit) -> code object // // Compile the source string (a Python module, statement or expression) -// into a code object that can be executed by exec() or eval(). -// The filename will be used for run-time error messages. -// The mode must be 'exec' to compile a module, 'single' to compile a -// single (interactive) statement, or 'eval' to compile an expression. +// into a code object that can be executed. +// +// srcDesc is used for run-time error messages and is typically a file system pathname, +// +// See py.CompileMode for compile mode options. +// // The flags argument, if present, controls which future statements influence // the compilation of the code. +// // The dont_inherit argument, if non-zero, stops the compilation inheriting // the effects of any future statements in effect in the code calling // compile; if absent or zero these statements do influence the compilation, // in addition to any features explicitly specified. -func Compile(str, filename, mode string, futureFlags int, dont_inherit bool) (py.Object, error) { +func Compile(src, srcDesc string, mode py.CompileMode, futureFlags int, dont_inherit bool) (*py.Code, error) { // Parse Ast - Ast, err := parser.ParseString(str, mode) + Ast, err := parser.Parse(bytes.NewBufferString(src), srcDesc, mode) if err != nil { return nil, err } // Make symbol table - SymTable, err := symtable.NewSymTable(Ast, filename) + SymTable, err := symtable.NewSymTable(Ast, srcDesc) if err != nil { return nil, err } c := newCompiler(nil, compilerScopeModule) - c.Filename = filename - err = c.compileAst(Ast, filename, futureFlags, dont_inherit, SymTable) + c.Filename = srcDesc + err = c.compileAst(Ast, srcDesc, futureFlags, dont_inherit, SymTable) if err != nil { return nil, err } @@ -209,6 +213,8 @@ func (c *compiler) compileAst(Ast ast.Ast, filename string, futureFlags int, don case *ast.Suite: panic("suite should not be possible") case *ast.Lambda: + code.Argcount = int32(len(node.Args.Args)) + code.Kwonlyargcount = int32(len(node.Args.Kwonlyargs)) // Make None the first constant as lambda can't have a docstring c.Const(py.None) code.Name = "" @@ -216,6 +222,8 @@ func (c *compiler) compileAst(Ast ast.Ast, filename string, futureFlags int, don c.Expr(node.Body) valueOnStack = true case *ast.FunctionDef: + code.Argcount = int32(len(node.Args.Args)) + code.Kwonlyargcount = int32(len(node.Args.Kwonlyargs)) code.Name = string(node.Name) c.setQualname() c.Stmts(c.docString(node.Body, true)) @@ -295,6 +303,7 @@ func (c *compiler) compileAst(Ast ast.Ast, filename string, futureFlags int, don code.Stacksize = int32(c.OpCodes.StackDepth()) code.Nlocals = int32(len(code.Varnames)) code.Lnotab = string(c.OpCodes.Lnotab()) + code.InitCell2arg() return nil } @@ -431,9 +440,11 @@ func (c *compiler) Jump(Op vm.OpCode, Dest *Label) { c.OpCodes.Add(instr) } -/* The test for LOCAL must come before the test for FREE in order to - handle classes where name is both local and free. The local var is - a method and the free var is a free var referenced within a method. +/* +The test for LOCAL must come before the test for FREE in order to + + handle classes where name is both local and free. The local var is + a method and the free var is a free var referenced within a method. */ func (c *compiler) getRefType(name string) symtable.Scope { if c.scopeType == compilerScopeClass && name == "__class__" { @@ -473,7 +484,8 @@ func (c *compiler) makeClosure(code *py.Code, args uint32, child *compiler, qual if reftype == symtable.ScopeCell { arg = c.FindId(name, c.Code.Cellvars) } else { /* (reftype == FREE) */ - arg = c.FindId(name, c.Code.Freevars) + // using CellAndFreeVars in closures requires skipping Cellvars + arg = len(c.Code.Cellvars) + c.FindId(name, c.Code.Freevars) } if arg < 0 { panic(fmt.Sprintf("compile: makeClosure: lookup %q in %q %v %v\nfreevars of %q: %v\n", name, c.SymTable.Name, reftype, arg, code.Name, code.Freevars)) @@ -662,27 +674,31 @@ func (c *compiler) class(Ast ast.Ast, class *ast.ClassDef) { } /* - Implements the with statement from PEP 343. - - The semantics outlined in that PEP are as follows: - - with EXPR as VAR: - BLOCK - - It is implemented roughly as: - - context = EXPR - exit = context.__exit__ # not calling it - value = context.__enter__() - try: - VAR = value # if VAR present in the syntax - BLOCK - finally: - if an exception was raised: - exc = copy of (exception, instance, traceback) - else: - exc = (None, None, None) - exit(*exc) +Implements the with statement from PEP 343. + +The semantics outlined in that PEP are as follows: + +with EXPR as VAR: + + BLOCK + +It is implemented roughly as: + +context = EXPR +exit = context.__exit__ # not calling it +value = context.__enter__() +try: + + VAR = value # if VAR present in the syntax + BLOCK + +finally: + + if an exception was raised: + exc = copy of (exception, instance, traceback) + else: + exc = (None, None, None) + exit(*exc) */ func (c *compiler) with(node *ast.With, pos int) { item := node.Items[pos] @@ -724,37 +740,38 @@ func (c *compiler) with(node *ast.With, pos int) { c.Op(vm.END_FINALLY) } -/* Code generated for "try: finally: " is as follows: - - SETUP_FINALLY L - - POP_BLOCK - LOAD_CONST - L: - END_FINALLY - - The special instructions use the block stack. Each block - stack entry contains the instruction that created it (here - SETUP_FINALLY), the level of the value stack at the time the - block stack entry was created, and a label (here L). - - SETUP_FINALLY: - Pushes the current value stack level and the label - onto the block stack. - POP_BLOCK: - Pops en entry from the block stack, and pops the value - stack until its level is the same as indicated on the - block stack. (The label is ignored.) - END_FINALLY: - Pops a variable number of entries from the *value* stack - and re-raises the exception they specify. The number of - entries popped depends on the (pseudo) exception type. - - The block stack is unwound when an exception is raised: - when a SETUP_FINALLY entry is found, the exception is pushed - onto the value stack (and the exception condition is cleared), - and the interpreter jumps to the label gotten from the block - stack. +/* +Code generated for "try: finally: " is as follows: + + SETUP_FINALLY L + + POP_BLOCK + LOAD_CONST + L: + END_FINALLY + + The special instructions use the block stack. Each block + stack entry contains the instruction that created it (here + SETUP_FINALLY), the level of the value stack at the time the + block stack entry was created, and a label (here L). + + SETUP_FINALLY: + Pushes the current value stack level and the label + onto the block stack. + POP_BLOCK: + Pops en entry from the block stack, and pops the value + stack until its level is the same as indicated on the + block stack. (The label is ignored.) + END_FINALLY: + Pops a variable number of entries from the *value* stack + and re-raises the exception they specify. The number of + entries popped depends on the (pseudo) exception type. + + The block stack is unwound when an exception is raised: + when a SETUP_FINALLY entry is found, the exception is pushed + onto the value stack (and the exception condition is cleared), + and the interpreter jumps to the label gotten from the block + stack. */ func (c *compiler) tryFinally(node *ast.Try) { end := new(Label) @@ -776,35 +793,36 @@ func (c *compiler) tryFinally(node *ast.Try) { } /* - Code generated for "try: S except E1 as V1: S1 except E2 as V2: S2 ...": - (The contents of the value stack is shown in [], with the top - at the right; 'tb' is trace-back info, 'val' the exception's - associated value, and 'exc' the exception.) - - Value stack Label Instruction Argument - [] SETUP_EXCEPT L1 - [] - [] POP_BLOCK - [] JUMP_FORWARD L0 - - [tb, val, exc] L1: DUP ) - [tb, val, exc, exc] ) - [tb, val, exc, exc, E1] COMPARE_OP EXC_MATCH ) only if E1 - [tb, val, exc, 1-or-0] POP_JUMP_IF_FALSE L2 ) - [tb, val, exc] POP - [tb, val] (or POP if no V1) - [tb] POP - [] - JUMP_FORWARD L0 - - [tb, val, exc] L2: DUP - .............................etc....................... - - [tb, val, exc] Ln+1: END_FINALLY # re-raise exception - - [] L0: - - Of course, parts are not generated if Vi or Ei is not present. +Code generated for "try: S except E1 as V1: S1 except E2 as V2: S2 ...": +(The contents of the value stack is shown in [], with the top +at the right; 'tb' is trace-back info, 'val' the exception's +associated value, and 'exc' the exception.) + +Value stack Label Instruction Argument +[] SETUP_EXCEPT L1 +[] +[] POP_BLOCK +[] JUMP_FORWARD L0 + +[tb, val, exc] L1: DUP ) +[tb, val, exc, exc] ) +[tb, val, exc, exc, E1] COMPARE_OP EXC_MATCH ) only if E1 +[tb, val, exc, 1-or-0] POP_JUMP_IF_FALSE L2 ) +[tb, val, exc] POP +[tb, val] (or POP if no V1) +[tb] POP +[] + + JUMP_FORWARD L0 + +[tb, val, exc] L2: DUP +.............................etc....................... + +[tb, val, exc] Ln+1: END_FINALLY # re-raise exception + +[] L0: + +Of course, parts are not generated if Vi or Ei is not present. */ func (c *compiler) tryExcept(node *ast.Try) { c.loops.Push(loop{Type: exceptLoop}) @@ -893,11 +911,13 @@ func (c *compiler) try(node *ast.Try) { } } -/* The IMPORT_NAME opcode was already generated. This function - merely needs to bind the result to a name. +/* +The IMPORT_NAME opcode was already generated. This function + + merely needs to bind the result to a name. - If there is a dot in name, we need to split it and emit a - LOAD_ATTR for each name. + If there is a dot in name, we need to split it and emit a + LOAD_ATTR for each name. */ func (c *compiler) importAs(name ast.Identifier, asname ast.Identifier) { attrs := strings.Split(string(name), ".") @@ -909,12 +929,14 @@ func (c *compiler) importAs(name ast.Identifier, asname ast.Identifier) { c.NameOp(string(asname), ast.Store) } -/* The Import node stores a module name like a.b.c as a single - string. This is convenient for all cases except - import a.b.c as d - where we need to parse that string to extract the individual - module names. - XXX Perhaps change the representation to make this case simpler? +/* +The Import node stores a module name like a.b.c as a single + + string. This is convenient for all cases except + import a.b.c as d + where we need to parse that string to extract the individual + module names. + XXX Perhaps change the representation to make this case simpler? */ func (c *compiler) import_(node *ast.Import) { //n = asdl_seq_LEN(s.v.Import.names); @@ -1203,6 +1225,7 @@ func (c *compiler) Stmt(stmt ast.Stmt) { l := c.loops.Top() if l == nil { c.panicSyntaxErrorf(node, loopError) + panic("impossible") } switch l.Type { case loopLoop: @@ -1342,12 +1365,16 @@ func (c *compiler) NameOp(name string, ctx ast.ExprContext) { default: panic("NameOp: ctx invalid for name variable") } - break } if op == 0 { panic("NameOp: Op not set") } - c.OpArg(op, c.Index(mangled, dict)) + i := c.Index(mangled, dict) + // using CellAndFreeVars in closures requires skipping Cellvars + if scope == symtable.ScopeFree { + i += uint32(len(c.Code.Cellvars)) + } + c.OpArg(op, i) } // Call a function which is already on the stack with n arguments already on the stack @@ -1390,7 +1417,9 @@ func (c *compiler) callHelper(n int, Args []ast.Expr, Keywords []*ast.Keyword, S c.OpArg(op, uint32(args+kwargs<<8)) } -/* List and set comprehensions and generator expressions work by creating a +/* + List and set comprehensions and generator expressions work by creating a + nested function to perform the actual iteration. This means that the iteration variables don't leak into the current scope. The defined function is called immediately following its definition, with the diff --git a/compile/compile_data_test.go b/compile/compile_data_test.go index edc53720..f0ee0422 100644 --- a/compile/compile_data_test.go +++ b/compile/compile_data_test.go @@ -12,7 +12,7 @@ import ( var compileTestData = []struct { in string - mode string // exec, eval or single + mode py.CompileMode out *py.Code exceptionType *py.Type errString string diff --git a/compile/compile_test.go b/compile/compile_test.go index 72d4a3f6..fe2c8671 100644 --- a/compile/compile_test.go +++ b/compile/compile_test.go @@ -10,7 +10,7 @@ package compile import ( "fmt" - "io/ioutil" + "io" "os/exec" "testing" @@ -118,7 +118,7 @@ func EqCodeCode(t *testing.T, name string, a, b string) { t.Errorf("%s code want %q, got %q", name, a, b) return } - stdoutData, err := ioutil.ReadAll(stdout) + stdoutData, err := io.ReadAll(stdout) if err != nil { t.Fatalf("Failed to read data: %v", err) } @@ -163,7 +163,7 @@ func EqCode(t *testing.T, name string, a, b *py.Code) { func TestCompile(t *testing.T) { for _, test := range compileTestData { // log.Printf(">>> %s", test.in) - codeObj, err := Compile(test.in, "", test.mode, 0, true) + code, err := Compile(test.in, "", test.mode, 0, true) if err != nil { if test.exceptionType == nil { t.Errorf("%s: Got exception %v when not expecting one", test.in, err) @@ -196,17 +196,12 @@ func TestCompile(t *testing.T) { } } else { if test.out == nil { - if codeObj != nil { - t.Errorf("%s: Expecting nil *py.Code but got %T", test.in, codeObj) + if code != nil { + t.Errorf("%s: Expecting nil *py.Code but got %T", test.in, code) } } else { - code, ok := codeObj.(*py.Code) - if !ok { - t.Errorf("%s: Expecting *py.Code but got %T", test.in, codeObj) - } else { - //t.Logf("Testing %q", test.in) - EqCode(t, test.in, test.out, code) - } + //t.Logf("Testing %q", test.in) + EqCode(t, test.in, test.out, code) } } } diff --git a/compile/instructions_test.go b/compile/instructions_test.go index d5fecfed..01c7db4e 100644 --- a/compile/instructions_test.go +++ b/compile/instructions_test.go @@ -53,7 +53,7 @@ func TestLnotab(t *testing.T) { }, } { got := test.instrs.Lnotab() - if bytes.Compare(test.want, got) != 0 { + if !bytes.Equal(test.want, got) { t.Errorf("%d: want %d got %d", i, test.want, got) } } diff --git a/compile/legacy.go b/compile/legacy.go index 6578b18f..c5f2571e 100644 --- a/compile/legacy.go +++ b/compile/legacy.go @@ -11,8 +11,8 @@ import ( "os/exec" "strings" - "github.com/go-python/gpython/marshal" "github.com/go-python/gpython/py" + "github.com/go-python/gpython/stdlib/marshal" ) // Compile with python3.4 - not used any more but keep for the moment! diff --git a/examples/embedding/README.md b/examples/embedding/README.md new file mode 100644 index 00000000..1dc5ae69 --- /dev/null +++ b/examples/embedding/README.md @@ -0,0 +1,102 @@ +## Embedding gpython + +This is an example demonstrating how to embed gpython into a Go application. + + +### Why embed gpython? + +Embedding a highly capable and familiar "interpreted" language allows your users +to easily augment app behavior, configuration, and customization -- all post-deployment. + +Have you ever discovered an exciting software project but lost interest when you had to also +learn its esoteric language schema? In an era of limited attention span, +most people are generally turned off if they have to learn a new language in addition to learning +to use your app. + +If you consider [why use Python](https://www.stxnext.com/what-is-python-used-for/), then perhaps also +consider that your users will be interested to hear that your software offers +even more value that it can be driven from a scripting language they already know. + +Python is widespread in finance, sciences, hobbyist programming and is often +endearingly regarded as most popular programming language for non-developers. +If your application can be driven by embedded Python, then chances are others will +feel excited and empowered that your project can be used out of the box +and feel like familiar territory. + +### But what about the lack of python modules? + +There are only be a small number of native modules available, but don't forget you have the entire +Go standard library and *any* Go package you can name at your fingertips to expose! +This plus multi-context capability gives gpython enormous potential on how it can +serve you. + +So basically, gpython is only off the table if you need to run python that makes heavy use of +modules that are only available in CPython. + +### Packing List + +| | | +|------------------------- | ------------------------------------------------------------------| +| `main.go` | if no args, runs in REPL mode, otherwise runs the given file | +| `lib/mylib.py` | models a library that your application would expose for users | +| `lib/REPL-startup.py` | invoked by `main.go` when starting REPL mode | +| `testdata/mylib-demo.py` | models a user-authored script that consumes `mylib` | +| `mylib.module.go` | Go implementation of `mylib_go` consumed by `mylib` | + + +### Invoking a Python Script + +```bash +$ cd examples/embedding/ +$ go build . +$ ./embedding ./testdata/mylib-demo.py +``` +``` +Welcome to a gpython embedded example, + where your wildest Go-based python dreams come true! + +========================================================== + Python 3.4 (github.com/go-python/gpython) + go1.17.6 on darwin amd64 +========================================================== + +Spring Break itinerary: + Stop 1: Miami, Florida | 7 nights + Stop 2: Mallorca, Spain | 3 nights + Stop 3: Ibiza, Spain | 14 nights + Stop 4: Monaco | 12 nights +### Made with Vacaton 1.0 by Fletch F. Fletcher + +I bet Monaco will be the best! +``` + +### REPL Mode + +```bash +$ ./embedding +``` +``` +======= Entering REPL mode, press Ctrl+D to exit ======= + +========================================================== + Python 3.4 (github.com/go-python/gpython) + go1.17.6 on darwin amd64 +========================================================== + +>>> v = Vacation("Spring Break", Stop("Florida", 3), Stop("Nice", 7)) +>>> print(str(v)) +Spring Break, 2 stop(s) +>>> v.PrintItinerary() +Spring Break itinerary: + Stop 1: Florida | 3 nights + Stop 2: Nice | 7 nights +### Made with Vacaton 1.0 by Fletch F. Fletcher +``` + +## Takeways + + - `main.go` demonstrates high-level convenience functions such as `py.RunFile()`. + - Embedding a Go `struct` as a Python object only requires that it implement `py.Object`, which is a single function: + `Type() *py.Type` + - See [py/run.go](https://github.com/go-python/gpython/tree/main/py/run.go) for more about interpreter instances and `py.Context` + - Helper functions are available in [py/util.go](https://github.com/go-python/gpython/tree/main/py/util.go) and your contributions are welcome! diff --git a/examples/embedding/lib/REPL-startup.py b/examples/embedding/lib/REPL-startup.py new file mode 100644 index 00000000..4c84fb03 --- /dev/null +++ b/examples/embedding/lib/REPL-startup.py @@ -0,0 +1,10 @@ +# Copyright 2022 The go-python Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# This file is called from main.go when in REPL mode + +# This is here to demonstrate making life easier for your users in REPL mode +# by doing pre-setup here so they don't have to import every time they start. +from mylib import * + diff --git a/examples/embedding/lib/mylib.py b/examples/embedding/lib/mylib.py new file mode 100644 index 00000000..b0105e7d --- /dev/null +++ b/examples/embedding/lib/mylib.py @@ -0,0 +1,53 @@ +# Copyright 2022 The go-python Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +import mylib_go as _go + +PY_VERSION = _go.PY_VERSION + + +print(''' +========================================================== + %s +========================================================== +''' % (PY_VERSION, )) + + +def Stop(location, num_nights = 2): + return _go.VacationStop_new(location, num_nights) + + +class Vacation: + + def __init__(self, tripName, *stops): + self._v, self._libVers = _go.Vacation_new() + self.tripName = tripName + self.AddStops(*stops) + + def __str__(self): + return "%s, %d stop(s)" % (self.tripName, self.NumStops()) + + def NumStops(self): + return self._v.num_stops() + + def GetStop(self, stop_num): + return self._v.get_stop(stop_num) + + def AddStops(self, *stops): + self._v.add_stops(stops) + + def PrintItinerary(self): + print(self.tripName, "itinerary:") + i = 1 + while 1: + + try: + stop = self.GetStop(i) + except IndexError: + break + + print(" Stop %d: %s" % (i, str(stop))) + i += 1 + + print("### Made with %s " % self._libVers) diff --git a/examples/embedding/main.go b/examples/embedding/main.go new file mode 100644 index 00000000..e4aadfef --- /dev/null +++ b/examples/embedding/main.go @@ -0,0 +1,54 @@ +// Copyright 2022 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "flag" + "fmt" + + // This initializes gpython for runtime execution and is essential. + // It defines forward-declared symbols and registers native built-in modules, such as sys and time. + _ "github.com/go-python/gpython/stdlib" + + // Commonly consumed gpython + "github.com/go-python/gpython/py" + "github.com/go-python/gpython/repl" + "github.com/go-python/gpython/repl/cli" +) + +func main() { + flag.Parse() + runWithFile(flag.Arg(0)) +} + +func runWithFile(pyFile string) error { + + // See type Context interface and related docs + ctx := py.NewContext(py.DefaultContextOpts()) + + // This drives modules being able to perform cleanup and release resources + defer ctx.Close() + + var err error + if len(pyFile) == 0 { + replCtx := repl.New(ctx) + + fmt.Print("\n======= Entering REPL mode, press Ctrl+D to exit =======\n") + + _, err = py.RunFile(ctx, "lib/REPL-startup.py", py.CompileOpts{}, replCtx.Module) + if err == nil { + cli.RunREPL(replCtx) + } + + } else { + _, err = py.RunFile(ctx, pyFile, py.CompileOpts{}, nil) + } + + if err != nil { + py.TracebackDump(err) + } + + return err +} diff --git a/examples/embedding/main_test.go b/examples/embedding/main_test.go new file mode 100644 index 00000000..651d2237 --- /dev/null +++ b/examples/embedding/main_test.go @@ -0,0 +1,15 @@ +// Copyright 2022 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "testing" + + "github.com/go-python/gpython/pytest" +) + +func TestEmbeddedExample(t *testing.T) { + pytest.RunScript(t, "./testdata/mylib-demo.py") +} diff --git a/examples/embedding/mylib.module.go b/examples/embedding/mylib.module.go new file mode 100644 index 00000000..1efb0fe8 --- /dev/null +++ b/examples/embedding/mylib.module.go @@ -0,0 +1,178 @@ +// Copyright 2022 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "fmt" + "runtime" + + "github.com/go-python/gpython/py" +) + +// These gpython py.Object type delcarations are the bridge between gpython and embedded Go types. +var ( + PyVacationStopType = py.NewType("Stop", "") + PyVacationType = py.NewType("Vacation", "") +) + +// init is where you register your embedded module and attach methods to your embedded class types. +func init() { + + // For each of your embedded python types, attach instance methods. + // When an instance method is invoked, the "self" py.Object is the instance. + PyVacationStopType.Dict["Set"] = py.MustNewMethod("Set", VacationStop_Set, 0, "") + PyVacationStopType.Dict["Get"] = py.MustNewMethod("Get", VacationStop_Get, 0, "") + PyVacationType.Dict["add_stops"] = py.MustNewMethod("Vacation.add_stops", Vacation_add_stops, 0, "") + PyVacationType.Dict["num_stops"] = py.MustNewMethod("Vacation.num_stops", Vacation_num_stops, 0, "") + PyVacationType.Dict["get_stop"] = py.MustNewMethod("Vacation.get_stop", Vacation_get_stop, 0, "") + + // Bind methods attached at the module (global) level. + // When these are invoked, the first py.Object param (typically "self") is the bound *Module instance. + methods := []*py.Method{ + py.MustNewMethod("VacationStop_new", VacationStop_new, 0, ""), + py.MustNewMethod("Vacation_new", Vacation_new, 0, ""), + } + + // Register a ModuleImpl instance used by the gpython runtime to instantiate new py.Module when first imported. + py.RegisterModule(&py.ModuleImpl{ + Info: py.ModuleInfo{ + Name: "mylib_go", + Doc: "Example embedded python module", + }, + Methods: methods, + Globals: py.StringDict{ + "PY_VERSION": py.String("Python 3.4 (github.com/go-python/gpython)"), + "GO_VERSION": py.String(fmt.Sprintf("%s on %s %s", runtime.Version(), runtime.GOOS, runtime.GOARCH)), + "MYLIB_VERS": py.String("Vacation 1.0 by Fletch F. Fletcher"), + }, + OnContextClosed: func(instance *py.Module) { + py.Println(instance, "<<< host py.Context of py.Module instance closing >>>\n+++") + }, + }) +} + +// VacationStop is an example Go struct to embed. +type VacationStop struct { + Desc py.String + NumNights py.Int +} + +// Type comprises the py.Object interface, allowing a Go struct to be cast as a py.Object. +// Instance methods of an type are then attached to this type object +func (stop *VacationStop) Type() *py.Type { + return PyVacationStopType +} + +func (stop *VacationStop) M__str__() (py.Object, error) { + line := fmt.Sprintf(" %-16v | %2v nights", stop.Desc, stop.NumNights) + return py.String(line), nil +} + +func (stop *VacationStop) M__repr__() (py.Object, error) { + return stop.M__str__() +} + +func VacationStop_new(module py.Object, args py.Tuple) (py.Object, error) { + stop := &VacationStop{} + VacationStop_Set(stop, args) + return stop, nil +} + +// VacationStop_Set is an embedded instance method of VacationStop +func VacationStop_Set(self py.Object, args py.Tuple) (py.Object, error) { + stop := self.(*VacationStop) + + // Check out other convenience functions in py/util.go + // Also available is py.ParseTuple(args, "si", ...) + err := py.LoadTuple(args, []interface{}{&stop.Desc, &stop.NumNights}) + if err != nil { + return nil, err + } + + /* Alternative util func is ParseTuple(): + var desc, nights py.Object + err := py.ParseTuple(args, "si", &desc, &nights) + if err != nil { + return nil, err + } + stop.Desc = desc.(py.String) + stop.NumNights = desc.(py.Int) + */ + + return py.None, nil +} + +// VacationStop_Get is an embedded instance method of VacationStop +func VacationStop_Get(self py.Object, args py.Tuple) (py.Object, error) { + stop := self.(*VacationStop) + + return py.Tuple{ + stop.Desc, + stop.NumNights, + }, nil +} + +type Vacation struct { + Stops []*VacationStop + MadeBy string +} + +func (v *Vacation) Type() *py.Type { + return PyVacationType +} + +func Vacation_new(module py.Object, args py.Tuple) (py.Object, error) { + v := &Vacation{} + + // For Module-bound methods, we have easy access to the parent Module + py.LoadAttr(module, "MYLIB_VERS", &v.MadeBy) + + ret := py.Tuple{ + v, + py.String(v.MadeBy), + } + return ret, nil +} + +func Vacation_num_stops(self py.Object, args py.Tuple) (py.Object, error) { + v := self.(*Vacation) + return py.Int(len(v.Stops)), nil +} + +func Vacation_get_stop(self py.Object, args py.Tuple) (py.Object, error) { + v := self.(*Vacation) + + // Check out other convenience functions in py/util.go + // If you would like to be a contributor for gpython, improving these or adding more is a great place to start! + stopNum, err := py.GetInt(args[0]) + if err != nil { + return nil, err + } + + if stopNum < 1 || int(stopNum) > len(v.Stops) { + return nil, py.ExceptionNewf(py.IndexError, "invalid stop index") + } + + return py.Object(v.Stops[stopNum-1]), nil +} + +func Vacation_add_stops(self py.Object, args py.Tuple) (py.Object, error) { + v := self.(*Vacation) + srcStops, ok := args[0].(py.Tuple) + if !ok { + return nil, py.ExceptionNewf(py.TypeError, "expected Tuple, got %T", args[0]) + } + + for _, arg := range srcStops { + stop, ok := arg.(*VacationStop) + if !ok { + return nil, py.ExceptionNewf(py.TypeError, "expected Stop, got %T", arg) + } + + v.Stops = append(v.Stops, stop) + } + + return py.None, nil +} diff --git a/examples/embedding/testdata/mylib-demo.py b/examples/embedding/testdata/mylib-demo.py new file mode 100644 index 00000000..1785c307 --- /dev/null +++ b/examples/embedding/testdata/mylib-demo.py @@ -0,0 +1,19 @@ +# Copyright 2022 The go-python Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +print(''' +Welcome to a gpython embedded example, + where your wildest Go-based python dreams come true!''') + +# This is a model for a public/user-side script that you or users would maintain, +# offering an open canvas to drive app behavior, customization, or anything you can dream up. +# +# Modules you offer for consumption can also serve to document such things. +from mylib import * + +springBreak = Vacation("Spring Break", Stop("Miami, Florida", 7), Stop("Mallorca, Spain", 3)) +springBreak.AddStops(Stop("Ibiza, Spain", 14), Stop("Monaco", 12)) +springBreak.PrintItinerary() + +print("\nI bet %s will be the best!\n" % springBreak.GetStop(4).Get()[0]) diff --git a/examples/embedding/testdata/mylib-demo_golden.txt b/examples/embedding/testdata/mylib-demo_golden.txt new file mode 100644 index 00000000..445e0a65 --- /dev/null +++ b/examples/embedding/testdata/mylib-demo_golden.txt @@ -0,0 +1,19 @@ + +Welcome to a gpython embedded example, + where your wildest Go-based python dreams come true! + +========================================================== + Python 3.4 (github.com/go-python/gpython) +========================================================== + +Spring Break itinerary: + Stop 1: Miami, Florida | 7 nights + Stop 2: Mallorca, Spain | 3 nights + Stop 3: Ibiza, Spain | 14 nights + Stop 4: Monaco | 12 nights +### Made with Vacation 1.0 by Fletch F. Fletcher + +I bet Monaco will be the best! + +<<< host py.Context of py.Module instance closing >>> ++++ diff --git a/examples/multi-context/main.go b/examples/multi-context/main.go new file mode 100644 index 00000000..f645e766 --- /dev/null +++ b/examples/multi-context/main.go @@ -0,0 +1,140 @@ +// Copyright 2022 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "fmt" + "log" + "runtime" + "strings" + "sync" + "time" + + // This initializes gpython for runtime execution and is critical. + // It defines forward-declared symbols and registers native built-in modules, such as sys and time. + _ "github.com/go-python/gpython/stdlib" + + // This is the primary import for gpython. + // It contains all symbols needed to fully compile and run python. + "github.com/go-python/gpython/py" +) + +func main() { + + // The total job count implies a fixed amount of work. + // The number of workers is how many py.Context (in concurrent goroutines) to pull jobs off the queue. + // One worker does all the work serially while N number of workers will (ideally) divides up. + totalJobs := 20 + + for i := 0; i < 10; i++ { + numWorkers := i + 1 + elapsed := RunMultiPi(numWorkers, totalJobs) + fmt.Printf("=====> %2d worker(s): %v\n\n", numWorkers, elapsed) + + // Give each trial a fresh start + runtime.GC() + } +} + +var jobScript = ` +pi = chud.pi_chudnovsky_bs(numDigits) +last_5 = pi % 100000 +print("%s: last 5 digits of %d is %d (job #%0d)" % (WORKER_ID, numDigits, last_5, jobID)) +` + +var jobSrcTemplate = ` +import pi_chudnovsky_bs as chud + +WORKER_ID = "{{WORKER_ID}}" + +print("%s ready!" % (WORKER_ID)) +` + +type worker struct { + name string + ctx py.Context + main *py.Module + job *py.Code +} + +func (w *worker) compileTemplate(pySrc string) { + pySrc = strings.Replace(pySrc, "{{WORKER_ID}}", w.name, -1) + + mainImpl := py.ModuleImpl{ + CodeSrc: pySrc, + } + + var err error + w.main, err = w.ctx.ModuleInit(&mainImpl) + if err != nil { + log.Fatal(err) + } +} + +func RunMultiPi(numWorkers, numTimes int) time.Duration { + var workersRunning sync.WaitGroup + + fmt.Printf("Starting %d worker(s) to calculate %d jobs...\n", numWorkers, numTimes) + + jobPipe := make(chan int) + go func() { + for i := 0; i < numTimes; i++ { + jobPipe <- i + 1 + } + close(jobPipe) + }() + + // Note that py.Code can be shared (accessed concurrently) since it is an inherently read-only object + jobCode, err := py.Compile(jobScript, "", py.ExecMode, 0, true) + if err != nil { + log.Fatal("jobScript failed to comple") + } + + workers := make([]worker, numWorkers) + for i := 0; i < numWorkers; i++ { + + opts := py.DefaultContextOpts() + + // Make sure our import statement will find pi_chudnovsky_bs + opts.SysPaths = append(opts.SysPaths, "..") + + workers[i] = worker{ + name: fmt.Sprintf("Worker #%d", i+1), + ctx: py.NewContext(opts), + job: jobCode, + } + + workersRunning.Add(1) + } + + startTime := time.Now() + + for i := range workers { + w := workers[i] + go func() { + + // Compiling can be concurrent since there is no associated py.Context + w.compileTemplate(jobSrcTemplate) + + for jobID := range jobPipe { + numDigits := 100000 + if jobID%2 == 0 { + numDigits *= 10 + } + py.SetAttrString(w.main.Globals, "numDigits", py.Int(numDigits)) + py.SetAttrString(w.main.Globals, "jobID", py.Int(jobID)) + w.ctx.RunCode(jobCode, w.main.Globals, w.main.Globals, nil) + } + workersRunning.Done() + + // This drives modules being able to perform cleanup and release resources + w.ctx.Close() + }() + } + + workersRunning.Wait() + + return time.Since(startTime) +} diff --git a/go.mod b/go.mod index 1bfb6b3f..c27f93bf 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,16 @@ module github.com/go-python/gpython +go 1.18 + +require ( + github.com/google/go-cmp v0.5.8 + github.com/gopherjs/gopherwasm v1.1.0 + github.com/peterh/liner v1.2.2 +) + require ( - github.com/gopherjs/gopherwasm v1.0.0 // indirect - github.com/peterh/liner v1.1.0 + github.com/gopherjs/gopherjs v0.0.0-20180825215210-0210a2f0f73c // indirect + github.com/mattn/go-runewidth v0.0.13 // indirect + github.com/rivo/uniseg v0.3.4 // indirect + golang.org/x/sys v0.0.0-20220817070843-5a390386f1f2 // indirect ) diff --git a/go.sum b/go.sum index c17fb31d..0d0cbc2f 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,17 @@ +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/gopherjs/gopherjs v0.0.0-20180825215210-0210a2f0f73c h1:16eHWuMGvCjSfgRJKqIzapE78onvvTbdi1rMkU00lZw= github.com/gopherjs/gopherjs v0.0.0-20180825215210-0210a2f0f73c/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gopherjs/gopherwasm v1.0.0 h1:32nge/RlujS1Im4HNCJPp0NbBOAeBXFuT1KonUuLl+Y= -github.com/gopherjs/gopherwasm v1.0.0/go.mod h1:SkZ8z7CWBz5VXbhJel8TxCmAcsQqzgWGR/8nMhyhZSI= -github.com/mattn/go-runewidth v0.0.3 h1:a+kO+98RDGEfo6asOGMmpodZq4FNtnGP54yps8BzLR4= +github.com/gopherjs/gopherwasm v1.1.0 h1:fA2uLoctU5+T3OhOn2vYP0DVT6pxc7xhTlBB1paATqQ= +github.com/gopherjs/gopherwasm v1.1.0/go.mod h1:SkZ8z7CWBz5VXbhJel8TxCmAcsQqzgWGR/8nMhyhZSI= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/peterh/liner v1.1.0 h1:f+aAedNJA6uk7+6rXsYBnhdo4Xux7ESLe+kcuVUF5os= -github.com/peterh/liner v1.1.0/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= +github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= +github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/peterh/liner v1.2.2 h1:aJ4AOodmL+JxOZZEL2u9iJf8omNRpqHc/EbrK+3mAXw= +github.com/peterh/liner v1.2.2/go.mod h1:xFwJyiKIXJZUKItq5dGHZSTBRAuG/CpeNpWLyiNRNwI= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.3.4 h1:3Z3Eu6FGHZWSfNKJTOUiPatWwfc7DzJRU04jFUqJODw= +github.com/rivo/uniseg v0.3.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220817070843-5a390386f1f2 h1:fqTvyMIIj+HRzMmnzr9NtpHP6uVpvB5fkHcgPDC4nu8= +golang.org/x/sys v0.0.0-20220817070843-5a390386f1f2/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/importlib/importlib.go b/importlib/importlib.go index e1d307db..58ccb966 100644 --- a/importlib/importlib.go +++ b/importlib/importlib.go @@ -7,15 +7,18 @@ package py import ( - "log" - - "github.com/go-python/gpython/marshal" + "github.com/go-python/gpython/py" ) // Load the frozen module func init() { - _, err := marshal.LoadFrozenModule("importlib", data) - log.Fatalf("Failed to load importlib: %v", err) + + py.RegisterModule(&py.ModuleImpl{ + Info: py.ModuleInfo{ + Name: "importlib", + }, + CodeBuf: data, + }) } // Auto-generated by Modules/_freeze_importlib.c diff --git a/main.go b/main.go index 06cba663..8be7ea2e 100644 --- a/main.go +++ b/main.go @@ -9,30 +9,19 @@ package main import ( "flag" "fmt" - "runtime/pprof" - - _ "github.com/go-python/gpython/builtin" - "github.com/go-python/gpython/repl/cli" - - //_ "github.com/go-python/gpython/importlib" - "io/ioutil" "log" "os" - "strings" + "runtime" + "runtime/pprof" - "github.com/go-python/gpython/compile" - "github.com/go-python/gpython/marshal" - _ "github.com/go-python/gpython/math" "github.com/go-python/gpython/py" - pysys "github.com/go-python/gpython/sys" - _ "github.com/go-python/gpython/time" - "github.com/go-python/gpython/vm" + "github.com/go-python/gpython/repl" + "github.com/go-python/gpython/repl/cli" + + _ "github.com/go-python/gpython/stdlib" ) -// Globals var ( - // Flags - debug = flag.Bool("d", false, "Print lots of debugging") cpuprofile = flag.String("cpuprofile", "", "Write cpu profile to file") ) @@ -47,27 +36,17 @@ Full options: flag.PrintDefaults() } -// Exit with the message -func fatal(message string, args ...interface{}) { - if !strings.HasSuffix(message, "\n") { - message += "\n" - } - syntaxError() - fmt.Fprintf(os.Stderr, message, args...) - os.Exit(1) -} - func main() { flag.Usage = syntaxError flag.Parse() - args := flag.Args() - py.MustGetModule("sys").Globals["argv"] = pysys.MakeArgv(args) - if len(args) == 0 { - cli.RunREPL() - return - } - prog := args[0] - // fmt.Printf("Running %q\n", prog) + xmain(flag.Args()) +} + +func xmain(args []string) { + opts := py.DefaultContextOpts() + opts.SysArgs = args + ctx := py.NewContext(opts) + defer ctx.Close() if *cpuprofile != "" { f, err := os.Create(*cpuprofile) @@ -81,41 +60,22 @@ func main() { defer pprof.StopCPUProfile() } - // FIXME should be using ImportModuleLevelObject() here - f, err := os.Open(prog) - if err != nil { - log.Fatalf("Failed to open %q: %v", prog, err) - } - var obj py.Object - if strings.HasSuffix(prog, ".pyc") { - obj, err = marshal.ReadPyc(f) - if err != nil { - log.Fatalf("Failed to marshal %q: %v", prog, err) - } - } else if strings.HasSuffix(prog, ".py") { - str, err := ioutil.ReadAll(f) - if err != nil { - log.Fatalf("Failed to read %q: %v", prog, err) - } - obj, err = compile.Compile(string(str), prog, "exec", 0, true) + // IF no args, enter REPL mode + if len(args) == 0 { + + fmt.Printf("Python 3.4.0 (%s, %s)\n", commit, date) + fmt.Printf("[Gpython %s]\n", version) + fmt.Printf("- os/arch: %s/%s\n", runtime.GOOS, runtime.GOARCH) + fmt.Printf("- go version: %s\n", runtime.Version()) + + replCtx := repl.New(ctx) + cli.RunREPL(replCtx) + + } else { + _, err := py.RunFile(ctx, args[0], py.CompileOpts{}, nil) if err != nil { - log.Fatalf("Can't compile %q: %v", prog, err) + py.TracebackDump(err) + os.Exit(1) } - } else { - log.Fatalf("Can't execute %q", prog) } - if err = f.Close(); err != nil { - log.Fatalf("Failed to close %q: %v", prog, err) - } - code := obj.(*py.Code) - module := py.NewModule("__main__", "", nil, nil) - module.Globals["__file__"] = py.String(prog) - res, err := vm.Run(module.Globals, module.Globals, code, nil) - if err != nil { - py.TracebackDump(err) - log.Fatal(err) - } - // fmt.Printf("Return = %v\n", res) - _ = res - } diff --git a/main_test.go b/main_test.go new file mode 100644 index 00000000..e41ed869 --- /dev/null +++ b/main_test.go @@ -0,0 +1,61 @@ +// Copyright 2022 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "bytes" + "flag" + "os" + "os/exec" + "path/filepath" + "testing" +) + +var regen = flag.Bool("regen", false, "regenerate golden files") + +func TestGPython(t *testing.T) { + + tmp, err := os.MkdirTemp("", "go-python-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmp) + + exe := filepath.Join(tmp, "out.exe") + cmd := exec.Command("go", "build", "-o", exe, ".") + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err = cmd.Run() + if err != nil { + t.Fatalf("failed to compile embedding example: %+v", err) + } + + got, err := exec.Command(exe, "testdata/hello.py").CombinedOutput() + if err != nil { + t.Fatalf("could not run gpython:\n%s\nerr: %+v", got, err) + } + + const fname = "testdata/hello_golden.txt" + + flag.Parse() + if *regen { + err = os.WriteFile(fname, got, 0644) + if err != nil { + t.Fatalf("could not write golden file: %+v", err) + } + } + + want, err := os.ReadFile(fname) + if err != nil { + t.Fatalf("could not read golden file: %+v", err) + } + if !bytes.Equal(got, want) { + t.Fatalf("stdout differ:\ngot:\n%s\nwant:\n%s\n", got, want) + } +} + +func TestRunFile(t *testing.T) { + xmain([]string{"./testdata/hello.py"}) +} diff --git a/parser/grammar_data_test.go b/parser/grammar_data_test.go index 4079325a..b0b53eeb 100644 --- a/parser/grammar_data_test.go +++ b/parser/grammar_data_test.go @@ -33,8 +33,8 @@ var grammarTestData = []struct { {"b'abc' b'''123'''", "eval", "Expression(body=Bytes(s=b'abc123'))", nil, ""}, {"1234", "eval", "Expression(body=Num(n=1234))", nil, ""}, {"01234", "eval", "", py.SyntaxError, "illegal decimal with leading zero"}, - {"1234d", "eval", "", py.SyntaxError, "invalid syntax"}, - {"1234d", "exec", "", py.SyntaxError, "invalid syntax"}, + {"1234d", "eval", "", py.SyntaxError, "unexpected EOF while parsing"}, + {"1234d", "exec", "", py.SyntaxError, "unexpected EOF while parsing"}, {"1234d", "single", "", py.SyntaxError, "unexpected EOF while parsing"}, {"0x1234", "eval", "Expression(body=Num(n=4660))", nil, ""}, {"12.34", "eval", "Expression(body=Num(n=12.34))", nil, ""}, @@ -325,10 +325,10 @@ var grammarTestData = []struct { {"pass\n", "single", "Interactive(body=[Pass()])", nil, ""}, {"if True:\n pass\n\n", "single", "Interactive(body=[If(test=NameConstant(value=True), body=[Pass()], orelse=[])])", nil, ""}, {"while True:\n pass\nelse:\n return\n", "single", "Interactive(body=[While(test=NameConstant(value=True), body=[Pass()], orelse=[Return(value=None)])])", nil, ""}, - {"a='potato", "eval", "", py.SyntaxError, "invalid syntax"}, + {"a='potato", "eval", "", py.SyntaxError, "unexpected EOF while parsing"}, {"a='potato", "exec", "", py.SyntaxError, "EOL while scanning string literal"}, {"a='potato", "single", "", py.SyntaxError, "EOL while scanning string literal"}, - {"a='''potato", "eval", "", py.SyntaxError, "invalid syntax"}, + {"a='''potato", "eval", "", py.SyntaxError, "unexpected EOF while parsing"}, {"a='''potato", "exec", "", py.SyntaxError, "EOF while scanning triple-quoted string literal"}, {"a='''potato", "single", "", py.SyntaxError, "EOF while scanning triple-quoted string literal"}, } diff --git a/parser/grammar_test.go b/parser/grammar_test.go index d3d48a50..a99beb95 100644 --- a/parser/grammar_test.go +++ b/parser/grammar_test.go @@ -21,7 +21,7 @@ var debugLevel = flag.Int("debugLevel", 0, "Debug level 0-4") func TestGrammar(t *testing.T) { SetDebug(*debugLevel) for _, test := range grammarTestData { - Ast, err := ParseString(test.in, test.mode) + Ast, err := ParseString(test.in, py.CompileMode(test.mode)) if err != nil { if test.exceptionType == nil { t.Errorf("%s: Got exception %v when not expecting one", test.in, err) diff --git a/parser/lexer.go b/parser/lexer.go index 6f9f1726..119a741b 100644 --- a/parser/lexer.go +++ b/parser/lexer.go @@ -62,7 +62,7 @@ type yyLex struct { // can be 'exec' if source consists of a sequence of statements, // 'eval' if it consists of a single expression, or 'single' if it // consists of a single interactive statement -func NewLex(r io.Reader, filename string, mode string) (*yyLex, error) { +func NewLex(r io.Reader, filename string, mode py.CompileMode) (*yyLex, error) { x := &yyLex{ reader: bufio.NewReader(r), filename: filename, @@ -70,12 +70,12 @@ func NewLex(r io.Reader, filename string, mode string) (*yyLex, error) { state: readString, } switch mode { - case "exec": + case py.ExecMode: x.queue(FILE_INPUT) x.exec = true - case "eval": + case py.EvalMode: x.queue(EVAL_INPUT) - case "single": + case py.SingleMode: x.queue(SINGLE_INPUT) x.interactive = true default: @@ -110,6 +110,9 @@ func (x *yyLex) dequeue() int { func (x *yyLex) refill() { var err error x.line, err = x.reader.ReadString('\n') + if strings.HasSuffix(x.line, "\r\n") { + x.line = x.line[:len(x.line)-2] + "\n" + } if yyDebug >= 2 { fmt.Printf("line = %q, err = %v\n", x.line, err) } @@ -916,7 +919,7 @@ func (x *yyLex) SyntaxErrorf(format string, a ...interface{}) { func (x *yyLex) ErrorReturn() error { if x.error { if x.errorString == "" { - if x.eof && x.interactive { + if x.eof { x.errorString = "unexpected EOF while parsing" } else { x.errorString = "invalid syntax" @@ -933,7 +936,7 @@ func SetDebug(level int) { } // Parse a file -func Parse(in io.Reader, filename string, mode string) (mod ast.Mod, err error) { +func Parse(in io.Reader, filename string, mode py.CompileMode) (mod ast.Mod, err error) { lex, err := NewLex(in, filename, mode) if err != nil { return nil, err @@ -952,12 +955,12 @@ func Parse(in io.Reader, filename string, mode string) (mod ast.Mod, err error) } // Parse a string -func ParseString(in string, mode string) (ast.Ast, error) { +func ParseString(in string, mode py.CompileMode) (ast.Ast, error) { return Parse(bytes.NewBufferString(in), "", mode) } // Lex a file only, returning a sequence of tokens -func Lex(in io.Reader, filename string, mode string) (lts LexTokens, err error) { +func Lex(in io.Reader, filename string, mode py.CompileMode) (lts LexTokens, err error) { lex, err := NewLex(in, filename, mode) if err != nil { return nil, err @@ -984,6 +987,6 @@ func Lex(in io.Reader, filename string, mode string) (lts LexTokens, err error) } // Lex a string -func LexString(in string, mode string) (lts LexTokens, err error) { +func LexString(in string, mode py.CompileMode) (lts LexTokens, err error) { return Lex(bytes.NewBufferString(in), "", mode) } diff --git a/parser/lexer_test.go b/parser/lexer_test.go index d8655605..5e044d39 100644 --- a/parser/lexer_test.go +++ b/parser/lexer_test.go @@ -7,8 +7,6 @@ package parser import ( "bytes" "fmt" - "log" - "math" "testing" "github.com/go-python/gpython/ast" @@ -196,7 +194,7 @@ func TestLex(t *testing.T) { for _, test := range []struct { in string errString string - mode string + mode py.CompileMode lts LexTokens }{ {"", "", "exec", LexTokens{ @@ -264,7 +262,7 @@ func TestLex(t *testing.T) { {"01", "illegal decimal with leading zero 1:0", "exec", LexTokens{ {FILE_INPUT, nil, ast.Pos{0, 0}}, }}, - {"1\n 2\n 3\n4\n", "", "exec", LexTokens{ + {"1\n 2\r\n 3\r\n4\n", "", "exec", LexTokens{ {FILE_INPUT, nil, ast.Pos{0, 0}}, {NUMBER, py.Int(1), ast.Pos{1, 0}}, {NEWLINE, nil, ast.Pos{1, 1}}, @@ -569,19 +567,6 @@ func TestLexerReadOperator(t *testing.T) { } } -// Whether two floats are more or less the same -func approxEq(a, b float64) bool { - log.Printf("ApproxEq(a = %#v, b = %#v)", a, b) - diff := a - b - log.Printf("ApproxEq(diff = %e)", diff) - if math.Abs(diff) > 1E-10 { - log.Printf("ApproxEq(false)") - return false - } - log.Printf("ApproxEq(true)") - return true -} - func TestLexerReadNumber(t *testing.T) { x := yyLex{} for _, test := range []struct { @@ -710,10 +695,10 @@ func TestLexerReadString(t *testing.T) { if testValueBytes, ok := test.value.(py.Bytes); !ok { t.Error("Expecting py.Bytes") } else { - equal = (bytes.Compare(valueBytes, testValueBytes) == 0) + equal = bytes.Equal(valueBytes, testValueBytes) } } else { - equal = (value == test.value) + equal = value == test.value } if token != test.token || !equal || x.line != test.out { diff --git a/parser/stringescape.go b/parser/stringescape.go index 5341ce44..c5c61ca3 100644 --- a/parser/stringescape.go +++ b/parser/stringescape.go @@ -18,7 +18,7 @@ func DecodeEscape(in *bytes.Buffer, byteMode bool) (out *bytes.Buffer, err error // Early exit if no escape sequences // NB in.Bytes() is cheap inBytes := in.Bytes() - if bytes.IndexRune(inBytes, '\\') < 0 { + if !bytes.ContainsRune(inBytes, '\\') { return in, nil } out = new(bytes.Buffer) @@ -135,7 +135,6 @@ func DecodeEscape(in *bytes.Buffer, byteMode bool) (out *bytes.Buffer, err error ignoreEscape = true default: ignoreEscape = true - break } // ignore unrecognised escape if ignoreEscape { diff --git a/parser/testparser/testparser.go b/parser/testparser/testparser.go index 6bf2d8e7..410ca043 100644 --- a/parser/testparser/testparser.go +++ b/parser/testparser/testparser.go @@ -7,7 +7,7 @@ package main import ( "flag" "fmt" - "io/ioutil" + "io" "log" "os" @@ -47,7 +47,7 @@ func main() { _, err = parser.Lex(in, path, "exec") } else if *compileFile { var input []byte - input, err = ioutil.ReadAll(in) + input, err = io.ReadAll(in) if err != nil { log.Fatalf("Failed to read %q: %v", path, err) } diff --git a/parser/y.go b/parser/y.go index 1f5bba73..39027c98 100644 --- a/parser/y.go +++ b/parser/y.go @@ -3,6 +3,7 @@ // license that can be found in the LICENSE file. // Code generated by goyacc -v y.output grammar.y. DO NOT EDIT. +// //line grammar.y:6 package parser diff --git a/py/args.go b/py/args.go index dcdfc392..04734ef3 100644 --- a/py/args.go +++ b/py/args.go @@ -368,9 +368,7 @@ // $ // // PyArg_ParseTupleAndKeywords() only: Indicates that the remaining -// arguments in the Python argument list are keyword-only. Currently, -// all keyword-only arguments must also be optional arguments, so | -// must always be specified before $ in the format string. +// arguments in the Python argument list are keyword-only. // // New in version 3.3. // @@ -416,8 +414,9 @@ func ParseTupleAndKeywords(args Tuple, kwargs StringDict, format string, kwlist if kwlist != nil && len(results) != len(kwlist) { return ExceptionNewf(TypeError, "Internal error: supply the same number of results and kwlist") } - min, max, name, ops := parseFormat(format) - err := checkNumberOfArgs(name, len(args)+len(kwargs), len(results), min, max) + var opsBuf [16]formatOp + min, name, kwOnly_i, ops := parseFormat(format, opsBuf[:0]) + err := checkNumberOfArgs(name, len(args)+len(kwargs), len(results), min, len(ops)) if err != nil { return err } @@ -434,46 +433,120 @@ func ParseTupleAndKeywords(args Tuple, kwargs StringDict, format string, kwlist found: } - // Create args tuple with all the arguments we have in - args = args.Copy() - for i, kw := range kwlist { - if value, ok := kwargs[kw]; ok { - if len(args) > i { + // Walk through all the results we want + for i, op := range ops { + + var ( + arg Object + kw string + ) + if i < len(kwlist) { + kw = kwlist[i] + arg = kwargs[kw] + } + + // Consume ordered args first -- they should not require keyword only or also be specified via keyword + if i < len(args) { + if i >= kwOnly_i { + return ExceptionNewf(TypeError, "%s() specifies argument '%s' that is keyword only", name, kw) + } + if arg != nil { return ExceptionNewf(TypeError, "%s() got multiple values for argument '%s'", name, kw) } - args = append(args, value) + arg = args[i] } - } + // Unspecified args retain their default value + if arg == nil { + continue + } - for i, arg := range args { - op := ops[i] result := results[i] - switch op { - case "O": + switch op.code { + case 'O': *result = arg - case "Z", "z": - if _, ok := arg.(NoneType); ok { - *result = arg - break + case 'Z': + switch op.modifier { + default: + return ExceptionNewf(TypeError, "%s() argument %d must be str or None, not %s", name, i+1, arg.Type().Name) + case '#', 0: + switch arg := arg.(type) { + case String, NoneType: + default: + return ExceptionNewf(TypeError, "%s() argument %d must be str or None, not %s", name, i+1, arg.Type().Name) + } } - fallthrough - case "U", "s": + *result = arg + case 'z': + switch op.modifier { + default: + switch arg := arg.(type) { + case String, NoneType: + // ok + default: + return ExceptionNewf(TypeError, "%s() argument %d must be str or None, not %s", name, i+1, arg.Type().Name) + } + case '#': + fallthrough // FIXME(sbinet): check for read-only? + case '*': + switch arg := arg.(type) { + case String, Bytes, NoneType: + // ok. + default: + return ExceptionNewf(TypeError, "%s() argument %d must be str, bytes-like or None, not %s", name, i+1, arg.Type().Name) + } + } + *result = arg + case 'U': if _, ok := arg.(String); !ok { return ExceptionNewf(TypeError, "%s() argument %d must be str, not %s", name, i+1, arg.Type().Name) } *result = arg - case "i": + case 's': + switch op.modifier { + default: + if _, ok := arg.(String); !ok { + return ExceptionNewf(TypeError, "%s() argument %d must be str, not %s", name, i+1, arg.Type().Name) + } + case '#': + fallthrough // FIXME(sbinet): check for read-only? + case '*': + switch arg := arg.(type) { + case String, Bytes: + // ok. + default: + return ExceptionNewf(TypeError, "%s() argument %d must be str or bytes-like, not %s", name, i+1, arg.Type().Name) + } + } + *result = arg + case 'y': + switch op.modifier { + default: + if _, ok := arg.(Bytes); !ok { + return ExceptionNewf(TypeError, "%s() argument %d must be bytes-like, not %s", name, i+1, arg.Type().Name) + } + case '#': + fallthrough // FIXME(sbinet): check for read-only? + case '*': + switch arg := arg.(type) { + case Bytes: + // ok. + default: + return ExceptionNewf(TypeError, "%s() argument %d must be bytes-like, not %s", name, i+1, arg.Type().Name) + } + } + *result = arg + case 'i', 'n': if _, ok := arg.(Int); !ok { return ExceptionNewf(TypeError, "%s() argument %d must be int, not %s", name, i+1, arg.Type().Name) } *result = arg - case "p": + case 'p': if _, ok := arg.(Bool); !ok { return ExceptionNewf(TypeError, "%s() argument %d must be bool, not %s", name, i+1, arg.Type().Name) } *result = arg - case "d": + case 'd': switch x := arg.(type) { case Int: *result = Float(x) @@ -495,30 +568,42 @@ func ParseTuple(args Tuple, format string, results ...*Object) error { return ParseTupleAndKeywords(args, nil, format, nil, results...) } +type formatOp struct { + code byte + modifier byte +} + // Parse the format -func parseFormat(format string) (min, max int, name string, ops []string) { +func parseFormat(format string, in []formatOp) (min int, name string, kwOnly_i int, ops []formatOp) { name = "function" min = -1 - for format != "" { - op := string(format[0]) - format = format[1:] - if len(format) > 1 && (format[1] == '*' || format[1] == '#') { - op += string(format[0]) - format = format[1:] + kwOnly_i = 0xFFFF + ops = in[:0] + + N := len(format) + for i := 0; i < N; { + op := formatOp{code: format[i]} + i++ + if i < N { + if mod := format[i]; mod == '*' || mod == '#' { + op.modifier = mod + i++ + } } - switch op { - case ":", ";": - name = format - format = "" - case "|": + switch op.code { + case ':', ';': + name = format[i:] + i = N + case '$': + kwOnly_i = len(ops) + case '|': min = len(ops) default: ops = append(ops, op) } } - max = len(ops) if min < 0 { - min = max + min = len(ops) } return } diff --git a/py/args_test.go b/py/args_test.go new file mode 100644 index 00000000..408ad342 --- /dev/null +++ b/py/args_test.go @@ -0,0 +1,328 @@ +// Copyright 2022 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package py + +import ( + "fmt" + "testing" +) + +func TestParseTupleAndKeywords(t *testing.T) { + for _, tc := range []struct { + args Tuple + kwargs StringDict + format string + kwlist []string + results []Object + err error + }{ + { + args: Tuple{String("a")}, + format: "O:func", + results: []Object{String("a")}, + }, + { + args: Tuple{None}, + format: "Z:func", + results: []Object{None}, + }, + { + args: Tuple{String("a")}, + format: "Z:func", + results: []Object{String("a")}, + }, + { + args: Tuple{Int(42)}, + format: "Z:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be str or None, not int'"), + }, + { + args: Tuple{None}, + format: "Z*:func", // FIXME(sbinet): invalid format. + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be str or None, not NoneType'"), + }, + { + args: Tuple{None}, + format: "Z#:func", + results: []Object{None}, + }, + { + args: Tuple{String("a")}, + format: "Z#:func", + results: []Object{String("a")}, + }, + { + args: Tuple{Int(42)}, + format: "Z#:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be str or None, not int'"), + }, + { + args: Tuple{None}, + format: "z:func", + results: []Object{None}, + }, + { + args: Tuple{String("a")}, + format: "z:func", + results: []Object{String("a")}, + }, + { + args: Tuple{Int(42)}, + format: "z:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be str or None, not int'"), + }, + { + args: Tuple{Bytes("a")}, + format: "z:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be str or None, not bytes'"), + }, + { + args: Tuple{None}, + format: "z*:func", + results: []Object{None}, + }, + { + args: Tuple{Bytes("a")}, + format: "z*:func", + results: []Object{Bytes("a")}, + }, + { + args: Tuple{String("a")}, + format: "z*:func", + results: []Object{String("a")}, + }, + { + args: Tuple{Int(42)}, + format: "z*:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be str, bytes-like or None, not int'"), + }, + { + args: Tuple{None}, + format: "z#:func", + results: []Object{None}, + }, + { + args: Tuple{Bytes("a")}, + format: "z#:func", + results: []Object{Bytes("a")}, + }, + { + args: Tuple{String("a")}, + format: "z#:func", + results: []Object{String("a")}, + }, + { + args: Tuple{Int(42)}, + format: "z#:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be str, bytes-like or None, not int'"), + }, + { + args: Tuple{String("a")}, + format: "s:func", + results: []Object{String("a")}, + }, + { + args: Tuple{None}, + format: "s:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be str, not NoneType'"), + }, + { + args: Tuple{Bytes("a")}, + format: "s:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be str, not bytes'"), + }, + { + args: Tuple{String("a")}, + format: "s#:func", + results: []Object{String("a")}, + }, + { + args: Tuple{Bytes("a")}, + format: "s#:func", + results: []Object{Bytes("a")}, + }, + { + args: Tuple{None}, + format: "s#:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be str or bytes-like, not NoneType'"), + }, + { + args: Tuple{String("a")}, + format: "s*:func", + results: []Object{String("a")}, + }, + { + args: Tuple{Bytes("a")}, + format: "s*:func", + results: []Object{Bytes("a")}, + }, + { + args: Tuple{None}, + format: "s*:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be str or bytes-like, not NoneType'"), + }, + { + args: Tuple{Bytes("a")}, + format: "y:func", + results: []Object{Bytes("a")}, + }, + { + args: Tuple{None}, + format: "y:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be bytes-like, not NoneType'"), + }, + { + args: Tuple{String("a")}, + format: "y:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be bytes-like, not str'"), + }, + { + args: Tuple{Bytes("a")}, + format: "y#:func", + results: []Object{Bytes("a")}, + }, + { + args: Tuple{String("a")}, + format: "y#:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be bytes-like, not str'"), + }, + { + args: Tuple{None}, + format: "y#:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be bytes-like, not NoneType'"), + }, + { + args: Tuple{Bytes("a")}, + format: "y*:func", + results: []Object{Bytes("a")}, + }, + { + args: Tuple{String("a")}, + format: "y*:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be bytes-like, not str'"), + }, + { + args: Tuple{None}, + format: "y*:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be bytes-like, not NoneType'"), + }, + { + args: Tuple{String("a")}, + format: "U:func", + results: []Object{String("a")}, + }, + { + args: Tuple{None}, + format: "U:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be str, not NoneType'"), + }, + { + args: Tuple{Bytes("a")}, + format: "U:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be str, not bytes'"), + }, + { + args: Tuple{Bytes("a")}, + format: "U*:func", // FIXME(sbinet): invalid format + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be str, not bytes'"), + }, + { + args: Tuple{Bytes("a")}, + format: "U#:func", // FIXME(sbinet): invalid format + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be str, not bytes'"), + }, + { + args: Tuple{Int(42)}, + format: "i:func", + results: []Object{Int(42)}, + }, + { + args: Tuple{None}, + format: "i:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be int, not NoneType'"), + }, + { + args: Tuple{Int(42)}, + format: "n:func", + results: []Object{Int(42)}, + }, + { + args: Tuple{None}, + format: "n:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be int, not NoneType'"), + }, + { + args: Tuple{Bool(true)}, + format: "p:func", + results: []Object{Bool(true)}, + }, + { + args: Tuple{None}, + format: "p:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be bool, not NoneType'"), + }, + { + args: Tuple{Float(42)}, + format: "d:func", + results: []Object{Float(42)}, + }, + { + args: Tuple{Int(42)}, + format: "d:func", + results: []Object{Float(42)}, + }, + { + args: Tuple{None}, + format: "p:func", + results: []Object{nil}, + err: fmt.Errorf("TypeError: 'func() argument 1 must be bool, not NoneType'"), + }, + } { + t.Run(tc.format, func(t *testing.T) { + results := make([]*Object, len(tc.results)) + for i := range tc.results { + results[i] = &tc.results[i] + } + err := ParseTupleAndKeywords(tc.args, tc.kwargs, tc.format, tc.kwlist, results...) + switch { + case err != nil && tc.err != nil: + if got, want := err.Error(), tc.err.Error(); got != want { + t.Fatalf("invalid error:\ngot= %s\nwant=%s", got, want) + } + case err != nil && tc.err == nil: + t.Fatalf("could not parse tuple+kwargs: %+v", err) + case err == nil && tc.err != nil: + t.Fatalf("expected an error (got=nil): %+v", tc.err) + case err == nil && tc.err == nil: + // ok. + } + // FIXME(sbinet): check results + }) + } +} diff --git a/py/arithmetic.go b/py/arithmetic.go index 199fceee..768764b8 100644 --- a/py/arithmetic.go +++ b/py/arithmetic.go @@ -147,24 +147,6 @@ func MakeFloat(a Object) (Object, error) { return nil, ExceptionNewf(TypeError, "unsupported operand type(s) for float: '%s'", a.Type().Name) } -// Iter the python Object returning an Object -// -// Will raise TypeError if Iter can't be run on this object -func Iter(a Object) (Object, error) { - - if A, ok := a.(I__iter__); ok { - res, err := A.M__iter__() - if err != nil { - return nil, err - } - if res != NotImplemented { - return res, nil - } - } - - return nil, ExceptionNewf(TypeError, "unsupported operand type(s) for iter: '%s'", a.Type().Name) -} - // Add two python objects together returning an Object // // Will raise TypeError if can't be add can't be run on these objects diff --git a/py/bigint.go b/py/bigint.go index b5e3ef36..bb195888 100644 --- a/py/bigint.go +++ b/py/bigint.go @@ -55,7 +55,7 @@ func BigIntCheckExact(obj Object) (*BigInt, error) { return bigInt, nil } -// Checks that obj is exactly a bigInd and returns an error if not +// Checks that obj is exactly a BigInt and returns an error if not func BigIntCheck(obj Object) (*BigInt, error) { // FIXME should be checking subclasses return BigIntCheckExact(obj) @@ -65,8 +65,8 @@ func BigIntCheck(obj Object) (*BigInt, error) { // Convert an Object to an BigInt // -// Retrurns ok as to whether the conversion worked or not -func convertToBigInt(other Object) (*BigInt, bool) { +// Returns ok as to whether the conversion worked or not +func ConvertToBigInt(other Object) (*BigInt, bool) { switch b := other.(type) { case Int: return (*BigInt)(big.NewInt(int64(b))), true @@ -173,7 +173,7 @@ func (a *BigInt) M__invert__() (Object, error) { } func (a *BigInt) M__add__(other Object) (Object, error) { - if b, ok := convertToBigInt(other); ok { + if b, ok := ConvertToBigInt(other); ok { return (*BigInt)(new(big.Int).Add((*big.Int)(a), (*big.Int)(b))).MaybeInt(), nil } return NotImplemented, nil @@ -188,14 +188,14 @@ func (a *BigInt) M__iadd__(other Object) (Object, error) { } func (a *BigInt) M__sub__(other Object) (Object, error) { - if b, ok := convertToBigInt(other); ok { + if b, ok := ConvertToBigInt(other); ok { return (*BigInt)(new(big.Int).Sub((*big.Int)(a), (*big.Int)(b))).MaybeInt(), nil } return NotImplemented, nil } func (a *BigInt) M__rsub__(other Object) (Object, error) { - if b, ok := convertToBigInt(other); ok { + if b, ok := ConvertToBigInt(other); ok { return (*BigInt)(new(big.Int).Sub((*big.Int)(b), (*big.Int)(a))).MaybeInt(), nil } return NotImplemented, nil @@ -206,7 +206,7 @@ func (a *BigInt) M__isub__(other Object) (Object, error) { } func (a *BigInt) M__mul__(other Object) (Object, error) { - if b, ok := convertToBigInt(other); ok { + if b, ok := ConvertToBigInt(other); ok { return (*BigInt)(new(big.Int).Mul((*big.Int)(a), (*big.Int)(b))).MaybeInt(), nil } return NotImplemented, nil @@ -306,14 +306,14 @@ func (a *BigInt) divMod(b *BigInt) (Object, Object, error) { } func (a *BigInt) M__divmod__(other Object) (Object, Object, error) { - if b, ok := convertToBigInt(other); ok { + if b, ok := ConvertToBigInt(other); ok { return a.divMod(b) } return NotImplemented, NotImplemented, nil } func (a *BigInt) M__rdivmod__(other Object) (Object, Object, error) { - if b, ok := convertToBigInt(other); ok { + if b, ok := ConvertToBigInt(other); ok { return b.divMod(a) } return NotImplemented, NotImplemented, nil @@ -343,18 +343,18 @@ func (a *BigInt) M__pow__(other, modulus Object) (Object, error) { var m *BigInt if modulus != None { var ok bool - if m, ok = convertToBigInt(modulus); !ok { + if m, ok = ConvertToBigInt(modulus); !ok { return NotImplemented, nil } } - if b, ok := convertToBigInt(other); ok { + if b, ok := ConvertToBigInt(other); ok { return a.pow(b, m) } return NotImplemented, nil } func (a *BigInt) M__rpow__(other Object) (Object, error) { - if b, ok := convertToBigInt(other); ok { + if b, ok := ConvertToBigInt(other); ok { return b.pow(a, nil) } return NotImplemented, nil @@ -365,7 +365,7 @@ func (a *BigInt) M__ipow__(other, modulus Object) (Object, error) { } func (a *BigInt) M__lshift__(other Object) (Object, error) { - if b, ok := convertToBigInt(other); ok { + if b, ok := ConvertToBigInt(other); ok { bb, err := b.GoInt() if err != nil { return nil, err @@ -379,7 +379,7 @@ func (a *BigInt) M__lshift__(other Object) (Object, error) { } func (a *BigInt) M__rlshift__(other Object) (Object, error) { - if b, ok := convertToBigInt(other); ok { + if b, ok := ConvertToBigInt(other); ok { aa, err := a.GoInt() if err != nil { return nil, err @@ -397,7 +397,7 @@ func (a *BigInt) M__ilshift__(other Object) (Object, error) { } func (a *BigInt) M__rshift__(other Object) (Object, error) { - if b, ok := convertToBigInt(other); ok { + if b, ok := ConvertToBigInt(other); ok { bb, err := b.GoInt() if err != nil { return nil, err @@ -411,7 +411,7 @@ func (a *BigInt) M__rshift__(other Object) (Object, error) { } func (a *BigInt) M__rrshift__(other Object) (Object, error) { - if b, ok := convertToBigInt(other); ok { + if b, ok := ConvertToBigInt(other); ok { aa, err := a.GoInt() if err != nil { return nil, err @@ -429,7 +429,7 @@ func (a *BigInt) M__irshift__(other Object) (Object, error) { } func (a *BigInt) M__and__(other Object) (Object, error) { - if b, ok := convertToBigInt(other); ok { + if b, ok := ConvertToBigInt(other); ok { return (*BigInt)(new(big.Int).And((*big.Int)(a), (*big.Int)(b))).MaybeInt(), nil } return NotImplemented, nil @@ -444,7 +444,7 @@ func (a *BigInt) M__iand__(other Object) (Object, error) { } func (a *BigInt) M__xor__(other Object) (Object, error) { - if b, ok := convertToBigInt(other); ok { + if b, ok := ConvertToBigInt(other); ok { return (*BigInt)(new(big.Int).Xor((*big.Int)(a), (*big.Int)(b))).MaybeInt(), nil } return NotImplemented, nil @@ -459,7 +459,7 @@ func (a *BigInt) M__ixor__(other Object) (Object, error) { } func (a *BigInt) M__or__(other Object) (Object, error) { - if b, ok := convertToBigInt(other); ok { + if b, ok := ConvertToBigInt(other); ok { return (*BigInt)(new(big.Int).Or((*big.Int)(a), (*big.Int)(b))).MaybeInt(), nil } return NotImplemented, nil @@ -498,7 +498,7 @@ func (a *BigInt) M__complex__() (Object, error) { } func (a *BigInt) M__round__(digits Object) (Object, error) { - if b, ok := convertToBigInt(digits); ok { + if b, ok := ConvertToBigInt(digits); ok { if (*big.Int)(b).Sign() >= 0 { return a, nil } @@ -528,42 +528,42 @@ func (a *BigInt) M__round__(digits Object) (Object, error) { // Rich comparison func (a *BigInt) M__lt__(other Object) (Object, error) { - if b, ok := convertToBigInt(other); ok { + if b, ok := ConvertToBigInt(other); ok { return NewBool((*big.Int)(a).Cmp((*big.Int)(b)) < 0), nil } return NotImplemented, nil } func (a *BigInt) M__le__(other Object) (Object, error) { - if b, ok := convertToBigInt(other); ok { + if b, ok := ConvertToBigInt(other); ok { return NewBool((*big.Int)(a).Cmp((*big.Int)(b)) <= 0), nil } return NotImplemented, nil } func (a *BigInt) M__eq__(other Object) (Object, error) { - if b, ok := convertToBigInt(other); ok { + if b, ok := ConvertToBigInt(other); ok { return NewBool((*big.Int)(a).Cmp((*big.Int)(b)) == 0), nil } return NotImplemented, nil } func (a *BigInt) M__ne__(other Object) (Object, error) { - if b, ok := convertToBigInt(other); ok { + if b, ok := ConvertToBigInt(other); ok { return NewBool((*big.Int)(a).Cmp((*big.Int)(b)) != 0), nil } return NotImplemented, nil } func (a *BigInt) M__gt__(other Object) (Object, error) { - if b, ok := convertToBigInt(other); ok { + if b, ok := ConvertToBigInt(other); ok { return NewBool((*big.Int)(a).Cmp((*big.Int)(b)) > 0), nil } return NotImplemented, nil } func (a *BigInt) M__ge__(other Object) (Object, error) { - if b, ok := convertToBigInt(other); ok { + if b, ok := ConvertToBigInt(other); ok { return NewBool((*big.Int)(a).Cmp((*big.Int)(b)) >= 0), nil } return NotImplemented, nil diff --git a/py/bool.go b/py/bool.go index d2e14326..413b25d4 100644 --- a/py/bool.go +++ b/py/bool.go @@ -52,7 +52,7 @@ func (a Bool) M__repr__() (Object, error) { // Convert an Object to an Bool // -// Retrurns ok as to whether the conversion worked or not +// Returns ok as to whether the conversion worked or not func convertToBool(other Object) (Bool, bool) { switch b := other.(type) { case Bool: @@ -93,6 +93,16 @@ func (a Bool) M__ne__(other Object) (Object, error) { return True, nil } +func notEq(eq Object, err error) (Object, error) { + if err != nil { + return nil, err + } + if eq == NotImplemented { + return eq, nil + } + return Not(eq) +} + // Check interface is satisfied var _ I__bool__ = Bool(false) var _ I__index__ = Bool(false) diff --git a/py/bytes.go b/py/bytes.go index b9ae3abe..787807ec 100644 --- a/py/bytes.go +++ b/py/bytes.go @@ -187,7 +187,7 @@ func (a Bytes) M__repr__() (Object, error) { // Convert an Object to an Bytes // -// Retrurns ok as to whether the conversion worked or not +// Returns ok as to whether the conversion worked or not func convertToBytes(other Object) (Bytes, bool) { switch b := other.(type) { case Bytes: @@ -214,14 +214,14 @@ func (a Bytes) M__le__(other Object) (Object, error) { func (a Bytes) M__eq__(other Object) (Object, error) { if b, ok := convertToBytes(other); ok { - return NewBool(bytes.Compare(a, b) == 0), nil + return NewBool(bytes.Equal(a, b)), nil } return NotImplemented, nil } func (a Bytes) M__ne__(other Object) (Object, error) { if b, ok := convertToBytes(other); ok { - return NewBool(bytes.Compare(a, b) != 0), nil + return NewBool(!bytes.Equal(a, b)), nil } return NotImplemented, nil } @@ -240,5 +240,61 @@ func (a Bytes) M__ge__(other Object) (Object, error) { return NotImplemented, nil } +func (a Bytes) M__add__(other Object) (Object, error) { + if b, ok := convertToBytes(other); ok { + o := make([]byte, len(a)+len(b)) + copy(o[:len(a)], a) + copy(o[len(a):], b) + return Bytes(o), nil + } + return NotImplemented, nil +} + +func (a Bytes) M__iadd__(other Object) (Object, error) { + if b, ok := convertToBytes(other); ok { + a = append(a, b...) + return a, nil + } + return NotImplemented, nil +} + +func (a Bytes) Replace(args Tuple) (Object, error) { + var ( + pyold Object = None + pynew Object = None + pycnt Object = Int(-1) + ) + err := ParseTuple(args, "yy|i:replace", &pyold, &pynew, &pycnt) + if err != nil { + return nil, err + } + + var ( + old = []byte(pyold.(Bytes)) + new = []byte(pynew.(Bytes)) + cnt = int(pycnt.(Int)) + ) + + return Bytes(bytes.Replace([]byte(a), old, new, cnt)), nil +} + // Check interface is satisfied -var _ richComparison = (Bytes)(nil) +var ( + _ richComparison = (Bytes)(nil) + _ I__add__ = (Bytes)(nil) + _ I__iadd__ = (Bytes)(nil) +) + +func init() { + BytesType.Dict["replace"] = MustNewMethod("replace", func(self Object, args Tuple) (Object, error) { + return self.(Bytes).Replace(args) + }, 0, `replace(self, old, new, count=-1) -> return a copy with all occurrences of substring old replaced by new. + + count + Maximum number of occurrences to replace. + -1 (the default value) means replace all occurrences. + +If the optional argument count is given, only the first count occurrences are +replaced.`) + +} diff --git a/py/call_iterator.go b/py/call_iterator.go new file mode 100644 index 00000000..36e85a79 --- /dev/null +++ b/py/call_iterator.go @@ -0,0 +1,51 @@ +// Copyright 2019 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// CallIterator objects + +package py + +// A python CallIterator object +type CallIterator struct { + callable Object + sentinel Object +} + +var CallIteratorType = NewType("callable_iterator", "callable_iterator type") + +// Type of this object +func (o *CallIterator) Type() *Type { + return CallIteratorType +} + +func (cit *CallIterator) M__iter__() (Object, error) { + return cit, nil +} + +// Get next one from the iteration +func (cit *CallIterator) M__next__() (Object, error) { + value, err := Call(cit.callable, nil, nil) + + if err != nil { + return nil, err + } + + if value == cit.sentinel { + return nil, StopIteration + } + + return value, nil +} + +// Define a new CallIterator +func NewCallIterator(callable Object, sentinel Object) *CallIterator { + c := &CallIterator{ + callable: callable, + sentinel: sentinel, + } + return c +} + +// Check interface is satisfied +var _ I_iterator = (*CallIterator)(nil) diff --git a/py/classmethod.go b/py/classmethod.go index 731184e4..8b6fbf8b 100644 --- a/py/classmethod.go +++ b/py/classmethod.go @@ -44,7 +44,9 @@ func (c *ClassMethod) GetDict() StringDict { // ClassMethodNew func ClassMethodNew(metatype *Type, args Tuple, kwargs StringDict) (res Object, err error) { - c := &ClassMethod{} + c := &ClassMethod{ + Dict: make(StringDict), + } err = UnpackTuple(args, kwargs, "classmethod", 1, 1, &c.Callable) if err != nil { return nil, err diff --git a/py/code.go b/py/code.go index 355e88e0..ad9a42af 100644 --- a/py/code.go +++ b/py/code.go @@ -97,7 +97,7 @@ const NAME_CHARS = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_abcdefghijklmnopqrstuvw // all_name_chars(s): true iff all chars in s are valid NAME_CHARS func all_name_chars(s String) bool { for _, c := range s { - if strings.IndexRune(NAME_CHARS, c) < 0 { + if !strings.ContainsRune(NAME_CHARS, c) { return false } } @@ -112,8 +112,6 @@ func NewCode(argcount int32, kwonlyargcount int32, filename_ Object, name_ Object, firstlineno int32, lnotab_ Object) *Code { - var cell2arg []byte - // Type assert the objects consts := consts_.(Tuple) namesTuple := names_.(Tuple) @@ -154,7 +152,6 @@ func NewCode(argcount int32, kwonlyargcount int32, // return nil; // } - n_cellvars := len(cellvars) intern_strings(namesTuple) intern_strings(varnamesTuple) intern_strings(freevarsTuple) @@ -167,13 +164,40 @@ func NewCode(argcount int32, kwonlyargcount int32, } } } + + co := &Code{ + Argcount: argcount, + Kwonlyargcount: kwonlyargcount, + Nlocals: nlocals, + Stacksize: stacksize, + Flags: flags, + Code: code, + Consts: consts, + Names: names, + Varnames: varnames, + Freevars: freevars, + Cellvars: cellvars, + Filename: filename, + Name: name, + Firstlineno: firstlineno, + Lnotab: lnotab, + Weakreflist: nil, + } + co.InitCell2arg() + return co +} + +// Create mapping between cells and arguments if needed. +func (co *Code) InitCell2arg() { + var cell2arg []byte + n_cellvars := len(co.Cellvars) /* Create mapping between cells and arguments if needed. */ if n_cellvars != 0 { - total_args := argcount + kwonlyargcount - if flags&CO_VARARGS != 0 { + total_args := co.Argcount + co.Kwonlyargcount + if co.Flags&CO_VARARGS != 0 { total_args++ } - if flags&CO_VARKEYWORDS != 0 { + if co.Flags&CO_VARKEYWORDS != 0 { total_args++ } used_cell2arg := false @@ -182,9 +206,9 @@ func NewCode(argcount int32, kwonlyargcount int32, cell2arg[i] = CO_CELL_NOT_AN_ARG } // Find cells which are also arguments. - for i, cell := range cellvars { + for i, cell := range co.Cellvars { for j := int32(0); j < total_args; j++ { - arg := varnames[j] + arg := co.Varnames[j] if cell == arg { cell2arg[i] = byte(j) used_cell2arg = true @@ -196,26 +220,7 @@ func NewCode(argcount int32, kwonlyargcount int32, cell2arg = nil } } - - return &Code{ - Argcount: argcount, - Kwonlyargcount: kwonlyargcount, - Nlocals: nlocals, - Stacksize: stacksize, - Flags: flags, - Code: code, - Consts: consts, - Names: names, - Varnames: varnames, - Freevars: freevars, - Cellvars: cellvars, - Cell2arg: cell2arg, - Filename: filename, - Name: name, - Firstlineno: firstlineno, - Lnotab: lnotab, - Weakreflist: nil, - } + co.Cell2arg = cell2arg } // Return number of free variables diff --git a/py/complex.go b/py/complex.go index 8f42a480..33c09fb3 100644 --- a/py/complex.go +++ b/py/complex.go @@ -42,7 +42,7 @@ func ComplexNew(metatype *Type, args Tuple, kwargs StringDict) (Object, error) { // Convert an Object to an Complex // -// Retrurns ok as to whether the conversion worked or not +// Returns ok as to whether the conversion worked or not func convertToComplex(other Object) (Complex, bool) { switch b := other.(type) { case Complex: @@ -152,13 +152,6 @@ func complexFloor(a Complex) Complex { return Complex(complex(math.Floor(real(a)), math.Floor(imag(a)))) } -// Floor divide two complex numbers -func complexFloorDiv(a, b Complex) Complex { - q := complexFloor(a / b) - r := a - q*b - return Complex(r) -} - func (a Complex) M__floordiv__(other Object) (Object, error) { if b, ok := convertToComplex(other); ok { return complexFloor(a / b), nil diff --git a/py/dict.go b/py/dict.go index a849973f..497da3a7 100644 --- a/py/dict.go +++ b/py/dict.go @@ -9,7 +9,9 @@ package py -import "bytes" +import ( + "bytes" +) const dictDoc = `dict() -> new empty dictionary dict(mapping) -> new dictionary initialized from a mapping object's @@ -22,16 +24,112 @@ dict(**kwargs) -> new dictionary initialized with the name=value pairs in the keyword argument list. For example: dict(one=1, two=2)` var ( - StringDictType = NewType("dict", dictDoc) + StringDictType = NewTypeX("dict", dictDoc, DictNew, nil) DictType = NewType("dict", dictDoc) expectingDict = ExceptionNewf(TypeError, "a dict is required") ) +func init() { + StringDictType.Dict["items"] = MustNewMethod("items", func(self Object, args Tuple) (Object, error) { + err := UnpackTuple(args, nil, "items", 0, 0) + if err != nil { + return nil, err + } + sMap := self.(StringDict) + o := make(Tuple, 0, len(sMap)) + for k, v := range sMap { + o = append(o, Tuple{String(k), v}) + } + return NewIterator(o), nil + }, 0, "items() -> list of D's (key, value) pairs, as 2-tuples") + + StringDictType.Dict["keys"] = MustNewMethod("keys", func(self Object, args Tuple) (Object, error) { + err := UnpackTuple(args, nil, "keys", 0, 0) + if err != nil { + return nil, err + } + sMap := self.(StringDict) + o := make(Tuple, 0, len(sMap)) + for k := range sMap { + o = append(o, String(k)) + } + return NewIterator(o), nil + }, 0, "keys() -> list of D's keys, as a list") + + StringDictType.Dict["values"] = MustNewMethod("values", func(self Object, args Tuple) (Object, error) { + err := UnpackTuple(args, nil, "values", 0, 0) + if err != nil { + return nil, err + } + sMap := self.(StringDict) + o := make(Tuple, 0, len(sMap)) + for _, v := range sMap { + o = append(o, v) + } + return NewIterator(o), nil + }, 0, "values() -> list of D's values, as a list") + + StringDictType.Dict["get"] = MustNewMethod("get", func(self Object, args Tuple) (Object, error) { + var length = len(args) + switch { + case length == 0: + return nil, ExceptionNewf(TypeError, "%s expected at least 1 arguments, got %d", "items()", length) + case length > 2: + return nil, ExceptionNewf(TypeError, "%s expected at most 2 arguments, got %d", "items()", length) + } + sMap := self.(StringDict) + if str, ok := args[0].(String); ok { + if res, ok := sMap[string(str)]; ok { + return res, nil + } + + switch length { + case 2: + return args[1], nil + default: + return None, nil + } + } + return nil, ExceptionNewf(KeyError, "%v", args[0]) + }, 0, "gets(key, default) -> If there is a val corresponding to key, return val, otherwise default") +} + // String to object dictionary // // Used for variables etc where the keys can only be strings type StringDict map[string]Object +// DictNew +func DictNew(metatype *Type, args Tuple, kwargs StringDict) (Object, error) { + if len(args) > 1 { + return nil, ExceptionNewf(TypeError, "dict expects at most one argument") + } + out := NewStringDict() + if len(args) == 1 { + arg := args[0] + seq, err := SequenceList(arg) + if err != nil { + return nil, err + } + for _, i := range seq.Items { + switch z := i.(type) { + case Tuple: + if zStr, ok := z[0].(String); ok { + out[string(zStr)] = z[1] + } + default: + return nil, ExceptionNewf(TypeError, "non-tuple sequence") + } + } + } + if len(kwargs) > 0 { + for k, v := range kwargs { + out[k] = v + } + } + return out, nil +} + // Type of this StringDict object func (o StringDict) Type() *Type { return StringDictType @@ -75,6 +173,10 @@ func (a StringDict) M__str__() (Object, error) { return a.M__repr__() } +func (a StringDict) M__len__() (Object, error) { + return Int(len(a)), nil +} + func (a StringDict) M__repr__() (Object, error) { var out bytes.Buffer out.WriteRune('{') @@ -100,6 +202,15 @@ func (a StringDict) M__repr__() (Object, error) { return String(out.String()), nil } +// Returns a list of keys from the dict +func (d StringDict) M__iter__() (Object, error) { + o := make(Tuple, 0, len(d)) + for k := range d { + o = append(o, String(k)) + } + return NewIterator(o), nil +} + func (d StringDict) M__getitem__(key Object) (Object, error) { str, ok := key.(String) if ok { @@ -111,6 +222,19 @@ func (d StringDict) M__getitem__(key Object) (Object, error) { return nil, ExceptionNewf(KeyError, "%v", key) } +func (d StringDict) M__delitem__(key Object) (Object, error) { + str, ok := key.(String) + if !ok { + return nil, ExceptionNewf(KeyError, "%v", key) + } + _, ok = d[string(str)] + if !ok { + return nil, ExceptionNewf(KeyError, "%v", key) + } + delete(d, string(str)) + return None, nil +} + func (d StringDict) M__setitem__(key, value Object) (Object, error) { str, ok := key.(String) if !ok { @@ -149,8 +273,29 @@ func (a StringDict) M__ne__(other Object) (Object, error) { if err != nil { return nil, err } + if res == NotImplemented { + return res, nil + } if res == True { return False, nil } return True, nil } + +func (a StringDict) M__contains__(other Object) (Object, error) { + key, ok := other.(String) + if !ok { + return nil, ExceptionNewf(KeyError, "FIXME can only have string keys!: %v", key) + } + + if _, ok := a[string(key)]; ok { + return True, nil + } + return False, nil +} + +func (d StringDict) GetDict() StringDict { + return d +} + +var _ IGetDict = (*StringDict)(nil) diff --git a/py/enumerate.go b/py/enumerate.go new file mode 100644 index 00000000..f66d0af7 --- /dev/null +++ b/py/enumerate.go @@ -0,0 +1,88 @@ +// Copyright 2018 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package py + +// A python Enumerate object +type Enumerate struct { + Iterable Object + Start Int +} + +// A python Enumerate iterator +type EnumerateIterator struct { + Enumerate + Index Int +} + +var EnumerateType = NewTypeX("enumerate", `enumerate(iterable, start=0) + +Return an enumerate object.`, + EnumerateNew, nil) + +var EnumerateIteratorType = NewType("enumerate_iterator", `enumerate_iterator object`) + +// Type of this object +func (e *Enumerate) Type() *Type { + return EnumerateType +} + +// Type of this object +func (ei *EnumerateIterator) Type() *Type { + return EnumerateIteratorType +} + +// EnumerateTypeNew +func EnumerateNew(metatype *Type, args Tuple, kwargs StringDict) (Object, error) { + var iterable Object + var start Object + err := UnpackTuple(args, kwargs, "enumerate", 1, 2, &iterable, &start) + if err != nil { + return nil, err + } + + if start == nil { + start = Int(0) + } + startIndex, err := Index(start) + if err != nil { + return nil, err + } + iter, err := Iter(iterable) + if err != nil { + return nil, err + } + + return &Enumerate{Iterable: iter, Start: startIndex}, nil +} + +// Enumerate iterator +func (e *Enumerate) M__iter__() (Object, error) { + return &EnumerateIterator{ + Enumerate: *e, + Index: e.Start, + }, nil +} + +// EnumerateIterator iterator +func (ei *EnumerateIterator) M__iter__() (Object, error) { + return ei, nil +} + +// EnumerateIterator iterator next +func (ei *EnumerateIterator) M__next__() (Object, error) { + value, err := Next(ei.Enumerate.Iterable) + if err != nil { + return nil, err + } + res := make(Tuple, 2) + res[0] = ei.Index + res[1] = value + ei.Index += 1 + return res, nil +} + +// Check interface is satisfied +var _ I__iter__ = (*Enumerate)(nil) +var _ I_iterator = (*EnumerateIterator)(nil) diff --git a/py/exception.go b/py/exception.go index 8dde2e67..73f92747 100644 --- a/py/exception.go +++ b/py/exception.go @@ -330,12 +330,14 @@ func ExceptionGivenMatches(err, exc Object) bool { // IsException matches the result of recover to an exception // -// For use to catch a single python exception from go code +// # For use to catch a single python exception from go code // // It can be an instance or the class itself func IsException(exception *Type, r interface{}) bool { var t *Type switch ex := r.(type) { + case ExceptionInfo: + t = ex.Type case *Exception: t = ex.Type() case *Type: @@ -360,6 +362,22 @@ func (e *Exception) M__getattr__(name string) (Object, error) { return e.Args, nil // FIXME All attributes are args! } +func (e *Exception) M__str__() (Object, error) { + msg := e.Args.(Tuple)[0] + return msg, nil +} + +func (e *Exception) M__repr__() (Object, error) { + msg := e.Args.(Tuple)[0].(String) + typ := e.Base.Name + return String(fmt.Sprintf("%s(%q)", typ, string(msg))), nil +} + // Check Interfaces -var _ error = (*Exception)(nil) -var _ error = (*ExceptionInfo)(nil) +var ( + _ error = (*ExceptionInfo)(nil) + + _ error = (*Exception)(nil) + _ I__str__ = (*Exception)(nil) + _ I__repr__ = (*Exception)(nil) +) diff --git a/py/file.go b/py/file.go index fa270865..3d9f0185 100644 --- a/py/file.go +++ b/py/file.go @@ -11,7 +11,6 @@ package py import ( "io" - "io/ioutil" "os" ) @@ -129,7 +128,7 @@ func (o *File) Read(args Tuple, kwargs StringDict) (Object, error) { return nil, ExceptionNewf(TypeError, "read() argument 1 must be int, not %s", arg.Type().Name) } - b, err := ioutil.ReadAll(r) + b, err := io.ReadAll(r) if err != nil { if err == io.EOF { return o.readResult(nil) @@ -167,71 +166,10 @@ func (o *File) M__exit__(exc_type, exc_value, traceback Object) (Object, error) } func OpenFile(filename, mode string, buffering int) (Object, error) { - var fileMode FileMode - var truncate bool - var exclusive bool - - for _, m := range mode { - switch m { - case 'r': - if fileMode&FileReadWrite != 0 { - return nil, ExceptionNewf(ValueError, "must have exactly one of create/read/write/append mode") - } - fileMode |= FileRead - - case 'w': - if fileMode&FileReadWrite != 0 { - return nil, ExceptionNewf(ValueError, "must have exactly one of create/read/write/append mode") - } - fileMode |= FileWrite - truncate = true - - case 'x': - if fileMode&FileReadWrite != 0 { - return nil, ExceptionNewf(ValueError, "must have exactly one of create/read/write/append mode") - } - fileMode |= FileWrite - exclusive = true - - case 'a': - if fileMode&FileReadWrite != 0 { - return nil, ExceptionNewf(ValueError, "must have exactly one of create/read/write/append mode") - } - fileMode |= FileWrite - truncate = false - - case '+': - if fileMode&FileReadWrite == 0 { - return nil, ExceptionNewf(ValueError, "Must have exactly one of create/read/write/append mode and at most one plus") - } - - truncate = (fileMode & FileWrite) != 0 - fileMode |= FileReadWrite - - case 'b': - if fileMode&FileReadWrite == 0 { - return nil, ExceptionNewf(ValueError, "Must have exactly one of create/read/write/append mode and at most one plus") - } - - if fileMode&FileText != 0 { - return nil, ExceptionNewf(ValueError, "can't have text and binary mode at once") - } - - fileMode |= FileBinary - - case 't': - if fileMode&FileReadWrite == 0 { - return nil, ExceptionNewf(ValueError, "Must have exactly one of create/read/write/append mode and at most one plus") - } - - if fileMode&FileBinary != 0 { - return nil, ExceptionNewf(ValueError, "can't have text and binary mode at once") - } - - fileMode |= FileText - } + fileMode, truncate, exclusive, err := FileModeFrom(mode) + if err != nil { + return nil, err } - var fmode int switch fileMode & FileReadWrite { @@ -281,3 +219,67 @@ func OpenFile(filename, mode string, buffering int) (Object, error) { // Check interface is satisfied var _ I__enter__ = (*File)(nil) var _ I__exit__ = (*File)(nil) + +func FileModeFrom(mode string) (perm FileMode, trunc, excl bool, err error) { + for _, m := range mode { + switch m { + case 'r': + if perm&FileReadWrite != 0 { + return 0, false, false, ExceptionNewf(ValueError, "must have exactly one of create/read/write/append mode") + } + perm |= FileRead + + case 'w': + if perm&FileReadWrite != 0 { + return 0, false, false, ExceptionNewf(ValueError, "must have exactly one of create/read/write/append mode") + } + perm |= FileWrite + trunc = true + + case 'x': + if perm&FileReadWrite != 0 { + return 0, false, false, ExceptionNewf(ValueError, "must have exactly one of create/read/write/append mode") + } + perm |= FileWrite + excl = true + + case 'a': + if perm&FileReadWrite != 0 { + return 0, false, false, ExceptionNewf(ValueError, "must have exactly one of create/read/write/append mode") + } + perm |= FileWrite + trunc = false + + case '+': + if perm&FileReadWrite == 0 { + return 0, false, false, ExceptionNewf(ValueError, "Must have exactly one of create/read/write/append mode and at most one plus") + } + + trunc = (perm & FileWrite) != 0 + perm |= FileReadWrite + + case 'b': + if perm&FileReadWrite == 0 { + return 0, false, false, ExceptionNewf(ValueError, "Must have exactly one of create/read/write/append mode and at most one plus") + } + + if perm&FileText != 0 { + return 0, false, false, ExceptionNewf(ValueError, "can't have text and binary mode at once") + } + + perm |= FileBinary + + case 't': + if perm&FileReadWrite == 0 { + return 0, false, false, ExceptionNewf(ValueError, "Must have exactly one of create/read/write/append mode and at most one plus") + } + + if perm&FileBinary != 0 { + return 0, false, false, ExceptionNewf(ValueError, "can't have text and binary mode at once") + } + + perm |= FileText + } + } + return perm, trunc, excl, nil +} diff --git a/py/filter.go b/py/filter.go new file mode 100644 index 00000000..447c4829 --- /dev/null +++ b/py/filter.go @@ -0,0 +1,72 @@ +// Copyright 2023 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package py + +// A python Filter object +type Filter struct { + it Object + fun Object +} + +var FilterType = NewTypeX("filter", `filter(function or None, iterable) --> filter object + +Return an iterator yielding those items of iterable for which function(item) +is true. If function is None, return the items that are true.`, + FilterTypeNew, nil) + +// Type of this object +func (f *Filter) Type() *Type { + return FilterType +} + +// FilterTypeNew +func FilterTypeNew(metatype *Type, args Tuple, kwargs StringDict) (res Object, err error) { + var fun, seq Object + var it Object + err = UnpackTuple(args, kwargs, "filter", 2, 2, &fun, &seq) + if err != nil { + return nil, err + } + it, err = Iter(seq) + if err != nil { + return nil, err + } + return &Filter{it: it, fun: fun}, nil +} + +func (f *Filter) M__iter__() (Object, error) { + return f, nil +} + +func (f *Filter) M__next__() (Object, error) { + var ok bool + for { + item, err := Next(f.it) + if err != nil { + return nil, err + } + // if (lz->func == Py_None || lz->func == (PyObject *)&PyBool_Type) + if _, _ok := f.fun.(Bool); _ok || f.fun == None { + ok, err = ObjectIsTrue(item) + } else { + var good Object + good, err = Call(f.fun, Tuple{item}, nil) + if err != nil { + return nil, err + } + ok, err = ObjectIsTrue(good) + } + if ok { + return item, nil + } + if err != nil { + return nil, err + } + } +} + +// Check interface is satisfied +var _ I__iter__ = (*Filter)(nil) +var _ I__next__ = (*Filter)(nil) diff --git a/py/float.go b/py/float.go index 94fddde3..c4bb96fd 100644 --- a/py/float.go +++ b/py/float.go @@ -48,6 +48,9 @@ func FloatNew(metatype *Type, args Tuple, kwargs StringDict) (Object, error) { } func (a Float) M__str__() (Object, error) { + if i := int64(a); Float(i) == a { + return String(fmt.Sprintf("%d.0", i)), nil + } return String(fmt.Sprintf("%g", a)), nil } @@ -115,7 +118,7 @@ var floatDivisionByZero = ExceptionNewf(ZeroDivisionError, "float division by ze // Convert an Object to an Float // -// Retrurns ok as to whether the conversion worked or not +// Returns ok as to whether the conversion worked or not func convertToFloat(other Object) (Float, bool) { switch b := other.(type) { case Float: @@ -391,6 +394,20 @@ func (a Float) M__ge__(other Object) (Object, error) { return NotImplemented, nil } +// Properties +func init() { + FloatType.Dict["is_integer"] = MustNewMethod("is_integer", func(self Object) (Object, error) { + if a, ok := convertToFloat(self); ok { + f, err := FloatAsFloat64(a) + if err != nil { + return nil, err + } + return NewBool(math.Floor(f) == f), nil + } + return cantConvert(self, "float") + }, 0, "is_integer() -> Return True if the float instance is finite with integral value, and False otherwise.") +} + // Check interface is satisfied var _ floatArithmetic = Float(0) var _ conversionBetweenTypes = Float(0) diff --git a/py/frame.go b/py/frame.go index f91438a0..3e5c9f99 100644 --- a/py/frame.go +++ b/py/frame.go @@ -26,6 +26,7 @@ type TryBlock struct { // A python Frame object type Frame struct { // Back *Frame // previous frame, or nil + Context Context // host module (state) context Code *Code // code segment Builtins StringDict // builtin symbol table Globals StringDict // global symbol table @@ -77,7 +78,7 @@ func (o *Frame) Type() *Type { } // Make a new frame for a code object -func NewFrame(globals, locals StringDict, code *Code, closure Tuple) *Frame { +func NewFrame(ctx Context, globals, locals StringDict, code *Code, closure Tuple) *Frame { nlocals := int(code.Nlocals) ncells := len(code.Cellvars) nfrees := len(code.Freevars) @@ -90,12 +91,13 @@ func NewFrame(globals, locals StringDict, code *Code, closure Tuple) *Frame { cellAndFreeVars := allocation[nlocals:varsize] return &Frame{ + Context: ctx, Globals: globals, Locals: locals, Code: code, LocalVars: localVars, CellAndFreeVars: cellAndFreeVars, - Builtins: Builtins.Globals, + Builtins: ctx.Store().Builtins.Globals, Localsplus: allocation, Stack: make([]Object, 0, code.Stacksize), } @@ -156,17 +158,18 @@ func (f *Frame) PopBlock() { } } -/* Convert between "fast" version of locals and dictionary version. +/* +Convert between "fast" version of locals and dictionary version. - map and values are input arguments. map is a tuple of strings. - values is an array of PyObject*. At index i, map[i] is the name of - the variable with value values[i]. The function copies the first - nmap variable from map/values into dict. If values[i] is NULL, - the variable is deleted from dict. + map and values are input arguments. map is a tuple of strings. + values is an array of PyObject*. At index i, map[i] is the name of + the variable with value values[i]. The function copies the first + nmap variable from map/values into dict. If values[i] is NULL, + the variable is deleted from dict. - If deref is true, then the values being copied are cell variables - and the value is extracted from the cell variable before being put - in dict. + If deref is true, then the values being copied are cell variables + and the value is extracted from the cell variable before being put + in dict. */ func map_to_dict(mapping []string, nmap int, dict StringDict, values []Object, deref bool) { for j := nmap - 1; j >= 0; j-- { @@ -187,25 +190,26 @@ func map_to_dict(mapping []string, nmap int, dict StringDict, values []Object, d } } -/* Copy values from the "locals" dict into the fast locals. +/* +Copy values from the "locals" dict into the fast locals. - dict is an input argument containing string keys representing - variables names and arbitrary PyObject* as values. + dict is an input argument containing string keys representing + variables names and arbitrary PyObject* as values. - mapping and values are input arguments. mapping is a tuple of strings. - values is an array of PyObject*. At index i, mapping[i] is the name of - the variable with value values[i]. The function copies the first - nmap variable from mapping/values into dict. If values[i] is nil, - the variable is deleted from dict. + mapping and values are input arguments. mapping is a tuple of strings. + values is an array of PyObject*. At index i, mapping[i] is the name of + the variable with value values[i]. The function copies the first + nmap variable from mapping/values into dict. If values[i] is nil, + the variable is deleted from dict. - If deref is true, then the values being copied are cell variables - and the value is extracted from the cell variable before being put - in dict. If clear is true, then variables in mapping but not in dict - are set to nil in mapping; if clear is false, variables missing in - dict are ignored. + If deref is true, then the values being copied are cell variables + and the value is extracted from the cell variable before being put + in dict. If clear is true, then variables in mapping but not in dict + are set to nil in mapping; if clear is false, variables missing in + dict are ignored. - Exceptions raised while modifying the dict are silently ignored, - because there is no good way to report them. + Exceptions raised while modifying the dict are silently ignored, + because there is no good way to report them. */ func dict_to_map(mapping []string, nmap int, dict StringDict, values []Object, deref bool, clear bool) { for j := nmap - 1; j >= 0; j-- { diff --git a/py/function.go b/py/function.go index 673c93d4..2c37499d 100644 --- a/py/function.go +++ b/py/function.go @@ -18,6 +18,7 @@ package py // A python Function object type Function struct { Code *Code // A code object, the __code__ attribute + Context Context // Host VM context Globals StringDict // A dictionary (other mappings won't do) Defaults Tuple // NULL or a tuple KwDefaults StringDict // NULL or a dict @@ -26,7 +27,6 @@ type Function struct { Name string // The __name__ attribute, a string object Dict StringDict // The __dict__ attribute, a dict or NULL Weakreflist List // List of weak references - Module Object // The __module__ attribute, can be anything Annotations StringDict // Annotations, a dict or NULL Qualname string // The qualified name } @@ -56,9 +56,8 @@ func (f *Function) GetDict() StringDict { // attribute. qualname should be a unicode object or ""; if "", the // __qualname__ attribute is set to the same value as its __name__ // attribute. -func NewFunction(code *Code, globals StringDict, qualname string) *Function { +func NewFunction(ctx Context, code *Code, globals StringDict, qualname string) *Function { var doc Object - var module Object = None if len(code.Consts) >= 1 { doc = code.Consts[0] if _, ok := doc.(String); !ok { @@ -68,28 +67,24 @@ func NewFunction(code *Code, globals StringDict, qualname string) *Function { doc = None } - // __module__: If module name is in globals, use it. Otherwise, use None. - if moduleobj, ok := globals["__name__"]; ok { - module = moduleobj - } - if qualname == "" { qualname = code.Name } return &Function{ Code: code, + Context: ctx, Qualname: qualname, Globals: globals, Name: code.Name, Doc: doc, - Module: module, + Dict: make(StringDict), } } // Call a function func (f *Function) M__call__(args Tuple, kwargs StringDict) (Object, error) { - result, err := VmEvalCodeEx(f.Code, f.Globals, NewStringDict(), args, kwargs, f.Defaults, f.KwDefaults, f.Closure) + result, err := VmEvalCode(f.Context, f.Code, f.Globals, NewStringDict(), args, kwargs, f.Defaults, f.KwDefaults, f.Closure) if err != nil { return nil, err } @@ -193,10 +188,6 @@ func init() { f.Dict = dict return nil }, - Fdel: func(self Object) error { - self.(*Function).Dict = nil - return nil - }, } FunctionType.Dict["__name__"] = &Property{ Fget: func(self Object) (Object, error) { diff --git a/py/gen.go b/py/gen.go index 4b32a883..671b0831 100644 --- a/py/gen.go +++ b/py/gen.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build ignore // +build ignore package main @@ -44,7 +45,6 @@ var data = Data{ {Name: "complex", Title: "MakeComplex", Operator: "complex", Unary: true, Conversion: "Complex"}, {Name: "int", Title: "MakeInt", Operator: "int", Unary: true, Conversion: "Int"}, {Name: "float", Title: "MakeFloat", Operator: "float", Unary: true, Conversion: "Float"}, - {Name: "iter", Title: "Iter", Operator: "iter", Unary: true}, }, BinaryOps: Ops{ {Name: "add", Title: "Add", Operator: "+", Binary: true}, diff --git a/py/import.go b/py/import.go index 3cd1fd64..709a4468 100644 --- a/py/import.go +++ b/py/import.go @@ -7,17 +7,19 @@ package py import ( - "io/ioutil" - "os" "path" - "path/filepath" "strings" ) -var ( - // This will become sys.path one day ;-) - modulePath = []string{"", "/usr/lib/python3.4", "/usr/local/lib/python3.4/dist-packages", "/usr/lib/python3/dist-packages"} -) +func Import(ctx Context, names ...string) error { + for _, name := range names { + _, err := ImportModuleLevelObject(ctx, name, nil, nil, nil, 0) + if err != nil { + return err + } + } + return nil +} // The workings of __import__ // @@ -78,9 +80,18 @@ var ( // // Changed in version 3.3: Negative values for level are no longer // supported (which also changes the default value to 0). -func ImportModuleLevelObject(name string, globals, locals StringDict, fromlist Tuple, level int) (Object, error) { +func ImportModuleLevelObject(ctx Context, name string, globals, locals StringDict, fromlist Tuple, level int) (Object, error) { // Module already loaded - return that - if module, ok := modules[name]; ok { + if module, err := ctx.GetModule(name); err == nil { + return module, nil + } + + // See if the module is a registered embeddded module that has not been loaded into this ctx yet. + if impl := GetModuleImpl(name); impl != nil { + module, err := ctx.ModuleInit(impl) + if err != nil { + return nil, err + } return module, nil } @@ -88,74 +99,24 @@ func ImportModuleLevelObject(name string, globals, locals StringDict, fromlist T return nil, ExceptionNewf(SystemError, "Relative import not supported yet") } + // Convert import's dot separators into path seps parts := strings.Split(name, ".") - pathParts := path.Join(parts...) + srcPathname := path.Join(parts...) - for _, mpath := range modulePath { - if mpath == "" { - mpathObj, ok := globals["__file__"] - if !ok { - var err error - mpath, err = os.Getwd() - if err != nil { - return nil, err - } - } else { - mpath = path.Dir(string(mpathObj.(String))) - } - } - fullPath := path.Join(mpath, pathParts) - // FIXME Read pyc/pyo too - fullPath, err := filepath.Abs(fullPath) - if err != nil { - continue - } - if fi, err := os.Stat(fullPath); err == nil && fi.IsDir() { - // FIXME this is a massive simplification! - fullPath = path.Join(fullPath, "__init__.py") - } else { - fullPath += ".py" - } - // Check if file exists - if _, err := os.Stat(fullPath); err == nil { - str, err := ioutil.ReadFile(fullPath) - if err != nil { - return nil, ExceptionNewf(OSError, "Couldn't read %q: %v", fullPath, err) - } - codeObj, err := Compile(string(str), fullPath, "exec", 0, true) - if err != nil { - return nil, err - } - code, ok := codeObj.(*Code) - if !ok { - return nil, ExceptionNewf(ImportError, "Compile didn't return code object") - } - module := NewModule(name, "", nil, nil) - _, err = VmRun(module.Globals, module.Globals, code, nil) - if err != nil { - return nil, err - } - module.Globals["__file__"] = String(fullPath) - return module, nil - } + opts := CompileOpts{ + UseSysPaths: true, } - return nil, ExceptionNewf(ImportError, "No module named '%s'", name) - // Convert to absolute path if relative - // Use __file__ from globals to work out what we are relative to - - // '' in path seems to mean use the current __file__ - - // Find a valid path which we need to check for the correct __init__.py in subdirectories etc - - // Look for .py and .pyc files - - // Make absolute module path too if we can for sys.modules - - //How do we uniquely identify modules? + if fromFile, ok := globals["__file__"]; ok { + opts.CurDir = path.Dir(string(fromFile.(String))) + } - // SystemError: Parent module '' not loaded, cannot perform relative import + module, err := RunFile(ctx, srcPathname, opts, name) + if err != nil { + return nil, err + } + return module, nil } // Straight port of the python code @@ -163,7 +124,7 @@ func ImportModuleLevelObject(name string, globals, locals StringDict, fromlist T // This calls functins from _bootstrap.py which is a frozen module // // Too much functionality for the moment -func XImportModuleLevelObject(nameObj, given_globals, locals, given_fromlist Object, level int) (Object, error) { +func XImportModuleLevelObject(ctx Context, nameObj, given_globals, locals, given_fromlist Object, level int) (Object, error) { var abs_name string var builtins_import Object var final_mod Object @@ -175,6 +136,7 @@ func XImportModuleLevelObject(nameObj, given_globals, locals, given_fromlist Obj var ok bool var name string var err error + store := ctx.Store() // Make sure to use default values so as to not have // PyObject_CallMethodObjArgs() truncate the parameter list because of a @@ -239,7 +201,7 @@ func XImportModuleLevelObject(nameObj, given_globals, locals, given_fromlist Obj } } - if _, ok = modules[string(Package)]; !ok { + if _, err = ctx.GetModule(string(Package)); err != nil { return nil, ExceptionNewf(SystemError, "Parent module %q not loaded, cannot perform relative import", Package) } } else { // level == 0 */ @@ -277,16 +239,16 @@ func XImportModuleLevelObject(nameObj, given_globals, locals, given_fromlist Obj // From this point forward, goto error_with_unlock! builtins_import, ok = globals["__import__"] if !ok { - builtins_import, ok = Builtins.Globals["__import__"] + builtins_import, ok = store.Builtins.Globals["__import__"] if !ok { return nil, ExceptionNewf(ImportError, "__import__ not found") } } - mod, ok = modules[abs_name] - if mod == None { + mod, err = ctx.GetModule(abs_name) + if err != nil || mod == None { return nil, ExceptionNewf(ImportError, "import of %q halted; None in sys.modules", abs_name) - } else if ok { + } else if err == nil { var value Object var err error initializing := false @@ -306,11 +268,11 @@ func XImportModuleLevelObject(nameObj, given_globals, locals, given_fromlist Obj } if initializing { // _bootstrap._lock_unlock_module() releases the import lock */ - value, err = Importlib.Call("_lock_unlock_module", Tuple{String(abs_name)}, nil) + _, err = store.Importlib.Call("_lock_unlock_module", Tuple{String(abs_name)}, nil) if err != nil { return nil, err } - } else { + // } else { // not initializing // FIXME locking // if _PyImport_ReleaseLock() < 0 { // return nil, ExceptionNewf(RuntimeError, "not holding the import lock") @@ -318,7 +280,7 @@ func XImportModuleLevelObject(nameObj, given_globals, locals, given_fromlist Obj } } else { // _bootstrap._find_and_load() releases the import lock - mod, err = Importlib.Call("_find_and_load", Tuple{String(abs_name), builtins_import}, nil) + mod, err = store.Importlib.Call("_find_and_load", Tuple{String(abs_name), builtins_import}, nil) if err != nil { return nil, err } @@ -345,8 +307,8 @@ func XImportModuleLevelObject(nameObj, given_globals, locals, given_fromlist Obj cut_off := len(name) - len(front) abs_name_len := len(abs_name) to_return := abs_name[:abs_name_len-cut_off] - final_mod, ok = modules[to_return] - if !ok { + final_mod, err = ctx.GetModule(to_return) + if err != nil { return nil, ExceptionNewf(KeyError, "%q not in sys.modules as expected", to_return) } } @@ -354,7 +316,7 @@ func XImportModuleLevelObject(nameObj, given_globals, locals, given_fromlist Obj final_mod = mod } } else { - final_mod, err = Importlib.Call("_handle_fromlist", Tuple{mod, fromlist, builtins_import}, nil) + final_mod, err = store.Importlib.Call("_handle_fromlist", Tuple{mod, fromlist, builtins_import}, nil) if err != nil { return nil, err } @@ -376,7 +338,7 @@ error: } // The actual import code -func BuiltinImport(self Object, args Tuple, kwargs StringDict, currentGlobal StringDict) (Object, error) { +func BuiltinImport(ctx Context, self Object, args Tuple, kwargs StringDict, currentGlobal StringDict) (Object, error) { kwlist := []string{"name", "globals", "locals", "fromlist", "level"} var name Object var globals Object = currentGlobal @@ -391,5 +353,5 @@ func BuiltinImport(self Object, args Tuple, kwargs StringDict, currentGlobal Str if fromlist == None { fromlist = Tuple{} } - return ImportModuleLevelObject(string(name.(String)), globals.(StringDict), locals.(StringDict), fromlist.(Tuple), int(level.(Int))) + return ImportModuleLevelObject(ctx, string(name.(String)), globals.(StringDict), locals.(StringDict), fromlist.(Tuple), int(level.(Int))) } diff --git a/py/int.go b/py/int.go index d9040fed..919c28aa 100644 --- a/py/int.go +++ b/py/int.go @@ -55,7 +55,7 @@ func (o Int) Type() *Type { func IntNew(metatype *Type, args Tuple, kwargs StringDict) (Object, error) { var xObj Object = Int(0) var baseObj Object - base := 0 + base := 10 err := ParseTupleAndKeywords(args, kwargs, "|OO:int", []string{"x", "base"}, &xObj, &baseObj) if err != nil { return nil, err @@ -216,7 +216,7 @@ func cantConvert(a Object, to string) (Object, error) { // Convert an Object to an Int // -// Retrurns ok as to whether the conversion worked or not +// Returns ok as to whether the conversion worked or not func convertToInt(other Object) (Int, bool) { switch b := other.(type) { case Int: @@ -254,7 +254,7 @@ func (a Int) M__pos__() (Object, error) { func (a Int) M__abs__() (Object, error) { if a == IntMin { - // FIXME upconvert + return a.M__neg__() } if a < 0 { return -a, nil diff --git a/py/internal.go b/py/internal.go index 7c3e999a..e649b299 100644 --- a/py/internal.go +++ b/py/internal.go @@ -87,10 +87,22 @@ func MakeGoInt64(a Object) (int64, error) { // // Will raise TypeError if Index can't be run on this object func Index(a Object) (Int, error) { - A, ok := a.(I__index__) - if ok { + if A, ok := a.(I__index__); ok { return A.M__index__() } + + if A, ok, err := TypeCall0(a, "__index__"); ok { + if err != nil { + return 0, err + } + + if res, ok := A.(Int); ok { + return res, nil + } + + return 0, ExceptionNewf(TypeError, "__index__ returned non-int: (type %s)", A.Type().Name) + } + return 0, ExceptionNewf(TypeError, "unsupported operand type(s) for index: '%s'", a.Type().Name) } @@ -418,3 +430,20 @@ func ReprAsString(self Object) (string, error) { } return string(str), nil } + +// Returns an iterator object +// +// Call __Iter__ Returns an iterator object +// +// If object is sequence object, create an iterator +func Iter(self Object) (res Object, err error) { + if I, ok := self.(I__iter__); ok { + return I.M__iter__() + } else if res, ok, err = TypeCall0(self, "__iter__"); ok { + return res, err + } + if ObjectIsSequence(self) { + return NewIterator(self), nil + } + return nil, ExceptionNewf(TypeError, "'%s' object is not iterable", self.Type().Name) +} diff --git a/py/iterator.go b/py/iterator.go index a2368da8..350700a9 100644 --- a/py/iterator.go +++ b/py/iterator.go @@ -8,8 +8,8 @@ package py // A python Iterator object type Iterator struct { - Pos int - Objs []Object + Pos int + Seq Object } var IteratorType = NewType("iterator", "iterator type") @@ -20,10 +20,10 @@ func (o *Iterator) Type() *Type { } // Define a new iterator -func NewIterator(Objs []Object) *Iterator { +func NewIterator(Seq Object) *Iterator { m := &Iterator{ - Pos: 0, - Objs: Objs, + Pos: 0, + Seq: Seq, } return m } @@ -33,13 +33,29 @@ func (it *Iterator) M__iter__() (Object, error) { } // Get next one from the iteration -func (it *Iterator) M__next__() (Object, error) { - if it.Pos >= len(it.Objs) { - return nil, StopIteration +func (it *Iterator) M__next__() (res Object, err error) { + if tuple, ok := it.Seq.(Tuple); ok { + if it.Pos >= len(tuple) { + return nil, StopIteration + } + res = tuple[it.Pos] + it.Pos++ + return res, nil + } + index := Int(it.Pos) + if I, ok := it.Seq.(I__getitem__); ok { + res, err = I.M__getitem__(index) + } else if res, ok, err = TypeCall1(it.Seq, "__getitem__", index); !ok { + return nil, ExceptionNewf(TypeError, "'%s' object is not iterable", it.Type().Name) + } + if err != nil { + if IsException(IndexError, err) { + return nil, StopIteration + } + return nil, err } - r := it.Objs[it.Pos] it.Pos++ - return r, nil + return res, nil } // Check interface is satisfied diff --git a/py/list.go b/py/list.go index c509c45b..9f6f62b0 100644 --- a/py/list.go +++ b/py/list.go @@ -6,6 +6,10 @@ package py +import ( + "sort" +) + var ListType = ObjectType.NewType("list", "list() -> new empty list\nlist(iterable) -> new list initialized from iterable's items", ListNew, nil) // FIXME lists are mutable so this should probably be struct { Tuple } then can use the sub methods on Tuple @@ -13,6 +17,59 @@ type List struct { Items []Object } +func init() { + // FIXME: all methods should be callable using list.method([], *args, **kwargs) or [].method(*args, **kwargs) + ListType.Dict["append"] = MustNewMethod("append", func(self Object, args Tuple) (Object, error) { + listSelf := self.(*List) + if len(args) != 1 { + return nil, ExceptionNewf(TypeError, "append() takes exactly one argument (%d given)", len(args)) + } + listSelf.Items = append(listSelf.Items, args[0]) + return NoneType{}, nil + }, 0, "append(item)") + + ListType.Dict["extend"] = MustNewMethod("extend", func(self Object, args Tuple) (Object, error) { + listSelf := self.(*List) + if len(args) != 1 { + return nil, ExceptionNewf(TypeError, "append() takes exactly one argument (%d given)", len(args)) + } + if oList, ok := args[0].(*List); ok { + listSelf.Items = append(listSelf.Items, oList.Items...) + } + return NoneType{}, nil + }, 0, "extend([item])") + + ListType.Dict["sort"] = MustNewMethod("sort", func(self Object, args Tuple, kwargs StringDict) (Object, error) { + const funcName = "sort" + l, isList := self.(*List) + if !isList { + // method called using `list.sort([], **kwargs)` + var o Object + err := UnpackTuple(args, nil, funcName, 1, 1, &o) + if err != nil { + return nil, err + } + var ok bool + l, ok = o.(*List) + if !ok { + return nil, ExceptionNewf(TypeError, "descriptor 'sort' requires a 'list' object but received a '%s'", o.Type()) + } + } else { + // method called using `[].sort(**kargs)` + err := UnpackTuple(args, nil, funcName, 0, 0) + if err != nil { + return nil, err + } + } + err := SortInPlace(l, kwargs, funcName) + if err != nil { + return nil, err + } + return NoneType{}, nil + }, 0, "sort(key=None, reverse=False)") + +} + // Type of this List object func (o *List) Type() *Type { return ListType @@ -63,6 +120,15 @@ func NewListFromItems(items []Object) *List { return l } +// Makes an argv into a tuple +func NewListFromStrings(items []string) *List { + l := NewListSized(len(items)) + for i, v := range items { + l.Items[i] = String(v) + } + return l +} + // Copy a list object func (l *List) Copy() *List { return NewListFromItems(l.Items) @@ -83,6 +149,13 @@ func (l *List) Extend(items []Object) { l.Items = append(l.Items, items...) } +// Extend the list with strings +func (l *List) ExtendWithStrings(items []string) { + for _, item := range items { + l.Items = append(l.Items, Object(String(item))) + } +} + // Extends the list with the sequence passed in func (l *List) ExtendSequence(seq Object) error { return Iterate(seq, func(item Object) bool { @@ -113,7 +186,7 @@ func (l *List) M__bool__() (Object, error) { } func (l *List) M__iter__() (Object, error) { - return NewIterator(l.Items), nil + return NewIterator(Tuple(l.Items)), nil } func (l *List) M__getitem__(key Object) (Object, error) { @@ -236,6 +309,9 @@ func (l *List) M__mul__(other Object) (Object, error) { if b, ok := convertToInt(other); ok { m := len(l.Items) n := int(b) * m + if n < 0 { + n = 0 + } newList := NewListSized(n) for i := 0; i < n; i += m { copy(newList.Items[i:i+m], l.Items) @@ -305,3 +381,126 @@ func (a *List) M__ne__(other Object) (Object, error) { } return False, nil } + +type sortable struct { + l *List + keyFunc Object + reverse bool + firstErr error +} + +type ptrSortable struct { + s *sortable +} + +func (s ptrSortable) Len() int { + return s.s.l.Len() +} + +func (s ptrSortable) Swap(i, j int) { + itemI, err := s.s.l.M__getitem__(Int(i)) + if err != nil { + if s.s.firstErr == nil { + s.s.firstErr = err + } + return + } + itemJ, err := s.s.l.M__getitem__(Int(j)) + if err != nil { + if s.s.firstErr == nil { + s.s.firstErr = err + } + return + } + _, err = s.s.l.M__setitem__(Int(i), itemJ) + if err != nil { + if s.s.firstErr == nil { + s.s.firstErr = err + } + } + _, err = s.s.l.M__setitem__(Int(j), itemI) + if err != nil { + if s.s.firstErr == nil { + s.s.firstErr = err + } + } +} + +func (s ptrSortable) Less(i, j int) bool { + itemI, err := s.s.l.M__getitem__(Int(i)) + if err != nil { + if s.s.firstErr == nil { + s.s.firstErr = err + } + return false + } + itemJ, err := s.s.l.M__getitem__(Int(j)) + if err != nil { + if s.s.firstErr == nil { + s.s.firstErr = err + } + return false + } + + if s.s.keyFunc != None { + itemI, err = Call(s.s.keyFunc, Tuple{itemI}, nil) + if err != nil { + if s.s.firstErr == nil { + s.s.firstErr = err + } + return false + } + itemJ, err = Call(s.s.keyFunc, Tuple{itemJ}, nil) + if err != nil { + if s.s.firstErr == nil { + s.s.firstErr = err + } + return false + } + } + + var cmpResult Object + if s.s.reverse { + cmpResult, err = Lt(itemJ, itemI) + } else { + cmpResult, err = Lt(itemI, itemJ) + } + + if err != nil { + if s.s.firstErr == nil { + s.s.firstErr = err + } + return false + } + + if boolResult, ok := cmpResult.(Bool); ok { + return bool(boolResult) + } + + return false +} + +// SortInPlace sorts the given List in place using a stable sort. +// kwargs can have the keys "key" and "reverse". +func SortInPlace(l *List, kwargs StringDict, funcName string) error { + var keyFunc Object + var reverse Object + err := ParseTupleAndKeywords(nil, kwargs, "|$OO:"+funcName, []string{"key", "reverse"}, &keyFunc, &reverse) + if err != nil { + return err + } + if keyFunc == nil { + keyFunc = None + } + if reverse == nil { + reverse = False + } + // FIXME: requires the same bool-check like CPython (or better "|$Op" that doesn't panic on nil). + ok, err := ObjectIsTrue(reverse) + if err != nil { + return err + } + s := ptrSortable{&sortable{l, keyFunc, ok, nil}} + sort.Stable(s) + return s.s.firstErr +} diff --git a/py/map.go b/py/map.go new file mode 100644 index 00000000..1c343538 --- /dev/null +++ b/py/map.go @@ -0,0 +1,60 @@ +// Copyright 2023 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package py + +// A python Map object +type Map struct { + iters Tuple + fun Object +} + +var MapType = NewTypeX("filter", `map(func, *iterables) --> map object + +Make an iterator that computes the function using arguments from +each of the iterables. Stops when the shortest iterable is exhausted.`, + MapTypeNew, nil) + +// Type of this object +func (m *Map) Type() *Type { + return FilterType +} + +// MapType +func MapTypeNew(metatype *Type, args Tuple, kwargs StringDict) (res Object, err error) { + numargs := len(args) + if numargs < 2 { + return nil, ExceptionNewf(TypeError, "map() must have at least two arguments.") + } + iters := make(Tuple, numargs-1) + for i := 1; i < numargs; i++ { + iters[i-1], err = Iter(args[i]) + if err != nil { + return nil, err + } + } + return &Map{iters: iters, fun: args[0]}, nil +} + +func (m *Map) M__iter__() (Object, error) { + return m, nil +} + +func (m *Map) M__next__() (Object, error) { + numargs := len(m.iters) + argtuple := make(Tuple, numargs) + + for i := 0; i < numargs; i++ { + val, err := Next(m.iters[i]) + if err != nil { + return nil, err + } + argtuple[i] = val + } + return Call(m.fun, argtuple, nil) +} + +// Check interface is satisfied +var _ I__iter__ = (*Map)(nil) +var _ I__next__ = (*Map)(nil) diff --git a/py/method.go b/py/method.go index 1f8ecb72..c8b0ab03 100644 --- a/py/method.go +++ b/py/method.go @@ -70,6 +70,8 @@ type Method struct { Flags int // Go function implementation method interface{} + // Parent module of this method + Module *Module } // Internal method types implemented within eval.go @@ -224,14 +226,14 @@ func newBoundMethod(name string, fn interface{}) (Object, error) { return f(a, b, c) } default: - return nil, fmt.Errorf("Unknown bound method type for %q: %T", name, fn) + return nil, fmt.Errorf("unknown bound method type for %q: %T", name, fn) } return m, nil } // Call a method func (m *Method) M__call__(args Tuple, kwargs StringDict) (Object, error) { - self := None // FIXME should be the module + self := Object(m.Module) if kwargs != nil { return m.CallWithKeywords(self, args, kwargs) } diff --git a/py/module.go b/py/module.go index ae2a2171..eccd12a7 100644 --- a/py/module.go +++ b/py/module.go @@ -6,24 +6,93 @@ package py -import "fmt" +import ( + "fmt" + "sync" +) + +type ModuleFlags int32 + +const ( + // ShareModule signals that an embedded module is threadsafe and read-only, meaninging it could be shared across multiple py.Context instances (for efficiency). + // Otherwise, ModuleImpl will create a separate py.Module instance for each py.Context that imports it. + // This should be used with extreme caution since any module mutation (write) means possible cross-context data corruption. + ShareModule ModuleFlags = 0x01 + + MainModuleName = "__main__" +) + +// ModuleInfo contains info and about a module and can specify flags that affect how it is imported into a py.Context +type ModuleInfo struct { + Name string // __name__ (if nil, "__main__" is used) + Doc string // __doc__ + FileDesc string // __file__ + Flags ModuleFlags +} -var ( +// ModuleImpl is used for modules that are ready to be imported into a py.Context. +// The model is that a ModuleImpl is read-only and instantiates a Module into a py.Context when imported. +// +// By convention, .Code is executed when a module instance is initialized. If nil, +// then .CodeBuf or .CodeSrc will be auto-compiled to set .Code. +type ModuleImpl struct { + Info ModuleInfo + Methods []*Method // Module-bound global method functions + Globals StringDict // Module-bound global variables + CodeSrc string // Module code body (source code to be compiled) + CodeBuf []byte // Module code body (serialized py.Code object) + Code *Code // Module code body + OnContextClosed func(*Module) // Callback for when a py.Context is closing to release resources +} + +// ModuleStore is a container of Module imported into an owning py.Context. +type ModuleStore struct { // Registry of installed modules - modules = make(map[string]*Module) + modules map[string]*Module // Builtin module Builtins *Module // this should be the frozen module importlib/_bootstrap.py generated // by Modules/_freeze_importlib.c into Python/importlib.h Importlib *Module -) +} + +func RegisterModule(module *ModuleImpl) { + gRuntime.RegisterModule(module) +} -// A python Module object +func GetModuleImpl(moduleName string) *ModuleImpl { + gRuntime.mu.RLock() + defer gRuntime.mu.RUnlock() + impl := gRuntime.ModuleImpls[moduleName] + return impl +} + +type Runtime struct { + mu sync.RWMutex + ModuleImpls map[string]*ModuleImpl +} + +var gRuntime = Runtime{ + ModuleImpls: make(map[string]*ModuleImpl), +} + +func (rt *Runtime) RegisterModule(impl *ModuleImpl) { + rt.mu.Lock() + defer rt.mu.Unlock() + rt.ModuleImpls[impl.Info.Name] = impl +} + +func NewModuleStore() *ModuleStore { + return &ModuleStore{ + modules: make(map[string]*Module), + } +} + +// Module is a runtime instance of a ModuleImpl bound to the py.Context that imported it. type Module struct { - Name string - Doc string - Globals StringDict - // dict Dict + ModuleImpl *ModuleImpl // Parent implementation of this Module instance + Globals StringDict // Initialized from ModuleImpl.Globals + Context Context // Parent context that "owns" this Module instance } var ModuleType = NewType("module", "module object") @@ -34,7 +103,11 @@ func (o *Module) Type() *Type { } func (m *Module) M__repr__() (Object, error) { - return String(fmt.Sprintf("", m.Name)), nil + name, ok := m.Globals["__name__"].(String) + if !ok { + name = "???" + } + return String(fmt.Sprintf("", string(name))), nil } // Get the Dict @@ -42,60 +115,82 @@ func (m *Module) GetDict() StringDict { return m.Globals } -// Define a new module -func NewModule(name, doc string, methods []*Method, globals StringDict) *Module { +// Calls a named method of a module +func (m *Module) Call(name string, args Tuple, kwargs StringDict) (Object, error) { + attr, err := GetAttrString(m, name) + if err != nil { + return nil, err + } + return Call(attr, args, kwargs) +} + +// Interfaces +var _ IGetDict = (*Module)(nil) + +// NewModule adds a new Module instance to this ModuleStore. +// Each given Method prototype is used to create a new "live" Method bound this the newly created Module. +// This func also sets appropriate module global attribs based on the given ModuleInfo (e.g. __name__). +func (store *ModuleStore) NewModule(ctx Context, impl *ModuleImpl) (*Module, error) { + name := impl.Info.Name + if name == "" { + name = MainModuleName + } m := &Module{ - Name: name, - Doc: doc, - Globals: globals.Copy(), + ModuleImpl: impl, + Globals: impl.Globals.Copy(), + Context: ctx, } // Insert the methods into the module dictionary - for _, method := range methods { - m.Globals[method.Name] = method + // Copy each method an insert each "live" with a ptr back to the module (which can also lead us to the host Context) + for _, method := range impl.Methods { + methodInst := new(Method) + *methodInst = *method + methodInst.Module = m + m.Globals[method.Name] = methodInst } // Set some module globals m.Globals["__name__"] = String(name) - m.Globals["__doc__"] = String(doc) + m.Globals["__doc__"] = String(impl.Info.Doc) m.Globals["__package__"] = None + if len(impl.Info.FileDesc) > 0 { + m.Globals["__file__"] = String(impl.Info.FileDesc) + } // Register the module - modules[name] = m + store.modules[name] = m // Make a note of some modules switch name { case "builtins": - Builtins = m + store.Builtins = m case "importlib": - Importlib = m + store.Importlib = m } - // fmt.Printf("Registering module %q\n", name) - return m + // fmt.Printf("Registered module %q\n", moduleName) + return m, nil } // Gets a module -func GetModule(name string) (*Module, error) { - m, ok := modules[name] +func (store *ModuleStore) GetModule(name string) (*Module, error) { + m, ok := store.modules[name] if !ok { - return nil, ExceptionNewf(ImportError, "Module %q not found", name) + return nil, ExceptionNewf(ImportError, "Module '%s' not found", name) } return m, nil } // Gets a module or panics -func MustGetModule(name string) *Module { - m, err := GetModule(name) +func (store *ModuleStore) MustGetModule(name string) *Module { + m, err := store.GetModule(name) if err != nil { panic(err) } return m } -// Calls a named method of a module -func (m *Module) Call(name string, args Tuple, kwargs StringDict) (Object, error) { - attr, err := GetAttrString(m, name) - if err != nil { - return nil, err +// OnContextClosed signals all module instances that the parent py.Context has closed +func (store *ModuleStore) OnContextClosed() { + for _, m := range store.modules { + if m.ModuleImpl.OnContextClosed != nil { + m.ModuleImpl.OnContextClosed(m) + } } - return Call(attr, args, kwargs) } - -// Interfaces -var _ IGetDict = (*Module)(nil) diff --git a/py/none.go b/py/none.go index 63a9952b..6c453b6c 100644 --- a/py/none.go +++ b/py/none.go @@ -33,7 +33,7 @@ func (a NoneType) M__repr__() (Object, error) { // Convert an Object to an NoneType // -// Retrurns ok as to whether the conversion worked or not +// Returns ok as to whether the conversion worked or not func convertToNoneType(other Object) (NoneType, bool) { switch b := other.(type) { case NoneType: diff --git a/py/object.go b/py/object.go index 55141cec..540ea0b6 100644 --- a/py/object.go +++ b/py/object.go @@ -23,24 +23,53 @@ func ObjectRepr(o Object) Object { } // Return whether the object is True or not -func ObjectIsTrue(o Object) bool { - if o == True { - return true +func ObjectIsTrue(o Object) (cmp bool, err error) { + switch o { + case True: + return true, nil + case False: + return false, nil + case None: + return false, nil } - if o == False { - return false + + var res Object + switch t := o.(type) { + case I__bool__: + res, err = t.M__bool__() + case I__len__: + res, err = t.M__len__() + case *Type: + var ok bool + if res, ok, err = TypeCall0(o, "__bool__"); ok { + break + } + if res, ok, err = TypeCall0(o, "__len__"); ok { + break + } + _ = ok // pass static-check + } + if err != nil { + return false, err } - if o == None { - return false + switch t := res.(type) { + case Bool: + return t == True, nil + case Int: + return t > 0, nil } + return true, nil +} - if I, ok := o.(I__bool__); ok { - cmp, err := I.M__bool__() - if err == nil && cmp == True { +// Return whether the object is a sequence +func ObjectIsSequence(o Object) bool { + switch t := o.(type) { + case I__getitem__: + return true + case *Type: + if t.GetAttrOrNil("__getitem__") != nil { return true - } else if err == nil && cmp == False { - return false } } return false diff --git a/py/py.go b/py/py.go index 0c48ee7b..2ebd5fcd 100644 --- a/py/py.go +++ b/py/py.go @@ -24,15 +24,10 @@ type IGoInt64 interface { GoInt64() (int64, error) } -// Some well known objects var ( // Set in vm/eval.go - to avoid circular import - VmRun func(globals, locals StringDict, code *Code, closure Tuple) (res Object, err error) - VmRunFrame func(frame *Frame) (res Object, err error) - VmEvalCodeEx func(co *Code, globals, locals StringDict, args []Object, kws StringDict, defs []Object, kwdefs StringDict, closure Tuple) (retval Object, err error) - - // See compile/compile.go - set to avoid circular import - Compile func(str, filename, mode string, flags int, dont_inherit bool) (Object, error) + VmEvalCode func(ctx Context, code *Code, globals, locals StringDict, args []Object, kws StringDict, defs []Object, kwdefs StringDict, closure Tuple) (retval Object, err error) + VmRunFrame func(frame *Frame) (res Object, err error) ) // Called to create a new instance of class cls. __new__() is a static method (special-cased so you need not declare it as such) that takes the class of which an instance was requested as its first argument. The remaining arguments are those passed to the object constructor expression (the call to the class). The return value of __new__() should be the new object instance (usually an instance of cls). @@ -44,7 +39,7 @@ var ( // If __new__() does not return an instance of cls, then the new instance’s __init__() method will not be invoked. // // __new__() is intended mainly to allow subclasses of immutable types (like int, str, or tuple) to customize instance creation. It is also commonly overridden in custom metaclasses in order to customize class creation. -//object.__new__(cls[, ...]) +// object.__new__(cls[, ...]) type I__new__ interface { M__new__(cls, args, kwargs Object) (Object, error) } @@ -57,7 +52,7 @@ type I__new__ interface { // [args...]). As a special constraint on constructors, no value may // be returned; doing so will cause a TypeError to be raised at // runtime. -//object.__init__(self[, ...]) +// object.__init__(self[, ...]) type I__init__ interface { M__init__(self, args, kwargs Object) (Object, error) } @@ -106,7 +101,7 @@ type I__init__ interface { // globals are deleted; if no other references to such globals exist, // this may help in assuring that imported modules are still available // at the time when the __del__() method is called. -//object.__del__(self) +// object.__del__(self) type I__del__ interface { M__del__() (Object, error) } @@ -123,7 +118,7 @@ type I__del__ interface { // // This is typically used for debugging, so it is important that the // representation is information-rich and unambiguous. -//object.__repr__(self) +// object.__repr__(self) type I__repr__ interface { M__repr__() (Object, error) } @@ -139,14 +134,14 @@ type I__repr__ interface { // // The default implementation defined by the built-in type object // calls object.__repr__(). -//object.__str__(self) +// object.__str__(self) type I__str__ interface { M__str__() (Object, error) } // Called by bytes() to compute a byte-string representation of an // object. This should return a bytes object. -//object.__bytes__(self) +// object.__bytes__(self) type I__bytes__ interface { M__bytes__() (Object, error) } @@ -164,7 +159,7 @@ type I__bytes__ interface { // standard formatting syntax. // // The return value must be a string object. -//object.__format__(self, format_spec) +// object.__format__(self, format_spec) type I__format__ interface { M__format__(format_spec Object) (Object, error) } @@ -201,32 +196,32 @@ type I__format__ interface { // // To automatically generate ordering operations from a single root // operation, see functools.total_ordering(). -//object.__lt__(self, other) +// object.__lt__(self, other) type I__lt__ interface { M__lt__(other Object) (Object, error) } -//object.__le__(self, other) +// object.__le__(self, other) type I__le__ interface { M__le__(other Object) (Object, error) } -//object.__eq__(self, other) +// object.__eq__(self, other) type I__eq__ interface { M__eq__(other Object) (Object, error) } -//object.__ne__(self, other) +// object.__ne__(self, other) type I__ne__ interface { M__ne__(other Object) (Object, error) } -//object.__gt__(self, other) +// object.__gt__(self, other) type I__gt__ interface { M__gt__(other Object) (Object, error) } -//object.__ge__(self, other) +// object.__ge__(self, other) type I__ge__ interface { M__ge__(other Object) (Object, error) } @@ -305,8 +300,8 @@ type richComparison interface { // // See also PYTHONHASHSEED. // -//Changed in version 3.3: Hash randomization is enabled by default. -//object.__hash__(self) +// Changed in version 3.3: Hash randomization is enabled by default. +// object.__hash__(self) type I__hash__ interface { M__hash__() (Object, error) } @@ -317,7 +312,7 @@ type I__hash__ interface { // considered true if its result is nonzero. If a class defines // neither __len__() nor __bool__(), all its instances are considered // true. -//object.__bool__(self) +// object.__bool__(self) type I__bool__ interface { M__bool__() (Object, error) } @@ -340,7 +335,7 @@ type I__bool__ interface { // instead inserting them in another object). See the // __getattribute__() method below for a way to actually get total // control over attribute access. -//object.__getattr__(self, name) +// object.__getattr__(self, name) type I__getattr__ interface { M__getattr__(name string) (Object, error) } @@ -355,10 +350,10 @@ type I__getattr__ interface { // same name to access any attributes it needs, for example, // object.__getattribute__(self, name). // -//Note This method may still be bypassed when looking up special -//methods as the result of implicit invocation via language syntax or -//built-in functions. See Special method lookup. -//object.__getattribute__(self, name) +// Note This method may still be bypassed when looking up special +// methods as the result of implicit invocation via language syntax or +// built-in functions. See Special method lookup. +// object.__getattribute__(self, name) type I__getattribute__ interface { M__getattribute__(name string) (Object, error) } @@ -371,7 +366,7 @@ type I__getattribute__ interface { // If __setattr__() wants to assign to an instance attribute, it // should call the base class method with the same name, for example, // object.__setattr__(self, name, value). -//object.__setattr__(self, name, value) +// object.__setattr__(self, name, value) type I__setattr__ interface { M__setattr__(name string, value Object) (Object, error) } @@ -379,7 +374,7 @@ type I__setattr__ interface { // Like __setattr__() but for attribute deletion instead of // assignment. This should only be implemented if del obj.name is // meaningful for the object. -//object.__delattr__(self, name) +// object.__delattr__(self, name) type I__delattr__ interface { M__delattr__(name string) (Object, error) } @@ -387,7 +382,7 @@ type I__delattr__ interface { // Called when dir() is called on the object. A sequence must be // returned. dir() converts the returned sequence to a list and sorts // it. -//object.__dir__(self) +// object.__dir__(self) type I__dir__ interface { M__dir__() (Object, error) } @@ -406,21 +401,21 @@ type I__dir__ interface { // attribute is accessed through the owner. This method should return // the (computed) attribute value or raise an AttributeError // exception. -//object.__get__(self, instance, owner) +// object.__get__(self, instance, owner) type I__get__ interface { M__get__(instance, owner Object) (Object, error) } // Called to set the attribute on an instance of the owner // class to a new value. -//object.__set__(self, instance, value) +// object.__set__(self, instance, value) type I__set__ interface { M__set__(instance, value Object) (Object, error) } // Called to delete the attribute on an instance instance of the owner // class. -//object.__delete__(self, instance) +// object.__delete__(self, instance) type I__delete__ interface { M__delete__(instance Object) (Object, error) } @@ -442,7 +437,7 @@ type I__delete__ interface { // Return true if instance should be considered a (direct or indirect) // instance of class. If defined, called to implement // isinstance(instance, class). -//object.__instancecheck__(self, instance) +// object.__instancecheck__(self, instance) type I__instancecheck__ interface { M__instancecheck__(instance Object) (Object, error) } @@ -450,7 +445,7 @@ type I__instancecheck__ interface { // Return true if subclass should be considered a (direct or indirect) // subclass of class. If defined, called to implement // issubclass(subclass, class). -//object.__subclasscheck__(self, subclass) +// object.__subclasscheck__(self, subclass) type I__subclasscheck__ interface { M__subclasscheck__(subclass Object) (Object, error) } @@ -458,7 +453,7 @@ type I__subclasscheck__ interface { // Called when the instance is “called” as a function; if this method // is defined, x(arg1, arg2, ...) is a shorthand for x.__call__(arg1, // arg2, ...). -//object.__call__(self[, args...]) +// object.__call__(self[, args...]) type I__call__ interface { M__call__(args Tuple, kwargs StringDict) (Object, error) } @@ -496,7 +491,7 @@ type I__call__ interface { // length of the object, an integer >= 0. Also, an object that doesn’t // define a __bool__() method and whose __len__() method returns zero // is considered to be false in a Boolean context. -//object.__len__(self) +// object.__len__(self) type I__len__ interface { M__len__() (Object, error) } @@ -507,7 +502,7 @@ type I__len__ interface { // is purely an optimization and is never required for correctness. // // New in version 3.4. -//object.__length_hint__(self) +// object.__length_hint__(self) type I__length_hint__ interface { M__length_hint__() (Object, error) } @@ -530,7 +525,7 @@ type I__length_hint__ interface { // // Note for loops expect that an IndexError will be raised for illegal // indexes to allow proper detection of the end of the sequence. -//object.__getitem__(self, key) +// object.__getitem__(self, key) type I__getitem__ interface { M__getitem__(key Object) (Object, error) } @@ -551,7 +546,7 @@ type I__setitem__ interface { // objects support removal of keys, or for sequences if elements can // be removed from the sequence. The same exceptions should be raised // for improper key values as for the __getitem__() method. -//object.__delitem__(self, key) +// object.__delitem__(self, key) type I__delitem__ interface { M__delitem__(key Object) (Object, error) } @@ -565,7 +560,7 @@ type I__delitem__ interface { // Iterator objects also need to implement this method; they are // required to return themselves. For more information on iterator // objects, see Iterator Types. -//object.__iter__(self) +// object.__iter__(self) type I__iter__ interface { M__iter__() (Object, error) } @@ -610,7 +605,7 @@ type I_generator interface { // should only provide __reversed__() if they can provide an // implementation that is more efficient than the one provided by // reversed(). -//object.__reversed__(self) +// object.__reversed__(self) type I__reversed__ interface { M__reversed__() (Object, error) } @@ -630,7 +625,7 @@ type I__reversed__ interface { // first tries iteration via __iter__(), then the old sequence // iteration protocol via __getitem__(), see this section in the // language reference. -//object.__contains__(self, item) +// object.__contains__(self, item) type I__contains__ interface { M__contains__(item Object) (Object, error) } @@ -1016,7 +1011,7 @@ type sequenceArithmetic interface { // Enter the runtime context related to this object. The with // statement will bind this method’s return value to the target(s) // specified in the as clause of the statement, if any. -//object.__enter__(self) +// object.__enter__(self) type I__enter__ interface { M__enter__() (Object, error) } @@ -1033,7 +1028,7 @@ type I__enter__ interface { // // Note that __exit__() methods should not reraise the passed-in // exception; this is the caller’s responsibility. -//object.__exit__(self, exc_type, exc_value, traceback) +// object.__exit__(self, exc_type, exc_value, traceback) type I__exit__ interface { M__exit__(exc_type, exc_value, traceback Object) (Object, error) } diff --git a/py/range.go b/py/range.go index e91349f4..1b261ccd 100644 --- a/py/range.go +++ b/py/range.go @@ -9,10 +9,10 @@ package py // A python Range object // FIXME one day support BigInts too! type Range struct { - Start Int - Stop Int - Step Int - //Length Object + Start Int + Stop Int + Step Int + Length Int } // A python Range iterator @@ -53,10 +53,12 @@ func RangeNew(metatype *Type, args Tuple, kwargs StringDict) (Object, error) { return nil, err } if len(args) == 1 { + length := computeRangeLength(0, startIndex, 1) return &Range{ - Start: Int(0), - Stop: startIndex, - Step: Int(1), + Start: Int(0), + Stop: startIndex, + Step: Int(1), + Length: length, }, nil } stopIndex, err := Index(stop) @@ -67,13 +69,33 @@ func RangeNew(metatype *Type, args Tuple, kwargs StringDict) (Object, error) { if err != nil { return nil, err } + length := computeRangeLength(startIndex, stopIndex, stepIndex) return &Range{ - Start: startIndex, - Stop: stopIndex, - Step: stepIndex, + Start: startIndex, + Stop: stopIndex, + Step: stepIndex, + Length: length, }, nil } +func (r *Range) M__getitem__(key Object) (Object, error) { + if slice, ok := key.(*Slice); ok { + return computeRangeSlice(r, slice) + } + + index, err := Index(key) + if err != nil { + return nil, err + } + index = computeNegativeIndex(index, r.Length) + + if index < 0 || index >= r.Length { + return nil, ExceptionNewf(IndexError, "range object index out of range") + } + result := computeItem(r, index) + return result, nil +} + // Make a range iterator from a range func (r *Range) M__iter__() (Object, error) { return &RangeIterator{ @@ -82,6 +104,18 @@ func (r *Range) M__iter__() (Object, error) { }, nil } +func (r *Range) M__str__() (Object, error) { + return r.M__repr__() +} + +func (r *Range) M__repr__() (Object, error) { + return r.repr() +} + +func (r *Range) M__len__() (Object, error) { + return r.Length, nil +} + // Range iterator func (it *RangeIterator) M__iter__() (Object, error) { return it, nil @@ -90,13 +124,169 @@ func (it *RangeIterator) M__iter__() (Object, error) { // Range iterator next func (it *RangeIterator) M__next__() (Object, error) { r := it.Index - if r >= it.Stop { + if it.Step >= 0 && r >= it.Stop { + return nil, StopIteration + } + + if it.Step < 0 && r <= it.Stop { return nil, StopIteration } it.Index += it.Step return r, nil } +func computeItem(r *Range, item Int) Int { + incr := item * r.Step + res := r.Start + incr + return res +} + +func computeRangeLength(start, stop, step Int) Int { + var lo, hi Int + if step > 0 { + lo = start + hi = stop + } else { + lo = stop + hi = start + step = (-step) + } + + if lo >= hi { + return Int(0) + } + res := (hi-lo-1)/step + 1 + return res +} + +func getIndexWithDefault(i Object, d Int) (Int, error) { + if i == None { + return d, nil + } else if res, err := Index(i); err != nil { + return 0, err + } else { + return res, nil + } +} + +func computeNegativeIndex(index, length Int) Int { + if index < 0 { + index += length + } + return index +} + +func computeBoundIndex(index, length Int) Int { + if index < 0 { + index = 0 + } else if index > length { + index = length + } + return index +} + +func computeRangeSlice(r *Range, s *Slice) (Object, error) { + start, err := getIndexWithDefault(s.Start, 0) + if err != nil { + return nil, err + } + stop, err := getIndexWithDefault(s.Stop, r.Length) + if err != nil { + return nil, err + } + step, err := getIndexWithDefault(s.Step, 1) + if err != nil { + return nil, err + } + + if step == 0 { + return nil, ExceptionNewf(ValueError, "slice step cannot be zero") + } + start = computeNegativeIndex(start, r.Length) + stop = computeNegativeIndex(stop, r.Length) + + start = computeBoundIndex(start, r.Length) + stop = computeBoundIndex(stop, r.Length) + + startIndex := computeItem(r, start) + stopIndex := computeItem(r, stop) + stepIndex := step * r.Step + + var sliceLength Int + if start < stop { + if stepIndex < 0 { + startIndex, stopIndex = stopIndex-1, startIndex-1 + } + } else { + if stepIndex < 0 { + startIndex, stopIndex = stopIndex+1, startIndex+1 + } + } + sliceLength = computeRangeLength(startIndex, stopIndex, stepIndex) + + return &Range{ + Start: startIndex, + Stop: stopIndex, + Step: stepIndex, + Length: sliceLength, + }, nil +} + // Check interface is satisfied +var _ I__getitem__ = (*Range)(nil) var _ I__iter__ = (*Range)(nil) var _ I_iterator = (*RangeIterator)(nil) + +func (a *Range) M__eq__(other Object) (Object, error) { + b, ok := other.(*Range) + if !ok { + return NotImplemented, nil + } + + if a.Length != b.Length { + return False, nil + } + + if a.Length == 0 { + return True, nil + } + if a.Start != b.Start { + return False, nil + } + + if a.Step == 1 { + return True, nil + } + if a.Step != b.Step { + return False, nil + } + + return True, nil +} + +func (a *Range) M__ne__(other Object) (Object, error) { + b, ok := other.(*Range) + if !ok { + return NotImplemented, nil + } + + if a.Length != b.Length { + return True, nil + } + + if a.Length == 0 { + return False, nil + } + if a.Start != b.Start { + return True, nil + } + + if a.Step == 1 { + return False, nil + } + if a.Step != b.Step { + return True, nil + } + + return False, nil +} diff --git a/py/range_repr110.go b/py/range_repr110.go new file mode 100644 index 00000000..2092ff30 --- /dev/null +++ b/py/range_repr110.go @@ -0,0 +1,40 @@ +// Copyright 2018 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.10 +// +build go1.10 + +// Range object + +package py + +import "strings" + +func (r *Range) repr() (Object, error) { + var b strings.Builder + b.WriteString("range(") + start, err := ReprAsString(r.Start) + if err != nil { + return nil, err + } + stop, err := ReprAsString(r.Stop) + if err != nil { + return nil, err + } + b.WriteString(start) + b.WriteString(", ") + b.WriteString(stop) + + if r.Step != 1 { + step, err := ReprAsString(r.Step) + if err != nil { + return nil, err + } + b.WriteString(", ") + b.WriteString(step) + } + b.WriteString(")") + + return String(b.String()), nil +} diff --git a/py/range_repr19.go b/py/range_repr19.go new file mode 100644 index 00000000..38c5d6d2 --- /dev/null +++ b/py/range_repr19.go @@ -0,0 +1,40 @@ +// Copyright 2018 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !go1.10 +// +build !go1.10 + +// Range object + +package py + +import "bytes" + +func (r *Range) repr() (Object, error) { + var b bytes.Buffer + b.WriteString("range(") + start, err := ReprAsString(r.Start) + if err != nil { + return nil, err + } + stop, err := ReprAsString(r.Stop) + if err != nil { + return nil, err + } + b.WriteString(start) + b.WriteString(", ") + b.WriteString(stop) + + if r.Step != 1 { + step, err := ReprAsString(r.Step) + if err != nil { + return nil, err + } + b.WriteString(", ") + b.WriteString(step) + } + b.WriteString(")") + + return String(b.String()), nil +} diff --git a/py/run.go b/py/run.go new file mode 100644 index 00000000..427cdbe6 --- /dev/null +++ b/py/run.go @@ -0,0 +1,182 @@ +// Copyright 2022 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package py + +type CompileMode string + +const ( + ExecMode CompileMode = "exec" // Compile a module + EvalMode CompileMode = "eval" // Compile an expression + SingleMode CompileMode = "single" // Compile a single (interactive) statement +) + +// Context is a gpython environment instance container, providing a high-level mechanism +// for multiple python interpreters to run concurrently without restriction. +// +// Context instances maintain completely independent environments, namely the modules that +// have been imported and their state. Modules imported into a Context are instanced +// from a parent ModuleImpl. For example, since Contexts each have their +// own sys module instance, each can set sys.path differently and independently. +// +// If you access a Context from multiple groutines, you are responsible that access is not concurrent, +// with the exception of Close() and Done(). +// +// See examples/multi-context and examples/embedding. +type Context interface { + + // Resolves then compiles (if applicable) the given file system pathname into a py.Code ready to be executed. + ResolveAndCompile(pathname string, opts CompileOpts) (CompileOut, error) + + // Creates a new py.Module instance and initializes ModuleImpl's code in the new module (if applicable). + ModuleInit(impl *ModuleImpl) (*Module, error) + + // RunCode is a lower-level invocation to execute the given py.Code. + // Blocks until execution is complete. + RunCode(code *Code, globals, locals StringDict, closure Tuple) (result Object, err error) + + // Returns the named module for this context (or an error if not found) + GetModule(moduleName string) (*Module, error) + + // Gereric access to this context's modules / state. + Store() *ModuleStore + + // Close signals this context is about to go out of scope and any internal resources should be released. + // Code execution on a py.Context that has been closed will result in an error. + Close() error + + // Done returns a signal that can be used to detect when this Context has fully closed / completed. + // If Close() is called while execution in progress, Done() will not signal until execution is complete. + Done() <-chan struct{} +} + +// CompileOpts specifies options for high-level compilation. +type CompileOpts struct { + UseSysPaths bool // If set, sys.path will be used to resolve relative pathnames + CurDir string // If non-empty, this is the path of the current working directory. If empty, os.Getwd() is used. +} + +// CompileOut the output of high-level compilation -- e.g. ResolveAndCompile() +type CompileOut struct { + SrcPathname string // Resolved pathname the .py file that was compiled (if applicable) + PycPathname string // Pathname of the .pyc file read and/or written (if applicable) + FileDesc string // Pathname to be used for a a module's "__file__" attrib + Code *Code // Read/Output code object ready for execution +} + +// DefaultCoreSysPaths specify default search paths for module sys +// This can be changed during runtime and plays nice with others using DefaultContextOpts() +var DefaultCoreSysPaths = []string{ + ".", + "lib", +} + +// DefaultAuxSysPaths are secondary default search paths for module sys. +// This can be changed during runtime and plays nice with others using DefaultContextOpts() +// They are separated from the default core paths since they the more likley thing you will want to completely replace when using gpython. +var DefaultAuxSysPaths = []string{ + "/usr/lib/python3.4", + "/usr/local/lib/python3.4/dist-packages", + "/usr/lib/python3/dist-packages", +} + +// ContextOpts specifies fundamental environment and input settings for creating a new py.Context +type ContextOpts struct { + SysArgs []string // sys.argv initializer + SysPaths []string // sys.path initializer +} + +var ( + // DefaultContextOpts should be the default opts created for py.NewContext. + // Calling this ensure that you future proof you code for suggested/default settings. + DefaultContextOpts = func() ContextOpts { + opts := ContextOpts{ + SysPaths: DefaultCoreSysPaths, + } + opts.SysPaths = append(opts.SysPaths, DefaultAuxSysPaths...) + return opts + } + + // NewContext is a high-level call to create a new gpython interpreter context. + // See type Context interface. + NewContext func(opts ContextOpts) Context + + // Compiles a python buffer into a py.Code object. + // Returns a py.Code object or otherwise an error. + Compile func(src, srcDesc string, mode CompileMode, flags int, dont_inherit bool) (*Code, error) +) + +// RunFile resolves the given pathname, compiles as needed, executes the code in the given module, and returns the Module to indicate success. +// +// See RunCode() for description of inModule. +func RunFile(ctx Context, pathname string, opts CompileOpts, inModule interface{}) (*Module, error) { + out, err := ctx.ResolveAndCompile(pathname, opts) + if err != nil { + return nil, err + } + + return RunCode(ctx, out.Code, out.FileDesc, inModule) +} + +// RunSrc compiles the given python buffer and executes it within the given module and returns the Module to indicate success. +// +// See RunCode() for description of inModule. +func RunSrc(ctx Context, pySrc string, pySrcDesc string, inModule interface{}) (*Module, error) { + if pySrcDesc == "" { + pySrcDesc = "" + } + code, err := Compile(pySrc+"\n", pySrcDesc, SingleMode, 0, true) + if err != nil { + return nil, err + } + + return RunCode(ctx, code, pySrcDesc, inModule) +} + +// RunCode executes the given code object within the given module and returns the Module to indicate success. +// +// If inModule is a *Module, then the code is run in that module. +// +// If inModule is nil, the code is run in a new __main__ module (and the new Module is returned). +// +// If inModule is a string, the code is run in a new module with the given name (and the new Module is returned). +func RunCode(ctx Context, code *Code, codeDesc string, inModule interface{}) (*Module, error) { + var ( + module *Module + moduleName string + err error + ) + + createNew := false + switch mod := inModule.(type) { + + case string: + moduleName = mod + createNew = true + case nil: + createNew = true + case *Module: + _, err = ctx.RunCode(code, mod.Globals, mod.Globals, nil) + module = mod + default: + err = ExceptionNewf(TypeError, "unsupported module type: %v", inModule) + } + + if err == nil && createNew { + moduleImpl := ModuleImpl{ + Info: ModuleInfo{ + Name: moduleName, + FileDesc: codeDesc, + }, + Code: code, + } + module, err = ctx.ModuleInit(&moduleImpl) + } + + if err != nil { + return nil, err + } + + return module, nil +} diff --git a/py/sequence.go b/py/sequence.go index 53e78295..9a2a57f2 100644 --- a/py/sequence.go +++ b/py/sequence.go @@ -43,9 +43,29 @@ func SequenceList(v Object) (*List, error) { } } +// Converts a sequence object v into a Set +func SequenceSet(v Object) (*Set, error) { + switch x := v.(type) { + case Tuple: + return NewSetFromItems(x), nil + case *List: + return NewSetFromItems(x.Items), nil + default: + s := NewSet() + err := Iterate(v, func(item Object) bool { + s.Add(item) + return false + }) + if err != nil { + return nil, err + } + return s, nil + } +} + // Call __next__ for the python object // -// Returns the next object +// # Returns the next object // // err == StopIteration or subclass when finished func Next(self Object) (obj Object, err error) { diff --git a/py/set.go b/py/set.go index cc27a464..718e286c 100644 --- a/py/set.go +++ b/py/set.go @@ -8,6 +8,8 @@ package py +import "bytes" + var SetType = NewTypeX("set", "set() -> new empty set object\nset(iterable) -> new set object\n\nBuild an unordered collection of unique elements.", SetNew, nil) type SetValue struct{} @@ -44,6 +46,17 @@ func NewSetFromItems(items []Object) *Set { return s } +func init() { + SetType.Dict["add"] = MustNewMethod("add", func(self Object, args Tuple) (Object, error) { + setSelf := self.(*Set) + if len(args) != 1 { + return nil, ExceptionNewf(TypeError, "append() takes exactly one argument (%d given)", len(args)) + } + setSelf.Add(args[0]) + return NoneType{}, nil + }, 0, "add(value)") +} + // Add an item to the set func (s *Set) Add(item Object) { s.items[item] = SetValue{} @@ -56,11 +69,10 @@ func SetNew(metatype *Type, args Tuple, kwargs StringDict) (Object, error) { if err != nil { return nil, err } - if iterable == nil { - return NewSet(), nil + if iterable != nil { + return SequenceSet(iterable) } - // FIXME should be able to initialise from an iterable! - return NewSetFromItems(iterable.(Tuple)), nil + return NewSet(), nil } var FrozenSetType = NewType("frozenset", "frozenset() -> empty frozenset object\nfrozenset(iterable) -> frozenset object\n\nBuild an immutable unordered collection of unique elements.") @@ -103,6 +115,25 @@ func (s *Set) M__bool__() (Object, error) { return NewBool(len(s.items) > 0), nil } +func (s *Set) M__repr__() (Object, error) { + var out bytes.Buffer + out.WriteRune('{') + spacer := false + for item := range s.items { + if spacer { + out.WriteString(", ") + } + str, err := ReprAsString(item) + if err != nil { + return nil, err + } + out.WriteString(str) + spacer = true + } + out.WriteRune('}') + return String(out.String()), nil +} + func (s *Set) M__iter__() (Object, error) { items := make(Tuple, 0, len(s.items)) for item := range s.items { @@ -111,6 +142,74 @@ func (s *Set) M__iter__() (Object, error) { return NewIterator(items), nil } +func (s *Set) M__and__(other Object) (Object, error) { + ret := NewSet() + b, ok := other.(*Set) + if !ok { + return nil, ExceptionNewf(TypeError, "unsupported operand type(s) for &: '%s' and '%s'", s.Type().Name, other.Type().Name) + } + for i := range b.items { + if _, ok := s.items[i]; ok { + ret.items[i] = SetValue{} + } + } + return ret, nil +} + +func (s *Set) M__or__(other Object) (Object, error) { + ret := NewSet() + b, ok := other.(*Set) + if !ok { + return nil, ExceptionNewf(TypeError, "unsupported operand type(s) for &: '%s' and '%s'", s.Type().Name, other.Type().Name) + } + for j := range s.items { + ret.items[j] = SetValue{} + } + for i := range b.items { + if _, ok := s.items[i]; !ok { + ret.items[i] = SetValue{} + } + } + return ret, nil +} + +func (s *Set) M__sub__(other Object) (Object, error) { + ret := NewSet() + b, ok := other.(*Set) + if !ok { + return nil, ExceptionNewf(TypeError, "unsupported operand type(s) for &: '%s' and '%s'", s.Type().Name, other.Type().Name) + } + for j := range s.items { + ret.items[j] = SetValue{} + } + for i := range b.items { + if _, ok := s.items[i]; ok { + delete(ret.items, i) + } + } + return ret, nil +} + +func (s *Set) M__xor__(other Object) (Object, error) { + ret := NewSet() + b, ok := other.(*Set) + if !ok { + return nil, ExceptionNewf(TypeError, "unsupported operand type(s) for &: '%s' and '%s'", s.Type().Name, other.Type().Name) + } + for j := range s.items { + ret.items[j] = SetValue{} + } + for i := range b.items { + _, ok := s.items[i] + if ok { + delete(ret.items, i) + } else { + ret.items[i] = SetValue{} + } + } + return ret, nil +} + // Check interface is satisfied var _ I__len__ = (*Set)(nil) var _ I__bool__ = (*Set)(nil) @@ -148,6 +247,9 @@ func (a *Set) M__ne__(other Object) (Object, error) { if err != nil { return nil, err } + if eq == NotImplemented { + return eq, nil + } if eq == True { return False, nil } diff --git a/py/slice.go b/py/slice.go index 53b21493..be5594d1 100644 --- a/py/slice.go +++ b/py/slice.go @@ -13,11 +13,11 @@ type Slice struct { Step Object } -var SliceType = NewType("slice", `slice(stop) -> slice object +var SliceType = NewTypeX("slice", `slice(stop) -> slice object "slice(stop) slice(start, stop[, step]) -Create a slice object. This is used for extended slicing (e.g. a[0:10:2]).`) +Create a slice object. This is used for extended slicing (e.g. a[0:10:2]).`, SliceNew, nil) // Type of this object func (o *Slice) Type() *Type { @@ -151,4 +151,50 @@ func (r *Slice) GetIndices(length int) (start, stop, step, slicelength int, err return } +func (a *Slice) M__eq__(other Object) (Object, error) { + b, ok := other.(*Slice) + if !ok { + return NotImplemented, nil + } + + if a.Start != b.Start { + return False, nil + } + + if a.Stop != b.Stop { + return False, nil + } + + if a.Step != b.Step { + return False, nil + } + + return True, nil +} + +func (a *Slice) M__ne__(other Object) (Object, error) { + return notEq(a.M__eq__(other)) +} + +func init() { + SliceType.Dict["start"] = &Property{ + Fget: func(self Object) (Object, error) { + selfSlice := self.(*Slice) + return selfSlice.Start, nil + }, + } + SliceType.Dict["stop"] = &Property{ + Fget: func(self Object) (Object, error) { + selfSlice := self.(*Slice) + return selfSlice.Stop, nil + }, + } + SliceType.Dict["step"] = &Property{ + Fget: func(self Object) (Object, error) { + selfSlice := self.(*Slice) + return selfSlice.Step, nil + }, + } +} + // Check interface is satisfied diff --git a/py/staticmethod.go b/py/staticmethod.go index 939c8cff..0ca07438 100644 --- a/py/staticmethod.go +++ b/py/staticmethod.go @@ -41,7 +41,9 @@ func (c *StaticMethod) GetDict() StringDict { // StaticMethodNew func StaticMethodNew(metatype *Type, args Tuple, kwargs StringDict) (res Object, err error) { - c := &StaticMethod{} + c := &StaticMethod{ + Dict: make(StringDict), + } err = UnpackTuple(args, kwargs, "staticmethod", 1, 1, &c.Callable) if err != nil { return nil, err diff --git a/py/string.go b/py/string.go index 9050f114..e470c01d 100644 --- a/py/string.go +++ b/py/string.go @@ -17,6 +17,7 @@ import ( "fmt" "strconv" "strings" + "unicode" "unicode/utf8" ) @@ -34,47 +35,17 @@ or repr(object). encoding defaults to sys.getdefaultencoding(). errors defaults to 'strict'.`, StrNew, nil) -// Type of this object -func (s String) Type() *Type { - return StringType -} - -// StrNew -func StrNew(metatype *Type, args Tuple, kwargs StringDict) (Object, error) { - var ( - sObj Object = String("") - encoding Object - errors Object - ) - // FIXME ignoring encoding and errors - err := ParseTupleAndKeywords(args, kwargs, "|OOO:str", []string{"bytes_or_buffer", "encoding", "errors"}, &sObj, &encoding, &errors) - if err != nil { - return nil, err - } - // FIXME ignoring encoding - // FIXME ignoring buffer protocol - return Str(sObj) -} - -// Intern s possibly returning a reference to an already interned string -func (s String) Intern() String { - // fmt.Printf("FIXME interning %q\n", s) - return s -} - -func (a String) M__str__() (Object, error) { - return a, nil -} - -func (a String) M__repr__() (Object, error) { - // FIXME combine this with parser/stringescape.go into file in py? +// Escape the py.String +func StringEscape(a String, ascii bool) string { s := string(a) var out bytes.Buffer quote := '\'' if strings.ContainsRune(s, '\'') && !strings.ContainsRune(s, '"') { quote = '"' } - out.WriteRune(quote) + if !ascii { + out.WriteRune(quote) + } for _, c := range s { switch { case c < 0x20: @@ -88,33 +59,210 @@ func (a String) M__repr__() (Object, error) { default: fmt.Fprintf(&out, `\x%02x`, c) } - case c < 0x7F: + case !ascii && c < 0x7F: if c == '\\' || (quote == '\'' && c == '\'') || (quote == '"' && c == '"') { out.WriteRune('\\') } out.WriteRune(c) case c < 0x100: - if strconv.IsPrint(c) { + if ascii || strconv.IsPrint(c) { out.WriteRune(c) } else { fmt.Fprintf(&out, "\\x%02x", c) } case c < 0x10000: - if strconv.IsPrint(c) { + if !ascii && strconv.IsPrint(c) { out.WriteRune(c) } else { fmt.Fprintf(&out, "\\u%04x", c) } default: - if strconv.IsPrint(c) { + if !ascii && strconv.IsPrint(c) { out.WriteRune(c) } else { fmt.Fprintf(&out, "\\U%08x", c) } } } - out.WriteRune(quote) - return String(out.String()), nil + if !ascii { + out.WriteRune(quote) + } + return out.String() +} + +// standard golang strings.Fields doesn't have a 'first N' argument +func fieldsN(s string, n int) []string { + out := []string{} + cur := []rune{} + for _, c := range s { + //until we have covered the first N elements, multiple white-spaces are 'merged' + if n < 0 || len(out) < n { + if unicode.IsSpace(c) { + if len(cur) > 0 { + out = append(out, string(cur)) + cur = []rune{} + } + } else { + cur = append(cur, c) + } + //until we see the next letter, after collecting the first N fields, continue to merge whitespaces + } else if len(out) == n && len(cur) == 0 { + if !unicode.IsSpace(c) { + cur = append(cur, c) + } + //now that enough words have been collected, just copy into the last element + } else { + cur = append(cur, c) + } + } + if len(cur) > 0 { + out = append(out, string(cur)) + } + return out +} + +func init() { + StringType.Dict["endswith"] = MustNewMethod("endswith", func(self Object, args Tuple) (Object, error) { + selfStr := string(self.(String)) + suffix := []string{} + if len(args) > 0 { + if s, ok := args[0].(String); ok { + suffix = append(suffix, string(s)) + } else if s, ok := args[0].(Tuple); ok { + for _, t := range s { + if v, ok := t.(String); ok { + suffix = append(suffix, string(v)) + } + } + } else { + return nil, ExceptionNewf(TypeError, "endswith first arg must be str, unicode, or tuple, not %s", args[0].Type()) + } + } else { + return nil, ExceptionNewf(TypeError, "endswith() takes at least 1 argument (0 given)") + } + for _, s := range suffix { + if strings.HasSuffix(selfStr, s) { + return Bool(true), nil + } + } + return Bool(false), nil + }, 0, "endswith(suffix[, start[, end]]) -> bool") + + StringType.Dict["find"] = MustNewMethod("find", func(self Object, args Tuple) (Object, error) { + return self.(String).find(args) + }, 0, `find(...) +S.find(sub[, start[, end]]) -> int + +Return the lowest index in S where substring sub is found, +such that sub is contained within S[start:end]. Optional +arguments start and end are interpreted as in slice notation. + +Return -1 on failure.`) + + StringType.Dict["replace"] = MustNewMethod("replace", func(self Object, args Tuple) (Object, error) { + return self.(String).Replace(args) + }, 0, `replace(self, old, new, count=-1) -> return a copy with all occurrences of substring old replaced by new. + + count + Maximum number of occurrences to replace. + -1 (the default value) means replace all occurrences. + +If the optional argument count is given, only the first count occurrences are +replaced.`) + + StringType.Dict["split"] = MustNewMethod("split", func(self Object, args Tuple, kwargs StringDict) (Object, error) { + return self.(String).Split(args, kwargs) + }, 0, "split(sub) -> split string with sub.") + + StringType.Dict["startswith"] = MustNewMethod("startswith", func(self Object, args Tuple) (Object, error) { + selfStr := string(self.(String)) + prefix := []string{} + if len(args) > 0 { + if s, ok := args[0].(String); ok { + prefix = append(prefix, string(s)) + } else if s, ok := args[0].(Tuple); ok { + for _, t := range s { + if v, ok := t.(String); ok { + prefix = append(prefix, string(v)) + } + } + } else { + return nil, ExceptionNewf(TypeError, "startswith first arg must be str, unicode, or tuple, not %s", args[0].Type()) + } + } else { + return nil, ExceptionNewf(TypeError, "startswith() takes at least 1 argument (0 given)") + } + if len(args) > 1 { + if s, ok := args[1].(Int); ok { + selfStr = selfStr[s:] + } + } + + for _, s := range prefix { + if strings.HasPrefix(selfStr, s) { + return Bool(true), nil + } + } + return Bool(false), nil + }, 0, "startswith(prefix[, start[, end]]) -> bool") + + StringType.Dict["strip"] = MustNewMethod("strip", func(self Object, args Tuple, kwargs StringDict) (Object, error) { + return self.(String).Strip(args) + }, 0, "strip(chars) -> replace chars from begining and end of string") + + StringType.Dict["rstrip"] = MustNewMethod("rstrip", func(self Object, args Tuple, kwargs StringDict) (Object, error) { + return self.(String).RStrip(args) + }, 0, "rstrip(chars) -> replace chars from end of string") + + StringType.Dict["lstrip"] = MustNewMethod("lstrip", func(self Object, args Tuple, kwargs StringDict) (Object, error) { + return self.(String).LStrip(args) + }, 0, "lstrip(chars) -> replace chars from begining of string") + + StringType.Dict["upper"] = MustNewMethod("upper", func(self Object, args Tuple, kwargs StringDict) (Object, error) { + return self.(String).Upper() + }, 0, "upper() -> a copy of the string converted to uppercase") + + StringType.Dict["lower"] = MustNewMethod("lower", func(self Object, args Tuple, kwargs StringDict) (Object, error) { + return self.(String).Lower() + }, 0, "lower() -> a copy of the string converted to lowercase") + +} + +// Type of this object +func (s String) Type() *Type { + return StringType +} + +// StrNew +func StrNew(metatype *Type, args Tuple, kwargs StringDict) (Object, error) { + var ( + sObj Object = String("") + encoding Object + errors Object + ) + // FIXME ignoring encoding and errors + err := ParseTupleAndKeywords(args, kwargs, "|OOO:str", []string{"bytes_or_buffer", "encoding", "errors"}, &sObj, &encoding, &errors) + if err != nil { + return nil, err + } + // FIXME ignoring encoding + // FIXME ignoring buffer protocol + return Str(sObj) +} + +// Intern s possibly returning a reference to an already interned string +func (s String) Intern() String { + // fmt.Printf("FIXME interning %q\n", s) + return s +} + +func (a String) M__str__() (Object, error) { + return a, nil +} + +func (a String) M__repr__() (Object, error) { + out := StringEscape(a, false) + return String(out), nil } func (s String) M__bool__() (Object, error) { @@ -172,7 +320,7 @@ func (a String) M__imul__(other Object) (Object, error) { // Convert an Object to an String // -// Retrurns ok as to whether the conversion worked or not +// Returns ok as to whether the conversion worked or not func convertToString(other Object) (String, bool) { switch b := other.(type) { case String: @@ -228,7 +376,6 @@ func (a String) M__ge__(other Object) (Object, error) { // % operator /* - 4.7.2. printf-style String Formatting Note The formatting operations described here exhibit a variety of @@ -462,13 +609,161 @@ func (s String) M__contains__(item Object) (Object, error) { return NewBool(strings.Contains(string(s), string(needle))), nil } +func (s String) find(args Tuple) (Object, error) { + var ( + pysub Object + pybeg Object = Int(0) + pyend Object = Int(s.len()) + pyfmt = "s|ii:find" + ) + err := ParseTuple(args, pyfmt, &pysub, &pybeg, &pyend) + if err != nil { + return nil, err + } + + var ( + beg = int(pybeg.(Int)) + end = int(pyend.(Int)) + size = s.len() + ) + if beg > size { + beg = size + } + if end < 0 { + end = size + } + if end > size { + end = size + } + + var ( + off = s.slice(0, beg, s.len()).len() + str = string(s.slice(beg, end, s.len())) + sub = string(pysub.(String)) + idx = strings.Index(str, sub) + ) + if idx < 0 { + return Int(idx), nil + } + return Int(off + String(str[:idx]).len()), nil +} + +func (s String) Split(args Tuple, kwargs StringDict) (Object, error) { + var ( + pyval Object = None + pymax Object = Int(-2) + pyfmt = "|Oi:split" + kwlst = []string{"sep", "maxsplit"} + ) + err := ParseTupleAndKeywords(args, kwargs, pyfmt, kwlst, &pyval, &pymax) + if err != nil { + return nil, err + } + + var ( + max = pymax.(Int) + vs []string + ) + switch v := pyval.(type) { + case String: + vs = strings.SplitN(string(s), string(v), int(max)+1) + case NoneType: + vs = fieldsN(string(s), int(max)) + default: + return nil, ExceptionNewf(TypeError, "Can't convert '%s' object to str implicitly", pyval.Type()) + } + o := List{} + for _, j := range vs { + o.Items = append(o.Items, String(j)) + } + return &o, nil +} + +func (s String) Replace(args Tuple) (Object, error) { + var ( + pyold Object = None + pynew Object = None + pycnt Object = Int(-1) + ) + err := ParseTuple(args, "ss|i:replace", &pyold, &pynew, &pycnt) + if err != nil { + return nil, err + } + + var ( + old = string(pyold.(String)) + new = string(pynew.(String)) + cnt = int(pycnt.(Int)) + ) + + return String(strings.Replace(string(s), old, new, cnt)), nil +} + +func stripFunc(args Tuple) (func(rune) bool, error) { + var ( + pyval Object = None + ) + err := ParseTuple(args, "|s", &pyval) + if err != nil { + return nil, err + } + f := unicode.IsSpace + switch v := pyval.(type) { + case String: + chars := []rune(string(v)) + f = func(s rune) bool { + for _, i := range chars { + if s == i { + return true + } + } + return false + } + } + return f, nil +} + +func (s String) Strip(args Tuple) (Object, error) { + f, err := stripFunc(args) + if err != nil { + return nil, err + } + return String(strings.TrimFunc(string(s), f)), nil +} + +func (s String) LStrip(args Tuple) (Object, error) { + f, err := stripFunc(args) + if err != nil { + return nil, err + } + return String(strings.TrimLeftFunc(string(s), f)), nil +} + +func (s String) RStrip(args Tuple) (Object, error) { + f, err := stripFunc(args) + if err != nil { + return nil, err + } + return String(strings.TrimRightFunc(string(s), f)), nil +} + +func (s String) Upper() (Object, error) { + return String(strings.ToUpper(string(s))), nil +} + +func (s String) Lower() (Object, error) { + return String(strings.ToLower(string(s))), nil +} + // Check stringerface is satisfied -var _ richComparison = String("") -var _ sequenceArithmetic = String("") -var _ I__mod__ = String("") -var _ I__rmod__ = String("") -var _ I__imod__ = String("") -var _ I__len__ = String("") -var _ I__bool__ = String("") -var _ I__getitem__ = String("") -var _ I__contains__ = String("") +var ( + _ richComparison = String("") + _ sequenceArithmetic = String("") + _ I__mod__ = String("") + _ I__rmod__ = String("") + _ I__imod__ = String("") + _ I__len__ = String("") + _ I__bool__ = String("") + _ I__getitem__ = String("") + _ I__contains__ = String("") +) diff --git a/py/string_test.go b/py/string_test.go new file mode 100644 index 00000000..053cd781 --- /dev/null +++ b/py/string_test.go @@ -0,0 +1,146 @@ +// Copyright 2022 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package py + +import "testing" + +func TestStringFind(t *testing.T) { + for _, tc := range []struct { + str string + sub string + beg int + end int + idx int + }{ + { + str: "hello world", + sub: "world", + idx: 6, + }, + { + str: "hello world", + sub: "o", + idx: 4, + }, + { + str: "hello world", + sub: "o", + beg: 5, + idx: 7, + }, + { + str: "hello world", + sub: "bye", + idx: -1, + }, + { + str: "Hello, 世界", + sub: "界", + idx: 8, + }, + { + str: "01234 6789", + sub: " ", + beg: 6, + idx: -1, + }, + { + str: "0123456789", + sub: "6", + beg: 1, + end: 6, + idx: -1, + }, + { + str: "0123456789", + sub: "6", + beg: 1, + end: 7, + idx: 6, + }, + { + str: "0123456789", + sub: "6", + beg: 1, + end: -1, + idx: 6, + }, + { + str: "0123456789", + sub: "6", + beg: 100, + end: -1, + idx: -1, + }, + { + str: "0123456789", + sub: "6", + beg: 2, + end: 1, + idx: -1, + }, + } { + t.Run(tc.str+":"+tc.sub, func(t *testing.T) { + beg := tc.beg + end := tc.end + if end == 0 { + end = len(tc.str) + } + idx, err := String(tc.str).find(Tuple{String(tc.sub), Int(beg), Int(end)}) + if err != nil { + t.Fatalf("invalid: %+v", err) + } + if got, want := int(idx.(Int)), tc.idx; got != want { + t.Fatalf("got=%d, want=%d", got, want) + } + }) + } +} + +func TestStringUpper(t *testing.T) { + tests := []struct { + name string + s String + want Object + }{{ + name: "abc", + s: String("abc"), + want: String("ABC")}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.s.Upper() + if err != nil { + t.Fatalf("Upper() error = %v", err) + } + if got.(String) != tt.want.(String) { + t.Fatalf("Upper() got = %v, want %v", got, tt.want) + } + }) + } +} + +func TestStringLower(t *testing.T) { + tests := []struct { + name string + s String + want Object + }{{ + name: "ABC", + s: String("ABC"), + want: String("abc")}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.s.Lower() + if err != nil { + t.Fatalf("Lower() error = %v", err) + } + if got.(String) != tt.want.(String) { + t.Fatalf("Lower() got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/py/tests/classmethod.py b/py/tests/classmethod.py new file mode 100644 index 00000000..db4a5ca6 --- /dev/null +++ b/py/tests/classmethod.py @@ -0,0 +1,19 @@ +# Copyright 2019 The go-python Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +doc="classmethod" + +class A: + @classmethod + def fn(cls, p): + assert cls is A + return p+1 + +a = A() +assert a.fn(1) == 2 + +a.x = 3 +assert a.x == 3 + +doc="finished" diff --git a/py/tests/dict.py b/py/tests/dict.py index 3581ba53..274f6c69 100644 --- a/py/tests/dict.py +++ b/py/tests/dict.py @@ -1,6 +1,7 @@ # Copyright 2018 The go-python Authors. All rights reserved. # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. +from libtest import assertRaises doc="str" assert str({}) == "{}" @@ -12,4 +13,83 @@ a = repr({"a":"b","c":5.5}) assert a == "{'a': 'b', 'c': 5.5}" or a == "{'c': 5.5, 'a': 'b'}" +doc="check __iter__" +a = {"a":"b","c":5.5} +l = list(iter(a)) +assert "a" in l +assert "c" in l +assert len(l) == 2 + +doc="check get" +a = {"a":1} +assert a.get('a') == 1 +assert a.get('a',100) == 1 +assert a.get('b') == None +assert a.get('b',1) == 1 +assert a.get('b',True) == True + +doc="check keys" +a = {"a":1} +assert list(a.keys()) == ["a"] + +doc="check values" +a = {"a":1} +assert list(a.values()) == [1] + +doc="check items" +a = {"a":"b","c":5.5} +for k, v in a.items(): + assert k in ["a", "c"] + if k == "a": + assert v == "b" + if k == "c": + assert v == 5.5 +assertRaises(TypeError, a.items, 'a') + +doc="del" +a = {'hello': 'world', 'hi': 'there'} +del a["hello"] +def doDel(d, key): + del d[key] +assertRaises(KeyError, lambda: doDel(a, "bob")) +assertRaises(KeyError, lambda: doDel(a, 123)) +assert not a.__contains__('hello') +assert a.__contains__('hi') + +doc="init" +a = dict( zip( "a,b,c".split(","), "1,2,3".split(",") ) ) +assert a["a"] == "1" +assert a["b"] == "2" +assert a["c"] == "3" + +a = dict(a="1", b="2", c="3") +assert a["a"] == "1" +assert a["b"] == "2" +assert a["c"] == "3" + +assertRaises(TypeError, dict, "a") +assertRaises(TypeError, dict, 1) +assertRaises(TypeError, dict, {"a":1}, {"b":2}) + +doc="__contain__" +a = {'hello': 'world'} +assert a.__contains__('hello') +assert not a.__contains__('world') + +doc="__eq__, __ne__" +a = {'a': 'b'} +assert a.__eq__(3) != True +assert a.__ne__(3) != False +assert a.__ne__(3) != True +assert a.__ne__(3) != False + +assert a.__ne__({}) == True +assert a.__eq__({'a': 'b'}) == True +assert a.__ne__({'a': 'b'}) == False + +doc="__len__" +a = {"a": "1", "b": "2"} +assert a.__len__() == 2 +assert len(a) == 2 + doc="finished" diff --git a/py/tests/filter.py b/py/tests/filter.py new file mode 100644 index 00000000..c42b2b63 --- /dev/null +++ b/py/tests/filter.py @@ -0,0 +1,65 @@ +# test_builtin.py:BuiltinTest.test_filter() +from libtest import assertRaises + +doc="filter" +class T0: + def __bool__(self): + return True +class T1: + def __len__(self): + return 1 +class T2: + def __bool__(self): + return False +class T3: + pass +t0, t1, t2, t3 = T0(), T1(), T2(), T3() +assert list(filter(None, [t0, t1, t2, t3])) == [t0, t1, t3] +assert list(filter(None, [1, [], 2, ''])) == [1, 2] + +class T3: + def __len__(self): + raise ValueError +t3 = T3() +assertRaises(ValueError, list, filter(None, [t3])) + +class Squares: + def __init__(self, max): + self.max = max + self.sofar = [] + + def __len__(self): return len(self.sofar) + + def __getitem__(self, i): + if not 0 <= i < self.max: raise IndexError + n = len(self.sofar) + while n <= i: + self.sofar.append(n*n) + n += 1 + return self.sofar[i] + +assert list(filter(lambda c: 'a' <= c <= 'z', 'Hello World')) == list('elloorld') +assert list(filter(None, [1, 'hello', [], [3], '', None, 9, 0])) == [1, 'hello', [3], 9] +assert list(filter(lambda x: x > 0, [1, -3, 9, 0, 2])) == [1, 9, 2] +assert list(filter(None, Squares(10))) == [1, 4, 9, 16, 25, 36, 49, 64, 81] +assert list(filter(lambda x: x%2, Squares(10))) == [1, 9, 25, 49, 81] +def identity(item): + return 1 +filter(identity, Squares(5)) +assertRaises(TypeError, filter) +class BadSeq(object): + def __getitem__(self, index): + if index<4: + return 42 + raise ValueError +assertRaises(ValueError, list, filter(lambda x: x, BadSeq())) +def badfunc(): + pass +assertRaises(TypeError, list, filter(badfunc, range(5))) + +# test bltinmodule.c::filtertuple() +assert list(filter(None, (1, 2))) == [1, 2] +assert list(filter(lambda x: x>=3, (1, 2, 3, 4))) == [3, 4] +assertRaises(TypeError, list, filter(42, (1, 2))) + +doc="finished" diff --git a/py/tests/float.py b/py/tests/float.py index 8b8a84af..93dd7fcc 100644 --- a/py/tests/float.py +++ b/py/tests/float.py @@ -14,4 +14,28 @@ assert float(" -1E400") == float("-inf") assertRaises(ValueError, float, "1 E200") +doc="repr" +assert repr(float("1.0")) == "1.0" +assert repr(float("1.")) == "1.0" +assert repr(float("1.1")) == "1.1" +assert repr(float("1.11")) == "1.11" +assert repr(float("-1.0")) == "-1.0" +assert repr(float("1.00101")) == "1.00101" +assert repr(float("1.00")) == "1.0" +assert repr(float("2.010")) == "2.01" + +doc="str" +assert str(float("1.0")) == "1.0" +assert str(float("1.")) == "1.0" +assert str(float("1.1")) == "1.1" +assert str(float("1.11")) == "1.11" +assert str(float("-1.0")) == "-1.0" +assert str(float("1.00101")) == "1.00101" +assert str(float("1.00")) == "1.0" +assert str(float("2.010")) == "2.01" + +doc="is_integer" +assert (1.0).is_integer() == True +assert (2.3).is_integer() == False + doc="finished" diff --git a/py/tests/function.py b/py/tests/function.py new file mode 100644 index 00000000..ba05142d --- /dev/null +++ b/py/tests/function.py @@ -0,0 +1,130 @@ +# Copyright 2019 The go-python Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +doc="function" + +def fn(p): + "docstring" + return p+1 + +assert fn(1) == 2 + +# FIXME this doesn't work yet +#assert fn.__doc__ == "docstring" +#fn.__doc__ = "hello" +#assert fn.__doc__ == "hello" + +assert str(type(fn)) == "" + +fn.x = 3 +assert fn.x == 3 + +def f2(p): + return p+2 + +doc="check __code__" +fn.__code__ = f2.__code__ +assert fn(1) == 3 +try: + fn.__code__ = "bad" +except TypeError: + pass +else: + assert False, "TypeError not raised" + +doc="check __defaults__" +def f3(p=2): + return p +assert f3.__defaults__ == (2,) +assert f3() == 2 +f3.__defaults__ = (10,) +assert f3() == 10 +assert f3.__defaults__ == (10,) +try: + f3.__defaults__ = "bad" +except TypeError: + pass +else: + assert False, "TypeError not raised" +del f3.__defaults__ +assert f3.__defaults__ == None or f3.__defaults__ == () + +doc="check __kwdefaults__" +def f4(*, b=2): + return b +assert f4.__kwdefaults__ == {"b":2} +assert f4() == 2 +f4.__kwdefaults__ = {"b":10} +assert f4() == 10 +assert f4.__kwdefaults__ == {"b":10} +try: + f4.__kwdefaults__ = "bad" +except TypeError: + pass +else: + assert False, "TypeError not raised" +del f4.__kwdefaults__ +assert f4.__kwdefaults__ == None or f4.__kwdefaults__ == {} + +doc="check __annotations__" +def f5(a: "potato") -> "sausage": + pass +assert f5.__annotations__ == {'a': 'potato', 'return': 'sausage'} +f5.__annotations__ = {'a': 'potato', 'return': 'SAUSAGE'} +assert f5.__annotations__ == {'a': 'potato', 'return': 'SAUSAGE'} +try: + f5.__annotations__ = "bad" +except TypeError: + pass +else: + assert False, "TypeError not raised" +del f5.__annotations__ +assert f5.__annotations__ == None or f5.__annotations__ == {} + +doc="check __dict__" +def f6(): + pass +assert f6.__dict__ == {} +f6.__dict__ = {'a': 'potato'} +assert f6.__dict__ == {'a': 'potato'} +try: + f6.__dict__ = "bad" +except TypeError: + pass +else: + assert False, "TypeError not raised" +try: + del f6.__dict__ +except (TypeError, AttributeError): + pass +else: + assert False, "Error not raised" + +doc="check __name__" +def f7(): + pass +assert f7.__name__ == "f7" +f7.__name__ = "new_name" +assert f7.__name__ == "new_name" +try: + f7.__name__ = 1 +except TypeError: + pass +else: + assert False, "TypeError not raised" + +doc="check __qualname__" +def f8(): + pass +assert f8.__qualname__ == "f8" +f8.__qualname__ = "new_qualname" +assert f8.__qualname__ == "new_qualname" +try: + f8.__qualname__ = 1 +except TypeError: + pass +else: + assert False, "TypeError not raised" + +doc="finished" diff --git a/py/tests/int.py b/py/tests/int.py index 758829ab..79356b3c 100644 --- a/py/tests/int.py +++ b/py/tests/int.py @@ -103,6 +103,7 @@ doc="sigils" assert int("7") == 7 +assert int("07") == 7 assert int("07", 10) == 7 assert int("F", 16) == 15 diff --git a/py/tests/iter.py b/py/tests/iter.py new file mode 100644 index 00000000..4eda19c9 --- /dev/null +++ b/py/tests/iter.py @@ -0,0 +1,33 @@ +# Copyright 2019 The go-python Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +doc="iter" +cnt = 0 +def f(): + global cnt + cnt += 1 + return cnt + +l = list(iter(f,20)) +assert len(l) == 19 +for idx, v in enumerate(l): + assert idx + 1 == v + +words1 = ['g', 'p', 'y', 't', 'h', 'o', 'n'] +words2 = list(iter(words1)) +for w1, w2 in zip(words1, words2): + assert w1 == w2 + +class SequenceClass: + def __init__(self, n): + self.n = n + def __getitem__(self, i): + if 0 <= i < self.n: + return i + else: + raise IndexError + +assert list(iter(SequenceClass(5))) == [0, 1, 2, 3, 4] + +doc="finished" \ No newline at end of file diff --git a/py/tests/list.py b/py/tests/list.py index 96500f34..b832b3df 100644 --- a/py/tests/list.py +++ b/py/tests/list.py @@ -1,17 +1,146 @@ -# Copyright 2018 The go-python Authors. All rights reserved. +# Copyright 2019 The go-python Authors. All rights reserved. # Use of this source code is governed by a BSD-style # license that can be found in the LICENSE file. +from libtest import assertRaises + doc="str" assert str([]) == "[]" assert str([1,2,3]) == "[1, 2, 3]" assert str([1,[2,3],4]) == "[1, [2, 3], 4]" assert str(["1",[2.5,17,[]]]) == "['1', [2.5, 17, []]]" +assert str([1, 1.0]) == "[1, 1.0]" doc="repr" assert repr([]) == "[]" assert repr([1,2,3]) == "[1, 2, 3]" assert repr([1,[2,3],4]) == "[1, [2, 3], 4]" assert repr(["1",[2.5,17,[]]]) == "['1', [2.5, 17, []]]" +assert repr([1, 1.0]) == "[1, 1.0]" + +doc="enumerate" +a = [e for e in enumerate([3,4,5,6,7], 4)] +idxs = [4, 5, 6, 7, 8] +values = [3, 4, 5, 6, 7] +for idx, value in enumerate(values): + assert idxs[idx] == a[idx][0] + assert values[idx] == a[idx][1] + +doc="append" +a = [1,2,3] +a.append(4) +assert repr(a) == "[1, 2, 3, 4]" +a = ['a', 'b', 'c'] +a.extend(['d', 'e', 'f']) +assert repr(a) == "['a', 'b', 'c', 'd', 'e', 'f']" +assertRaises(TypeError, lambda: [].append()) + +doc="mul" +a = [1, 2, 3] +assert a * 2 == [1, 2, 3, 1, 2, 3] +assert a * 0 == [] +assert a * -1 == [] + +doc="sort" +# [].sort +a = [3, 1.1, 1, 2] +s1 = list(a) +s1.sort() +assert s1 == [1, 1.1, 2, 3] +s1.sort() # sort a sorted list +assert s1 == [1, 1.1, 2, 3] +s2 = list(a) +s2.sort(reverse=True) +assert s2 == [3, 2, 1.1, 1] +s2.sort() # sort a reversed list +assert s2 == [1, 1.1, 2, 3] +s3 = list(a) +s3.sort(key=lambda l: l+1) # test lambda key +assert s3 == [1, 1.1, 2, 3] +s4 = [2.0, 2, 1, 1.0] +s4.sort(key=lambda l: 0) # test stability +assert s4 == [2.0, 2, 1, 1.0] +assert [type(t) for t in s4] == [float, int, int, float] +s4 = [2.0, 2, 1, 1.0] +s4.sort() # test stability +assert s4 == [1, 1.0, 2.0, 2] +assert [type(t) for t in s4] == [int, float, float, int] +s5 = [2.0, "abc"] +assertRaises(TypeError, lambda: s5.sort()) +s5 = [] +s5.sort() +assert s5 == [] +s5 = [0] +s5.sort() +assert s5 == [0] +s5 = [0, 1] +# Sorting a list of len >= 2 with uncallable key must fail on all Python implementations. +assertRaises(TypeError, lambda: s5.sort(key=1)) + +# list.sort([]) +a = [3, 1.1, 1, 2] +s1 = list(a) +assert list.sort(s1) is None +assert s1 == [1, 1.1, 2, 3] +assert list.sort(s1) is None # sort a sorted list +assert s1 == [1, 1.1, 2, 3] +s2 = list(a) +list.sort(s2, reverse=True) +assert s2 == [3, 2, 1.1, 1] +list.sort(s2) # sort a reversed list +assert s2 == [1, 1.1, 2, 3] +s3 = list(a) +list.sort(s3, key=lambda l: l+1) # test lambda key +assert s3 == [1, 1.1, 2, 3] +s4 = [2.0, 2, 1, 1.0] +list.sort(s4, key=lambda l: 0) # test stability +assert s4 == [2.0, 2, 1, 1.0] +assert [type(t) for t in s4] == [float, int, int, float] +s4 = [2.0, 2, 1, 1.0] +list.sort(s4) # test stability +assert s4 == [1, 1.0, 2.0, 2] +assert [type(t) for t in s4] == [int, float, float, int] +s5 = [2.0, "abc"] +assertRaises(TypeError, lambda: list.sort(s5)) +s5 = [] +list.sort(s5) +assert s5 == [] +s5 = [0] +list.sort(s5) +assert s5 == [0] +s5 = [0, 1] +# Sorting a list of len >= 2 with uncallable key must fail on all Python implementations. +assertRaises(TypeError, lambda: list.sort(s5, key=1)) +assertRaises(TypeError, lambda: list.sort(1)) + +class Index: + def __index__(self): + return 1 + +a = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] +b = Index() +assert a[b] == 1 +assert a[b:10] == a[1:10] +assert a[10:b:-1] == a[10:1:-1] + +class NonIntegerIndex: + def __index__(self): + return 1.1 + +a = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] +b = NonIntegerIndex() +try: + a[b] +except TypeError: + pass +else: + assert False, "TypeError not raised" + +try: + a[b:10] +except TypeError: + pass +else: + assert False, "TypeError not raised" doc="finished" diff --git a/py/tests/map.py b/py/tests/map.py new file mode 100644 index 00000000..3e5a4e1e --- /dev/null +++ b/py/tests/map.py @@ -0,0 +1,54 @@ +# test_builtin.py:BuiltinTest.test_map() +from libtest import assertRaises + +doc="map" +class Squares: + def __init__(self, max): + self.max = max + self.sofar = [] + + def __len__(self): return len(self.sofar) + + def __getitem__(self, i): + if not 0 <= i < self.max: raise IndexError + n = len(self.sofar) + while n <= i: + self.sofar.append(n*n) + n += 1 + return self.sofar[i] + +assert list(map(lambda x: x*x, range(1,4))) == [1, 4, 9] +try: + from math import sqrt +except ImportError: + def sqrt(x): + return pow(x, 0.5) +assert list(map(lambda x: list(map(sqrt, x)), [[16, 4], [81, 9]])) == [[4.0, 2.0], [9.0, 3.0]] +assert list(map(lambda x, y: x+y, [1,3,2], [9,1,4])) == [10, 4, 6] + +def plus(*v): + accu = 0 + for i in v: accu = accu + i + return accu +assert list(map(plus, [1, 3, 7])) == [1, 3, 7] +assert list(map(plus, [1, 3, 7], [4, 9, 2])) == [1+4, 3+9, 7+2] +assert list(map(plus, [1, 3, 7], [4, 9, 2], [1, 1, 0])) == [1+4+1, 3+9+1, 7+2+0] +assert list(map(int, Squares(10))) == [0, 1, 4, 9, 16, 25, 36, 49, 64, 81] +def Max(a, b): + if a is None: + return b + if b is None: + return a + return max(a, b) +assert list(map(Max, Squares(3), Squares(2))) == [0, 1] +assertRaises(TypeError, map) +assertRaises(TypeError, map, lambda x: x, 42) +class BadSeq: + def __iter__(self): + raise ValueError + yield None +assertRaises(ValueError, list, map(lambda x: x, BadSeq())) +def badfunc(x): + raise RuntimeError +assertRaises(RuntimeError, list, map(badfunc, range(5))) +doc="finished" diff --git a/py/tests/range.py b/py/tests/range.py new file mode 100644 index 00000000..c027b4fe --- /dev/null +++ b/py/tests/range.py @@ -0,0 +1,139 @@ +# Copyright 2018 The go-python Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +doc="range" +a = range(255) +b = [e for e in a] +assert len(a) == len(b) +a = range(5, 100, 5) +b = [e for e in a] +assert len(a) == len(b) +a = range(100 ,0, 1) +b = [e for e in a] +assert len(a) == len(b) + +a = range(100, 0, -1) +b = [e for e in a] +assert len(a) == 100 +assert len(b) == 100 + +doc="range_get_item" +a = range(3) +assert a[2] == 2 +assert a[1] == 1 +assert a[0] == 0 +assert a[-1] == 2 +assert a[-2] == 1 +assert a[-3] == 0 + +b = range(0, 10, 2) +assert b[4] == 8 +assert b[3] == 6 +assert b[2] == 4 +assert b[1] == 2 +assert b[0] == 0 +assert b[-4] == 2 +assert b[-3] == 4 +assert b[-2] == 6 +assert b[-1] == 8 + +doc="range_eq" +assert range(10) == range(0, 10) +assert not range(10) == 3 +assert range(20) != range(10) +assert range(100, 200, 1) == range(100, 200) +assert range(0, 10, 3) == range(0, 12, 3) +assert range(2000, 100) == range(3, 1) +assert range(0, 10, -3) == range(0, 12, -3) +assert not range(0, 20, 2) == range(0, 20, 4) +try: + range('3', 10) == range(2) +except TypeError: + pass +else: + assert False, "TypeError not raised" + +doc="range_ne" +assert range(10, 0, -3) != range(12, 0, -3) +assert range(10) != 3 +assert not range(100, 200, 1) != range(100, 200) +assert range(0, 10) != range(0, 12) +assert range(0, 10) != range(0, 10, 2) +assert range(0, 20, 2) != range(0, 21, 2) +assert range(0, 20, 2) != range(0, 20, 4) +assert not range(0, 20, 3) != range(0, 20, 3) +try: + range('3', 10) != range(2) +except TypeError: + pass +else: + assert False, "TypeError not raised" + +doc="range_str" +assert str(range(10)) == 'range(0, 10)' +assert str(range(10, 0, 3)) == 'range(10, 0, 3)' +assert str(range(0, 3)) == 'range(0, 3)' +assert str(range(10, 3, -2)) == 'range(10, 3, -2)' + +doc="range_slice" +a = range(10) +assert a[::-1][0] == 9 +assert a[::-1][9] == 0 +assert a[0:3][0] == 0 +assert a[0:3][2] == 2 +assert a[-3:10][0] == 7 +assert a[-100:13][0] == 0 +assert a[-100:13][9] == 9 + +try: + a[0:3][3] +except IndexError: + pass +else: + assert False, "IndexError not raised" +try: + a[100:13][0] +except IndexError: + pass +else: + assert False, "IndexError not raised" +try: + a[0:3:0] +except ValueError: + pass +else: + assert False, "ValueError not raised" + +doc="range_index" +class Index: + def __index__(self): + return 1 + +a = range(10) +b = Index() +assert a[b] == 1 +assert a[b:10] == a[1:10] +assert a[10:b:-1] == a[10:1:-1] + +class NonIntegerIndex: + def __index__(self): + return 1.1 + +a = range(10) +b = NonIntegerIndex() +try: + a[b] +except TypeError: + pass +else: + assert False, "TypeError not raised" + +try: + a[b:10] +except TypeError: + pass +else: + assert False, "TypeError not raised" + +doc="finished" diff --git a/py/tests/set.py b/py/tests/set.py new file mode 100644 index 00000000..3eeaf1d3 --- /dev/null +++ b/py/tests/set.py @@ -0,0 +1,109 @@ +# Copyright 2019 The go-python Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +from libtest import assertRaises + +doc="__and__" +a = {1, 2, 3} +b = {2, 3, 4, 5} +c = a.__and__(b) +assert 2 in c +assert 3 in c + +d = a & b +assert 2 in d +assert 3 in d + +doc="__or__" +a = {1, 2, 3} +b = {2, 3, 4, 5} +c = a.__or__(b) +assert 1 in c +assert 2 in c +assert 3 in c +assert 4 in c +assert 5 in c + +d = a | b +assert 1 in c +assert 2 in c +assert 3 in c +assert 4 in c +assert 5 in c + +doc="__sub__" +a = {1, 2, 3} +b = {2, 3, 4, 5} +c = a.__sub__(b) +d = b.__sub__(a) +assert 1 in c +assert 4 in d +assert 5 in d + +e = a - b +f = b - a +assert 1 in c +assert 4 in d +assert 5 in d + +doc="__xor__" +a = {1, 2, 3} +b = {2, 3, 4, 5} +c = a.__xor__(b) +assert 1 in c +assert 4 in c +assert 5 in c + +d = a ^ b +assert 1 in c + +doc="__repr__" +a = {1, 2, 3} +b = a.__repr__() +assert "{" in b +assert "1" in b +assert "2" in b +assert "3" in b +assert "}" in b + +doc="set" +a = set([1,2,3]) +b = set("set") +c = set((4,5)) +assert len(a) == 3 +assert len(b) == 3 +assert len(c) == 2 +assert 1 in a +assert 2 in a +assert 3 in a +assert "s" in b +assert "e" in b +assert "t" in b +assert 4 in c +assert 5 in c + +doc="add" +a = set() +a.add(1) +a.add(2) +a.add(3) +assert len(a) == 3 +assert 1 in a +assert 2 in a +assert 3 in a +assert 4 not in a +assertRaises(TypeError, lambda: a.add()) + +doc="__eq__, __ne__" +a = set([1,2,3]) +assert a.__eq__(3) != True +assert a.__ne__(3) != False +assert a.__ne__(3) != True +assert a.__ne__(3) != False # This part should be changed in comparison with NotImplemented + +assert a.__ne__(set()) == True +assert a.__eq__({1,2,3}) == True +assert a.__ne__({1,2,3}) == False + +doc="finished" diff --git a/py/tests/slice.py b/py/tests/slice.py new file mode 100644 index 00000000..24b31677 --- /dev/null +++ b/py/tests/slice.py @@ -0,0 +1,27 @@ +# Copyright 2019 The go-python Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +doc="slice" +a = slice(10) +assert a.start == None +assert a.stop == 10 +assert a.step == None + +a = slice(0, 10, 1) +assert a.start == 0 +assert a.stop == 10 +assert a.step == 1 + +assert slice(1).__eq__(slice(1)) +assert slice(1) != slice(2) +assert slice(1) == slice(None, 1, None) +assert slice(0, 0, 0) == slice(0, 0, 0) + +assert slice(0, 0, 1) != slice(0, 0, 0) +assert slice(0, 1, 0) != slice(0, 0, 0) +assert slice(1, 0, 0) != slice(0, 0, 0) +assert slice(0).__ne__(slice(1)) +assert slice(0, None, 3).__ne__(slice(0, 0, 3)) + +doc="finished" \ No newline at end of file diff --git a/py/tests/staticmethod.py b/py/tests/staticmethod.py new file mode 100644 index 00000000..f93aa757 --- /dev/null +++ b/py/tests/staticmethod.py @@ -0,0 +1,18 @@ +# Copyright 2019 The go-python Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +doc="staticmethod" + +class A: + @staticmethod + def fn(p): + return p+1 + +a = A() +assert a.fn(1) == 2 + +a.x = 3 +assert a.x == 3 + +doc="finished" diff --git a/py/tests/string.py b/py/tests/string.py index c837a441..f2ad6e9b 100644 --- a/py/tests/string.py +++ b/py/tests/string.py @@ -59,6 +59,17 @@ class C(): assert not( 1 == "potato") assert 1 != "potato" +doc="startswith" +assert "HELLO THERE".startswith("HELL") +assert not "HELLO THERE".startswith("THERE") +assert "HELLO".startswith("LLO", 2) +assert "HELLO THERE".startswith(("HERE", "HELL")) + +doc="endswith" +assert "HELLO THERE".endswith("HERE") +assert not "HELLO THERE".endswith("HELL") +assert "HELLO THERE".endswith(("HELL", "HERE")) + doc="bool" assert "true" assert not "" @@ -99,6 +110,14 @@ class C(): asc="hello" uni="£100世界𠜎" # 1,2,3,4 byte unicode characters +doc="split" +assert ["0","1","2","4"] == list("0,1,2,4".split(",")) +assert [""] == list("".split(",")) +assert ['a', 'd,c'] == list("a,d,c".split(",",1)) +assert ['a', 'd', 'b'] == list(" a d b ".split()) +assert ['a', 'd b '] == list(" a d b ".split(None, 1)) +assertRaisesText(TypeError, "Can't convert 'int' object to str implicitly", lambda: "0,1,2,4".split(1)) + doc="ascii len" assert len(asc) == 5 @@ -867,4 +886,54 @@ def index(s, i): assert uni[7:7:2] == '' assert uni[7:7:3] == '' +doc="string strip methods" +a = " adfasd " +assert a.rstrip() == " adfasd" +assert a.lstrip() == "adfasd " +assert a.strip() == "adfasd" + +a = " a bada a" +assert a.rstrip("a ") == " a bad" +assert a.lstrip("a ") == "bada a" +assert a.strip("a ") == "bad" + +doc="upper" +a = "abc" +assert a.upper() == "ABC" + +doc="lower" +a = "ABC" +assert a.lower() == "abc" + +class Index: + def __index__(self): + return 1 + +a = '012345678910' +b = Index() +assert a[b] == '1' +assert a[b:10] == a[1:10] +assert a[10:b:-1] == a[10:1:-1] + +class NonIntegerIndex: + def __index__(self): + return 1.1 + +a = '012345678910' +b = NonIntegerIndex() +try: + a[b] +except TypeError: + pass +else: + assert False, "TypeError not raised" + +try: + a[b:10] +except TypeError: + pass +else: + assert False, "TypeError not raised" + + doc="finished" diff --git a/py/tests/tuple.py b/py/tests/tuple.py index 49bb2ed5..6bf05948 100644 --- a/py/tests/tuple.py +++ b/py/tests/tuple.py @@ -7,11 +7,49 @@ assert str((1,2,3)) == "(1, 2, 3)" assert str((1,(2,3),4)) == "(1, (2, 3), 4)" assert str(("1",(2.5,17,()))) == "('1', (2.5, 17, ()))" +assert str((1, 1.0)) == "(1, 1.0)" doc="repr" assert repr(()) == "()" assert repr((1,2,3)) == "(1, 2, 3)" assert repr((1,(2,3),4)) == "(1, (2, 3), 4)" assert repr(("1",(2.5,17,()))) == "('1', (2.5, 17, ()))" +assert repr((1, 1.0)) == "(1, 1.0)" + +doc="mul" +a = (1, 2, 3) +assert a * 2 == (1, 2, 3, 1, 2, 3) +assert a * 0 == () +assert a * -1 == () + +class Index: + def __index__(self): + return 1 + +a = (0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10) +b = Index() +assert a[b] == 1 +assert a[b:10] == a[1:10] +assert a[10:b:-1] == a[10:1:-1] + +class NonIntegerIndex: + def __index__(self): + return 1.1 + +a = (0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10) +b = NonIntegerIndex() +try: + a[b] +except TypeError: + pass +else: + assert False, "TypeError not raised" + +try: + a[b:10] +except TypeError: + pass +else: + assert False, "TypeError not raised" doc="finished" diff --git a/py/tests/zip.py b/py/tests/zip.py new file mode 100644 index 00000000..adc68d9a --- /dev/null +++ b/py/tests/zip.py @@ -0,0 +1,13 @@ +# Copyright 2018 The go-python Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +doc="zip" +a = [1, 2, 3, 4, 5] +b = [10, 11, 12] +c = [e for e in zip(a, b)] +assert len(c) == 3 +for idx, e in enumerate(c): + assert a[idx] == c[idx][0] + assert b[idx] == c[idx][1] +doc="finished" \ No newline at end of file diff --git a/py/traceback.go b/py/traceback.go index 7cad8f04..6dcc9ce6 100644 --- a/py/traceback.go +++ b/py/traceback.go @@ -52,7 +52,7 @@ RuntimeError: this is the error message func (tb *Traceback) TracebackDump(w io.Writer) { for ; tb != nil; tb = tb.Next { fmt.Fprintf(w, " File %q, line %d, in %s\n", tb.Frame.Code.Filename, tb.Lineno, tb.Frame.Code.Name) - fmt.Fprintf(w, " %s\n", "FIXME line of source goes here") + //fmt.Fprintf(w, " %s\n", "FIXME line of source goes here") } } @@ -64,10 +64,10 @@ func TracebackDump(err interface{}) { case *ExceptionInfo: e.TracebackDump(os.Stderr) case *Exception: - fmt.Fprintf(os.Stderr, "Exception %#v\n", e) + fmt.Fprintf(os.Stderr, "Exception %v\n", e) fmt.Fprintf(os.Stderr, "-- No traceback available --\n") default: - fmt.Fprintf(os.Stderr, "Error %#v\n", err) + fmt.Fprintf(os.Stderr, "Error %v\n", err) fmt.Fprintf(os.Stderr, "-- No traceback available --\n") } } diff --git a/py/tuple.go b/py/tuple.go index d4292ed9..e65cf765 100644 --- a/py/tuple.go +++ b/py/tuple.go @@ -113,6 +113,7 @@ func (a Tuple) M__add__(other Object) (Object, error) { copy(newTuple[len(b):], b) return newTuple, nil } + return NotImplemented, nil } @@ -131,6 +132,9 @@ func (l Tuple) M__mul__(other Object) (Object, error) { if b, ok := convertToInt(other); ok { m := len(l) n := int(b) * m + if n < 0 { + n = 0 + } newTuple := make(Tuple, n) for i := 0; i < n; i += m { copy(newTuple[i:i+m], l) diff --git a/py/type.go b/py/type.go index a9f835a4..db31ba61 100644 --- a/py/type.go +++ b/py/type.go @@ -306,7 +306,7 @@ func (t *Type) NewTypeFlags(Name string, Doc string, New NewFunc, Init InitFunc, Dict: StringDict{}, Bases: Tuple{t}, } - TypeDelayReady(t) + TypeDelayReady(tt) return tt } @@ -458,7 +458,7 @@ func (t *Type) Lookup(name string) Object { // // Doesn't call __getattr__ etc // -// Returns nil if not found +// # Returns nil if not found // // Doesn't look in the instance dictionary // @@ -478,7 +478,7 @@ func (t *Type) NativeGetAttrOrNil(name string) Object { // // Doesn't call __getattr__ etc // -// Returns nil if not found +// # Returns nil if not found // // FIXME this isn't totally correct! // as we are ignoring getattribute etc @@ -552,7 +552,8 @@ func TypeCall2(self Object, name string, arg1, arg2 Object) (Object, bool, error // Two variants: // // - lookup_maybe() returns nil without raising an exception -// when the _PyType_Lookup() call fails; +// +// when the _PyType_Lookup() call fails; // // - lookup_method() always raises an exception upon errors. func lookup_maybe(self Object, attr string) Object { @@ -569,14 +570,14 @@ func lookup_maybe(self Object, attr string) Object { return res } -func lookup_method(self Object, attr string) Object { - res := lookup_maybe(self, attr) - if res == nil { - // FIXME PyErr_SetObject(PyExc_AttributeError, attrid->object); - return ExceptionNewf(AttributeError, "'%s' object has no attribute '%s'", self.Type().Name, attr) - } - return res -} +// func lookup_method(self Object, attr string) Object { +// res := lookup_maybe(self, attr) +// if res == nil { +// // FIXME PyErr_SetObject(PyExc_AttributeError, attrid->object); +// return ExceptionNewf(AttributeError, "'%s' object has no attribute '%s'", self.Type().Name, attr) +// } +// return res +// } // Method resolution order algorithm C3 described in // "A Monotonic Superclass Linearization for Dylan", @@ -954,26 +955,26 @@ func add_subclass(base, t *Type) { // return result; } -func remove_subclass(base, t *Type) { - // Py_ssize_t i; - // PyObject *list, *ref; - - // list = base->tp_subclasses; - // if (list == nil) { - // return; - // } - // assert(PyList_Check(list)); - // i = PyList_GET_SIZE(list); - // while (--i >= 0) { - // ref = PyList_GET_ITEM(list, i); - // assert(PyWeakref_CheckRef(ref)); - // if (PyWeakref_GET_OBJECT(ref) == (PyObject*)type) { - // /* this can't fail, right? */ - // PySequence_DelItem(list, i); - // return; - // } - // } -} +// func remove_subclass(base, t *Type) { +// // Py_ssize_t i; +// // PyObject *list, *ref; +// +// // list = base->tp_subclasses; +// // if (list == nil) { +// // return; +// // } +// // assert(PyList_Check(list)); +// // i = PyList_GET_SIZE(list); +// // while (--i >= 0) { +// // ref = PyList_GET_ITEM(list, i); +// // assert(PyWeakref_CheckRef(ref)); +// // if (PyWeakref_GET_OBJECT(ref) == (PyObject*)type) { +// // /* this can't fail, right? */ +// // PySequence_DelItem(list, i); +// // return; +// // } +// // } +// } // Ready the type for use // @@ -1092,7 +1093,7 @@ func (t *Type) Ready() error { // if the type dictionary doesn't contain a __doc__, set it from // the tp_doc slot. - if _, ok := t.Dict["__doc__"]; ok { + if _, ok := t.Dict["__doc__"]; !ok { if t.Doc != "" { t.Dict["__doc__"] = String(t.Doc) } else { diff --git a/py/util.go b/py/util.go new file mode 100644 index 00000000..e2904187 --- /dev/null +++ b/py/util.go @@ -0,0 +1,236 @@ +// Copyright 2022 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package py + +import ( + "errors" + "strconv" + "strings" +) + +var ( + ErrUnsupportedObjType = errors.New("unsupported obj type") +) + +// GetLen is a high-level convenience function that returns the length of the given Object. +func GetLen(obj Object) (Int, error) { + getlen, ok := obj.(I__len__) + if !ok { + return 0, nil + } + + lenObj, err := getlen.M__len__() + if err != nil { + return 0, err + } + + return GetInt(lenObj) +} + +// GetInt is a high-level convenience function that converts the given value to an int. +func GetInt(obj Object) (Int, error) { + toIdx, ok := obj.(I__index__) + if !ok { + _, err := cantConvert(obj, "int") + return 0, err + } + + return toIdx.M__index__() +} + +// LoadTuple attempts to convert each element of the given list and store into each destination value (based on its type). +func LoadTuple(args Tuple, vars []interface{}) error { + + if len(args) > len(vars) { + return ExceptionNewf(RuntimeError, "%d args given, expected %d", len(args), len(vars)) + } + + if len(vars) > len(args) { + vars = vars[:len(args)] + } + + for i, rval := range vars { + err := loadValue(args[i], rval) + if err == ErrUnsupportedObjType { + return ExceptionNewf(TypeError, "arg %d has unsupported object type: %s", i, args[i].Type().Name) + } + } + + return nil +} + +// LoadAttr gets the named attribute and attempts to store it into the given destination value (based on its type). +func LoadAttr(obj Object, attrName string, dst interface{}) error { + attr, err := GetAttrString(obj, attrName) + if err != nil { + return err + } + err = loadValue(attr, dst) + if err == ErrUnsupportedObjType { + return ExceptionNewf(TypeError, "attribute \"%s\" has unsupported object type: %s", attrName, attr.Type().Name) + } + return nil +} + +// LoadIntsFromList extracts a list of ints contained given a py.List or py.Tuple +func LoadIntsFromList(list Object) ([]int64, error) { + N, err := GetLen(list) + if err != nil { + return nil, err + } + + getter, ok := list.(I__getitem__) + if !ok { + return nil, nil + } + + if N <= 0 { + return nil, nil + } + + intList := make([]int64, N) + for i := Int(0); i < N; i++ { + item, err := getter.M__getitem__(i) + if err != nil { + return nil, err + } + + var intVal Int + intVal, err = GetInt(item) + if err != nil { + return nil, err + } + + intList[i] = int64(intVal) + } + + return intList, nil +} + +func loadValue(src Object, data interface{}) error { + var ( + v_str string + v_float float64 + v_int int64 + ) + + haveStr := false + + switch v := src.(type) { + case Bool: + if v { + v_int = 1 + v_float = 1 + v_str = "True" + } else { + v_str = "False" + } + haveStr = true + case Int: + v_int = int64(v) + v_float = float64(v) + case Float: + v_int = int64(v) + v_float = float64(v) + case String: + v_str = string(v) + haveStr = true + case NoneType: + // No-op + default: + return ErrUnsupportedObjType + } + + switch dst := data.(type) { + case *Int: + *dst = Int(v_int) + case *bool: + *dst = v_int != 0 + case *int8: + *dst = int8(v_int) + case *uint8: + *dst = uint8(v_int) + case *int16: + *dst = int16(v_int) + case *uint16: + *dst = uint16(v_int) + case *int32: + *dst = int32(v_int) + case *uint32: + *dst = uint32(v_int) + case *int: + *dst = int(v_int) + case *uint: + *dst = uint(v_int) + case *int64: + *dst = v_int + case *uint64: + *dst = uint64(v_int) + case *float32: + if haveStr { + v_float, _ = strconv.ParseFloat(v_str, 32) + } + *dst = float32(v_float) + case *float64: + if haveStr { + v_float, _ = strconv.ParseFloat(v_str, 64) + } + *dst = v_float + case *Float: + if haveStr { + v_float, _ = strconv.ParseFloat(v_str, 64) + } + *dst = Float(v_float) + case *string: + *dst = v_str + case *String: + *dst = String(v_str) + // case []uint64: + // for i := range data { + // dst[i] = order.Uint64(bs[8*i:]) + // } + // case []float32: + // for i := range data { + // dst[i] = math.Float32frombits(order.Uint32(bs[4*i:])) + // } + // case []float64: + // for i := range data { + // dst[i] = math.Float64frombits(order.Uint64(bs[8*i:])) + // } + + default: + return ExceptionNewf(NotImplementedError, "%s", "unsupported Go data type") + } + return nil +} + +// Println prints the provided strings to gpython's stdout. +func Println(self Object, args ...string) bool { + sysModule, err := self.(*Module).Context.GetModule("sys") + if err != nil { + return false + } + stdout := sysModule.Globals["stdout"] + write, err := GetAttrString(stdout, "write") + if err != nil { + return false + } + call, ok := write.(I__call__) + if !ok { + return false + } + for _, v := range args { + if !strings.Contains(v, "\n") { + v += " " + } + _, err := call.M__call__(Tuple{String(v)}, nil) + if err != nil { + return false + } + + } + _, err = call.M__call__(Tuple{String("\n")}, nil) // newline + return err == nil +} diff --git a/py/zip.go b/py/zip.go new file mode 100644 index 00000000..eee9dc27 --- /dev/null +++ b/py/zip.go @@ -0,0 +1,66 @@ +// Copyright 2018 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package py + +// A python Zip object +type Zip struct { + itTuple Tuple + size int +} + +// // A python ZipIterator iterator +// type ZipIterator struct { +// zip Zip +// } + +var ZipType = NewTypeX("zip", `zip(iter1 [,iter2 [...]]) --> zip object + +Return a zip object whose .__next__() method returns a tuple where +the i-th element comes from the i-th iterable argument. The .__next__() +method continues until the shortest iterable in the argument sequence +is exhausted and then it raises StopIteration.`, + ZipTypeNew, nil) + +// Type of this object +func (z *Zip) Type() *Type { + return ZipType +} + +// ZipTypeNew +func ZipTypeNew(metatype *Type, args Tuple, kwargs StringDict) (Object, error) { + tupleSize := len(args) + itTuple := make(Tuple, tupleSize) + for i := 0; i < tupleSize; i++ { + item := args[i] + iter, err := Iter(item) + if err != nil { + return nil, ExceptionNewf(TypeError, "zip argument #%d must support iteration", i+1) + } + itTuple[i] = iter + } + + return &Zip{itTuple: itTuple, size: tupleSize}, nil +} + +// Zip iterator +func (z *Zip) M__iter__() (Object, error) { + return z, nil +} + +func (z *Zip) M__next__() (Object, error) { + result := make(Tuple, z.size) + for i := 0; i < z.size; i++ { + value, err := Next(z.itTuple[i]) + if err != nil { + return nil, err + } + result[i] = value + } + return result, nil +} + +// Check interface is satisfied +var _ I__iter__ = (*Zip)(nil) +var _ I__next__ = (*Zip)(nil) diff --git a/pytest/pytest.go b/pytest/pytest.go index 54d4b980..7c331d26 100644 --- a/pytest/pytest.go +++ b/pytest/pytest.go @@ -5,21 +5,30 @@ package pytest import ( - "io/ioutil" + "bytes" + "flag" + "fmt" + "io" "os" "path" + "path/filepath" "strings" + "sync/atomic" "testing" - _ "github.com/go-python/gpython/builtin" "github.com/go-python/gpython/compile" "github.com/go-python/gpython/py" - _ "github.com/go-python/gpython/sys" - "github.com/go-python/gpython/vm" + "github.com/google/go-cmp/cmp" + + _ "github.com/go-python/gpython/stdlib" ) -// Run the code in str -func Run(t *testing.T, prog string) { +var RegenTestData = flag.Bool("regen", false, "Regenerate golden files from current testdata.") + +var gContext = py.NewContext(py.DefaultContextOpts()) + +// Compile the program in the file prog to code in the module that is returned +func compileProgram(t testing.TB, prog string) (*py.Module, *py.Code) { f, err := os.Open(prog) if err != nil { t.Fatalf("%s: Open failed: %v", prog, err) @@ -30,39 +39,48 @@ func Run(t *testing.T, prog string) { } }() - str, err := ioutil.ReadAll(f) + str, err := io.ReadAll(f) if err != nil { t.Fatalf("%s: ReadAll failed: %v", prog, err) } + return CompileSrc(t, gContext, string(str), prog) +} - obj, err := compile.Compile(string(str), prog, "exec", 0, true) +func CompileSrc(t testing.TB, ctx py.Context, pySrc string, prog string) (*py.Module, *py.Code) { + code, err := compile.Compile(string(pySrc), prog, py.ExecMode, 0, true) if err != nil { t.Fatalf("%s: Compile failed: %v", prog, err) } - code := obj.(*py.Code) - module := py.NewModule("__main__", "", nil, nil) - module.Globals["__file__"] = py.String(prog) + module, err := ctx.Store().NewModule(ctx, &py.ModuleImpl{ + Info: py.ModuleInfo{ + FileDesc: prog, + }, + }) + if err != nil { + t.Fatalf("%s: NewModule failed: %v", prog, err) + } + + return module, code +} - _, err = vm.Run(module.Globals, module.Globals, code, nil) +// Run the code in the module +func run(t testing.TB, module *py.Module, code *py.Code) { + _, err := gContext.RunCode(code, module.Globals, module.Globals, nil) if err != nil { - if wantErr, ok := module.Globals["err"]; ok { - wantErrObj, ok := wantErr.(py.Object) - if !ok { - t.Fatalf("%s: want err is not py.Object: %#v", prog, wantErr) - } + if wantErrObj, ok := module.Globals["err"]; ok { gotExc, ok := err.(py.ExceptionInfo) if !ok { - t.Fatalf("%s: got err is not ExceptionInfo: %#v", prog, err) + t.Fatalf("got err is not ExceptionInfo: %#v", err) } if gotExc.Value.Type() != wantErrObj.Type() { - t.Fatalf("%s: Want exception %v got %v", prog, wantErrObj, gotExc.Value) + t.Fatalf("Want exception %v got %v", wantErrObj, gotExc.Value) } - t.Logf("%s: matched exception", prog) + // t.Logf("matched exception") return } else { py.TracebackDump(err) - t.Fatalf("%s: Run failed: %v at %q", prog, err, module.Globals["doc"]) + t.Fatalf("Run failed: %v at %q", err, module.Globals["doc"]) } } @@ -70,28 +88,197 @@ func Run(t *testing.T, prog string) { if doc, ok := module.Globals["doc"]; ok { if docStr, ok := doc.(py.String); ok { if string(docStr) != "finished" { - t.Fatalf("%s: Didn't finish at %q", prog, docStr) + t.Fatalf("Didn't finish at %q", docStr) } } else { - t.Fatalf("%s: Set doc variable to non string: %#v", prog, doc) + t.Fatalf("Set doc variable to non string: %#v", doc) } } else { - t.Fatalf("%s: Didn't set doc variable at all", prog) + t.Fatalf("Didn't set doc variable at all") } } -// Runs the tests in the directory passed in -func RunTests(t *testing.T, testDir string) { - files, err := ioutil.ReadDir(testDir) +// find the python files in the directory passed in +func findFiles(t testing.TB, testDir string) (names []string) { + files, err := os.ReadDir(testDir) if err != nil { t.Fatalf("ReadDir failed: %v", err) } for _, f := range files { name := f.Name() if !strings.HasPrefix(name, "lib") && strings.HasSuffix(name, ".py") { - name := path.Join(testDir, name) - t.Logf("%s: Running", name) - Run(t, name) + names = append(names, name) + } + } + return names +} + +// RunTests runs the tests in the directory passed in +func RunTests(t *testing.T, testDir string) { + for _, name := range findFiles(t, testDir) { + t.Run(name, func(t *testing.T) { + module, code := compileProgram(t, path.Join(testDir, name)) + run(t, module, code) + }) + } +} + +// RunBenchmarks runs the benchmarks in the directory passed in +func RunBenchmarks(b *testing.B, testDir string) { + for _, name := range findFiles(b, testDir) { + module, code := compileProgram(b, path.Join(testDir, name)) + b.Run(name, func(b *testing.B) { + for i := 0; i < b.N; i++ { + run(b, module, code) + } + }) + } +} + +// RunScript runs the provided path to a script. +// RunScript captures the stdout and stderr while executing the script +// and compares it to a golden file, blocking until completion. +// +// RunScript("./testdata/foo.py") +// +// will compare the output with "./testdata/foo_golden.txt". +func RunScript(t *testing.T, fname string) { + + RunTestTasks(t, []*Task{ + { + PyFile: fname, + }, + }) +} + +// RunTestTasks runs each given task in a newly created py.Context concurrently. +// If a fatal error is encountered, the given testing.T is signaled. +func RunTestTasks(t *testing.T, tasks []*Task) { + onCompleted := make(chan *Task) + + numTasks := len(tasks) + for ti := 0; ti < numTasks; ti++ { + task := tasks[ti] + go func() { + err := task.run() + task.Err = err + onCompleted <- task + }() + } + + tasks = tasks[:0] + for ti := 0; ti < numTasks; ti++ { + task := <-onCompleted + if task.Err != nil { + t.Error(task.Err) + } + tasks = append(tasks, task) + } +} + +var ( + taskCounter int32 +) + +type Task struct { + num int32 // Assigned when this task is run + ID string // unique key identifying this task. If empty, autogenerated from the basename of PyFile + PyFile string // If set, this file pathname is executed in a newly created ctx + PyTask func(ctx py.Context) error // If set, a new created ctx is created and this blocks until completion + GoldFile string // Filename containing the "gold standard" stdout+stderr. If empty, autogenerated from PyFile or ID + Err error // Non-nil if a fatal error is encountered with this task +} + +func (task *Task) run() error { + fileBase := "" + + opts := py.DefaultContextOpts() + if task.PyFile != "" { + opts.SysArgs = []string{task.PyFile} + if task.ID == "" { + ext := filepath.Ext(task.PyFile) + fileBase = task.PyFile[0 : len(task.PyFile)-len(ext)] + } + } + + task.num = atomic.AddInt32(&taskCounter, 1) + if task.ID == "" { + if fileBase == "" { + task.ID = fmt.Sprintf("task-%04d", atomic.AddInt32(&taskCounter, 1)) + } else { + task.ID = strings.TrimPrefix(fileBase, "./") } } + + if task.GoldFile == "" { + task.GoldFile = fileBase + "_golden.txt" + } + + ctx := py.NewContext(opts) + defer ctx.Close() + + sys := ctx.Store().MustGetModule("sys") + tmp, err := os.MkdirTemp("", "gpython-pytest-") + if err != nil { + return err + } + defer os.RemoveAll(tmp) + + out, err := os.Create(filepath.Join(tmp, "combined")) + if err != nil { + return fmt.Errorf("could not create stdout+stderr output file: %w", err) + } + defer out.Close() + + sys.Globals["stdout"] = &py.File{File: out, FileMode: py.FileWrite} + sys.Globals["stderr"] = &py.File{File: out, FileMode: py.FileWrite} + + if task.PyFile != "" { + _, err := py.RunFile(ctx, task.PyFile, py.CompileOpts{}, nil) + if err != nil { + return fmt.Errorf("could not run target script %q: %w", task.PyFile, err) + } + } + + if task.PyTask != nil { + err := task.PyTask(ctx) + if err != nil { + return fmt.Errorf("PyTask %q failed: %w", task.ID, err) + } + } + + // Close the ctx explicitly as it may legitimately generate output + ctx.Close() + <-ctx.Done() + + err = out.Close() + if err != nil { + return fmt.Errorf("could not close output file: %w", err) + } + + got, err := os.ReadFile(out.Name()) + if err != nil { + return fmt.Errorf("could not read script output file: %w", err) + } + + if *RegenTestData { + err := os.WriteFile(task.GoldFile, got, 0644) + if err != nil { + return fmt.Errorf("could not write golden output %q: %w", task.GoldFile, err) + } + } + + want, err := os.ReadFile(task.GoldFile) + if err != nil { + return fmt.Errorf("could not read golden output %q: %w", task.GoldFile, err) + } + + diff := cmp.Diff(string(want), string(got)) + if !bytes.Equal(got, want) { + out := fileBase + ".txt" + _ = os.WriteFile(out, got, 0644) + return fmt.Errorf("output differ: -- (-ref +got)\n%s", diff) + } + + return nil } diff --git a/pytest/pytest_test.go b/pytest/pytest_test.go new file mode 100644 index 00000000..15e136bf --- /dev/null +++ b/pytest/pytest_test.go @@ -0,0 +1,33 @@ +// Copyright 2018 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package pytest + +import ( + "testing" +) + +func TestCompileSrc(t *testing.T) { + for _, tc := range []struct { + name string + code string + }{ + { + name: "hello", + code: `print("hello")`, + }, + } { + t.Run(tc.name, func(t *testing.T) { + _, _ = CompileSrc(t, gContext, tc.code, tc.name) + }) + } +} + +func TestRunTests(t *testing.T) { + RunTests(t, "./testdata/tests") +} + +func TestRunScript(t *testing.T) { + RunScript(t, "./testdata/hello.py") +} diff --git a/pytest/testdata/hello.py b/pytest/testdata/hello.py new file mode 100644 index 00000000..fdbccfc1 --- /dev/null +++ b/pytest/testdata/hello.py @@ -0,0 +1,7 @@ +# Copyright 2022 The go-python Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +print("hello") +print("world") +print("bye.") diff --git a/pytest/testdata/hello_golden.txt b/pytest/testdata/hello_golden.txt new file mode 100644 index 00000000..e4226353 --- /dev/null +++ b/pytest/testdata/hello_golden.txt @@ -0,0 +1,3 @@ +hello +world +bye. diff --git a/pytest/testdata/tests/libtest.py b/pytest/testdata/tests/libtest.py new file mode 100644 index 00000000..003eb3db --- /dev/null +++ b/pytest/testdata/tests/libtest.py @@ -0,0 +1,11 @@ +# Copyright 2022 The go-python Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +""" +Simple test harness +""" + +def testFunc(): + return + diff --git a/pytest/testdata/tests/module.py b/pytest/testdata/tests/module.py new file mode 100644 index 00000000..4151c996 --- /dev/null +++ b/pytest/testdata/tests/module.py @@ -0,0 +1,12 @@ +# Copyright 2022 The go-python Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +from libtest import testFunc + +doc="module" +assert True +assert not False +assert testFunc() is None + +doc="finished" diff --git a/repl/cli/cli.go b/repl/cli/cli.go index 1affc2c5..6648094a 100644 --- a/repl/cli/cli.go +++ b/repl/cli/cli.go @@ -12,7 +12,6 @@ import ( "os/user" "path/filepath" - "github.com/go-python/gpython/py" "github.com/go-python/gpython/repl" "github.com/peterh/liner" ) @@ -36,7 +35,6 @@ type readline struct { *liner.State repl *repl.REPL historyFile string - module *py.Module prompt string } @@ -119,18 +117,20 @@ func (rl *readline) Print(out string) { } // RunREPL starts the REPL loop -func RunREPL() { - repl := repl.New() - rl := newReadline(repl) - repl.SetUI(rl) +func RunREPL(replCtx *repl.REPL) { + if replCtx == nil { + replCtx = repl.New(nil) + } + rl := newReadline(replCtx) + replCtx.SetUI(rl) defer rl.Close() err := rl.ReadHistory() if err != nil { - fmt.Printf("Failed to open history: %v\n", err) + if !os.IsNotExist(err) { + fmt.Printf("Failed to open history: %v\n", err) + } } - fmt.Printf("Gpython 3.4.0\n") - for { line, err := rl.Prompt(rl.prompt) if err != nil { diff --git a/repl/repl.go b/repl/repl.go index a9e50862..f6639b25 100644 --- a/repl/repl.go +++ b/repl/repl.go @@ -10,7 +10,6 @@ import ( "sort" "strings" - "github.com/go-python/gpython/compile" "github.com/go-python/gpython/py" "github.com/go-python/gpython/vm" ) @@ -23,7 +22,8 @@ const ( // Repl state type REPL struct { - module *py.Module + Context py.Context + Module *py.Module prog string continuation bool previous string @@ -39,15 +39,23 @@ type UI interface { Print(string) } -// New create a new REPL and initialises the state machine -func New() *REPL { +// New create a new REPL and initializes the state machine +func New(ctx py.Context) *REPL { + if ctx == nil { + ctx = py.NewContext(py.DefaultContextOpts()) + } + r := &REPL{ - module: py.NewModule("__main__", "", nil, nil), + Context: ctx, prog: "", continuation: false, previous: "", } - r.module.Globals["__file__"] = py.String(r.prog) + r.Module, _ = ctx.ModuleInit(&py.ModuleImpl{ + Info: py.ModuleInfo{ + FileDesc: r.prog, + }, + }) return r } @@ -76,15 +84,19 @@ func (r *REPL) Run(line string) { if toCompile == "" { return } - obj, err := compile.Compile(toCompile+"\n", r.prog, "single", 0, true) + code, err := py.Compile(toCompile+"\n", r.prog, py.SingleMode, 0, true) if err != nil { // Detect that we should start a continuation line // FIXME detect EOF properly! errText := err.Error() if strings.Contains(errText, "unexpected EOF while parsing") || strings.Contains(errText, "EOF while scanning triple-quoted string literal") { - r.continuation = true - r.previous += string(line) + "\n" - r.term.SetPrompt(ContinuationPrompt) + stripped := strings.TrimSpace(toCompile) + isComment := len(stripped) > 0 && stripped[0] == '#' + if !isComment { + r.continuation = true + r.previous += string(line) + "\n" + r.term.SetPrompt(ContinuationPrompt) + } return } } @@ -95,8 +107,7 @@ func (r *REPL) Run(line string) { r.term.Print(fmt.Sprintf("Compile error: %v", err)) return } - code := obj.(*py.Code) - _, err = vm.Run(r.module.Globals, r.module.Globals, code, nil) + _, err = r.Context.RunCode(code, r.Module.Globals, r.Module.Globals, nil) if err != nil { py.TracebackDump(err) } @@ -125,8 +136,8 @@ func (r *REPL) Completer(line string, pos int) (head string, completions []strin } } } - match(r.module.Globals) - match(py.Builtins.Globals) + match(r.Module.Globals) + match(r.Context.Store().Builtins.Globals) sort.Strings(completions) return head, completions, tail } diff --git a/repl/repl_test.go b/repl/repl_test.go index e5324519..b154bfec 100644 --- a/repl/repl_test.go +++ b/repl/repl_test.go @@ -6,10 +6,7 @@ import ( "testing" // import required modules - _ "github.com/go-python/gpython/builtin" - _ "github.com/go-python/gpython/math" - _ "github.com/go-python/gpython/sys" - _ "github.com/go-python/gpython/time" + _ "github.com/go-python/gpython/stdlib" ) type replTest struct { @@ -28,17 +25,18 @@ func (rt *replTest) Print(out string) { } func (rt *replTest) assert(t *testing.T, what, wantPrompt, wantOut string) { + t.Helper() if rt.prompt != wantPrompt { - t.Errorf("%s: Prompt wrong, want %q got %q", what, wantPrompt, rt.prompt) + t.Errorf("%s: Prompt wrong:\ngot= %q\nwant=%q", what, rt.prompt, wantPrompt) } if rt.out != wantOut { - t.Errorf("%s: Output wrong, want %q got %q", what, wantOut, rt.out) + t.Errorf("%s: Output wrong:\ngot= %q\nwant=%q", what, rt.out, wantOut) } rt.out = "" } func TestREPL(t *testing.T) { - r := New() + r := New(nil) rt := &replTest{} r.SetUI(rt) @@ -66,11 +64,19 @@ func TestREPL(t *testing.T) { rt.assert(t, "multi#5", NormalPrompt, "45") r.Run("if") - rt.assert(t, "compileError", NormalPrompt, "Compile error: \n File \"\", line 1, offset 2\n if\n\n\nSyntaxError: 'invalid syntax'") + rt.assert(t, "compileError", NormalPrompt, "Compile error: \n File \"\", line 1, offset 2\n if\n\n\nSyntaxError: 'invalid syntax'") + + // test comments in the REPL work properly + r.Run("# this is a comment") + rt.assert(t, "comment", NormalPrompt, "") + r.Run("a = 42") + rt.assert(t, "comment continuation", NormalPrompt, "") + r.Run("a") + rt.assert(t, "comment check", NormalPrompt, "42") } func TestCompleter(t *testing.T) { - r := New() + r := New(nil) rt := &replTest{} r.SetUI(rt) @@ -113,13 +119,13 @@ func TestCompleter(t *testing.T) { t.Run(fmt.Sprintf("line=%q,pos=%d)", test.line, test.pos), func(t *testing.T) { gotHead, gotCompletions, gotTail := r.Completer(test.line, test.pos) if test.wantHead != gotHead { - t.Errorf("head: want %q got %q", test.wantHead, gotHead) + t.Errorf("invalid head:\ngot= %q\nwant=%q", gotHead, test.wantHead) } if !reflect.DeepEqual(test.wantCompletions, gotCompletions) { - t.Errorf("completions: want %#v got %#v", test.wantCompletions, gotCompletions) + t.Errorf("invalid completions:\ngot= %#v\nwant=%#v", gotCompletions, test.wantCompletions) } if test.wantTail != gotTail { - t.Errorf("tail: want %q got %q", test.wantTail, gotTail) + t.Errorf("invalid tail:\ngot= %q\nwant=%q", gotTail, test.wantTail) } }) } diff --git a/repl/web/main.go b/repl/web/main.go index bad66f16..f0cebfd8 100644 --- a/repl/web/main.go +++ b/repl/web/main.go @@ -4,6 +4,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build js // +build js package main @@ -12,14 +13,11 @@ import ( "log" "runtime" + "github.com/go-python/gpython/repl" "github.com/gopherjs/gopherwasm/js" // gopherjs to wasm converter shim // import required modules - _ "github.com/go-python/gpython/builtin" - _ "github.com/go-python/gpython/math" - "github.com/go-python/gpython/repl" - _ "github.com/go-python/gpython/sys" - _ "github.com/go-python/gpython/time" + _ "github.com/go-python/gpython/stdlib" ) // Implement the replUI interface @@ -77,7 +75,7 @@ func main() { node.Get("classList").Call("add", "active") // Make a repl referring to an empty term for the moment - REPL := repl.New() + REPL := repl.New(nil) cb := js.NewCallback(func(args []js.Value) { REPL.Run(args[0].String()) }) diff --git a/repl/web/serve.go b/repl/web/serve.go index ee9a6b78..30e38e44 100644 --- a/repl/web/serve.go +++ b/repl/web/serve.go @@ -1,4 +1,5 @@ -//+build none +//go:build none +// +build none package main diff --git a/stdlib/array/array.go b/stdlib/array/array.go new file mode 100644 index 00000000..b9e682ae --- /dev/null +++ b/stdlib/array/array.go @@ -0,0 +1,762 @@ +// Copyright 2023 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package array provides the implementation of the python's 'array' module. +package array + +import ( + "fmt" + "reflect" + "strings" + + "github.com/go-python/gpython/py" +) + +// FIXME(sbinet): consider creating an "array handler" type for each of the typecodes +// and make the handler a field of the "array" type. +// or make "array" an interface ? + +// array provides efficient manipulation of C-arrays (as Go slices). +type array struct { + descr byte // typecode of elements + esize int // element size in bytes + data any + + append func(v py.Object) (py.Object, error) + extend func(seq py.Object) (py.Object, error) +} + +// Type of this StringDict object +func (*array) Type() *py.Type { + return ArrayType +} + +var ( + _ py.Object = (*array)(nil) + _ py.I__getitem__ = (*array)(nil) + _ py.I__setitem__ = (*array)(nil) + _ py.I__len__ = (*array)(nil) + _ py.I__repr__ = (*array)(nil) + _ py.I__str__ = (*array)(nil) +) + +var ( + typecodes = py.String("bBuhHiIlLqQfd") + ArrayType = py.ObjectType.NewType("array.array", array_doc, array_new, nil) + + descr2esize = map[byte]int{ + 'b': 1, + 'B': 1, + 'u': 2, + 'h': 2, + 'H': 2, + 'i': 2, + 'I': 2, + 'l': 8, + 'L': 8, + 'q': 8, + 'Q': 8, + 'f': 4, + 'd': 8, + } +) + +func init() { + py.RegisterModule(&py.ModuleImpl{ + Info: py.ModuleInfo{ + Name: "array", + Doc: "This module defines an object type which can efficiently represent\n" + + "an array of basic values: characters, integers, floating point\n" + + "numbers. Arrays are sequence types and behave very much like lists,\n" + + "except that the type of objects stored in them is constrained.\n", + }, + Methods: []*py.Method{}, + Globals: py.StringDict{ + "typecodes": typecodes, + "array": ArrayType, + "ArrayType": ArrayType, + }, + }) + + ArrayType.Dict["itemsize"] = &py.Property{ + Fget: func(self py.Object) (py.Object, error) { + arr := self.(*array) + return py.Int(arr.esize), nil + }, + Doc: "the size, in bytes, of one array item", + } + + ArrayType.Dict["typecode"] = &py.Property{ + Fget: func(self py.Object) (py.Object, error) { + arr := self.(*array) + return py.String(arr.descr), nil + }, + Doc: "the typecode character used to create the array", + } + + ArrayType.Dict["append"] = py.MustNewMethod("append", array_append, 0, array_append_doc) + ArrayType.Dict["extend"] = py.MustNewMethod("extend", array_extend, 0, array_extend_doc) +} + +const array_doc = `array(typecode [, initializer]) -> array + +Return a new array whose items are restricted by typecode, and +initialized from the optional initializer value, which must be a list, +string or iterable over elements of the appropriate type. + +Arrays represent basic values and behave very much like lists, except +the type of objects stored in them is constrained. The type is specified +at object creation time by using a type code, which is a single character. +The following type codes are defined: + + Type code C Type Minimum size in bytes + 'b' signed integer 1 + 'B' unsigned integer 1 + 'u' Unicode character 2 (see note) + 'h' signed integer 2 + 'H' unsigned integer 2 + 'i' signed integer 2 + 'I' unsigned integer 2 + 'l' signed integer 4 + 'L' unsigned integer 4 + 'q' signed integer 8 (see note) + 'Q' unsigned integer 8 (see note) + 'f' floating point 4 + 'd' floating point 8 + +NOTE: The 'u' typecode corresponds to Python's unicode character. On +narrow builds this is 2-bytes on wide builds this is 4-bytes. + +NOTE: The 'q' and 'Q' type codes are only available if the platform +C compiler used to build Python supports 'long long', or, on Windows, +'__int64'. + +Methods: + +append() -- append a new item to the end of the array +buffer_info() -- return information giving the current memory info +byteswap() -- byteswap all the items of the array +count() -- return number of occurrences of an object +extend() -- extend array by appending multiple elements from an iterable +fromfile() -- read items from a file object +fromlist() -- append items from the list +frombytes() -- append items from the string +index() -- return index of first occurrence of an object +insert() -- insert a new item into the array at a provided position +pop() -- remove and return item (default last) +remove() -- remove first occurrence of an object +reverse() -- reverse the order of the items in the array +tofile() -- write all items to a file object +tolist() -- return the array converted to an ordinary list +tobytes() -- return the array converted to a string + +Attributes: + +typecode -- the typecode character used to create the array +itemsize -- the length in bytes of one array item + +` + +func array_new(metatype *py.Type, args py.Tuple, kwargs py.StringDict) (py.Object, error) { + switch n := len(args); n { + case 0: + return nil, py.ExceptionNewf(py.TypeError, "array() takes at least 1 argument (0 given)") + case 1, 2: + // ok + default: + return nil, py.ExceptionNewf(py.TypeError, "array() takes at most 2 arguments (%d given)", n) + } + + if len(kwargs) != 0 { + return nil, py.ExceptionNewf(py.TypeError, "array.array() takes no keyword arguments") + } + + descr, ok := args[0].(py.String) + if !ok { + return nil, py.ExceptionNewf(py.TypeError, "array() argument 1 must be a unicode character, not %s", args[0].Type().Name) + } + + if len(descr) != 1 { + return nil, py.ExceptionNewf(py.TypeError, "array() argument 1 must be a unicode character, not str") + } + + if !strings.ContainsAny(string(descr), string(typecodes)) { + ts := new(strings.Builder) + for i, v := range typecodes { + if i > 0 { + switch { + case i == len(typecodes)-1: + ts.WriteString(" or ") + default: + ts.WriteString(", ") + } + } + ts.WriteString(string(v)) + } + return nil, py.ExceptionNewf(py.ValueError, "bad typecode (must be %s)", ts) + } + + arr := &array{ + descr: descr[0], + esize: descr2esize[descr[0]], + } + + switch descr[0] { + case 'u': + var data []rune + arr.data = data + arr.append = arr.appendRune + arr.extend = arr.extendRune + case 'b': + var data []int8 + arr.data = data + arr.append = arr.appendI8 + arr.extend = arr.extendI8 + case 'h': + var data []int16 + arr.data = data + arr.append = arr.appendI16 + arr.extend = arr.extendI16 + case 'i': + var data []int32 + arr.data = data + arr.append = arr.appendI32 + arr.extend = arr.extendI32 + case 'l', 'q': + var data []int64 + arr.data = data + arr.append = arr.appendI64 + arr.extend = arr.extendI64 + case 'B': + var data []uint8 + arr.data = data + arr.append = arr.appendU8 + arr.extend = arr.extendU8 + case 'H': + var data []uint16 + arr.data = data + arr.append = arr.appendU16 + arr.extend = arr.extendU16 + case 'I': + var data []uint32 + arr.data = data + arr.append = arr.appendU32 + arr.extend = arr.extendU32 + case 'L', 'Q': + var data []uint64 + arr.data = data + arr.append = arr.appendU64 + arr.extend = arr.extendU64 + case 'f': + var data []float32 + arr.data = data + arr.append = arr.appendF32 + arr.extend = arr.extendF32 + case 'd': + var data []float64 + arr.data = data + arr.append = arr.appendF64 + arr.extend = arr.extendF64 + } + + if len(args) == 2 { + _, err := arr.extend(args[1]) + if err != nil { + return nil, err + } + } + + return arr, nil +} + +const array_append_doc = `Append new value v to the end of the array.` + +func array_append(self py.Object, args py.Tuple) (py.Object, error) { + arr, ok := self.(*array) + if !ok { + return nil, py.ExceptionNewf(py.TypeError, "expected an array, got '%s'", self.Type().Name) + } + if len(args) != 1 { + return nil, py.ExceptionNewf(py.TypeError, "array.append() takes exactly one argument (%d given)", len(args)) + } + + return arr.append(args[0]) +} + +const array_extend_doc = `Append items to the end of the array.` + +func array_extend(self py.Object, args py.Tuple) (py.Object, error) { + arr, ok := self.(*array) + if !ok { + return nil, py.ExceptionNewf(py.TypeError, "expected an array, got '%s'", self.Type().Name) + } + if len(args) == 0 { + return nil, py.ExceptionNewf(py.TypeError, "extend() takes exactly 1 positional argument (%d given)", len(args)) + } + if len(args) != 1 { + return nil, py.ExceptionNewf(py.TypeError, "extend() takes at most 1 argument (%d given)", len(args)) + } + + return arr.extend(args[0]) +} + +func (arr *array) M__repr__() (py.Object, error) { + o := new(strings.Builder) + o.WriteString("array('" + string(arr.descr) + "'") + if data := reflect.ValueOf(arr.data); arr.data != nil && data.Len() > 0 { + switch arr.descr { + case 'u': + o.WriteString(", '") + o.WriteString(string(arr.data.([]rune))) + o.WriteString("'") + default: + o.WriteString(", [") + for i := 0; i < data.Len(); i++ { + if i > 0 { + o.WriteString(", ") + } + // FIXME(sbinet): we don't get exactly the same display wrt CPython for float32 + fmt.Fprintf(o, "%v", data.Index(i)) + } + o.WriteString("]") + } + } + o.WriteString(")") + return py.String(o.String()), nil +} + +func (arr *array) M__str__() (py.Object, error) { + return arr.M__repr__() +} + +func (arr *array) M__len__() (py.Object, error) { + if arr.data == nil { + return py.Int(0), nil + } + sli := reflect.ValueOf(arr.data) + return py.Int(sli.Len()), nil +} + +func (arr *array) M__getitem__(k py.Object) (py.Object, error) { + switch k := k.(type) { + case py.Int: + var ( + sli = reflect.ValueOf(arr.data) + i = int(k) + ) + if i < 0 { + i = sli.Len() + i + } + if i < 0 || sli.Len() <= i { + return nil, py.ExceptionNewf(py.IndexError, "array index out of range") + } + switch arr.descr { + case 'b', 'h', 'i', 'l', 'q': + return py.Int(sli.Index(i).Int()), nil + case 'B', 'H', 'I', 'L', 'Q': + return py.Int(sli.Index(i).Uint()), nil + case 'u': + return py.String([]rune{rune(sli.Index(i).Int())}), nil + case 'f', 'd': + return py.Float(sli.Index(i).Float()), nil + } + case *py.Slice: + return nil, py.NotImplementedError + default: + return nil, py.ExceptionNewf(py.TypeError, "array indices must be integers") + } + panic("impossible") +} + +func (arr *array) M__setitem__(k, v py.Object) (py.Object, error) { + switch k := k.(type) { + case py.Int: + var ( + sli = reflect.ValueOf(arr.data) + i = int(k) + ) + if i < 0 { + i = sli.Len() + i + } + if i < 0 || sli.Len() <= i { + return nil, py.ExceptionNewf(py.IndexError, "array index out of range") + } + switch arr.descr { + case 'b', 'h', 'i', 'l', 'q': + vv, ok := v.(py.Int) + if !ok { + return nil, py.ExceptionNewf(py.TypeError, "'%s' object cannot be interpreted as an integer", v.Type().Name) + } + sli.Index(i).SetInt(int64(vv)) + case 'B', 'H', 'I', 'L', 'Q': + vv, ok := v.(py.Int) + if !ok { + return nil, py.ExceptionNewf(py.TypeError, "'%s' object cannot be interpreted as an integer", v.Type().Name) + } + sli.Index(i).SetUint(uint64(vv)) + case 'u': + vv, ok := v.(py.Int) + if !ok { + return nil, py.ExceptionNewf(py.TypeError, "array item must be unicode character") + } + sli.Index(i).SetInt(int64(vv)) + case 'f', 'd': + var vv float64 + switch v := v.(type) { + case py.Int: + vv = float64(v) + case py.Float: + vv = float64(v) + default: + return nil, py.ExceptionNewf(py.TypeError, "must be real number, not %s", v.Type().Name) + } + sli.Index(i).SetFloat(vv) + } + return py.None, nil + case *py.Slice: + return nil, py.NotImplementedError + default: + return nil, py.ExceptionNewf(py.TypeError, "array indices must be integers") + } + panic("impossible") +} + +func (arr *array) appendRune(v py.Object) (py.Object, error) { + str, ok := v.(py.String) + if !ok { + return nil, py.ExceptionNewf(py.TypeError, "array item must be unicode character") + } + + arr.data = append(arr.data.([]rune), []rune(str)...) + return py.None, nil +} + +func (arr *array) appendI8(v py.Object) (py.Object, error) { + vv, err := asInt(v) + if err != nil { + return nil, err + } + arr.data = append(arr.data.([]int8), int8(vv)) + return py.None, nil +} + +func (arr *array) appendI16(v py.Object) (py.Object, error) { + vv, err := asInt(v) + if err != nil { + return nil, err + } + arr.data = append(arr.data.([]int16), int16(vv)) + return py.None, nil +} + +func (arr *array) appendI32(v py.Object) (py.Object, error) { + vv, err := asInt(v) + if err != nil { + return nil, err + } + arr.data = append(arr.data.([]int32), int32(vv)) + return py.None, nil +} + +func (arr *array) appendI64(v py.Object) (py.Object, error) { + vv, err := asInt(v) + if err != nil { + return nil, err + } + arr.data = append(arr.data.([]int64), int64(vv)) + return py.None, nil +} + +func (arr *array) appendU8(v py.Object) (py.Object, error) { + vv, err := asInt(v) + if err != nil { + return nil, err + } + arr.data = append(arr.data.([]uint8), uint8(vv)) + return py.None, nil +} + +func (arr *array) appendU16(v py.Object) (py.Object, error) { + vv, err := asInt(v) + if err != nil { + return nil, err + } + arr.data = append(arr.data.([]uint16), uint16(vv)) + return py.None, nil +} + +func (arr *array) appendU32(v py.Object) (py.Object, error) { + vv, err := asInt(v) + if err != nil { + return nil, err + } + arr.data = append(arr.data.([]uint32), uint32(vv)) + return py.None, nil +} + +func (arr *array) appendU64(v py.Object) (py.Object, error) { + vv, err := asInt(v) + if err != nil { + return nil, err + } + arr.data = append(arr.data.([]uint64), uint64(vv)) + return py.None, nil +} + +func (arr *array) appendF32(v py.Object) (py.Object, error) { + vv, err := py.FloatAsFloat64(v) + if err != nil { + return nil, err + } + arr.data = append(arr.data.([]float32), float32(vv)) + return py.None, nil +} + +func (arr *array) appendF64(v py.Object) (py.Object, error) { + vv, err := py.FloatAsFloat64(v) + if err != nil { + return nil, err + } + arr.data = append(arr.data.([]float64), float64(vv)) + return py.None, nil +} + +func (arr *array) extendRune(arg py.Object) (py.Object, error) { + itr, err := py.Iter(arg) + if err != nil { + return nil, err + } + + nxt := itr.(py.I__next__) + + for { + o, err := nxt.M__next__() + if err == py.StopIteration { + break + } + _, err = arr.appendRune(o) + if err != nil { + return nil, err + } + } + return py.None, nil +} + +func (arr *array) extendI8(arg py.Object) (py.Object, error) { + itr, err := py.Iter(arg) + if err != nil { + return nil, err + } + + nxt := itr.(py.I__next__) + + for { + o, err := nxt.M__next__() + if err == py.StopIteration { + break + } + _, err = arr.appendI8(o) + if err != nil { + return nil, err + } + } + return py.None, nil +} + +func (arr *array) extendI16(arg py.Object) (py.Object, error) { + itr, err := py.Iter(arg) + if err != nil { + return nil, err + } + + nxt := itr.(py.I__next__) + + for { + o, err := nxt.M__next__() + if err == py.StopIteration { + break + } + _, err = arr.appendI16(o) + if err != nil { + return nil, err + } + } + return py.None, nil +} + +func (arr *array) extendI32(arg py.Object) (py.Object, error) { + itr, err := py.Iter(arg) + if err != nil { + return nil, err + } + + nxt := itr.(py.I__next__) + + for { + o, err := nxt.M__next__() + if err == py.StopIteration { + break + } + _, err = arr.appendI32(o) + if err != nil { + return nil, err + } + } + return py.None, nil +} + +func (arr *array) extendI64(arg py.Object) (py.Object, error) { + itr, err := py.Iter(arg) + if err != nil { + return nil, err + } + + nxt := itr.(py.I__next__) + + for { + o, err := nxt.M__next__() + if err == py.StopIteration { + break + } + _, err = arr.appendI64(o) + if err != nil { + return nil, err + } + } + return py.None, nil +} + +func (arr *array) extendU8(arg py.Object) (py.Object, error) { + itr, err := py.Iter(arg) + if err != nil { + return nil, err + } + + nxt := itr.(py.I__next__) + + for { + o, err := nxt.M__next__() + if err == py.StopIteration { + break + } + _, err = arr.appendU8(o) + if err != nil { + return nil, err + } + } + return py.None, nil +} + +func (arr *array) extendU16(arg py.Object) (py.Object, error) { + itr, err := py.Iter(arg) + if err != nil { + return nil, err + } + + nxt := itr.(py.I__next__) + + for { + o, err := nxt.M__next__() + if err == py.StopIteration { + break + } + _, err = arr.appendU16(o) + if err != nil { + return nil, err + } + } + return py.None, nil +} + +func (arr *array) extendU32(arg py.Object) (py.Object, error) { + itr, err := py.Iter(arg) + if err != nil { + return nil, err + } + + nxt := itr.(py.I__next__) + + for { + o, err := nxt.M__next__() + if err == py.StopIteration { + break + } + _, err = arr.appendU32(o) + if err != nil { + return nil, err + } + } + return py.None, nil +} + +func (arr *array) extendU64(arg py.Object) (py.Object, error) { + itr, err := py.Iter(arg) + if err != nil { + return nil, err + } + + nxt := itr.(py.I__next__) + + for { + o, err := nxt.M__next__() + if err == py.StopIteration { + break + } + _, err = arr.appendU64(o) + if err != nil { + return nil, err + } + } + return py.None, nil +} + +func (arr *array) extendF32(arg py.Object) (py.Object, error) { + itr, err := py.Iter(arg) + if err != nil { + return nil, err + } + + nxt := itr.(py.I__next__) + + for { + o, err := nxt.M__next__() + if err == py.StopIteration { + break + } + _, err = arr.appendF32(o) + if err != nil { + return nil, err + } + } + return py.None, nil +} + +func (arr *array) extendF64(arg py.Object) (py.Object, error) { + itr, err := py.Iter(arg) + if err != nil { + return nil, err + } + + nxt := itr.(py.I__next__) + + for { + o, err := nxt.M__next__() + if err == py.StopIteration { + break + } + _, err = arr.appendF64(o) + if err != nil { + return nil, err + } + } + return py.None, nil +} + +func asInt(o py.Object) (int64, error) { + v, ok := o.(py.Int) + if !ok { + return 0, py.ExceptionNewf(py.TypeError, "unsupported operand type(s) for int: '%s'", o.Type().Name) + } + return int64(v), nil +} diff --git a/stdlib/array/array_test.go b/stdlib/array/array_test.go new file mode 100644 index 00000000..a34ed842 --- /dev/null +++ b/stdlib/array/array_test.go @@ -0,0 +1,15 @@ +// Copyright 2023 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package array_test + +import ( + "testing" + + "github.com/go-python/gpython/pytest" +) + +func TestArray(t *testing.T) { + pytest.RunScript(t, "./testdata/test.py") +} diff --git a/stdlib/array/testdata/test.py b/stdlib/array/testdata/test.py new file mode 100644 index 00000000..906ed898 --- /dev/null +++ b/stdlib/array/testdata/test.py @@ -0,0 +1,173 @@ +# Copyright 2023 The go-python Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +import array + +print("globals:") +for name in ("typecodes", "array"): + v = getattr(array, name) + print("\narray.%s:\n%s" % (name,repr(v))) + pass + +def assertEqual(x, y): + assert x == y, "got: %s, want: %s" % (repr(x), repr(y)) + +assertEqual(array.typecodes, 'bBuhHiIlLqQfd') + +for i, typ in enumerate(array.typecodes): + print("") + print("typecode '%s'" % (typ,)) + if typ == 'u': + arr = array.array(typ, "?世界!") + if typ in "bhilq": + arr = array.array(typ, [-1, -2, -3, -4]) + if typ in "BHILQ": + arr = array.array(typ, [+1, +2, +3, +4]) + if typ in "fd": + arr = array.array(typ, [-1.0, -2.0, -3.0, -4.0]) + print(" array: %s ## repr" % (repr(arr),)) + print(" array: %s ## str" % (str(arr),)) + print(" itemsize: %s" % (arr.itemsize,)) + print(" typecode: %s" % (arr.typecode,)) + print(" len: %s" % (len(arr),)) + print(" arr[0]: %s" % (arr[0],)) + print(" arr[-1]: %s" % (arr[-1],)) + try: + arr[-10] + print(" ERROR1: expected an exception") + except: + print(" caught an exception [ok]") + + try: + arr[10] + print(" ERROR2: expected an exception") + except: + print(" caught an exception [ok]") + arr[-2] = 33 + if typ in "fd": + arr[-2] = 0.3 + print(" arr[-2]: %s" % (arr[-2],)) + + try: + arr[-10] = 2 + print(" ERROR3: expected an exception") + except: + print(" caught an exception [ok]") + + if typ in "bhilqfd": + arr.extend([-5,-6]) + if typ in "BHILQ": + arr.extend([5,6]) + if typ == 'u': + arr.extend("he") + print(" array: %s" % (repr(arr),)) + print(" len: %s" % (len(arr),)) + + if typ in "bhilqfd": + arr.append(-7) + if typ in "BHILQ": + arr.append(7) + if typ == 'u': + arr.append("l") + print(" array: %s" % (repr(arr),)) + print(" len: %s" % (len(arr),)) + + try: + arr.append() + print(" ERROR4: expected an exception") + except: + print(" caught an exception [ok]") + try: + arr.append([]) + print(" ERROR5: expected an exception") + except: + print(" caught an exception [ok]") + try: + arr.append(1, 2) + print(" ERROR6: expected an exception") + except: + print(" caught an exception [ok]") + try: + arr.append(None) + print(" ERROR7: expected an exception") + except: + print(" caught an exception [ok]") + + try: + arr.extend() + print(" ERROR8: expected an exception") + except: + print(" caught an exception [ok]") + try: + arr.extend(None) + print(" ERROR9: expected an exception") + except: + print(" caught an exception [ok]") + try: + arr.extend([1,None]) + print(" ERROR10: expected an exception") + except: + print(" caught an exception [ok]") + try: + arr.extend(1,None) + print(" ERROR11: expected an exception") + except: + print(" caught an exception [ok]") + + try: + arr[0] = object() + print(" ERROR12: expected an exception") + except: + print(" caught an exception [ok]") + pass + +print("\n") +print("## testing array.array(...)") +try: + arr = array.array() + print("ERROR1: expected an exception") +except: + print("caught an exception [ok]") + +try: + arr = array.array(b"d") + print("ERROR2: expected an exception") +except: + print("caught an exception [ok]") + +try: + arr = array.array("?") + print("ERROR3: expected an exception") +except: + print("caught an exception [ok]") + +try: + arr = array.array("dd") + print("ERROR4: expected an exception") +except: + print("caught an exception [ok]") + +try: + arr = array.array("d", initializer=[1,2]) + print("ERROR5: expected an exception") +except: + print("caught an exception [ok]") + +try: + arr = array.array("d", [1], []) + print("ERROR6: expected an exception") +except: + print("caught an exception [ok]") + +try: + arr = array.array("d", 1) + print("ERROR7: expected an exception") +except: + print("caught an exception [ok]") + +try: + arr = array.array("d", ["a","b"]) + print("ERROR8: expected an exception") +except: + print("caught an exception [ok]") diff --git a/stdlib/array/testdata/test_golden.txt b/stdlib/array/testdata/test_golden.txt new file mode 100644 index 00000000..09e9db3d --- /dev/null +++ b/stdlib/array/testdata/test_golden.txt @@ -0,0 +1,356 @@ +globals: + +array.typecodes: +'bBuhHiIlLqQfd' + +array.array: + + +typecode 'b' + array: array('b', [-1, -2, -3, -4]) ## repr + array: array('b', [-1, -2, -3, -4]) ## str + itemsize: 1 + typecode: b + len: 4 + arr[0]: -1 + arr[-1]: -4 + caught an exception [ok] + caught an exception [ok] + arr[-2]: 33 + caught an exception [ok] + array: array('b', [-1, -2, 33, -4, -5, -6]) + len: 6 + array: array('b', [-1, -2, 33, -4, -5, -6, -7]) + len: 7 + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + +typecode 'B' + array: array('B', [1, 2, 3, 4]) ## repr + array: array('B', [1, 2, 3, 4]) ## str + itemsize: 1 + typecode: B + len: 4 + arr[0]: 1 + arr[-1]: 4 + caught an exception [ok] + caught an exception [ok] + arr[-2]: 33 + caught an exception [ok] + array: array('B', [1, 2, 33, 4, 5, 6]) + len: 6 + array: array('B', [1, 2, 33, 4, 5, 6, 7]) + len: 7 + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + +typecode 'u' + array: array('u', '?世界!') ## repr + array: array('u', '?世界!') ## str + itemsize: 2 + typecode: u + len: 4 + arr[0]: ? + arr[-1]: ! + caught an exception [ok] + caught an exception [ok] + arr[-2]: ! + caught an exception [ok] + array: array('u', '?世!!he') + len: 6 + array: array('u', '?世!!hel') + len: 7 + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + +typecode 'h' + array: array('h', [-1, -2, -3, -4]) ## repr + array: array('h', [-1, -2, -3, -4]) ## str + itemsize: 2 + typecode: h + len: 4 + arr[0]: -1 + arr[-1]: -4 + caught an exception [ok] + caught an exception [ok] + arr[-2]: 33 + caught an exception [ok] + array: array('h', [-1, -2, 33, -4, -5, -6]) + len: 6 + array: array('h', [-1, -2, 33, -4, -5, -6, -7]) + len: 7 + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + +typecode 'H' + array: array('H', [1, 2, 3, 4]) ## repr + array: array('H', [1, 2, 3, 4]) ## str + itemsize: 2 + typecode: H + len: 4 + arr[0]: 1 + arr[-1]: 4 + caught an exception [ok] + caught an exception [ok] + arr[-2]: 33 + caught an exception [ok] + array: array('H', [1, 2, 33, 4, 5, 6]) + len: 6 + array: array('H', [1, 2, 33, 4, 5, 6, 7]) + len: 7 + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + +typecode 'i' + array: array('i', [-1, -2, -3, -4]) ## repr + array: array('i', [-1, -2, -3, -4]) ## str + itemsize: 2 + typecode: i + len: 4 + arr[0]: -1 + arr[-1]: -4 + caught an exception [ok] + caught an exception [ok] + arr[-2]: 33 + caught an exception [ok] + array: array('i', [-1, -2, 33, -4, -5, -6]) + len: 6 + array: array('i', [-1, -2, 33, -4, -5, -6, -7]) + len: 7 + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + +typecode 'I' + array: array('I', [1, 2, 3, 4]) ## repr + array: array('I', [1, 2, 3, 4]) ## str + itemsize: 2 + typecode: I + len: 4 + arr[0]: 1 + arr[-1]: 4 + caught an exception [ok] + caught an exception [ok] + arr[-2]: 33 + caught an exception [ok] + array: array('I', [1, 2, 33, 4, 5, 6]) + len: 6 + array: array('I', [1, 2, 33, 4, 5, 6, 7]) + len: 7 + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + +typecode 'l' + array: array('l', [-1, -2, -3, -4]) ## repr + array: array('l', [-1, -2, -3, -4]) ## str + itemsize: 8 + typecode: l + len: 4 + arr[0]: -1 + arr[-1]: -4 + caught an exception [ok] + caught an exception [ok] + arr[-2]: 33 + caught an exception [ok] + array: array('l', [-1, -2, 33, -4, -5, -6]) + len: 6 + array: array('l', [-1, -2, 33, -4, -5, -6, -7]) + len: 7 + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + +typecode 'L' + array: array('L', [1, 2, 3, 4]) ## repr + array: array('L', [1, 2, 3, 4]) ## str + itemsize: 8 + typecode: L + len: 4 + arr[0]: 1 + arr[-1]: 4 + caught an exception [ok] + caught an exception [ok] + arr[-2]: 33 + caught an exception [ok] + array: array('L', [1, 2, 33, 4, 5, 6]) + len: 6 + array: array('L', [1, 2, 33, 4, 5, 6, 7]) + len: 7 + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + +typecode 'q' + array: array('q', [-1, -2, -3, -4]) ## repr + array: array('q', [-1, -2, -3, -4]) ## str + itemsize: 8 + typecode: q + len: 4 + arr[0]: -1 + arr[-1]: -4 + caught an exception [ok] + caught an exception [ok] + arr[-2]: 33 + caught an exception [ok] + array: array('q', [-1, -2, 33, -4, -5, -6]) + len: 6 + array: array('q', [-1, -2, 33, -4, -5, -6, -7]) + len: 7 + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + +typecode 'Q' + array: array('Q', [1, 2, 3, 4]) ## repr + array: array('Q', [1, 2, 3, 4]) ## str + itemsize: 8 + typecode: Q + len: 4 + arr[0]: 1 + arr[-1]: 4 + caught an exception [ok] + caught an exception [ok] + arr[-2]: 33 + caught an exception [ok] + array: array('Q', [1, 2, 33, 4, 5, 6]) + len: 6 + array: array('Q', [1, 2, 33, 4, 5, 6, 7]) + len: 7 + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + +typecode 'f' + array: array('f', [-1, -2, -3, -4]) ## repr + array: array('f', [-1, -2, -3, -4]) ## str + itemsize: 4 + typecode: f + len: 4 + arr[0]: -1 + arr[-1]: -4 + caught an exception [ok] + caught an exception [ok] + arr[-2]: 0.30000001192092896 + caught an exception [ok] + array: array('f', [-1, -2, 0.3, -4, -5, -6]) + len: 6 + array: array('f', [-1, -2, 0.3, -4, -5, -6, -7]) + len: 7 + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + +typecode 'd' + array: array('d', [-1, -2, -3, -4]) ## repr + array: array('d', [-1, -2, -3, -4]) ## str + itemsize: 8 + typecode: d + len: 4 + arr[0]: -1 + arr[-1]: -4 + caught an exception [ok] + caught an exception [ok] + arr[-2]: 0.3 + caught an exception [ok] + array: array('d', [-1, -2, 0.3, -4, -5, -6]) + len: 6 + array: array('d', [-1, -2, 0.3, -4, -5, -6, -7]) + len: 7 + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + caught an exception [ok] + + +## testing array.array(...) +caught an exception [ok] +caught an exception [ok] +caught an exception [ok] +caught an exception [ok] +caught an exception [ok] +caught an exception [ok] +caught an exception [ok] +caught an exception [ok] diff --git a/stdlib/binascii/binascii.go b/stdlib/binascii/binascii.go new file mode 100644 index 00000000..f39af440 --- /dev/null +++ b/stdlib/binascii/binascii.go @@ -0,0 +1,235 @@ +// Copyright 2022 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package binascii provides the implementation of the python's 'binascii' module. +package binascii + +import ( + "bytes" + "encoding/base64" + "encoding/hex" + "errors" + "hash/crc32" + "io" + "mime/quotedprintable" + + "github.com/go-python/gpython/py" +) + +var ( + Incomplete = py.ExceptionType.NewType("binascii.Incomplete", "", nil, nil) + Error = py.ValueError.NewType("binascii.Error", "", nil, nil) +) + +func init() { + py.RegisterModule(&py.ModuleImpl{ + Info: py.ModuleInfo{ + Name: "binascii", + Doc: "Conversion between binary data and ASCII", + }, + Methods: []*py.Method{ + py.MustNewMethod("a2b_base64", a2b_base64, 0, "Decode a line of base64 data."), + py.MustNewMethod("b2a_base64", b2a_base64, 0, "Base64-code line of data."), + py.MustNewMethod("a2b_hex", a2b_hex, 0, a2b_hex_doc), + py.MustNewMethod("b2a_hex", b2a_hex, 0, b2a_hex_doc), + py.MustNewMethod("crc32", crc32_, 0, "Compute CRC-32 incrementally."), + py.MustNewMethod("unhexlify", a2b_hex, 0, unhexlify_doc), + py.MustNewMethod("hexlify", b2a_hex, 0, hexlify_doc), + py.MustNewMethod("a2b_qp", a2b_qp, 0, a2b_qp_doc), + py.MustNewMethod("b2a_qp", b2a_qp, 0, b2a_qp_doc), + }, + Globals: py.StringDict{ + "Incomplete": Incomplete, + "Error": Error, + }, + }) +} + +func b2a_base64(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Object, error) { + var ( + pydata py.Object + pynewl py.Object = py.True + ) + err := py.ParseTupleAndKeywords(args, kwargs, "y*|p:binascii.b2a_base64", []string{"data", "newline"}, &pydata, &pynewl) + if err != nil { + return nil, err + } + + var ( + buf = []byte(pydata.(py.Bytes)) + newline = bool(pynewl.(py.Bool)) + ) + + out := base64.StdEncoding.EncodeToString(buf) + if newline { + out += "\n" + } + return py.Bytes(out), nil +} + +func a2b_base64(self py.Object, args py.Tuple) (py.Object, error) { + var pydata py.Object + err := py.ParseTuple(args, "s:binascii.a2b_base64", &pydata) + if err != nil { + return nil, err + } + + out, err := base64.StdEncoding.DecodeString(string(pydata.(py.String))) + if err != nil { + return nil, py.ExceptionNewf(Error, "could not decode base64 data: %+v", err) + } + + return py.Bytes(out), nil +} + +func crc32_(self py.Object, args py.Tuple) (py.Object, error) { + var ( + pydata py.Object + pycrc py.Object = py.Int(0) + ) + + err := py.ParseTuple(args, "y*|i:binascii.crc32", &pydata, &pycrc) + if err != nil { + return nil, err + } + + crc := crc32.Update(uint32(pycrc.(py.Int)), crc32.IEEETable, []byte(pydata.(py.Bytes))) + return py.Int(crc), nil + +} + +const a2b_hex_doc = `Binary data of hexadecimal representation. + +hexstr must contain an even number of hex digits (upper or lower case). +This function is also available as "unhexlify()".` + +func a2b_hex(self py.Object, args py.Tuple) (py.Object, error) { + var ( + hexErr hex.InvalidByteError + pydata py.Object + src string + ) + err := py.ParseTuple(args, "s*:binascii.a2b_hex", &pydata) + if err != nil { + return nil, err + } + + switch v := pydata.(type) { + case py.String: + src = string(v) + case py.Bytes: + src = string(v) + } + + o, err := hex.DecodeString(src) + if err != nil { + switch { + case errors.Is(err, hex.ErrLength): + return nil, py.ExceptionNewf(Error, "Odd-length string") + case errors.As(err, &hexErr): + return nil, py.ExceptionNewf(Error, "Non-hexadecimal digit found") + default: + return nil, py.ExceptionNewf(Error, "could not decode hex data: %+v", err) + } + } + + return py.Bytes(o), nil +} + +const b2a_hex_doc = `Hexadecimal representation of binary data. + +The return value is a bytes object. This function is also +available as "hexlify()".` + +func b2a_hex(self py.Object, args py.Tuple) (py.Object, error) { + var pydata py.Object + err := py.ParseTuple(args, "y*:binascii.b2a_hex", &pydata) + if err != nil { + return nil, err + } + + o := hex.EncodeToString([]byte(pydata.(py.Bytes))) + return py.Bytes(o), nil +} + +const unhexlify_doc = `Binary data of hexadecimal representation. + +hexstr must contain an even number of hex digits (upper or lower case).` + +const hexlify_doc = `Hexadecimal representation of binary data. + +The return value is a bytes object. This function is also +available as "b2a_hex()".` + +const a2b_qp_doc = `Decode a string of qp-encoded data.` + +func a2b_qp(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Object, error) { + var ( + pydata py.Object + pyhdr py.Object = py.Bool(false) + ) + err := py.ParseTupleAndKeywords(args, kwargs, "y*|p:binascii.a2b_qp", []string{"data", "header"}, &pydata, &pyhdr) + if err != nil { + return nil, err + } + + // TODO(sbinet) + if pyhdr.(py.Bool) { + return nil, py.NotImplementedError + } + + o := new(bytes.Buffer) + r := quotedprintable.NewReader(bytes.NewReader([]byte(pydata.(py.Bytes)))) + _, err = io.Copy(o, r) + if err != nil { + // FIXME(sbinet): decorate the error somehow? + return nil, err + } + return py.Bytes(o.Bytes()), nil +} + +const b2a_qp_doc = `Encode a string using quoted-printable encoding. + +On encoding, when istext is set, newlines are not encoded, and white +space at end of lines is. When istext is not set, \r and \n (CR/LF) +are both encoded. When quotetabs is set, space and tabs are encoded.` + +func b2a_qp(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Object, error) { + var ( + pydata py.Object + pyqtabs py.Object = py.Bool(false) + pyistxt py.Object = py.Bool(true) + pyhdr py.Object = py.Bool(false) + ) + err := py.ParseTupleAndKeywords(args, kwargs, "y*|ppp:binascii.b2a_qp", []string{"data", "quotetabs", "istext", "header"}, &pydata, &pyqtabs, &pyistxt, &pyhdr) + if err != nil { + return nil, err + } + + if pyqtabs.(py.Bool) { + return nil, py.NotImplementedError + } + + if !pyistxt.(py.Bool) { + return nil, py.NotImplementedError + } + + if pyhdr.(py.Bool) { + return nil, py.NotImplementedError + } + + o := new(bytes.Buffer) + w := quotedprintable.NewWriter(o) + _, err = w.Write([]byte(pydata.(py.Bytes))) + if err != nil { + // FIXME(sbinet): decorate the error somehow? + return nil, err + } + err = w.Close() + if err != nil { + // FIXME(sbinet): decorate the error somehow? + return nil, err + } + return py.Bytes(o.Bytes()), nil +} diff --git a/stdlib/binascii/binascii_test.go b/stdlib/binascii/binascii_test.go new file mode 100644 index 00000000..bd40c782 --- /dev/null +++ b/stdlib/binascii/binascii_test.go @@ -0,0 +1,15 @@ +// Copyright 2022 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package binascii_test + +import ( + "testing" + + "github.com/go-python/gpython/pytest" +) + +func TestBinascii(t *testing.T) { + pytest.RunScript(t, "./testdata/test.py") +} diff --git a/stdlib/binascii/testdata/test.py b/stdlib/binascii/testdata/test.py new file mode 100644 index 00000000..840c8672 --- /dev/null +++ b/stdlib/binascii/testdata/test.py @@ -0,0 +1,72 @@ +# Copyright 2022 The go-python Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +import binascii + +print("globals:") +for name in ("Error", "Incomplete"): + v = getattr(binascii, name) + print("\nbinascii.%s:\n%s" % (name,repr(v))) + +def assertEqual(x, y): + assert x == y, "got: %s, want: %s" % (repr(x), repr(y)) + +rawdata = b"The quick brown fox jumps over the lazy dog.\r\n" +rawdata += bytes(range(256)) +rawdata += b"\r\nHello world.\n" + +## base64 +assertEqual(binascii.b2a_base64(b'hello world!'), b'aGVsbG8gd29ybGQh\n') +assertEqual(binascii.b2a_base64(b'hello\nworld!'), b'aGVsbG8Kd29ybGQh\n') +assertEqual(binascii.b2a_base64(b'hello world!', newline=False), b'aGVsbG8gd29ybGQh') +assertEqual(binascii.b2a_base64(b'hello\nworld!', newline=False), b'aGVsbG8Kd29ybGQh') +assertEqual(binascii.a2b_base64("aGVsbG8gd29ybGQh\n"), b'hello world!') + +assertEqual(binascii.b2a_base64(rawdata), b"VGhlIHF1aWNrIGJyb3duIGZveCBqdW1wcyBvdmVyIHRoZSBsYXp5IGRvZy4NCgABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fICEiIyQlJicoKSorLC0uLzAxMjM0NTY3ODk6Ozw9Pj9AQUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVpbXF1eX2BhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ent8fX5/gIGCg4SFhoeIiYqLjI2Oj5CRkpOUlZaXmJmam5ydnp+goaKjpKWmp6ipqqusra6vsLGys7S1tre4ubq7vL2+v8DBwsPExcbHyMnKy8zNzs/Q0dLT1NXW19jZ2tvc3d7f4OHi4+Tl5ufo6err7O3u7/Dx8vP09fb3+Pn6+/z9/v8NCkhlbGxvIHdvcmxkLgo=\n") + +try: + binascii.b2a_base64("string") + print("expected an exception") +except TypeError as e: + print("expected an exception:", e) + pass + +## crc32 +assertEqual(binascii.crc32(b'hello world!'), 62177901) +assertEqual(binascii.crc32(b'hello world!', 0), 62177901) +assertEqual(binascii.crc32(b'hello world!', 42), 4055036404) + +## hex +assertEqual(binascii.b2a_hex(b'hello world!'), b'68656c6c6f20776f726c6421') +assertEqual(binascii.a2b_hex(b'68656c6c6f20776f726c6421'), b'hello world!') +assertEqual(binascii.hexlify(b'hello world!'), b'68656c6c6f20776f726c6421') +assertEqual(binascii.unhexlify(b'68656c6c6f20776f726c6421'), b'hello world!') + +assertEqual(binascii.b2a_hex(rawdata), b'54686520717569636b2062726f776e20666f78206a756d7073206f76657220746865206c617a7920646f672e0d0a000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f202122232425262728292a2b2c2d2e2f303132333435363738393a3b3c3d3e3f404142434445464748494a4b4c4d4e4f505152535455565758595a5b5c5d5e5f606162636465666768696a6b6c6d6e6f707172737475767778797a7b7c7d7e7f808182838485868788898a8b8c8d8e8f909192939495969798999a9b9c9d9e9fa0a1a2a3a4a5a6a7a8a9aaabacadaeafb0b1b2b3b4b5b6b7b8b9babbbcbdbebfc0c1c2c3c4c5c6c7c8c9cacbcccdcecfd0d1d2d3d4d5d6d7d8d9dadbdcdddedfe0e1e2e3e4e5e6e7e8e9eaebecedeeeff0f1f2f3f4f5f6f7f8f9fafbfcfdfeff0d0a48656c6c6f20776f726c642e0a') + +try: + binascii.a2b_hex(b'123') + print("expected an exception") +except binascii.Error as e: + print("expected an exception:",e) + pass + +try: + binascii.a2b_hex(b'hell') + print("expected an exception") +except binascii.Error as e: + print("expected an exception:",e) + pass + +## quotedprintable +assertEqual(binascii.b2a_qp(b'hello world! = \t'), b'hello world! =3D =09') +assertEqual(binascii.a2b_qp(b'hello world! =3D =09'), b'hello world! = \t') +## ## TODO +## #assertEqual(binascii.a2b_qp(b'hello world!', header=True), b'hello world!') +## assertEqual(binascii.a2b_qp(rawdata, header=False), b'The quick brown fox jumps over the lazy dog.\r\n\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff\r\nHello world.\n') +## assertEqual(binascii.b2a_qp(binascii.a2b_qp(rawdata)), b'The quick brown fox jumps over the lazy dog.\r\n=00=01=02=03=04=05=06=07=08=09\r\n=0B=0C\r=0E=0F=10=11=12=13=14=15=16=17=18=19=1A=1B=1C=1D=1E=1F !"#$%&\'()*+,-=\r\n./0123456789:;<=3D>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuv=\r\nwxyz{|}~=7F=80=81=82=83=84=85=86=87=88=89=8A=8B=8C=8D=8E=8F=90=91=92=93=94=\r\n=95=96=97=98=99=9A=9B=9C=9D=9E=9F=A0=A1=A2=A3=A4=A5=A6=A7=A8=A9=AA=AB=AC=AD=\r\n=AE=AF=B0=B1=B2=B3=B4=B5=B6=B7=B8=B9=BA=BB=BC=BD=BE=BF=C0=C1=C2=C3=C4=C5=C6=\r\n=C7=C8=C9=CA=CB=CC=CD=CE=CF=D0=D1=D2=D3=D4=D5=D6=D7=D8=D9=DA=DB=DC=DD=DE=DF=\r\n=E0=E1=E2=E3=E4=E5=E6=E7=E8=E9=EA=EB=EC=ED=EE=EF=F0=F1=F2=F3=F4=F5=F6=F7=F8=\r\n=F9=FA=FB=FC=FD=FE=FF\r\nHello world.\r\n') +## ## TODO +## #assertEqual(binascii.a2b_qp(rawdata, header=True), b'The quick brown fox jumps over the lazy dog.\r\n\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^ `abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff\r\nHello world.\n') + +print("done.") diff --git a/stdlib/binascii/testdata/test_golden.txt b/stdlib/binascii/testdata/test_golden.txt new file mode 100644 index 00000000..29f03223 --- /dev/null +++ b/stdlib/binascii/testdata/test_golden.txt @@ -0,0 +1,11 @@ +globals: + +binascii.Error: + + +binascii.Incomplete: + +expected an exception: binascii.b2a_base64() argument 1 must be bytes-like, not str +expected an exception: Odd-length string +expected an exception: Non-hexadecimal digit found +done. diff --git a/builtin/builtin.go b/stdlib/builtin/builtin.go similarity index 69% rename from builtin/builtin.go rename to stdlib/builtin/builtin.go index 05896c78..290cb939 100644 --- a/builtin/builtin.go +++ b/stdlib/builtin/builtin.go @@ -6,6 +6,9 @@ package builtin import ( + "fmt" + "math/big" + "strconv" "unicode/utf8" "github.com/go-python/gpython/compile" @@ -24,12 +27,12 @@ func init() { py.MustNewMethod("abs", builtin_abs, 0, abs_doc), py.MustNewMethod("all", builtin_all, 0, all_doc), py.MustNewMethod("any", builtin_any, 0, any_doc), - // py.MustNewMethod("ascii", builtin_ascii, 0, ascii_doc), - // py.MustNewMethod("bin", builtin_bin, 0, bin_doc), + py.MustNewMethod("ascii", builtin_ascii, 0, ascii_doc), + py.MustNewMethod("bin", builtin_bin, 0, bin_doc), // py.MustNewMethod("callable", builtin_callable, 0, callable_doc), py.MustNewMethod("chr", builtin_chr, 0, chr_doc), py.MustNewMethod("compile", builtin_compile, 0, compile_doc), - // py.MustNewMethod("delattr", builtin_delattr, 0, delattr_doc), + py.MustNewMethod("delattr", builtin_delattr, 0, delattr_doc), // py.MustNewMethod("dir", builtin_dir, 0, dir_doc), py.MustNewMethod("divmod", builtin_divmod, 0, divmod_doc), py.MustNewMethod("eval", py.InternalMethodEval, 0, eval_doc), @@ -39,26 +42,26 @@ func init() { py.MustNewMethod("globals", py.InternalMethodGlobals, 0, globals_doc), py.MustNewMethod("hasattr", builtin_hasattr, 0, hasattr_doc), // py.MustNewMethod("hash", builtin_hash, 0, hash_doc), - // py.MustNewMethod("hex", builtin_hex, 0, hex_doc), + py.MustNewMethod("hex", builtin_hex, 0, hex_doc), // py.MustNewMethod("id", builtin_id, 0, id_doc), // py.MustNewMethod("input", builtin_input, 0, input_doc), - // py.MustNewMethod("isinstance", builtin_isinstance, 0, isinstance_doc), + py.MustNewMethod("isinstance", builtin_isinstance, 0, isinstance_doc), // py.MustNewMethod("issubclass", builtin_issubclass, 0, issubclass_doc), - // py.MustNewMethod("iter", builtin_iter, 0, iter_doc), + py.MustNewMethod("iter", builtin_iter, 0, iter_doc), py.MustNewMethod("len", builtin_len, 0, len_doc), py.MustNewMethod("locals", py.InternalMethodLocals, 0, locals_doc), - // py.MustNewMethod("max", builtin_max, 0, max_doc), - // py.MustNewMethod("min", builtin_min, 0, min_doc), + py.MustNewMethod("max", builtin_max, 0, max_doc), + py.MustNewMethod("min", builtin_min, 0, min_doc), py.MustNewMethod("next", builtin_next, 0, next_doc), py.MustNewMethod("open", builtin_open, 0, open_doc), - // py.MustNewMethod("oct", builtin_oct, 0, oct_doc), + py.MustNewMethod("oct", builtin_oct, 0, oct_doc), py.MustNewMethod("ord", builtin_ord, 0, ord_doc), py.MustNewMethod("pow", builtin_pow, 0, pow_doc), py.MustNewMethod("print", builtin_print, 0, print_doc), py.MustNewMethod("repr", builtin_repr, 0, repr_doc), py.MustNewMethod("round", builtin_round, 0, round_doc), py.MustNewMethod("setattr", builtin_setattr, 0, setattr_doc), - // py.MustNewMethod("sorted", builtin_sorted, 0, sorted_doc), + py.MustNewMethod("sorted", builtin_sorted, 0, sorted_doc), py.MustNewMethod("sum", builtin_sum, 0, sum_doc), // py.MustNewMethod("vars", builtin_vars, 0, vars_doc), } @@ -74,25 +77,25 @@ func init() { "classmethod": py.ClassMethodType, "complex": py.ComplexType, "dict": py.StringDictType, // FIXME - // "enumerate": py.EnumType, - // "filter": py.FilterType, - "float": py.FloatType, - "frozenset": py.FrozenSetType, + "enumerate": py.EnumerateType, + "filter": py.FilterType, + "float": py.FloatType, + "frozenset": py.FrozenSetType, // "property": py.PropertyType, - "int": py.IntType, // FIXME LongType? - "list": py.ListType, - // "map": py.MapType, + "int": py.IntType, // FIXME LongType? + "list": py.ListType, + "map": py.MapType, "object": py.ObjectType, "range": py.RangeType, // "reversed": py.ReversedType, - "set": py.SetType, - // "slice": py.SliceType, + "set": py.SetType, + "slice": py.SliceType, "staticmethod": py.StaticMethodType, "str": py.StringType, // "super": py.SuperType, "tuple": py.TupleType, "type": py.TypeType, - // "zip": py.ZipType, + "zip": py.ZipType, // Exceptions "ArithmeticError": py.ArithmeticError, @@ -160,7 +163,16 @@ func init() { "Warning": py.Warning, "ZeroDivisionError": py.ZeroDivisionError, } - py.NewModule("builtins", builtin_doc, methods, globals) + + py.RegisterModule(&py.ModuleImpl{ + Info: py.ModuleInfo{ + Name: "builtins", + Doc: builtin_doc, + Flags: py.ShareModule, + }, + Methods: methods, + Globals: globals, + }) } const print_doc = `print(value, ..., sep=' ', end='\\n', file=sys.stdout, flush=False) @@ -176,18 +188,22 @@ func builtin_print(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Obje var ( sepObj py.Object = py.String(" ") endObj py.Object = py.String("\n") - file py.Object = py.MustGetModule("sys").Globals["stdout"] flush py.Object ) + sysModule, err := self.(*py.Module).Context.GetModule("sys") + if err != nil { + return nil, err + } + stdout := sysModule.Globals["stdout"] kwlist := []string{"sep", "end", "file", "flush"} - err := py.ParseTupleAndKeywords(nil, kwargs, "|ssOO:print", kwlist, &sepObj, &endObj, &file, &flush) + err = py.ParseTupleAndKeywords(nil, kwargs, "|ssOO:print", kwlist, &sepObj, &endObj, &stdout, &flush) if err != nil { return nil, err } sep := sepObj.(py.String) end := endObj.(py.String) - write, err := py.GetAttrString(file, "write") + write, err := py.GetAttrString(stdout, "write") if err != nil { return nil, err } @@ -217,7 +233,7 @@ func builtin_print(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Obje } if shouldFlush, _ := py.MakeBool(flush); shouldFlush == py.True { - fflush, err := py.GetAttrString(file, "flush") + fflush, err := py.GetAttrString(stdout, "flush") if err == nil { return py.Call(fflush, nil, nil) } @@ -277,7 +293,11 @@ func builtin_all(self, seq py.Object) (py.Object, error) { } return nil, err } - if !py.ObjectIsTrue(item) { + ok, err := py.ObjectIsTrue(item) + if err != nil { + return nil, err + } + if !ok { return py.False, nil } } @@ -302,13 +322,58 @@ func builtin_any(self, seq py.Object) (py.Object, error) { } return nil, err } - if py.ObjectIsTrue(item) { + ok, err := py.ObjectIsTrue(item) + if err != nil { + return nil, err + } + if ok { return py.True, nil } } return py.False, nil } +const ascii_doc = `Return an ASCII-only representation of an object. + +As repr(), return a string containing a printable representation of an +object, but escape the non-ASCII characters in the string returned by +repr() using \\x, \\u or \\U escapes. This generates a string similar +to that returned by repr() in Python 2. +` + +func builtin_ascii(self, o py.Object) (py.Object, error) { + reprObj, err := py.Repr(o) + if err != nil { + return nil, err + } + repr := reprObj.(py.String) + out := py.StringEscape(repr, true) + return py.String(out), err +} + +const bin_doc = `Return the binary representation of an integer. + +>>> bin(2796202) +'0b1010101010101010101010' +` + +func builtin_bin(self, o py.Object) (py.Object, error) { + bigint, ok := py.ConvertToBigInt(o) + if !ok { + return nil, py.ExceptionNewf(py.TypeError, "'%s' object cannot be interpreted as an integer", o.Type().Name) + } + + value := (*big.Int)(bigint) + var out string + if value.Sign() < 0 { + value = new(big.Int).Abs(value) + out = fmt.Sprintf("-0b%b", value) + } else { + out = fmt.Sprintf("0b%b", value) + } + return py.String(out), nil +} + const round_doc = `round(number[, ndigits]) -> number Round a number to a given precision in decimal digits (default 0 digits). @@ -406,7 +471,7 @@ func builtin___build_class__(self py.Object, args py.Tuple, kwargs py.StringDict } // fmt.Printf("Calling %v with %v and %v\n", fn.Name, fn.Globals, ns) // fmt.Printf("Code = %#v\n", fn.Code) - cell, err = py.VmRun(fn.Globals, ns, fn.Code, fn.Closure) + cell, err = fn.Context.RunCode(fn.Code, fn.Globals, ns, fn.Closure) if err != nil { return nil, err } @@ -528,6 +593,56 @@ func builtin_open(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Objec int(buffering.(py.Int))) } +const oct_doc = `oct(number) -> string + +Return the octal representation of an integer. + + >>> oct(342391) + '0o1234567' +` + +func builtin_oct(self, v py.Object) (py.Object, error) { + var ( + i int64 + err error + ) + switch v := v.(type) { + case *py.BigInt: + vv := (*big.Int)(v) + neg := false + if vv.Cmp(big.NewInt(0)) == -1 { + neg = true + } + str := vv.Text(8) + if neg { + str = "-0o" + str[1:] + } else { + str = "0o" + str + } + return py.String(str), nil + case py.IGoInt64: + i, err = v.GoInt64() + case py.IGoInt: + var vv int + vv, err = v.GoInt() + i = int64(vv) + default: + return nil, py.ExceptionNewf(py.TypeError, "'%s' object cannot be interpreted as an integer", v.Type().Name) + } + + if err != nil { + return nil, err + } + + str := strconv.FormatInt(i, 8) + if i < 0 { + str = "-0o" + str[1:] + } else { + str = "0o" + str + } + return py.String(str), nil +} + const ord_doc = `ord(c) -> integer Return the integer ordinal of a one-character string.` @@ -633,6 +748,27 @@ func source_as_string(cmd py.Object, funcname, what string /*, PyCompilerFlags * // return nil; } +const delattr_doc = `Deletes the named attribute from the given object. + +delattr(x, 'y') is equivalent to "del x.y" +` + +func builtin_delattr(self py.Object, args py.Tuple) (py.Object, error) { + var v py.Object + var name py.Object + + err := py.UnpackTuple(args, nil, "delattr", 2, 2, &v, &name) + if err != nil { + return nil, err + } + + err = py.DeleteAttr(v, name) + if err != nil { + return nil, err + } + return py.None, nil +} + const compile_doc = `compile(source, filename, mode[, flags[, dont_inherit]]) -> code object Compile the source string (a Python module, statement or expression) @@ -685,9 +821,9 @@ func builtin_compile(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Ob return nil, py.ExceptionNewf(py.ValueError, "compile(): invalid optimize value") } - if dont_inherit.(py.Int) != 0 { - // PyEval_MergeCompilerFlags(&cf) - } + // if dont_inherit.(py.Int) != 0 { + // PyEval_MergeCompilerFlags(&cf) + // } // switch string(startstr.(py.String)) { // case "exec": @@ -718,7 +854,7 @@ func builtin_compile(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Ob return nil, err } // result = py.CompileStringExFlags(str, filename, start[mode], &cf, optimize) - result, err = compile.Compile(str, string(filename.(py.String)), string(startstr.(py.String)), int(supplied_flags.(py.Int)), dont_inherit.(py.Int) != 0) + result, err = compile.Compile(str, string(filename.(py.String)), py.CompileMode(startstr.(py.String)), int(supplied_flags.(py.Int)), dont_inherit.(py.Int) != 0) if err != nil { return nil, err } @@ -762,6 +898,129 @@ object. The globals and locals are dictionaries, defaulting to the current globals and locals. If only globals is given, locals defaults to it.` +const hex_doc = `hex(number) -> string + +Return the hexadecimal representation of an integer. + + >>> hex(12648430) + '0xc0ffee' +` + +func builtin_hex(self, v py.Object) (py.Object, error) { + var ( + i int64 + err error + ) + switch v := v.(type) { + case *py.BigInt: + // test bigint first to make sure we correctly handle the case + // where int64 isn't large enough. + vv := (*big.Int)(v) + neg := false + if vv.Cmp(big.NewInt(0)) == -1 { + neg = true + } + str := vv.Text(16) + if neg { + str = "-0x" + str[1:] + } else { + str = "0x" + str + } + return py.String(str), nil + case py.IGoInt64: + i, err = v.GoInt64() + case py.IGoInt: + var vv int + vv, err = v.GoInt() + i = int64(vv) + default: + return nil, py.ExceptionNewf(py.TypeError, "'%s' object cannot be interpreted as an integer", v.Type().Name) + } + + if err != nil { + return nil, err + } + + str := strconv.FormatInt(i, 16) + if i < 0 { + str = "-0x" + str[1:] + } else { + str = "0x" + str + } + return py.String(str), nil +} + +const isinstance_doc = `isinstance(obj, class_or_tuple) -> bool + +Return whether an object is an instance of a class or of a subclass thereof. + +A tuple, as in isinstance(x, (A, B, ...)), may be given as the target to +check against. This is equivalent to isinstance(x, A) or isinstance(x, B) +or ... etc. +` + +func isinstance(obj py.Object, classOrTuple py.Object) (py.Bool, error) { + switch class_tuple := classOrTuple.(type) { + case py.Tuple: + for idx := range class_tuple { + res, _ := isinstance(obj, class_tuple[idx]) + if res { + return res, nil + } + } + return false, nil + default: + if classOrTuple.Type().ObjectType != py.TypeType { + return false, py.ExceptionNewf(py.TypeError, "isinstance() arg 2 must be a type or tuple of types") + } + return obj.Type() == classOrTuple, nil + } +} + +func builtin_isinstance(self py.Object, args py.Tuple) (py.Object, error) { + var obj py.Object + var classOrTuple py.Object + err := py.UnpackTuple(args, nil, "isinstance", 2, 2, &obj, &classOrTuple) + if err != nil { + return nil, err + } + + return isinstance(obj, classOrTuple) +} + +const iter_doc = `iter(iterable) -> iterator +iter(callable, sentinel) -> iterator + +Get an iterator from an object. In the first form, the argument must +supply its own iterator, or be a sequence. +In the second form, the callable is called until it returns the sentinel. +` + +func builtin_iter(self py.Object, args py.Tuple) (py.Object, error) { + nArgs := len(args) + if nArgs < 1 { + return nil, py.ExceptionNewf(py.TypeError, + "iter expected at least 1 arguments, got %d", + nArgs) + } else if nArgs > 2 { + return nil, py.ExceptionNewf(py.TypeError, + "iter expected at most 2 arguments, got %d", + nArgs) + } + + v := args[0] + if nArgs == 1 { + return py.Iter(v) + } + _, ok := v.(*py.Function) + sentinel := args[1] + if !ok { + return nil, py.ExceptionNewf(py.TypeError, + "iter(v, w): v must be callable") + } + return py.NewCallIterator(v, sentinel), nil +} + // For code see vm/builtin.go const len_doc = `len(object) -> integer @@ -772,6 +1031,135 @@ func builtin_len(self, v py.Object) (py.Object, error) { return py.Len(v) } +const max_doc = ` +max(iterable, *[, default=obj, key=func]) -> value +max(arg1, arg2, *args, *[, key=func]) -> value + +With a single iterable argument, return its biggest item. The +default keyword-only argument specifies an object to return if +the provided iterable is empty. +With two or more arguments, return the largest argument.` + +func builtin_max(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Object, error) { + return min_max(args, kwargs, "max") +} + +const min_doc = ` +min(iterable, *[, default=obj, key=func]) -> value +min(arg1, arg2, *args, *[, key=func]) -> value + +With a single iterable argument, return its smallest item. The +default keyword-only argument specifies an object to return if +the provided iterable is empty. +With two or more arguments, return the smallest argument.` + +func builtin_min(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Object, error) { + return min_max(args, kwargs, "min") +} + +func min_max(args py.Tuple, kwargs py.StringDict, name string) (py.Object, error) { + kwlist := []string{"key", "default"} + positional := len(args) + var format string + var values py.Object + var cmp func(a py.Object, b py.Object) (py.Object, error) + if name == "min" { + format = "|$OO:min" + cmp = py.Le + } else if name == "max" { + format = "|$OO:max" + cmp = py.Ge + } + var defaultValue py.Object + var keyFunc py.Object + var maxVal, maxItem py.Object + var kf *py.Function + + if positional > 1 { + values = args + } else { + err := py.UnpackTuple(args, nil, name, 1, 1, &values) + if err != nil { + return nil, err + } + } + err := py.ParseTupleAndKeywords(nil, kwargs, format, kwlist, &keyFunc, &defaultValue) + if err != nil { + return nil, err + } + if keyFunc == py.None { + keyFunc = nil + } + if keyFunc != nil { + var ok bool + kf, ok = keyFunc.(*py.Function) + if !ok { + return nil, py.ExceptionNewf(py.TypeError, "'%s' object is not callable", keyFunc.Type()) + } + } + if defaultValue != nil { + maxItem = defaultValue + if keyFunc != nil { + maxVal, err = py.Call(kf, py.Tuple{defaultValue}, nil) + if err != nil { + return nil, err + } + } else { + maxVal = defaultValue + } + } + iter, err := py.Iter(values) + if err != nil { + return nil, err + } + + for { + item, err := py.Next(iter) + if err != nil { + if py.IsException(py.StopIteration, err) { + break + } + return nil, err + } + if maxVal == nil { + if keyFunc != nil { + maxVal, err = py.Call(kf, py.Tuple{item}, nil) + if err != nil { + return nil, err + } + } else { + maxVal = item + } + maxItem = item + } else { + var compareVal py.Object + if keyFunc != nil { + compareVal, err = py.Call(kf, py.Tuple{item}, nil) + if err != nil { + return nil, err + } + } else { + compareVal = item + } + changed, err := cmp(compareVal, maxVal) + if err != nil { + return nil, err + } + if changed == py.True { + maxVal = compareVal + maxItem = item + } + } + + } + + if maxItem == nil { + return nil, py.ExceptionNewf(py.ValueError, "%s() arg is an empty sequence", name) + } + + return maxItem, nil +} + const chr_doc = `chr(i) -> Unicode character Return a Unicode string of one character with ordinal i; 0 <= i <= 0x10ffff.` @@ -848,3 +1236,28 @@ func builtin_sum(self py.Object, args py.Tuple) (py.Object, error) { } return start, nil } + +const sorted_doc = `sorted(iterable, key=None, reverse=False) + +Return a new list containing all items from the iterable in ascending order. + +A custom key function can be supplied to customize the sort order, and the +reverse flag can be set to request the result in descending order.` + +func builtin_sorted(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Object, error) { + const funcName = "sorted" + var iterable py.Object + err := py.UnpackTuple(args, nil, funcName, 1, 1, &iterable) + if err != nil { + return nil, err + } + l, err := py.SequenceList(iterable) + if err != nil { + return nil, err + } + err = py.SortInPlace(l, kwargs, funcName) + if err != nil { + return nil, err + } + return l, nil +} diff --git a/builtin/builtin_test.go b/stdlib/builtin/builtin_test.go similarity index 100% rename from builtin/builtin_test.go rename to stdlib/builtin/builtin_test.go diff --git a/stdlib/builtin/tests/builtin.py b/stdlib/builtin/tests/builtin.py new file mode 100644 index 00000000..ae4e8a5f --- /dev/null +++ b/stdlib/builtin/tests/builtin.py @@ -0,0 +1,472 @@ +# Copyright 2019 The go-python Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +from libtest import assertRaises + +doc="abs" +assert abs(0) == 0 +assert abs(-0) == 0 +assert abs(0.0) == 0.0 +assert abs(-0.0) == 0.0 +assert abs(10) == 10 +assert abs(-10) == 10 +assert abs(12.3) == 12.3 +assert abs(-12.3) == 12.3 +assert abs(1 << 63) == 1 << 63 +assert abs(-1 << 63) == 1 << 63 +assert abs(-(1 << 63)) == 1 << 63 +assert abs(1 << 66) == 1 << 66 +assert abs(-1 << 66) == 1 << 66 +assert abs(-(1 << 66)) == 1 << 66 + +doc="all" +assert all((0,0,0)) == False +assert all((1,1,0)) == False +assert all(["hello", "world"]) == True +assert all([]) == True + +doc="any" +assert any((0,0,0)) == False +assert any((1,1,0)) == True +assert any(["hello", "world"]) == True +assert any([]) == False + +doc="ascii" +assert ascii('hello world') == "'hello world'" +assert ascii('안녕 세상') == "'\\uc548\\ub155 \\uc138\\uc0c1'" +assert ascii(chr(0x10001)) == "'\\U00010001'" +assert ascii('안녕 gpython') == "'\\uc548\\ub155 gpython'" + +doc="bin" +assert bin(False) == '0b0' +assert bin(True) == '0b1' +assert bin(0) == '0b0' +assert bin(1) == '0b1' +assert bin(-1) == '-0b1' +assert bin(10) == '0b1010' +assert bin(-10) == '-0b1010' +assert bin(2**32) == '0b100000000000000000000000000000000' +assert bin(2**32-1) == '0b11111111111111111111111111111111' +assert bin(-(2**32)) == '-0b100000000000000000000000000000000' +assert bin(-(2**32-1)) == '-0b11111111111111111111111111111111' + +doc="chr" +assert chr(65) == "A" +assert chr(163) == "£" +assert chr(0x263A) == "☺" + +doc="compile" +code = compile("pass", "", "exec") +assert code is not None +# FIXME + +doc="divmod" +assert divmod(34,7) == (4, 6) + +doc="enumerate" +a = [3, 4, 5, 6, 7] +for idx, value in enumerate(a): + assert value == a[idx] + +doc="eval" +# smoke test only - see vm/tests/builtin.py for more tests +assert eval("1+2") == 3 + +doc="exec" +# smoke test only - see vm/tests/builtin.py for more tests +glob = {"a":100} +assert exec("b = a+100", glob) == None +assert glob["b"] == 200 + +doc="getattr" +class C: + def __init__(self): + self.potato = 42 +c = C() +assert getattr(c, "potato") == 42 +assert getattr(c, "potato", 43) == 42 +assert getattr(c, "sausage", 43) == 43 + +doc="globals" +a = 1 +assert globals()["a"] == 1 + +doc="hasattr" +assert hasattr(c, "potato") +assert not hasattr(c, "sausage") + +doc="len" +assert len(()) == 0 +assert len((1,2,3)) == 3 +assert len("hello") == 5 +assert len("£☺") == 2 + +doc="locals" +def fn(x): + assert locals()["x"] == 1 +fn(1) + +def func(p): + return p[1] + +doc="min" +values = (1,2,3) +v = min(values) +assert v == 1 +v = min(4,5,6) +assert v == 4 +v = min((), default=-1) +assert v == -1 +v = min([], default=-1) +assert v == -1 +v = min([], key=func, default=(1,3)) +assert v == (1,3) +v = min([(1,3), (2,1)], key=func) +assert v == (2,1) +ok = False +try: + min([(1,3), (2,1)], key=3) +except TypeError: + ok = True +assert ok, "TypeError not raised" +ok = False +try: + min([]) +except ValueError: + ok = True +assert ok, "ValueError not raised" + +doc="max" +values = (1,2,3) +v = max(values) +assert v == 3 +v = max(4,5,6) +assert v == 6 +v = max((), default=-1) +assert v == -1 +v = max([], default=-1) +assert v == -1 +v = max([], key=func, default=(1,3)) +assert v == (1,3) +v = max([(1,3), (2,1)], key=func) +assert v == (1,3) +ok = False +try: + max([(1,3), (2,1)], key=3) +except TypeError: + ok = True +assert ok, "TypeError not raised" +ok = False +try: + max([]) +except ValueError: + ok = True +assert ok, "ValueError not raised" + +doc="hex" +assert hex( 0)=="0x0", "hex(0)" +assert hex( 1)=="0x1", "hex(1)" +assert hex(42)=="0x2a", "hex(42)" +assert hex( -0)=="0x0", "hex(-0)" +assert hex( -1)=="-0x1", "hex(-1)" +assert hex(-42)=="-0x2a", "hex(-42)" +assert hex( 1<<64) == "0x10000000000000000", "hex(1<<64)" +assert hex(-1<<64) == "-0x10000000000000000", "hex(-1<<64)" +assert hex( 1<<128) == "0x100000000000000000000000000000000", "hex(1<<128)" +assert hex(-1<<128) == "-0x100000000000000000000000000000000", "hex(-1<<128)" +assertRaises(TypeError, hex, 10.0) ## TypeError: 'float' object cannot be interpreted as an integer +assertRaises(TypeError, hex, float(0)) ## TypeError: 'float' object cannot be interpreted as an integer + +doc="isinstance" +class A: + pass +a = A() +assert isinstance(1, (str, tuple, int)) +assert isinstance(a, (str, (tuple, (A, )))) +assertRaises(TypeError, isinstance, 1, (A, ), "foo") +assertRaises(TypeError, isinstance, 1, [A, "foo"]) + +doc="iter" +cnt = 0 +def f(): + global cnt + cnt += 1 + return cnt + +l = list(iter(f,20)) +assert len(l) == 19 +for idx, v in enumerate(l): + assert idx + 1 == v + +words1 = ['g', 'p', 'y', 't', 'h', 'o', 'n'] +words2 = list(iter(words1)) +for w1, w2 in zip(words1, words2): + assert w1 == w2 + +ok = False +try: + iter() +except TypeError: + ok = True +finally: + assert ok, "TypeError not raised" + ok = False + +try: + l = [1, 2, 3] + iter(l, 2) +except TypeError: + ok = True +finally: + assert ok, "TypeError not raised" + ok = False + +try: + iter(f, 2, 3) +except TypeError: + ok = True +finally: + assert ok, "TypeError not raised" + ok = False + + +doc="next no default" +def gen(): + yield 1 + yield 2 +g = gen() +assert next(g) == 1 +assert next(g) == 2 +ok = False +try: + next(g) +except StopIteration: + ok = True +assert ok, "StopIteration not raised" + +doc="next with default" +g = gen() +assert next(g, 42) == 1 +assert next(g, 42) == 2 +assert next(g, 42) == 42 +assert next(g, 42) == 42 + +doc="next no default with exception" +def gen2(): + yield 1 + raise ValueError("potato") +g = gen2() +assert next(g) == 1 +ok = False +try: + next(g) +except ValueError: + ok = True +assert ok, "ValueError not raised" + +doc="next with default and exception" +g = gen2() +assert next(g, 42) == 1 +ok = False +try: + next(g) +except ValueError: + ok = True +assert ok, "ValueError not raised" + +doc="oct" +assert oct(0) == '0o0' +assert oct(100) == '0o144' +assert oct(-100) == '-0o144' +assertRaises(TypeError, oct, ()) + +doc="ord" +assert 65 == ord("A") +assert 163 == ord("£") +assert 0x263A == ord("☺") +assert 65 == ord(b"A") +ok = False +try: + ord("AA") +except TypeError as e: + if e.args[0] != "ord() expected a character, but string of length 2 found": + raise + ok = True +assert ok, "TypeError not raised" +try: + ord(None) +except TypeError as e: + if e.args[0] != "ord() expected string of length 1, but NoneType found": + raise + ok = True +assert ok, "TypeError not raised" + +doc="open" +assert open(__file__) is not None + +doc="pow" +assert pow(2, 10) == 1024 +assert pow(2, 10, 17) == 4 + +doc="repr" +assert repr(5) == "5" +assert repr("hello") == "'hello'" + +doc="print" +ok = False +try: + print("hello", sep=1, end="!") +except TypeError as e: + if e.args[0] != "print() argument 1 must be str, not int": + raise + ok = True +assert ok, "TypeError not raised" + +try: + print("hello", sep=",", end=1) +except TypeError as e: + if e.args[0] != "print() argument 2 must be str, not int": + raise + ok = True +assert ok, "TypeError not raised" + +try: + print("hello", sep=",", end="!", file=1) +except AttributeError as e: + if e.args[0] != "'int' has no attribute 'write'": + raise + ok = True +assert ok, "AttributeError not raised" + +with open("testfile", "w") as f: + print("hello", "world", end="!\n", file=f, sep=", ") + print("hells", "bells", end="...", file=f) + print(" ~", "Brother ", "Foo", "bar", file=f, end="", sep="") + +with open("testfile", "r") as f: + assert f.read() == "hello, world!\nhells bells... ~Brother Foobar" + +with open("testfile", "w") as f: + print(1,2,3,sep=",", flush=False, end=",\n", file=f) + print("4",5, file=f, end="!", flush=True, sep=",") + +with open("testfile", "r") as f: + assert f.read() == "1,2,3,\n4,5!" + +doc="round" +assert round(1.1) == 1.0 + +doc="setattr" +class C: pass +c = C() +assert not hasattr(c, "potato") +setattr(c, "potato", "spud") +assert getattr(c, "potato") == "spud" +assert c.potato == "spud" +delattr(c, "potato") +assert not hasattr(c, "potato") +ok = False +try: + delattr(c, "potato") +except AttributeError as e: + ok = True +finally: + assert ok + +doc="sorted" +a = [3, 1.1, 1, 2] +assert sorted(a) == [1, 1.1, 2, 3] +assert sorted(sorted(a)) == [1, 1.1, 2, 3] +assert sorted(a, reverse=True) == [3, 2, 1.1, 1] +assert sorted(a, key=lambda l: l+1) == [1, 1.1, 2, 3] +s = [2.0, 2, 1, 1.0] +assert sorted(s, key=lambda l: 0) == [2.0, 2, 1, 1.0] +assert [type(t) for t in sorted(s, key=lambda l: 0)] == [float, int, int, float] +assert sorted(s) == [1, 1.0, 2.0, 2] +assert [type(t) for t in sorted(s)] == [int, float, float, int] + +try: + sorted([2.0, "abc"]) +except TypeError: + pass +else: + assert False + +assert sorted([]) == [] +assert sorted([0]) == [0] +s = [0, 1] +try: + # Sorting a list of len >= 2 with uncallable key must fail on all Python implementations. + sorted(s, key=1) +except TypeError: + pass +else: + assert False + +try: + sorted(1) +except TypeError: + pass +else: + assert False + +try: + sorted() +except TypeError: + pass +else: + assert False + +doc="sum" +assert sum([1,2,3]) == 6 +assert sum([1,2,3], 3) == 9 +assert sum((1,2,3)) == 6 +assert sum((1,2,3), 3) == 9 +assert sum((1, 2.5, 3)) == 6.5 +assert sum((1, 2.5, 3), 3) == 9.5 + +try: + sum([1,2,3], 'hi') +except TypeError as e: + if e.args[0] != "sum() can't sum strings [use ''.join(seq) instead]": + raise + ok = True +assert ok, "TypeError not raised" + +try: + sum([1,2,3], b'hi') +except TypeError as e: + if e.args[0] != "sum() can't sum bytes [use b''.join(seq) instead]": + raise + ok = True +assert ok, "TypeError not raised" + +try: + sum(['h', 'i']) +except TypeError as e: + if e.args[0] != "unsupported operand type(s) for +: 'int' and 'str'": + raise + ok = True +assert ok, "TypeError not raised" + +doc="zip" +ok = False +a = [3, 4, 5, 6, 7] +b = [8, 9, 10, 11, 12] +assert [e for e in zip(a, b)] == [(3,8), (4,9), (5,10), (6,11), (7,12)] +try: + zip(1,2,3) +except TypeError as e: + if e.args[0] != "zip argument #1 must support iteration": + raise + ok = True +assert ok, "TypeError not raised" + +doc="__import__" +lib = __import__("lib") +assert lib.libfn() == 42 +assert lib.libvar == 43 +assert lib.libclass().method() == 44 + +doc="finished" diff --git a/builtin/tests/lib.py b/stdlib/builtin/tests/lib.py similarity index 100% rename from builtin/tests/lib.py rename to stdlib/builtin/tests/lib.py diff --git a/stdlib/builtin/tests/libtest.py b/stdlib/builtin/tests/libtest.py new file mode 100644 index 00000000..feebd7ee --- /dev/null +++ b/stdlib/builtin/tests/libtest.py @@ -0,0 +1,17 @@ +# Copyright 2018 The go-python Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +""" +Simple test harness +""" + +def assertRaises(expecting, fn, *args, **kwargs): + """Check the exception was raised - don't check the text""" + try: + fn(*args, **kwargs) + except expecting as e: + pass + else: + assert False, "%s not raised" % (expecting,) + diff --git a/stdlib/glob/glob.go b/stdlib/glob/glob.go new file mode 100644 index 00000000..173a2493 --- /dev/null +++ b/stdlib/glob/glob.go @@ -0,0 +1,64 @@ +// Copyright 2022 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package glob provides the implementation of the python's 'glob' module. +package glob + +import ( + "path/filepath" + + "github.com/go-python/gpython/py" +) + +func init() { + py.RegisterModule(&py.ModuleImpl{ + Info: py.ModuleInfo{ + Name: "glob", + Doc: "Filename globbing utility.", + }, + Methods: []*py.Method{ + py.MustNewMethod("glob", glob, 0, glob_doc), + }, + }) +} + +const glob_doc = `Return a list of paths matching a pathname pattern. +The pattern may contain simple shell-style wildcards a la +fnmatch. However, unlike fnmatch, filenames starting with a +dot are special cases that are not matched by '*' and '?' +patterns.` + +func glob(self py.Object, args py.Tuple) (py.Object, error) { + var ( + pypathname py.Object + ) + err := py.ParseTuple(args, "s*:glob", &pypathname) + if err != nil { + return nil, err + } + + var ( + pathname string + cnv func(v string) py.Object + ) + switch n := pypathname.(type) { + case py.String: + pathname = string(n) + cnv = func(v string) py.Object { return py.String(v) } + case py.Bytes: + pathname = string(n) + cnv = func(v string) py.Object { return py.Bytes(v) } + } + matches, err := filepath.Glob(pathname) + if err != nil { + return nil, err + } + + lst := py.List{Items: make([]py.Object, len(matches))} + for i, v := range matches { + lst.Items[i] = cnv(v) + } + + return &lst, nil +} diff --git a/stdlib/glob/glob_test.go b/stdlib/glob/glob_test.go new file mode 100644 index 00000000..c90cb091 --- /dev/null +++ b/stdlib/glob/glob_test.go @@ -0,0 +1,15 @@ +// Copyright 2022 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package glob_test + +import ( + "testing" + + "github.com/go-python/gpython/pytest" +) + +func TestGlob(t *testing.T) { + pytest.RunScript(t, "./testdata/test.py") +} diff --git a/stdlib/glob/testdata/test.py b/stdlib/glob/testdata/test.py new file mode 100644 index 00000000..042b6b3b --- /dev/null +++ b/stdlib/glob/testdata/test.py @@ -0,0 +1,58 @@ +# Copyright 2022 The go-python Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +import glob + +def norm(vs): + if len(vs) == 0: + return vs + if type(vs[0]) == type(""): + return normStr(vs) + return normBytes(vs) + +def normStr(vs): + from os import sep + x = [] + for v in vs: + x.append(v.replace('/', sep)) + return x + +def normBytes(vs): + from os import sep + x = [] + for v in vs: + x.append(v.replace(b'/', bytes(sep, encoding="utf-8"))) + return x + +def assertEqual(x, y): + xx = norm(x) + yy = norm(y) + assert xx == yy, "got: %s, want: %s" % (repr(x), repr(y)) + + +## test strings +assertEqual(glob.glob('*'), ["glob.go", "glob_test.go", "testdata"]) +assertEqual(glob.glob('*test*'), ["glob_test.go", "testdata"]) +assertEqual(glob.glob('*/test*'), ["testdata/test.py", "testdata/test_golden.txt"]) +assertEqual(glob.glob('*/test*_*'), ["testdata/test_golden.txt"]) +assertEqual(glob.glob('*/t??t*_*'), ["testdata/test_golden.txt"]) +assertEqual(glob.glob('*/t[e]?t*_*'), ["testdata/test_golden.txt"]) +assertEqual(glob.glob('*/t[oe]?t*_*'), ["testdata/test_golden.txt"]) +assertEqual(glob.glob('*/t[o]?t*_*'), []) + +## FIXME(sbinet) +## assertEqual(glob.glob('*/t[!o]?t*_*'), ["testdata/test_golden.txt"]) + +## test bytes +assertEqual(glob.glob(b'*'), [b"glob.go", b"glob_test.go", b"testdata"]) +assertEqual(glob.glob(b'*test*'), [b"glob_test.go", b"testdata"]) +assertEqual(glob.glob(b'*/test*'), [b"testdata/test.py", b"testdata/test_golden.txt"]) +assertEqual(glob.glob(b'*/test*_*'), [b"testdata/test_golden.txt"]) +assertEqual(glob.glob(b'*/t??t*_*'), [b"testdata/test_golden.txt"]) +assertEqual(glob.glob(b'*/t[e]?t*_*'), [b"testdata/test_golden.txt"]) +assertEqual(glob.glob(b'*/t[oe]?t*_*'), [b"testdata/test_golden.txt"]) +assertEqual(glob.glob(b'*/t[o]?t*_*'), []) + +## FIXME(sbinet) +## assertEqual(glob.glob(b'*/t[!o]?t*_*'), [b"testdata/test_golden.txt"]) diff --git a/stdlib/glob/testdata/test_golden.txt b/stdlib/glob/testdata/test_golden.txt new file mode 100644 index 00000000..e69de29b diff --git a/marshal/marshal.go b/stdlib/marshal/marshal.go similarity index 96% rename from marshal/marshal.go rename to stdlib/marshal/marshal.go index dd039d5c..937008cc 100644 --- a/marshal/marshal.go +++ b/stdlib/marshal/marshal.go @@ -6,7 +6,6 @@ package marshal import ( - "bytes" "encoding/binary" "errors" "fmt" @@ -15,7 +14,6 @@ import ( "strconv" "github.com/go-python/gpython/py" - "github.com/go-python/gpython/vm" ) const ( @@ -448,29 +446,12 @@ func ReadPyc(r io.Reader) (obj py.Object, err error) { } // FIXME do something with timestamp & length? if header.Magic>>16 != 0x0a0d { - return nil, errors.New("Bad magic in .pyc file") + return nil, errors.New("bad magic in .pyc file") } // fmt.Printf("header = %v\n", header) return ReadObject(r) } -// Unmarshals a frozen module -func LoadFrozenModule(name string, data []byte) (*py.Module, error) { - r := bytes.NewBuffer(data) - obj, err := ReadObject(r) - if err != nil { - return nil, err - } - code := obj.(*py.Code) - module := py.NewModule(name, "", nil, nil) - _, err = vm.Run(module.Globals, module.Globals, code, nil) - if err != nil { - py.TracebackDump(err) - return nil, err - } - return module, nil -} - const dump_doc = `dump(value, file[, version]) Write the value on the open file. The value must be a supported type. @@ -634,5 +615,14 @@ func init() { globals := py.StringDict{ "version": py.Int(MARSHAL_VERSION), } - py.NewModule("marshal", module_doc, methods, globals) + + py.RegisterModule(&py.ModuleImpl{ + Info: py.ModuleInfo{ + Name: "marshal", + Doc: module_doc, + Flags: py.ShareModule, + }, + Globals: globals, + Methods: methods, + }) } diff --git a/math/math.go b/stdlib/math/math.go similarity index 87% rename from math/math.go rename to stdlib/math/math.go index 88d60fb3..6c6c32d9 100644 --- a/math/math.go +++ b/stdlib/math/math.go @@ -62,59 +62,57 @@ raised for division by zero and mod by zero. */ /* - In general, on an IEEE-754 platform the aim is to follow the C99 - standard, including Annex 'F', whenever possible. Where the - standard recommends raising the 'divide-by-zero' or 'invalid' - floating-point exceptions, Python should raise a ValueError. Where - the standard recommends raising 'overflow', Python should raise an - OverflowError. In all other circumstances a value should be - returned. +In general, on an IEEE-754 platform the aim is to follow the C99 +standard, including Annex 'F', whenever possible. Where the +standard recommends raising the 'divide-by-zero' or 'invalid' +floating-point exceptions, Python should raise a ValueError. Where +the standard recommends raising 'overflow', Python should raise an +OverflowError. In all other circumstances a value should be +returned. */ var ( EDOM = py.ExceptionNewf(py.ValueError, "math domain error") ERANGE = py.ExceptionNewf(py.OverflowError, "math range error") ) -// panic if ok is false -func assert(ok bool) { - if !ok { - panic("assertion failed") - } -} - // isFinite is true if x is not Nan or +/-Inf func isFinite(x float64) bool { return !(math.IsInf(x, 0) || math.IsNaN(x)) } /* - math_1 is used to wrap a libm function f that takes a float64 - arguments and returns a float64. - - The error reporting follows these rules, which are designed to do - the right thing on C89/C99 platforms and IEEE 754/non IEEE 754 - platforms. - - - a NaN result from non-NaN inputs causes ValueError to be raised - - an infinite result from finite inputs causes OverflowError to be - raised if can_overflow is 1, or raises ValueError if can_overflow - is 0. - - if the result is finite and errno == EDOM then ValueError is - raised - - if the result is finite and nonzero and errno == ERANGE then - OverflowError is raised - - The last rule is used to catch overflow on platforms which follow - C89 but for which HUGE_VAL is not an infinity. - - For the majority of one-argument functions these rules are enough - to ensure that Python's functions behave as specified in 'Annex F' - of the C99 standard, with the 'invalid' and 'divide-by-zero' - floating-point exceptions mapping to Python's ValueError and the - 'overflow' floating-point exception mapping to OverflowError. - math_1 only works for functions that don't have singularities *and* - the possibility of overflow; fortunately, that covers everything we - care about right now. +math_1 is used to wrap a libm function f that takes a float64 +arguments and returns a float64. + +The error reporting follows these rules, which are designed to do +the right thing on C89/C99 platforms and IEEE 754/non IEEE 754 +platforms. + +- a NaN result from non-NaN inputs causes ValueError to be raised +- an infinite result from finite inputs causes OverflowError to be + + raised if can_overflow is 1, or raises ValueError if can_overflow + is 0. + +- if the result is finite and errno == EDOM then ValueError is + + raised + +- if the result is finite and nonzero and errno == ERANGE then + + OverflowError is raised + +The last rule is used to catch overflow on platforms which follow +C89 but for which HUGE_VAL is not an infinity. + +For the majority of one-argument functions these rules are enough +to ensure that Python's functions behave as specified in 'Annex F' +of the C99 standard, with the 'invalid' and 'divide-by-zero' +floating-point exceptions mapping to Python's ValueError and the +'overflow' floating-point exception mapping to OverflowError. +math_1 only works for functions that don't have singularities *and* +the possibility of overflow; fortunately, that covers everything we +care about right now. */ func math_1_to_whatever(arg py.Object, fn func(float64) float64, can_overflow bool) (float64, error) { x, err := py.FloatAsFloat64(arg) @@ -140,9 +138,12 @@ func checkResult(x, r float64, can_overflow bool) (float64, error) { return r, nil } -/* variant of math_1, to be used when the function being wrapped is known to - set errno properly (that is, errno = EDOM for invalid or divide-by-zero, - errno = ERANGE for overflow). */ +/* +variant of math_1, to be used when the function being wrapped is known to + + set errno properly (that is, errno = EDOM for invalid or divide-by-zero, + errno = ERANGE for overflow). +*/ func math_1a(arg py.Object, fn func(float64) float64) (py.Object, error) { x, err := py.FloatAsFloat64(arg) if err != nil { @@ -153,30 +154,35 @@ func math_1a(arg py.Object, fn func(float64) float64) (py.Object, error) { } /* - math_2 is used to wrap a libm function f that takes two float64 - arguments and returns a float64. - - The error reporting follows these rules, which are designed to do - the right thing on C89/C99 platforms and IEEE 754/non IEEE 754 - platforms. - - - a NaN result from non-NaN inputs causes ValueError to be raised - - an infinite result from finite inputs causes OverflowError to be - raised. - - if the result is finite and errno == EDOM then ValueError is - raised - - if the result is finite and nonzero and errno == ERANGE then - OverflowError is raised - - The last rule is used to catch overflow on platforms which follow - C89 but for which HUGE_VAL is not an infinity. - - For most two-argument functions (copysign, fmod, hypot, atan2) - these rules are enough to ensure that Python's functions behave as - specified in 'Annex F' of the C99 standard, with the 'invalid' and - 'divide-by-zero' floating-point exceptions mapping to Python's - ValueError and the 'overflow' floating-point exception mapping to - OverflowError. +math_2 is used to wrap a libm function f that takes two float64 +arguments and returns a float64. + +The error reporting follows these rules, which are designed to do +the right thing on C89/C99 platforms and IEEE 754/non IEEE 754 +platforms. + +- a NaN result from non-NaN inputs causes ValueError to be raised +- an infinite result from finite inputs causes OverflowError to be + + raised. + +- if the result is finite and errno == EDOM then ValueError is + + raised + +- if the result is finite and nonzero and errno == ERANGE then + + OverflowError is raised + +The last rule is used to catch overflow on platforms which follow +C89 but for which HUGE_VAL is not an infinity. + +For most two-argument functions (copysign, fmod, hypot, atan2) +these rules are enough to ensure that Python's functions behave as +specified in 'Annex F' of the C99 standard, with the 'invalid' and +'divide-by-zero' floating-point exceptions mapping to Python's +ValueError and the 'overflow' floating-point exception mapping to +OverflowError. */ func math_1(arg py.Object, fn func(float64) float64, can_overflow bool) (py.Object, error) { f, err := math_1_to_whatever(arg, fn, can_overflow) @@ -446,34 +452,35 @@ const math_tanh_doc = "tanh(x)\n\nReturn the hyperbolic tangent of x." accurate result returned by sum(itertools.chain(seq1, seq2)). */ -/* Full precision summation of a sequence of floats. - - def msum(iterable): - partials = [] # sorted, non-overlapping partial sums - for x in iterable: - i = 0 - for y in partials: - if abs(x) < abs(y): - x, y = y, x - hi = x + y - lo = y - (hi - x) - if lo: - partials[i] = lo - i += 1 - x = hi - partials[i:] = [x] - return sum_exact(partials) - - Rounded x+y stored in hi with the roundoff stored in lo. Together hi+lo - are exactly equal to x+y. The inner loop applies hi/lo summation to each - partial so that the list of partial sums remains exact. - - Sum_exact() adds the partial sums exactly and correctly rounds the final - result (using the round-half-to-even rule). The items in partials remain - non-zero, non-special, non-overlapping and strictly increasing in - magnitude, but possibly not all having the same sign. - - Depends on IEEE 754 arithmetic guarantees and half-even rounding. +/* +Full precision summation of a sequence of floats. + + def msum(iterable): + partials = [] # sorted, non-overlapping partial sums + for x in iterable: + i = 0 + for y in partials: + if abs(x) < abs(y): + x, y = y, x + hi = x + y + lo = y - (hi - x) + if lo: + partials[i] = lo + i += 1 + x = hi + partials[i:] = [x] + return sum_exact(partials) + + Rounded x+y stored in hi with the roundoff stored in lo. Together hi+lo + are exactly equal to x+y. The inner loop applies hi/lo summation to each + partial so that the list of partial sums remains exact. + + Sum_exact() adds the partial sums exactly and correctly rounds the final + result (using the round-half-to-even rule). The items in partials remain + non-zero, non-special, non-overlapping and strictly increasing in + magnitude, but possibly not all having the same sign. + + Depends on IEEE 754 arithmetic guarantees and half-even rounding. */ func math_fsum(self py.Object, seq py.Object) (py.Object, error) { const NUM_PARTIALS = 32 /* initial partials array size, on stack */ @@ -939,14 +946,17 @@ const math_modf_doc = `modf(x) Return the fractional and integer parts of x. Both results carry the sign of x and are floats.` -/* A decent logarithm is easy to compute even for huge ints, but libm can't - do that by itself -- loghelper can. func is log or log10, and name is - "log" or "log10". Note that overflow of the result isn't possible: an int - can contain no more than INT_MAX * SHIFT bits, so has value certainly less - than 2**(2**64 * 2**16) == 2**2**80, and log2 of that is 2**80, which is - small enough to fit in an IEEE single. log and log10 are even smaller. - However, intermediate overflow is possible for an int if the number of bits - in that int is larger than PY_SSIZE_T_MAX. */ +/* +A decent logarithm is easy to compute even for huge ints, but libm can't + + do that by itself -- loghelper can. func is log or log10, and name is + "log" or "log10". Note that overflow of the result isn't possible: an int + can contain no more than INT_MAX * SHIFT bits, so has value certainly less + than 2**(2**64 * 2**16) == 2**2**80, and log2 of that is 2**80, which is + small enough to fit in an IEEE single. log and log10 are even smaller. + However, intermediate overflow is possible for an int if the number of bits + in that int is larger than PY_SSIZE_T_MAX. +*/ func loghelper(arg py.Object, fn func(float64) float64, fnname string) (py.Object, error) { /* If it is int, do it ourselves. */ if xBig, err := py.BigIntCheck(arg); err == nil { @@ -1087,10 +1097,12 @@ func math_hypot(self py.Object, args py.Tuple) (py.Object, error) { const math_hypot_doc = `hypot(x, y) Return the Euclidean distance, sqrt(x*x + y*y).` -/* pow can't use math_2, but needs its own wrapper: the problem is - that an infinite result can arise either as a result of overflow - (in which case OverflowError should be raised) or as a result of - e.g. 0.**-5. (for which ValueError needs to be raised.) +/* +pow can't use math_2, but needs its own wrapper: the problem is + + that an infinite result can arise either as a result of overflow + (in which case OverflowError should be raised) or as a result of + e.g. 0.**-5. (for which ValueError needs to be raised.) */ func math_pow(self py.Object, args py.Tuple) (py.Object, error) { var ox, oy py.Object @@ -1333,9 +1345,17 @@ func init() { py.MustNewMethod("trunc", math_trunc, 0, math_trunc_doc), py.MustNewMethod("to_ulps", math_to_ulps, 0, math_to_ulps_doc), } - globals := py.StringDict{ - "pi": py.Float(math.Pi), - "e": py.Float(math.E), - } - py.NewModule("math", math_doc, methods, globals) + + py.RegisterModule(&py.ModuleImpl{ + Info: py.ModuleInfo{ + Name: "math", + Doc: math_doc, + Flags: py.ShareModule, + }, + Methods: methods, + Globals: py.StringDict{ + "pi": py.Float(math.Pi), + "e": py.Float(math.E), + }, + }) } diff --git a/math/math_test.go b/stdlib/math/math_test.go similarity index 100% rename from math/math_test.go rename to stdlib/math/math_test.go diff --git a/math/tests/libtest.py b/stdlib/math/tests/libtest.py similarity index 100% rename from math/tests/libtest.py rename to stdlib/math/tests/libtest.py diff --git a/math/tests/libulp.py b/stdlib/math/tests/libulp.py similarity index 100% rename from math/tests/libulp.py rename to stdlib/math/tests/libulp.py diff --git a/math/tests/mathtests.py b/stdlib/math/tests/mathtests.py similarity index 100% rename from math/tests/mathtests.py rename to stdlib/math/tests/mathtests.py diff --git a/math/tests/testcases.py b/stdlib/math/tests/testcases.py similarity index 100% rename from math/tests/testcases.py rename to stdlib/math/tests/testcases.py diff --git a/stdlib/os/os.go b/stdlib/os/os.go new file mode 100644 index 00000000..c37fce19 --- /dev/null +++ b/stdlib/os/os.go @@ -0,0 +1,576 @@ +// Copyright 2022 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package os implements the Python os module. +package os + +import ( + "os" + "os/exec" + "runtime" + "strconv" + "strings" + + "github.com/go-python/gpython/py" +) + +var ( + osSep = py.String("/") + osName = py.String("posix") + osPathsep = py.String(":") + osLinesep = py.String("\n") + osDefpath = py.String(":/bin:/usr/bin") + osDevnull = py.String("/dev/null") + + osAltsep py.Object = py.None +) + +func initGlobals() { + switch runtime.GOOS { + case "android": + osName = py.String("java") + case "windows": + osSep = py.String(`\`) + osName = py.String("nt") + osPathsep = py.String(";") + osLinesep = py.String("\r\n") + osDefpath = py.String(`C:\bin`) + osDevnull = py.String("nul") + osAltsep = py.String("/") + } +} + +func init() { + initGlobals() + + methods := []*py.Method{ + py.MustNewMethod("_exit", _exit, 0, "Immediate program termination."), + py.MustNewMethod("close", closefd, 0, closefd_doc), + py.MustNewMethod("fdopen", fdopen, 0, fdopen_doc), + py.MustNewMethod("getcwd", getCwd, 0, "Get the current working directory"), + py.MustNewMethod("getcwdb", getCwdb, 0, "Get the current working directory in a byte slice"), + py.MustNewMethod("chdir", chdir, 0, "Change the current working directory"), + py.MustNewMethod("getenv", getenv, 0, "Return the value of the environment variable key if it exists, or default if it doesn’t. key, default and the result are str."), + py.MustNewMethod("getpid", getpid, 0, "Return the current process id."), + py.MustNewMethod("listdir", listDir, 0, listDir_doc), + py.MustNewMethod("makedirs", makedirs, 0, makedirs_doc), + py.MustNewMethod("mkdir", mkdir, 0, mkdir_doc), + py.MustNewMethod("putenv", putenv, 0, "Set the environment variable named key to the string value."), + py.MustNewMethod("remove", remove, 0, remove_doc), + py.MustNewMethod("removedirs", removedirs, 0, removedirs_doc), + py.MustNewMethod("rmdir", rmdir, 0, rmdir_doc), + py.MustNewMethod("system", system, 0, "Run shell commands, prints stdout directly to default"), + py.MustNewMethod("unsetenv", unsetenv, 0, "Unset (delete) the environment variable named key."), + } + globals := py.StringDict{ + "error": py.OSError, + "environ": getEnvVariables(), + "sep": osSep, + "name": osName, + "curdir": py.String("."), + "pardir": py.String(".."), + "extsep": py.String("."), + "altsep": osAltsep, + "pathsep": osPathsep, + "linesep": osLinesep, + "defpath": osDefpath, + "devnull": osDevnull, + } + + py.RegisterModule(&py.ModuleImpl{ + Info: py.ModuleInfo{ + Name: "os", + Doc: "Miscellaneous operating system interfaces", + }, + Methods: methods, + Globals: globals, + }) +} + +// getEnvVariables returns the dictionary of environment variables. +func getEnvVariables() py.StringDict { + vs := os.Environ() + dict := py.NewStringDictSized(len(vs)) + for _, evar := range vs { + key_value := strings.SplitN(evar, "=", 2) // returns a []string containing [key,value] + dict.M__setitem__(py.String(key_value[0]), py.String(key_value[1])) + } + + return dict +} + +const closefd_doc = `Close a file descriptor` + +func closefd(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Object, error) { + var ( + pyfd py.Object + ) + err := py.ParseTupleAndKeywords(args, kwargs, "i", []string{"fd"}, &pyfd) + if err != nil { + return nil, err + } + + var ( + fd = uintptr(pyfd.(py.Int)) + name = strconv.Itoa(int(fd)) + ) + + f := os.NewFile(fd, name) + if f == nil { + return nil, py.ExceptionNewf(py.OSError, "Bad file descriptor") + } + + err = f.Close() + if err != nil { + return nil, err + } + + return py.None, nil +} + +const fdopen_doc = `# Supply os.fdopen()` + +func fdopen(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Object, error) { + var ( + pyfd py.Object + pymode py.Object = py.String("r") + pybuffering py.Object = py.Int(-1) + pyencoding py.Object = py.None + ) + err := py.ParseTupleAndKeywords( + args, kwargs, + "i|s#is#", []string{"fd", "mode", "buffering", "encoding"}, + &pyfd, &pymode, &pybuffering, &pyencoding, + ) + if err != nil { + return nil, err + } + + // FIXME(sbinet): handle buffering + // FIXME(sbinet): handle encoding + + var ( + fd = uintptr(pyfd.(py.Int)) + name = strconv.Itoa(int(fd)) + mode string + ) + + switch v := pymode.(type) { + case py.String: + mode = string(v) + case py.Bytes: + mode = string(v) + } + + perm, _, _, err := py.FileModeFrom(mode) + if err != nil { + return nil, err + } + + f := os.NewFile(fd, name) + if f == nil { + return nil, py.ExceptionNewf(py.OSError, "Bad file descriptor") + } + + return &py.File{f, perm}, nil +} + +// getCwd returns the current working directory. +func getCwd(self py.Object, args py.Tuple) (py.Object, error) { + dir, err := os.Getwd() + if err != nil { + return nil, py.ExceptionNewf(py.OSError, "Unable to get current working directory.") + } + return py.String(dir), nil +} + +// getCwdb returns the current working directory as a byte list. +func getCwdb(self py.Object, args py.Tuple) (py.Object, error) { + dir, err := os.Getwd() + if err != nil { + return nil, py.ExceptionNewf(py.OSError, "Unable to get current working directory.") + } + return py.Bytes(dir), nil +} + +// chdir changes the current working directory to the provided path. +func chdir(self py.Object, args py.Tuple) (py.Object, error) { + if len(args) == 0 { + return nil, py.ExceptionNewf(py.TypeError, "Missing required argument 'path' (pos 1)") + } + dir, ok := args[0].(py.String) + if !ok { + return nil, py.ExceptionNewf(py.TypeError, "str expected, not "+args[0].Type().Name) + } + err := os.Chdir(string(dir)) + if err != nil { + return nil, py.ExceptionNewf(py.NotADirectoryError, "Couldn't change cwd; "+err.Error()) + } + return py.None, nil +} + +// getenv returns the value of the environment variable key. +// If no such environment variable exists and a default value was provided, that value is returned. +func getenv(self py.Object, args py.Tuple) (py.Object, error) { + if len(args) < 1 { + return nil, py.ExceptionNewf(py.TypeError, "missing one required argument: 'name:str'") + } + k, ok := args[0].(py.String) + if !ok { + return nil, py.ExceptionNewf(py.TypeError, "str expected (pos 1), not "+args[0].Type().Name) + } + v, ok := os.LookupEnv(string(k)) + if ok { + return py.String(v), nil + } + if len(args) == 2 { + return args[1], nil + } + return py.None, nil +} + +// getpid returns the current process id. +func getpid(self py.Object, args py.Tuple) (py.Object, error) { + return py.Int(os.Getpid()), nil +} + +const listDir_doc = ` +Return a list containing the names of the files in the directory. + +path can be specified as either str, bytes. If path is bytes, the filenames + returned will also be bytes; in all other circumstances + the filenames returned will be str. +If path is None, uses the path='.'. + +The list is in arbitrary order. It does not include the special +entries '.' and '..' even if they are present in the directory. +` + +func listDir(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Object, error) { + var ( + path py.Object = py.None + ) + err := py.ParseTupleAndKeywords(args, kwargs, "|z*:listdir", []string{"path"}, &path) + if err != nil { + return nil, err + } + + if path == py.None { + cwd, err := os.Getwd() + if err != nil { + return nil, py.ExceptionNewf(py.OSError, "cannot get cwd, error %s", err.Error()) + } + path = py.String(cwd) + } + + dirName := "" + returnsBytes := false + switch v := path.(type) { + case py.String: + dirName = string(v) + case py.Bytes: + dirName = string(v) + returnsBytes = true + default: + return nil, py.ExceptionNewf(py.TypeError, "str or bytes expected, not %T", path) + } + + dirEntries, err := os.ReadDir(dirName) + if err != nil { + return nil, py.ExceptionNewf(py.OSError, "cannot read directory %s, error %s", dirName, err.Error()) + } + result := py.NewListSized(len(dirEntries)) + for i, dirEntry := range dirEntries { + if returnsBytes { + result.Items[i] = py.Bytes(dirEntry.Name()) + } else { + result.Items[i] = py.String(dirEntry.Name()) + } + } + return result, nil +} + +const makedirs_doc = `makedirs(name [, mode=0o777][, exist_ok=False]) + +Super-mkdir; create a leaf directory and all intermediate ones. Works like +mkdir, except that any intermediate path segment (not just the rightmost) +will be created if it does not exist. If the target directory already +exists, raise an OSError if exist_ok is False. Otherwise no exception is +raised. This is recursive.` + +func makedirs(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Object, error) { + var ( + pypath py.Object + pymode py.Object = py.Int(0o777) + pyok py.Object = py.False + ) + err := py.ParseTupleAndKeywords( + args, kwargs, + "s#|ip:makedirs", []string{"path", "mode", "exist_ok"}, + &pypath, &pymode, &pyok, + ) + if err != nil { + return nil, err + } + + var ( + path = "" + mode = os.FileMode(pymode.(py.Int)) + ) + switch v := pypath.(type) { + case py.String: + path = string(v) + case py.Bytes: + path = string(v) + } + + if pyok.(py.Bool) == py.False { + // check if leaf exists. + _, err := os.Stat(path) + // FIXME(sbinet): handle other errors. + if err == nil { + return nil, py.ExceptionNewf(py.FileExistsError, "File exists: '%s'", path) + } + } + + err = os.MkdirAll(path, mode) + if err != nil { + return nil, err + } + + return py.None, nil +} + +const mkdir_doc = `Create a directory. + +If dir_fd is not None, it should be a file descriptor open to a directory, + and path should be relative; path will then be relative to that directory. +dir_fd may not be implemented on your platform. + If it is unavailable, using it will raise a NotImplementedError. + +The mode argument is ignored on Windows.` + +func mkdir(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Object, error) { + var ( + pypath py.Object + pymode py.Object = py.Int(511) + pydirfd py.Object = py.None + ) + err := py.ParseTupleAndKeywords( + args, kwargs, + "s#|ii:mkdir", []string{"path", "mode", "dir_fd"}, + &pypath, &pymode, &pydirfd, + ) + if err != nil { + return nil, err + } + + var ( + path = "" + mode = os.FileMode(pymode.(py.Int)) + ) + switch v := pypath.(type) { + case py.String: + path = string(v) + case py.Bytes: + path = string(v) + } + + if pydirfd != py.None { + // FIXME(sbinet) + return nil, py.ExceptionNewf(py.NotImplementedError, "mkdir(dir_fd=XXX) not implemented") + } + + err = os.Mkdir(path, mode) + if err != nil { + return nil, err + } + + return py.None, nil +} + +// putenv sets the value of an environment variable named by the key. +func putenv(self py.Object, args py.Tuple) (py.Object, error) { + if len(args) != 2 { + return nil, py.ExceptionNewf(py.TypeError, "missing required arguments: 'key:str' and 'value:str'") + } + k, ok := args[0].(py.String) + if !ok { + return nil, py.ExceptionNewf(py.TypeError, "str expected (pos 1), not "+args[0].Type().Name) + } + v, ok := args[1].(py.String) + if !ok { + return nil, py.ExceptionNewf(py.TypeError, "str expected (pos 2), not "+args[1].Type().Name) + } + err := os.Setenv(string(k), string(v)) + if err != nil { + return nil, py.ExceptionNewf(py.OSError, "Unable to set enviroment variable") + } + return py.None, nil +} + +// Unset (delete) the environment variable named key. +func unsetenv(self py.Object, args py.Tuple) (py.Object, error) { + if len(args) != 1 { + return nil, py.ExceptionNewf(py.TypeError, "missing one required argument: 'key:str'") + } + k, ok := args[0].(py.String) + if !ok { + return nil, py.ExceptionNewf(py.TypeError, "str expected (pos 1), not "+args[0].Type().Name) + } + err := os.Unsetenv(string(k)) + if err != nil { + return nil, py.ExceptionNewf(py.OSError, "Unable to unset enviroment variable") + } + return py.None, nil +} + +// os._exit() immediate program termination; unlike sys.exit(), which raises a SystemExit, this function will termninate the program immediately. +func _exit(self py.Object, args py.Tuple) (py.Object, error) { // can never return + if len(args) == 0 { + os.Exit(0) + } + arg, ok := args[0].(py.Int) + if !ok { + return nil, py.ExceptionNewf(py.TypeError, "expected int (pos 1), not "+args[0].Type().Name) + } + os.Exit(int(arg)) + return nil, nil +} + +const remove_doc = `Remove a file (same as unlink()). + +If dir_fd is not None, it should be a file descriptor open to a directory, + and path should be relative; path will then be relative to that directory. +dir_fd may not be implemented on your platform. + If it is unavailable, using it will raise a NotImplementedError.` + +func remove(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Object, error) { + var ( + pypath py.Object + pydir py.Object = py.None + ) + err := py.ParseTupleAndKeywords(args, kwargs, "s#|i:remove", []string{"path", "dir_fd"}, &pypath, &pydir) + if err != nil { + return nil, err + } + + if pydir != py.None { + // FIXME(sbinet) ? + return nil, py.ExceptionNewf(py.NotImplementedError, "remove(dir_fd=XXX) not implemented") + } + + var name string + switch v := pypath.(type) { + case py.String: + name = string(v) + case py.Bytes: + name = string(v) + } + + err = os.Remove(name) + if err != nil { + return nil, err + } + + return py.None, nil +} + +const removedirs_doc = `removedirs(name) + +Super-rmdir; remove a leaf directory and all empty intermediate +ones. Works like rmdir except that, if the leaf directory is +successfully removed, directories corresponding to rightmost path +segments will be pruned away until either the whole path is +consumed or an error occurs. Errors during this latter phase are +ignored -- they generally mean that a directory was not empty.` + +func removedirs(self py.Object, args py.Tuple) (py.Object, error) { + var pypath py.Object + err := py.ParseTuple(args, "s#:rmdir", &pypath) + if err != nil { + return nil, err + } + + var name string + switch v := pypath.(type) { + case py.String: + name = string(v) + case py.Bytes: + name = string(v) + } + + err = os.RemoveAll(name) + if err != nil { + return nil, err + } + + return py.None, nil +} + +const rmdir_doc = `Remove a directory. + +If dir_fd is not None, it should be a file descriptor open to a directory, + and path should be relative; path will then be relative to that directory. +dir_fd may not be implemented on your platform. + If it is unavailable, using it will raise a NotImplementedError.` + +func rmdir(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Object, error) { + var ( + pypath py.Object + pydir py.Object = py.None + ) + err := py.ParseTupleAndKeywords(args, kwargs, "s#|i:rmdir", []string{"path", "dir_fd"}, &pypath, &pydir) + if err != nil { + return nil, err + } + + if pydir != py.None { + // FIXME(sbinet) ? + return nil, py.ExceptionNewf(py.NotImplementedError, "rmdir(dir_fd=XXX) not implemented") + } + + var name string + switch v := pypath.(type) { + case py.String: + name = string(v) + case py.Bytes: + name = string(v) + } + + err = os.Remove(name) + if err != nil { + return nil, err + } + + return py.None, nil +} + +// os.system(command string) this function runs a shell command and directs the output to standard output. +func system(self py.Object, args py.Tuple) (py.Object, error) { + if len(args) != 1 { + return nil, py.ExceptionNewf(py.TypeError, "missing one required argument: 'command:str'") + } + arg, ok := args[0].(py.String) + if !ok { + return nil, py.ExceptionNewf(py.TypeError, "str expected (pos 1), not "+args[0].Type().Name) + } + + var command *exec.Cmd + if runtime.GOOS != "windows" { + command = exec.Command("/bin/sh", "-c", string(arg)) + } else { + command = exec.Command("cmd.exe", string(arg)) + } + outb, err := command.CombinedOutput() // - commbinedoutput to get both stderr and stdout - + if err != nil { + return nil, py.ExceptionNewf(py.OSError, err.Error()) + } + ok = py.Println(self, string(outb)) + if !ok { + return py.Int(1), nil + } + + return py.Int(0), nil +} diff --git a/stdlib/os/os_test.go b/stdlib/os/os_test.go new file mode 100644 index 00000000..8ae63d09 --- /dev/null +++ b/stdlib/os/os_test.go @@ -0,0 +1,15 @@ +// Copyright 2022 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package os_test + +import ( + "testing" + + "github.com/go-python/gpython/pytest" +) + +func TestOs(t *testing.T) { + pytest.RunScript(t, "./testdata/test.py") +} diff --git a/stdlib/os/testdata/test.py b/stdlib/os/testdata/test.py new file mode 100644 index 00000000..7b5b6815 --- /dev/null +++ b/stdlib/os/testdata/test.py @@ -0,0 +1,196 @@ +# Copyright 2022 The go-python Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +import os + +print("test os") +print("os.error: ", os.error) +print("os.getenv($GPYTHON_TEST_HOME)=", os.getenv("GPYTHON_TEST_HOME")) +os.putenv("GPYTHON_TEST_HOME", "/home/go") +print("os.environ($GPYTHON_TEST_HOME)=", os.environ.get("GPYTHON_TEST_HOME")) +print("os.getenv($GPYTHON_TEST_HOME)=", os.getenv("GPYTHON_TEST_HOME")) +os.unsetenv("GPYTHON_TEST_HOME") +print("os.unsetenv($GPYTHON_TEST_HOME)=", os.getenv("GPYTHON_TEST_HOME")) + +if not os.error is OSError: + print("os.error is not OSError!") +else: + print("os.error is OSError [OK]") + +## FIXME(sbinet): check returned value with a known one +## (ie: when os.mkdir is implemented) +if os.getcwd() == None: + print("os.getcwd() == None !") +else: + print("os.getcwd() != None [OK]") + +## FIXME(sbinet): check returned value with a known one +## (ie: when os.mkdir is implemented) +if os.getcwdb() == None: + print("os.getcwdb() == None !") +else: + print("os.getcwdb() != None [OK]") + +print("os.system('echo hello')...") +if os.name != "nt": + os.system('echo hello') +else: ## FIXME(sbinet): find a way to test this nicely + print("hello\n") + +if os.getpid() > 1: + print("os.getpid is greater than 1 [OK]") +else: + print("invalid os.getpid: ", os.getpid()) + +orig = os.getcwd() +testdir = "/" +if os.name == "nt": + testdir = "C:\\" +os.chdir(testdir) +if os.getcwd() != testdir: + print("invalid getcwd() after os.chdir:",os.getcwd()) +else: + print("os.chdir(testdir) [OK]") +os.chdir(orig) + +try: + os.chdir(1) + print("expected an error with os.chdir(1)") +except TypeError: + print("os.chdir(1) failed [OK]") + +try: + os.environ.get(15) + print("expected an error with os.environ.get(15)") +except KeyError: + print("os.environ.get(15) failed [OK]") + +try: + os.putenv() + print("expected an error with os.putenv()") +except TypeError: + print("os.putenv() failed [OK]") + +try: + os.unsetenv() + print("expected an error with os.unsetenv()") +except TypeError: + print("os.unsetenv() failed [OK]") + +try: + os.getenv() + print("expected an error with os.getenv()") +except TypeError: + print("os.getenv() failed [OK]") + +try: + os.unsetenv("FOO", "BAR") + print("expected an error with os.unsetenv(\"FOO\", \"BAR\")") +except TypeError: + print("os.unsetenv(\"FOO\", \"BAR\") failed [OK]") + +if bytes(os.getcwd(), "utf-8") == os.getcwdb(): + print('bytes(os.getcwd(), "utf-8") == os.getcwdb() [OK]') +else: + print('expected: bytes(os.getcwd(), "utf-8") == os.getcwdb()') + +golden = { + "posix": { + "sep": "/", + "pathsep": ":", + "linesep": "\n", + "devnull": "/dev/null", + "altsep": None + }, + "nt": { + "sep": "\\", + "pathsep": ";", + "linesep": "\r\n", + "devnull": "nul", + "altsep": "/" + }, +}[os.name] + +for k in ("sep", "pathsep", "linesep", "devnull", "altsep"): + if getattr(os, k) != golden[k]: + print("invalid os."+k+": got=",getattr(os,k),", want=", golden[k]) + else: + print("os."+k+": [OK]") + +## close +import tempfile +fd, tmp = tempfile.mkstemp() +os.close(fd=fd) +os.remove(tmp) +try: + os.close(-1) + print("closing a bad file descriptor should have failed") +except Exception as e: + print("caught: %s [OK]" % e) + +## fdopen +import tempfile +fd, tmp = tempfile.mkstemp() +f = os.fdopen(fd, "w+") +## if f.name != str(fd): +## print("invalid fd-name:", f.name) +f.close() +os.remove(tmp) + +## mkdir,rmdir,remove,removedirs +import tempfile +try: + top = tempfile.mkdtemp(prefix="gpython-os-test-") + dir1 = top + os.sep + "dir1" + dir2 = top + os.sep + "dir2" + dir11 = top + os.sep + "dir1" + os.sep + "dir11" + fname = dir2 + os.sep + "foo.txt" + os.mkdir(dir1) + os.rmdir(dir1) + os.mkdir(dir1) + os.mkdir(dir2) + os.mkdir(dir11) + print(os.listdir(bytes(top, "utf-8"))) + orig = os.getcwd() + os.chdir(top) + print(os.listdir()) + os.chdir(orig) + os.removedirs(dir1) + try: + os.mkdir(dir11) + print("creating nested dirs with os.mkdir should have failed") + except SystemError as e: + print("caught: SystemError - no such file or directory [OK]") + except Exception as e: + print("caught: %s" % e) + + os.makedirs(dir11) + try: + os.makedirs(dir11) + print("creating already existing dirs should have failed") + except FileExistsError as e: + print("caught: FileExistsError [OK]") + except Exception as e: + print("INVALID error caught: %s" % e) + os.makedirs(dir11, exist_ok=True) + + with open(fname, "w+") as f: + pass + try: + os.rmdir(dir2) + print("removing a non-empty directory should have failed") + except SystemError as e: + print("caught: SystemError - directory not empty [OK]") + except Exception as e: + print("INVALID error caught: %s" % e) + os.remove(fname) + os.rmdir(dir2) + print(os.listdir(top)) +except Exception as e: + print("could not create/remove directories: %s" % e) +finally: + os.removedirs(top) + print("os.{mkdir,rmdir,remove,removedirs} worked as expected") + +print("OK") diff --git a/stdlib/os/testdata/test_golden.txt b/stdlib/os/testdata/test_golden.txt new file mode 100644 index 00000000..fd5fc39b --- /dev/null +++ b/stdlib/os/testdata/test_golden.txt @@ -0,0 +1,35 @@ +test os +os.error: +os.getenv($GPYTHON_TEST_HOME)= None +os.environ($GPYTHON_TEST_HOME)= None +os.getenv($GPYTHON_TEST_HOME)= /home/go +os.unsetenv($GPYTHON_TEST_HOME)= None +os.error is OSError [OK] +os.getcwd() != None [OK] +os.getcwdb() != None [OK] +os.system('echo hello')... +hello + +os.getpid is greater than 1 [OK] +os.chdir(testdir) [OK] +os.chdir(1) failed [OK] +os.environ.get(15) failed [OK] +os.putenv() failed [OK] +os.unsetenv() failed [OK] +os.getenv() failed [OK] +os.unsetenv("FOO", "BAR") failed [OK] +bytes(os.getcwd(), "utf-8") == os.getcwdb() [OK] +os.sep: [OK] +os.pathsep: [OK] +os.linesep: [OK] +os.devnull: [OK] +os.altsep: [OK] +caught: OSError: 'Bad file descriptor' [OK] +[b'dir1', b'dir2'] +['dir1', 'dir2'] +caught: SystemError - no such file or directory [OK] +caught: FileExistsError [OK] +caught: SystemError - directory not empty [OK] +['dir1'] +os.{mkdir,rmdir,remove,removedirs} worked as expected +OK diff --git a/stdlib/stdlib.go b/stdlib/stdlib.go new file mode 100644 index 00000000..7d1fb811 --- /dev/null +++ b/stdlib/stdlib.go @@ -0,0 +1,298 @@ +// Copyright 2022 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package stdlib provides the bootstrap code to wire in all the stdlib +// (python) modules into a gpython context and VM. +package stdlib + +import ( + "bytes" + "os" + "path" + "path/filepath" + "strings" + "sync" + + "github.com/go-python/gpython/py" + "github.com/go-python/gpython/stdlib/marshal" + "github.com/go-python/gpython/vm" + + _ "github.com/go-python/gpython/stdlib/array" + _ "github.com/go-python/gpython/stdlib/binascii" + _ "github.com/go-python/gpython/stdlib/builtin" + _ "github.com/go-python/gpython/stdlib/glob" + _ "github.com/go-python/gpython/stdlib/math" + _ "github.com/go-python/gpython/stdlib/os" + _ "github.com/go-python/gpython/stdlib/string" + _ "github.com/go-python/gpython/stdlib/sys" + _ "github.com/go-python/gpython/stdlib/tempfile" + _ "github.com/go-python/gpython/stdlib/time" +) + +func init() { + // Assign the base-level py.Context creation function while also preventing an import cycle. + py.NewContext = NewContext +} + +// context implements interface py.Context +type context struct { + store *py.ModuleStore + opts py.ContextOpts + closeOnce sync.Once + closing bool + closed bool + running sync.WaitGroup + done chan struct{} +} + +// NewContext creates a new gpython interpreter instance context. +// +// See interface py.Context defined in py/run.go +func NewContext(opts py.ContextOpts) py.Context { + ctx := &context{ + opts: opts, + done: make(chan struct{}), + closing: false, + closed: false, + } + + ctx.store = py.NewModuleStore() + + py.Import(ctx, "builtins", "sys") + + sys_mod := ctx.Store().MustGetModule("sys") + sys_mod.Globals["argv"] = py.NewListFromStrings(opts.SysArgs) + sys_mod.Globals["path"] = py.NewListFromStrings(opts.SysPaths) + + return ctx +} + +// ModuleInit digests a ModuleImpl, compiling and marshalling as needed, creating a new Module instance in this Context. +func (ctx *context) ModuleInit(impl *py.ModuleImpl) (*py.Module, error) { + err := ctx.pushBusy() + defer ctx.popBusy() + if err != nil { + return nil, err + } + + if impl.Code == nil && len(impl.CodeSrc) > 0 { + impl.Code, err = py.Compile(string(impl.CodeSrc), impl.Info.FileDesc, py.ExecMode, 0, true) + if err != nil { + return nil, err + } + } + + if impl.Code == nil && len(impl.CodeBuf) > 0 { + codeBuf := bytes.NewBuffer(impl.CodeBuf) + obj, err := marshal.ReadObject(codeBuf) + if err != nil { + return nil, err + } + impl.Code, _ = obj.(*py.Code) + if impl.Code == nil { + return nil, py.ExceptionNewf(py.AssertionError, "Embedded code did not produce a py.Code object") + } + } + + module, err := ctx.Store().NewModule(ctx, impl) + if err != nil { + return nil, err + } + + if impl.Code != nil { + _, err = ctx.RunCode(impl.Code, module.Globals, module.Globals, nil) + if err != nil { + return nil, err + } + } + + return module, nil +} + +// See interface py.Context defined in py/run.go +func (ctx *context) ResolveAndCompile(pathname string, opts py.CompileOpts) (py.CompileOut, error) { + err := ctx.pushBusy() + defer ctx.popBusy() + if err != nil { + return py.CompileOut{}, err + } + + tryPaths := defaultPaths + if opts.UseSysPaths { + tryPaths = ctx.Store().MustGetModule("sys").Globals["path"].(*py.List).Items + } + + out := py.CompileOut{} + + err = resolveRunPath(pathname, opts, tryPaths, func(fpath string) (bool, error) { + + stat, err := os.Stat(fpath) + if err == nil && stat.IsDir() { + // FIXME this is a massive simplification! + fpath = path.Join(fpath, "__init__.py") + _, err = os.Stat(fpath) + } + + ext := strings.ToLower(filepath.Ext(fpath)) + if ext == "" && os.IsNotExist(err) { + fpath += ".py" + ext = ".py" + _, err = os.Stat(fpath) + } + + // Keep searching while we get FNFs, stop on an error + if err != nil { + if os.IsNotExist(err) { + return true, nil + } + err = py.ExceptionNewf(py.OSError, "Error accessing %q: %v", fpath, err) + return false, err + } + + switch ext { + case ".py": + var pySrc []byte + pySrc, err = os.ReadFile(fpath) + if err != nil { + return false, py.ExceptionNewf(py.OSError, "Error reading %q: %v", fpath, err) + } + + out.Code, err = py.Compile(string(pySrc), fpath, py.ExecMode, 0, true) + if err != nil { + return false, err + } + out.SrcPathname = fpath + case ".pyc": + file, err := os.Open(fpath) + if err != nil { + return false, py.ExceptionNewf(py.OSError, "Error opening %q: %v", fpath, err) + } + defer file.Close() + codeObj, err := marshal.ReadPyc(file) + if err != nil { + return false, py.ExceptionNewf(py.ImportError, "Failed to marshal %q: %v", fpath, err) + } + out.Code, _ = codeObj.(*py.Code) + out.PycPathname = fpath + } + + out.FileDesc = fpath + return false, nil + }) + + if out.Code == nil && err == nil { + err = py.ExceptionNewf(py.AssertionError, "Missing code object") + } + + if err != nil { + return py.CompileOut{}, err + } + + return out, nil +} + +func (ctx *context) pushBusy() error { + if ctx.closed { + return py.ExceptionNewf(py.RuntimeError, "Context closed") + } + ctx.running.Add(1) + return nil +} + +func (ctx *context) popBusy() { + ctx.running.Done() +} + +// See interface py.Context defined in py/run.go +func (ctx *context) Close() error { + ctx.closeOnce.Do(func() { + ctx.closing = true + ctx.running.Wait() + ctx.closed = true + + // Give each module a chance to release resources + ctx.store.OnContextClosed() + close(ctx.done) + }) + return nil +} + +// See interface py.Context defined in py/run.go +func (ctx *context) Done() <-chan struct{} { + return ctx.done +} + +var defaultPaths = []py.Object{ + py.String("."), +} + +func resolveRunPath(runPath string, opts py.CompileOpts, pathObjs []py.Object, tryPath func(pyPath string) (bool, error)) error { + runPath = strings.TrimSuffix(runPath, "/") + + var ( + err error + cwd string + cont = true + ) + + for _, pathObj := range pathObjs { + pathStr, ok := pathObj.(py.String) + if !ok { + continue + } + + // If an absolute path, just try that. + // Otherwise, check from the passed current dir then check from the current working dir. + fpath := path.Join(string(pathStr), runPath) + if filepath.IsAbs(fpath) { + cont, err = tryPath(fpath) + } else { + if len(opts.CurDir) > 0 { + subPath := path.Join(opts.CurDir, fpath) + cont, err = tryPath(subPath) + } + if cont && err == nil { + if cwd == "" { + cwd, _ = os.Getwd() + } + subPath := path.Join(cwd, fpath) + cont, err = tryPath(subPath) + } + } + if !cont { + break + } + } + + if err != nil { + return err + } + + if cont { + return py.ExceptionNewf(py.FileNotFoundError, "Failed to resolve %q", runPath) + } + + return err +} + +// See interface py.Context defined in py/run.go +func (ctx *context) RunCode(code *py.Code, globals, locals py.StringDict, closure py.Tuple) (py.Object, error) { + err := ctx.pushBusy() + defer ctx.popBusy() + if err != nil { + return nil, err + } + + return vm.EvalCode(ctx, code, globals, locals, nil, nil, nil, nil, closure) +} + +// See interface py.Context defined in py/run.go +func (ctx *context) GetModule(moduleName string) (*py.Module, error) { + return ctx.store.GetModule(moduleName) +} + +// See interface py.Context defined in py/run.go +func (ctx *context) Store() *py.ModuleStore { + return ctx.store +} diff --git a/stdlib/string/string.go b/stdlib/string/string.go new file mode 100644 index 00000000..314fdd09 --- /dev/null +++ b/stdlib/string/string.go @@ -0,0 +1,117 @@ +// Copyright 2022 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package string provides the implementation of the python's 'string' module. +package string + +import ( + "strings" + + "github.com/go-python/gpython/py" +) + +func init() { + py.RegisterModule(&py.ModuleImpl{ + Info: py.ModuleInfo{ + Name: "string", + Doc: module_doc, + }, + Methods: []*py.Method{ + py.MustNewMethod("capwords", capwords, 0, capwords_doc), + }, + Globals: py.StringDict{ + "whitespace": whitespace, + "ascii_lowercase": ascii_lowercase, + "ascii_uppercase": ascii_uppercase, + "ascii_letters": ascii_letters, + "digits": digits, + "hexdigits": hexdigits, + "octdigits": octdigits, + "punctuation": punctuation, + "printable": printable, + }, + }) +} + +const module_doc = `A collection of string constants. + +Public module variables: + +whitespace -- a string containing all ASCII whitespace +ascii_lowercase -- a string containing all ASCII lowercase letters +ascii_uppercase -- a string containing all ASCII uppercase letters +ascii_letters -- a string containing all ASCII letters +digits -- a string containing all ASCII decimal digits +hexdigits -- a string containing all ASCII hexadecimal digits +octdigits -- a string containing all ASCII octal digits +punctuation -- a string containing all ASCII punctuation characters +printable -- a string containing all ASCII characters considered printable +` + +var ( + whitespace = py.String(" \t\n\r\x0b\x0c") + ascii_lowercase = py.String("abcdefghijklmnopqrstuvwxyz") + ascii_uppercase = py.String("ABCDEFGHIJKLMNOPQRSTUVWXYZ") + ascii_letters = ascii_lowercase + ascii_uppercase + digits = py.String("0123456789") + hexdigits = py.String("0123456789abcdefABCDEF") + octdigits = py.String("01234567") + punctuation = py.String("!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~") + printable = py.String("0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~ \t\n\r\x0b\x0c") +) + +const capwords_doc = `capwords(s [,sep]) -> string + +Split the argument into words using split, capitalize each +word using capitalize, and join the capitalized words using +join. If the optional second argument sep is absent or None, +runs of whitespace characters are replaced by a single space +and leading and trailing whitespace are removed, otherwise +sep is used to split and join the words.` + +func capwords(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Object, error) { + var ( + pystr py.Object + pysep py.Object = py.None + ) + err := py.ParseTupleAndKeywords(args, kwargs, "s|z", []string{"s", "sep"}, &pystr, &pysep) + if err != nil { + return nil, err + } + + pystr = py.String(strings.ToLower(string(pystr.(py.String)))) + pyvs, err := pystr.(py.String).Split(py.Tuple{pysep}, nil) + if err != nil { + return nil, err + } + + var ( + lst = pyvs.(*py.List).Items + vs = make([]string, len(lst)) + sep = "" + title = func(s string) string { + if s == "" { + return s + } + return strings.ToUpper(s[:1]) + s[1:] + } + ) + + switch pysep { + case py.None: + for i := range vs { + v := string(lst[i].(py.String)) + vs[i] = title(strings.Trim(v, string(whitespace))) + } + sep = " " + default: + sep = string(pysep.(py.String)) + for i := range vs { + v := string(lst[i].(py.String)) + vs[i] = title(v) + } + } + + return py.String(strings.Join(vs, sep)), nil +} diff --git a/stdlib/string/string_test.go b/stdlib/string/string_test.go new file mode 100644 index 00000000..0ae79ef5 --- /dev/null +++ b/stdlib/string/string_test.go @@ -0,0 +1,15 @@ +// Copyright 2022 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package string_test + +import ( + "testing" + + "github.com/go-python/gpython/pytest" +) + +func TestString(t *testing.T) { + pytest.RunScript(t, "./testdata/test.py") +} diff --git a/stdlib/string/testdata/test.py b/stdlib/string/testdata/test.py new file mode 100644 index 00000000..2bec0736 --- /dev/null +++ b/stdlib/string/testdata/test.py @@ -0,0 +1,32 @@ +# Copyright 2022 The go-python Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +import string + +print("globals:") +for name in ("whitespace", + "ascii_lowercase", + "ascii_uppercase", + "ascii_letters", + "digits", + "hexdigits", + "octdigits", + "punctuation", + "printable"): + v = getattr(string, name) + print("\nstring.%s:\n%s" % (name,repr(v))) + +def assertEqual(x, y): + assert x == y, "got: %s, want: %s" % (repr(x), repr(y)) + +assertEqual(string.capwords('abc def ghi'), 'Abc Def Ghi') +assertEqual(string.capwords('abc\tdef\nghi'), 'Abc Def Ghi') +assertEqual(string.capwords('abc\t def \nghi'), 'Abc Def Ghi') +assertEqual(string.capwords('ABC DEF GHI'), 'Abc Def Ghi') +assertEqual(string.capwords('ABC-DEF-GHI', '-'), 'Abc-Def-Ghi') +assertEqual(string.capwords('ABC-def DEF-ghi GHI'), 'Abc-def Def-ghi Ghi') +assertEqual(string.capwords(' aBc DeF '), 'Abc Def') +assertEqual(string.capwords('\taBc\tDeF\t'), 'Abc Def') +assertEqual(string.capwords('\taBc\tDeF\t', '\t'), '\tAbc\tDef\t') + diff --git a/stdlib/string/testdata/test_golden.txt b/stdlib/string/testdata/test_golden.txt new file mode 100644 index 00000000..95a3e2ef --- /dev/null +++ b/stdlib/string/testdata/test_golden.txt @@ -0,0 +1,28 @@ +globals: + +string.whitespace: +' \t\n\r\x0b\x0c' + +string.ascii_lowercase: +'abcdefghijklmnopqrstuvwxyz' + +string.ascii_uppercase: +'ABCDEFGHIJKLMNOPQRSTUVWXYZ' + +string.ascii_letters: +'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' + +string.digits: +'0123456789' + +string.hexdigits: +'0123456789abcdefABCDEF' + +string.octdigits: +'01234567' + +string.punctuation: +'!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~' + +string.printable: +'0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~ \t\n\r\x0b\x0c' diff --git a/sys/sys.go b/stdlib/sys/sys.go similarity index 96% rename from sys/sys.go rename to stdlib/sys/sys.go index fcd1f0df..3a2318eb 100644 --- a/sys/sys.go +++ b/stdlib/sys/sys.go @@ -19,6 +19,7 @@ package sys import ( "os" + "runtime" "github.com/go-python/gpython/py" ) @@ -347,10 +348,10 @@ func sys_setrecursionlimit(self py.Object, args py.Tuple) (py.Object, error) { return nil, py.NotImplementedError } -const hash_info_doc = `hash_info - -A struct sequence providing parameters used for computing -numeric hashes. The attributes are read only.` +// const hash_info_doc = `hash_info +// +// A struct sequence providing parameters used for computing +// numeric hashes. The attributes are read only.` // PyStructSequence_Field hash_info_fields[] = { // {"width", "width of the type used for hashing, in bits"}, @@ -539,9 +540,9 @@ func sys_clear_type_cache(self py.Object, args py.Tuple) (py.Object, error) { return nil, py.NotImplementedError } -const flags__doc__ = `sys.flags - -Flags provided through command line arguments or environment vars.` +// const flags__doc__ = `sys.flags +// +// Flags provided through command line arguments or environment vars.` // PyTypeObject FlagsType; @@ -603,9 +604,9 @@ Flags provided through command line arguments or environment vars.` // return seq; // } -const version_info__doc__ = `sys.version_info - -Version information as a named tuple.` +//const version_info__doc__ = `sys.version_info +// +//Version information as a named tuple.` // PyStructSequence_Field version_info_fields[] = { // {"major", "Major release number"}, @@ -652,18 +653,34 @@ func init() { py.MustNewMethod("call_tracing", sys_call_tracing, 0, call_tracing_doc), py.MustNewMethod("_debugmallocstats", sys_debugmallocstats, 0, debugmallocstats_doc), } - argv := MakeArgv(os.Args[1:]) - stdin, stdout, stderr := &py.File{os.Stdin, py.FileRead}, - &py.File{os.Stdout, py.FileWrite}, - &py.File{os.Stderr, py.FileWrite} + + stdin := &py.File{File: os.Stdin, FileMode: py.FileRead} + stdout := &py.File{File: os.Stdout, FileMode: py.FileWrite} + stderr := &py.File{File: os.Stderr, FileMode: py.FileWrite} + + executable, err := os.Executable() + if err != nil { + switch runtime.GOOS { + case "js", "wasip1": + // These platforms don't implement os.Executable (at least as of Go + // 1.21), see https://github.com/tailscale/tailscale/pull/8325 + executable = "gpython" + default: + panic(err) + } + } + globals := py.StringDict{ - "argv": argv, + "path": py.NewList(), + "argv": py.NewListFromStrings(os.Args[1:]), "stdin": stdin, "stdout": stdout, "stderr": stderr, "__stdin__": stdin, "__stdout__": stdout, "__stderr__": stderr, + "executable": py.String(executable), + //"version": py.Int(MARSHAL_VERSION), // /* stdin/stdout/stderr are now set by pythonrun.c */ @@ -787,14 +804,14 @@ func init() { // SET_SYS_FROM_STRING("thread_info", PyThread_GetInfo()); // #endif } - py.NewModule("sys", module_doc, methods, globals) -} -// Makes an argv into a tuple -func MakeArgv(pyargs []string) py.Object { - argv := py.NewListSized(len(pyargs)) - for i, v := range pyargs { - argv.Items[i] = py.String(v) - } - return argv + py.RegisterModule(&py.ModuleImpl{ + Info: py.ModuleInfo{ + Name: "sys", + Doc: module_doc, + }, + Methods: methods, + Globals: globals, + }) + } diff --git a/stdlib/tempfile/tempfile.go b/stdlib/tempfile/tempfile.go new file mode 100644 index 00000000..c31612da --- /dev/null +++ b/stdlib/tempfile/tempfile.go @@ -0,0 +1,283 @@ +// Copyright 2022 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package tempfile provides the implementation of the python's 'tempfile' module. +package tempfile + +import ( + "fmt" + "os" + + "github.com/go-python/gpython/py" +) + +var ( + gblTempDir py.Object = py.None +) + +const tempfile_doc = `Temporary files. + +This module provides generic, low- and high-level interfaces for +creating temporary files and directories. All of the interfaces +provided by this module can be used without fear of race conditions +except for 'mktemp'. 'mktemp' is subject to race conditions and +should not be used; it is provided for backward compatibility only. + +The default path names are returned as str. If you supply bytes as +input, all return values will be in bytes. Ex: + + >>> tempfile.mkstemp() + (4, '/tmp/tmptpu9nin8') + >>> tempfile.mkdtemp(suffix=b'') + b'/tmp/tmppbi8f0hy' + +This module also provides some data items to the user: + + TMP_MAX - maximum number of names that will be tried before + giving up. + tempdir - If this is set to a string before the first use of + any routine from this module, it will be considered as + another candidate location to store temporary files.` + +func init() { + py.RegisterModule(&py.ModuleImpl{ + Info: py.ModuleInfo{ + Name: "tempfile", + Doc: tempfile_doc, + }, + Methods: []*py.Method{ + py.MustNewMethod("gettempdir", gettempdir, 0, gettempdir_doc), + py.MustNewMethod("gettempdirb", gettempdirb, 0, gettempdirb_doc), + py.MustNewMethod("mkdtemp", mkdtemp, 0, mkdtemp_doc), + py.MustNewMethod("mkstemp", mkstemp, 0, mkstemp_doc), + }, + Globals: py.StringDict{ + "tempdir": gblTempDir, + }, + }) +} + +const gettempdir_doc = `Returns tempfile.tempdir as str.` + +func gettempdir(self py.Object) (py.Object, error) { + // FIXME(sbinet): lock access to glbTempDir? + if gblTempDir != py.None { + switch dir := gblTempDir.(type) { + case py.String: + return dir, nil + case py.Bytes: + return py.String(dir), nil + default: + return nil, py.ExceptionNewf(py.TypeError, "expected str, bytes or os.PathLike object, not %s", dir.Type().Name) + } + } + return py.String(os.TempDir()), nil +} + +const gettempdirb_doc = `Returns tempfile.tempdir as bytes.` + +func gettempdirb(self py.Object) (py.Object, error) { + // FIXME(sbinet): lock access to glbTempDir? + if gblTempDir != py.None { + switch dir := gblTempDir.(type) { + case py.String: + return py.Bytes(dir), nil + case py.Bytes: + return dir, nil + default: + return nil, py.ExceptionNewf(py.TypeError, "expected str, bytes or os.PathLike object, not %s", dir.Type().Name) + } + } + return py.Bytes(os.TempDir()), nil +} + +const mkdtemp_doc = `mkdtemp(suffix=None, prefix=None, dir=None) + User-callable function to create and return a unique temporary + directory. The return value is the pathname of the directory. + + Arguments are as for mkstemp, except that the 'text' argument is + not accepted. + + The directory is readable, writable, and searchable only by the + creating user. + + Caller is responsible for deleting the directory when done with it.` + +func mkdtemp(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Object, error) { + var ( + pysuffix py.Object = py.None + pyprefix py.Object = py.None + pydir py.Object = py.None + ) + err := py.ParseTupleAndKeywords(args, kwargs, + "|z#z#z#:mkdtemp", + []string{"suffix", "prefix", "dir"}, + &pysuffix, &pyprefix, &pydir, + ) + if err != nil { + return nil, err + } + + str := func(v py.Object, typ *uint8) string { + switch v := v.(type) { + case py.Bytes: + *typ = 2 + return string(v) + case py.String: + *typ = 1 + return string(v) + case py.NoneType: + *typ = 0 + return "" + default: + panic(fmt.Errorf("tempfile: invalid type %T (v=%+v)", v, v)) + } + } + + var ( + t1, t2, t3 uint8 + + suffix = str(pysuffix, &t1) + prefix = str(pyprefix, &t2) + dir = str(pydir, &t3) + pattern = prefix + "*" + suffix + ) + + cmp := func(t1, t2 uint8) bool { + if t1 > 0 && t2 > 0 { + return t1 == t2 + } + return true + } + + if !cmp(t1, t2) || !cmp(t1, t3) || !cmp(t2, t3) { + return nil, py.ExceptionNewf(py.TypeError, "Can't mix bytes and non-bytes in path components") + } + + tmp, err := os.MkdirTemp(dir, pattern) + if err != nil { + return nil, err + } + + typ := t1 + if typ == 0 { + typ = t2 + } + if typ == 0 { + typ = t3 + } + + switch typ { + case 2: + return py.Bytes(tmp), nil + default: + return py.String(tmp), nil + } +} + +const mkstemp_doc = `mkstemp(suffix=None, prefix=None, dir=None, text=False) + +User-callable function to create and return a unique temporary +file. The return value is a pair (fd, name) where fd is the +file descriptor returned by os.open, and name is the filename. + +If 'suffix' is not None, the file name will end with that suffix, +otherwise there will be no suffix. + +If 'prefix' is not None, the file name will begin with that prefix, +otherwise a default prefix is used. + +If 'dir' is not None, the file will be created in that directory, +otherwise a default directory is used. + +If 'text' is specified and true, the file is opened in text +mode. Else (the default) the file is opened in binary mode. + +If any of 'suffix', 'prefix' and 'dir' are not None, they must be the +same type. If they are bytes, the returned name will be bytes; str +otherwise. + +The file is readable and writable only by the creating user ID. +If the operating system uses permission bits to indicate whether a +file is executable, the file is executable by no one. The file +descriptor is not inherited by children of this process. + +Caller is responsible for deleting the file when done with it.` + +func mkstemp(self py.Object, args py.Tuple, kwargs py.StringDict) (py.Object, error) { + var ( + pysuffix py.Object = py.None + pyprefix py.Object = py.None + pydir py.Object = py.None + pytext py.Object = py.False // FIXME(sbinet): can we do something with that? + ) + + err := py.ParseTupleAndKeywords(args, kwargs, + "|z#z#z#p:mkstemp", + []string{"suffix", "prefix", "dir", "text"}, + &pysuffix, &pyprefix, &pydir, &pytext, + ) + if err != nil { + return nil, err + } + + str := func(v py.Object, typ *uint8) string { + switch v := v.(type) { + case py.Bytes: + *typ = 2 + return string(v) + case py.String: + *typ = 1 + return string(v) + case py.NoneType: + *typ = 0 + return "" + default: + panic(fmt.Errorf("tempfile: invalid type %T (v=%+v)", v, v)) + } + } + + var ( + t1, t2, t3 uint8 + + suffix = str(pysuffix, &t1) + prefix = str(pyprefix, &t2) + dir = str(pydir, &t3) + pattern = prefix + "*" + suffix + ) + + cmp := func(t1, t2 uint8) bool { + if t1 > 0 && t2 > 0 { + return t1 == t2 + } + return true + } + + if !cmp(t1, t2) || !cmp(t1, t3) || !cmp(t2, t3) { + return nil, py.ExceptionNewf(py.TypeError, "Can't mix bytes and non-bytes in path components") + } + + f, err := os.CreateTemp(dir, pattern) + if err != nil { + return nil, err + } + + typ := t1 + if typ == 0 { + typ = t2 + } + if typ == 0 { + typ = t3 + } + + tuple := py.Tuple{py.Int(f.Fd())} + switch typ { + case 2: + tuple = append(tuple, py.Bytes(f.Name())) + default: + tuple = append(tuple, py.String(f.Name())) + } + + return tuple, nil +} diff --git a/stdlib/tempfile/tempfile_test.go b/stdlib/tempfile/tempfile_test.go new file mode 100644 index 00000000..33a75e98 --- /dev/null +++ b/stdlib/tempfile/tempfile_test.go @@ -0,0 +1,15 @@ +// Copyright 2022 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package tempfile_test + +import ( + "testing" + + "github.com/go-python/gpython/pytest" +) + +func TestTempfile(t *testing.T) { + pytest.RunScript(t, "./testdata/test.py") +} diff --git a/stdlib/tempfile/testdata/test.py b/stdlib/tempfile/testdata/test.py new file mode 100644 index 00000000..0fdecc9d --- /dev/null +++ b/stdlib/tempfile/testdata/test.py @@ -0,0 +1,110 @@ +# Copyright 2022 The go-python Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +import tempfile +import os + +print("test tempfile") + +if not tempfile.tempdir is None: + print("tempfile.tempdir is not None: %s" % (tempfile.tempdir,)) +else: + print("tempfile.tempdir is None [OK]") + +v = tempfile.gettempdir() +if type(v) != type(""): + print("tempfile.gettempdir() returned %s (type=%s)" % (v, type(v))) + +v = tempfile.gettempdirb() +if type(v) != type(b""): + print("tempfile.gettempdirb() returned %s (type=%s)" % (v, type(v))) + +## mkdtemp +try: + tmp = tempfile.mkdtemp() + os.rmdir(tmp) + print("mkdtemp() [OK]") +except Exception as e: + print("could not create tmp dir w/ mkdtemp(): %s" % e) + +try: + tmp = tempfile.mkdtemp(prefix="prefix-", suffix="-suffix") + os.rmdir(tmp) + print("mkdtemp(prefix='prefix-', suffix='-suffix') [OK]") +except Exception as e: + print("could not create tmp dir w/ mkdtemp(prefix='prefix-', suffix='-suffix'): %s" % e) + +try: + top = tempfile.mkdtemp(prefix="prefix-", suffix="-suffix") + tmp = tempfile.mkdtemp(prefix="prefix-", suffix="-suffix", dir=top) + os.rmdir(tmp) + os.rmdir(top) + print("mkdtemp(prefix='prefix-', suffix='-suffix', dir=top) [OK]") +except Exception as e: + print("could not create tmp dir w/ mkdtemp(prefix='prefix-', suffix='-suffix', dir=top): %s" % e) + +try: + top = tempfile.mkdtemp(prefix="prefix-", suffix="-suffix") + tmp = tempfile.mkdtemp(prefix="prefix-", suffix="-suffix", dir=top) + os.removedirs(top) + print("mkdtemp(prefix='prefix-', suffix='-suffix', dir=top) [OK]") +except Exception as e: + print("could not create tmp dir w/ mkdtemp(prefix='prefix-', suffix='-suffix', dir=top): %s" % e) + +try: + tmp = tempfile.mkdtemp(prefix=b"prefix-", suffix="-suffix") + print("missing exception!") + os.rmdir(tmp) +except TypeError as e: + print("caught: %s [OK]" % e) +except Exception as e: + print("INVALID error caught: %s" % e) + +def remove(fd, name): + os.close(fd) + os.remove(name) + +## mkstemp +try: + fd, tmp = tempfile.mkstemp() + remove(fd, tmp) + print("mkstemp() [OK]") +except Exception as e: + print("could not create tmp dir w/ mkstemp(): %s" % e) + +try: + fd, tmp = tempfile.mkstemp(prefix="prefix-", suffix="-suffix") + remove(fd, tmp) + print("mkstemp(prefix='prefix-', suffix='-suffix') [OK]") +except Exception as e: + print("could not create tmp dir w/ mkstemp(prefix='prefix-', suffix='-suffix'): %s" % e) + +try: + top = tempfile.mkdtemp(prefix="prefix-", suffix="-suffix") + fd, tmp = tempfile.mkstemp(prefix="prefix-", suffix="-suffix", dir=top) + remove(fd, tmp) + os.remove(top) + print("mkstemp(prefix='prefix-', suffix='-suffix', dir=top) [OK]") +except Exception as e: + print("could not create tmp dir w/ mkstemp(prefix='prefix-', suffix='-suffix', dir=top): %s" % e) + +try: + top = tempfile.mkdtemp(prefix="prefix-", suffix="-suffix") + fd, tmp = tempfile.mkstemp(prefix="prefix-", suffix="-suffix", dir=top) + os.fdopen(fd).close() ## needed on Windows. + os.removedirs(top) + print("mkstemp(prefix='prefix-', suffix='-suffix', dir=top) [OK]") +except Exception as e: + print("could not create tmp dir w/ mkstemp(prefix='prefix-', suffix='-suffix', dir=top): %s" % e) + +try: + fd, tmp = tempfile.mkstemp(prefix=b"prefix-", suffix="-suffix") + print("missing exception!") + remove(fd, tmp) +except TypeError as e: + print("caught: %s [OK]" % e) +except Exception as e: + print("INVALID error caught: %s" % e) + +print("OK") diff --git a/stdlib/tempfile/testdata/test_golden.txt b/stdlib/tempfile/testdata/test_golden.txt new file mode 100644 index 00000000..ff7814de --- /dev/null +++ b/stdlib/tempfile/testdata/test_golden.txt @@ -0,0 +1,13 @@ +test tempfile +tempfile.tempdir is None [OK] +mkdtemp() [OK] +mkdtemp(prefix='prefix-', suffix='-suffix') [OK] +mkdtemp(prefix='prefix-', suffix='-suffix', dir=top) [OK] +mkdtemp(prefix='prefix-', suffix='-suffix', dir=top) [OK] +caught: TypeError: "Can't mix bytes and non-bytes in path components" [OK] +mkstemp() [OK] +mkstemp(prefix='prefix-', suffix='-suffix') [OK] +mkstemp(prefix='prefix-', suffix='-suffix', dir=top) [OK] +mkstemp(prefix='prefix-', suffix='-suffix', dir=top) [OK] +caught: TypeError: "Can't mix bytes and non-bytes in path components" [OK] +OK diff --git a/stdlib/time/testdata/test.py b/stdlib/time/testdata/test.py new file mode 100644 index 00000000..1f8536cd --- /dev/null +++ b/stdlib/time/testdata/test.py @@ -0,0 +1,49 @@ +# Copyright 2022 The go-python Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +import time + +now = time.time() +now = time.time_ns() +now = time.clock() + +def notimplemented(fn, *args, **kwargs): + try: + fn(*args, **kwargs) + print("error for %s(%s, %s)" % (fn,args,kwargs)) + except NotImplementedError: + pass + +notimplemented(time.clock_gettime) +notimplemented(time.clock_settime) + +print("# sleep") +time.sleep(0.1) +try: + time.sleep(-1) + print("no error sleep(-1)") +except ValueError as e: + print("caught error: %s" % (e,)) + pass +try: + time.sleep("1") + print("no error sleep('1')") +except TypeError as e: + print("caught error: %s" % (e,)) + pass + +notimplemented(time.gmtime) +notimplemented(time.localtime) +notimplemented(time.asctime) +notimplemented(time.ctime) +notimplemented(time.mktime, 1) +notimplemented(time.strftime) +notimplemented(time.strptime) +notimplemented(time.tzset) +notimplemented(time.monotonic) +notimplemented(time.process_time) +notimplemented(time.perf_counter) +notimplemented(time.get_clock_info) + +print("OK") diff --git a/stdlib/time/testdata/test_golden.txt b/stdlib/time/testdata/test_golden.txt new file mode 100644 index 00000000..cb12313f --- /dev/null +++ b/stdlib/time/testdata/test_golden.txt @@ -0,0 +1,4 @@ +# sleep +caught error: ValueError: 'sleep length must be non-negative' +caught error: TypeError: 'sleep() argument 1 must be float, not str' +OK diff --git a/time/time.go b/stdlib/time/time.go similarity index 98% rename from time/time.go rename to stdlib/time/time.go index 84eb8288..81d50271 100644 --- a/time/time.go +++ b/stdlib/time/time.go @@ -18,7 +18,16 @@ Return the current time in seconds since the Epoch. Fractions of a second may be present if the system clock provides them.` func time_time(self py.Object) (py.Object, error) { - return py.Float(time.Now().UnixNano()) / 1E9, nil + return py.Float(time.Now().UnixNano()) / 1e9, nil +} + +const time_ns_doc = `time_ns() -> int + +Return the current time in nanoseconds since the Epoch. +` + +func time_time_ns(self py.Object) (py.Object, error) { + return py.Int(time.Now().UnixNano()), nil } // func floatclock(_Py_clock_info_t *info) (py.Object, error) { @@ -141,7 +150,7 @@ func time_sleep(self py.Object, args py.Tuple) (py.Object, error) { if secs < 0 { return nil, py.ExceptionNewf(py.ValueError, "sleep length must be non-negative") } - time.Sleep(time.Duration(secs * 1E9)) + time.Sleep(time.Duration(secs * py.Float(time.Second))) return py.None, nil } @@ -979,6 +988,7 @@ func PyInit_timezone(m py.Object) { func init() { methods := []*py.Method{ py.MustNewMethod("time", time_time, 0, time_doc), + py.MustNewMethod("time_ns", time_time_ns, 0, time_ns_doc), py.MustNewMethod("clock", time_clock, 0, clock_doc), py.MustNewMethod("clock_gettime", time_clock_gettime, 0, clock_gettime_doc), py.MustNewMethod("clock_settime", time_clock_settime, 0, clock_settime_doc), @@ -997,11 +1007,15 @@ func init() { py.MustNewMethod("perf_counter", time_perf_counter, 0, perf_counter_doc), py.MustNewMethod("get_clock_info", time_get_clock_info, 0, get_clock_info_doc), } - globals := py.StringDict{ - //"version": py.Int(MARSHAL_VERSION), - } - py.NewModule("time", module_doc, methods, globals) + py.RegisterModule(&py.ModuleImpl{ + Info: py.ModuleInfo{ + Name: "time", + Doc: module_doc, + }, + Methods: methods, + Globals: py.StringDict{}, + }) } const module_doc = `This module provides various functions to manipulate time values. @@ -1037,6 +1051,7 @@ tzname -- tuple of (standard time zone name, DST time zone name) Functions: time() -- return current time in seconds since the Epoch as a float +time_ns() -- return current time in nanoseconds since the Epoch clock() -- return CPU time since process start as a float sleep() -- delay for a number of seconds given as a float gmtime() -- convert seconds since Epoch to UTC tuple diff --git a/stdlib/time/time_test.go b/stdlib/time/time_test.go new file mode 100644 index 00000000..0afb30c0 --- /dev/null +++ b/stdlib/time/time_test.go @@ -0,0 +1,15 @@ +// Copyright 2022 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package time_test + +import ( + "testing" + + "github.com/go-python/gpython/pytest" +) + +func TestTime(t *testing.T) { + pytest.RunScript(t, "./testdata/test.py") +} diff --git a/symtable/symtable.go b/symtable/symtable.go index daf846be..b7d36b4d 100644 --- a/symtable/symtable.go +++ b/symtable/symtable.go @@ -17,7 +17,6 @@ package symtable import ( "fmt" - "log" "sort" "strings" @@ -229,13 +228,10 @@ func (st *SymTable) Parse(Ast ast.Ast) { cur, ok := st.Symbols[string(name)] if ok { if (cur.Flags & DefLocal) != 0 { - // FIXME this should be a warning - log.Printf("name '%s' is assigned to before nonlocal declaration", name) - + st.panicSyntaxErrorf(node, "name '%s' is assigned to before nonlocal declaration", name) } if (cur.Flags & DefUse) != 0 { - // FIXME this should be a warning - log.Printf("name '%s' is used prior to nonlocal declaration", name) + st.panicSyntaxErrorf(node, "name '%s' is used prior to nonlocal declaration", name) } } st.AddDef(node, name, DefNonlocal) @@ -245,13 +241,11 @@ func (st *SymTable) Parse(Ast ast.Ast) { cur, ok := st.Symbols[string(name)] if ok { if (cur.Flags & DefLocal) != 0 { - // FIXME this should be a warning - log.Printf("name '%s' is assigned to before global declaration", name) + st.panicSyntaxErrorf(node, "name '%s' is assigned to before global declaration", name) } if (cur.Flags & DefUse) != 0 { - // FIXME this should be a warning - log.Printf("name '%s' is used prior to global declaration", name) + st.panicSyntaxErrorf(node, "name '%s' is used prior to global declaration", name) } } st.AddDef(node, name, DefGlobal) @@ -557,11 +551,12 @@ func (s StringSet) Contains(k string) bool { global: set of all symbol names explicitly declared as global */ -/* Decide on scope of name, given flags. +/* +Decide on scope of name, given flags. - The namespace dictionaries may be modified to record information - about the new name. For example, a new global will add an entry to - global. A name that was global can be changed to local. + The namespace dictionaries may be modified to record information + about the new name. For example, a new global will add an entry to + global. A name that was global can be changed to local. */ func (st *SymTable) AnalyzeName(scopes Scopes, name string, symbol Symbol, bound, local, free, global StringSet) { flags := symbol.Flags @@ -624,12 +619,14 @@ func (st *SymTable) AnalyzeName(scopes Scopes, name string, symbol Symbol, bound scopes[name] = ScopeGlobalImplicit } -/* If a name is defined in free and also in locals, then this block - provides the binding for the free variable. The name should be - marked CELL in this block and removed from the free list. +/* +If a name is defined in free and also in locals, then this block + + provides the binding for the free variable. The name should be + marked CELL in this block and removed from the free list. - Note that the current block's free variables are included in free. - That's safe because no name can be free and local in the same scope. + Note that the current block's free variables are included in free. + That's safe because no name can be free and local in the same scope. */ func AnalyzeCells(scopes Scopes, free StringSet) { for name, scope := range scopes { @@ -697,24 +694,25 @@ func (symbols Symbols) Update(scopes Scopes, bound, free StringSet, classflag bo } } -/* Make final symbol table decisions for block of ste. - - Arguments: - st -- current symtable entry (input/output) - bound -- set of variables bound in enclosing scopes (input). bound - is nil for module blocks. - free -- set of free variables in enclosed scopes (output) - globals -- set of declared global variables in enclosing scopes (input) - - The implementation uses two mutually recursive functions, - analyze_block() and analyze_child_block(). analyze_block() is - responsible for analyzing the individual names defined in a block. - analyze_child_block() prepares temporary namespace dictionaries - used to evaluated nested blocks. - - The two functions exist because a child block should see the name - bindings of its enclosing blocks, but those bindings should not - propagate back to a parent block. +/* +Make final symbol table decisions for block of ste. + + Arguments: + st -- current symtable entry (input/output) + bound -- set of variables bound in enclosing scopes (input). bound + is nil for module blocks. + free -- set of free variables in enclosed scopes (output) + globals -- set of declared global variables in enclosing scopes (input) + + The implementation uses two mutually recursive functions, + analyze_block() and analyze_child_block(). analyze_block() is + responsible for analyzing the individual names defined in a block. + analyze_child_block() prepares temporary namespace dictionaries + used to evaluated nested blocks. + + The two functions exist because a child block should see the name + bindings of its enclosing blocks, but those bindings should not + propagate back to a parent block. */ func (st *SymTable) AnalyzeBlock(bound, free, global StringSet) { local := make(StringSet) // collect new names bound in block diff --git a/symtable/symtable_data_test.go b/symtable/symtable_data_test.go index a09e5d85..2ac2754a 100644 --- a/symtable/symtable_data_test.go +++ b/symtable/symtable_data_test.go @@ -12,7 +12,7 @@ import ( var symtableTestData = []struct { in string - mode string // exec, eval or single + mode py.CompileMode out *SymTable exceptionType *py.Type errString string @@ -568,112 +568,8 @@ var symtableTestData = []struct { }, }, }, nil, ""}, - {"def inner():\n print(x)\n global x\n", "exec", &SymTable{ - Type: ModuleBlock, - Name: "top", - Lineno: 0, - Unoptimized: optTopLevel, - Nested: false, - Free: false, - ChildFree: false, - Generator: false, - Varargs: false, - Varkeywords: false, - ReturnsValue: false, - NeedsClassClosure: false, - Varnames: []string{}, - Symbols: Symbols{ - "inner": Symbol{ - Flags: DefLocal, - Scope: ScopeLocal, - }, - "x": Symbol{ - Flags: DefGlobal, - Scope: ScopeGlobalExplicit, - }, - }, - Children: Children{ - &SymTable{ - Type: FunctionBlock, - Name: "inner", - Lineno: 1, - Unoptimized: 0, - Nested: false, - Free: false, - ChildFree: false, - Generator: false, - Varargs: false, - Varkeywords: false, - ReturnsValue: false, - NeedsClassClosure: false, - Varnames: []string{}, - Symbols: Symbols{ - "print": Symbol{ - Flags: DefUse, - Scope: ScopeGlobalImplicit, - }, - "x": Symbol{ - Flags: DefGlobal | DefUse, - Scope: ScopeGlobalExplicit, - }, - }, - Children: Children{}, - }, - }, - }, nil, ""}, - {"def fn(a):\n b = 6\n global b\n b = a", "exec", &SymTable{ - Type: ModuleBlock, - Name: "top", - Lineno: 0, - Unoptimized: optTopLevel, - Nested: false, - Free: false, - ChildFree: false, - Generator: false, - Varargs: false, - Varkeywords: false, - ReturnsValue: false, - NeedsClassClosure: false, - Varnames: []string{}, - Symbols: Symbols{ - "b": Symbol{ - Flags: DefGlobal, - Scope: ScopeGlobalExplicit, - }, - "fn": Symbol{ - Flags: DefLocal, - Scope: ScopeLocal, - }, - }, - Children: Children{ - &SymTable{ - Type: FunctionBlock, - Name: "fn", - Lineno: 1, - Unoptimized: 0, - Nested: false, - Free: false, - ChildFree: false, - Generator: false, - Varargs: false, - Varkeywords: false, - ReturnsValue: false, - NeedsClassClosure: false, - Varnames: []string{"a"}, - Symbols: Symbols{ - "a": Symbol{ - Flags: DefParam | DefUse, - Scope: ScopeLocal, - }, - "b": Symbol{ - Flags: DefGlobal | DefLocal, - Scope: ScopeGlobalExplicit, - }, - }, - Children: Children{}, - }, - }, - }, nil, ""}, + {"def inner():\n print(x)\n global x\n", "exec", nil, py.SyntaxError, "name 'x' is used prior to global declaration"}, + {"def fn(a):\n b = 6\n global b\n b = a", "exec", nil, py.SyntaxError, "name 'b' is assigned to before global declaration"}, {"def fn(a=b,c=1):\n return a+b", "exec", &SymTable{ Type: ModuleBlock, Name: "top", @@ -817,154 +713,8 @@ var symtableTestData = []struct { }, nil, ""}, {"def fn(a):\n nonlocal b\n ", "exec", nil, py.SyntaxError, "no binding for nonlocal 'b' found"}, {"def outer():\n def inner():\n nonlocal x\n x = 2", "exec", nil, py.SyntaxError, "no binding for nonlocal 'x' found"}, - {"def outer():\n x = 1\n def inner():\n print(x)\n nonlocal x\n", "exec", &SymTable{ - Type: ModuleBlock, - Name: "top", - Lineno: 0, - Unoptimized: optTopLevel, - Nested: false, - Free: false, - ChildFree: true, - Generator: false, - Varargs: false, - Varkeywords: false, - ReturnsValue: false, - NeedsClassClosure: false, - Varnames: []string{}, - Symbols: Symbols{ - "outer": Symbol{ - Flags: DefLocal, - Scope: ScopeLocal, - }, - }, - Children: Children{ - &SymTable{ - Type: FunctionBlock, - Name: "outer", - Lineno: 1, - Unoptimized: 0, - Nested: false, - Free: false, - ChildFree: true, - Generator: false, - Varargs: false, - Varkeywords: false, - ReturnsValue: false, - NeedsClassClosure: false, - Varnames: []string{}, - Symbols: Symbols{ - "inner": Symbol{ - Flags: DefLocal, - Scope: ScopeLocal, - }, - "x": Symbol{ - Flags: DefLocal, - Scope: ScopeCell, - }, - }, - Children: Children{ - &SymTable{ - Type: FunctionBlock, - Name: "inner", - Lineno: 3, - Unoptimized: 0, - Nested: true, - Free: true, - ChildFree: false, - Generator: false, - Varargs: false, - Varkeywords: false, - ReturnsValue: false, - NeedsClassClosure: false, - Varnames: []string{}, - Symbols: Symbols{ - "print": Symbol{ - Flags: DefUse, - Scope: ScopeGlobalImplicit, - }, - "x": Symbol{ - Flags: DefNonlocal | DefUse, - Scope: ScopeFree, - }, - }, - Children: Children{}, - }, - }, - }, - }, - }, nil, ""}, - {"def outer():\n x = 1\n def inner():\n x = 2\n nonlocal x", "exec", &SymTable{ - Type: ModuleBlock, - Name: "top", - Lineno: 0, - Unoptimized: optTopLevel, - Nested: false, - Free: false, - ChildFree: true, - Generator: false, - Varargs: false, - Varkeywords: false, - ReturnsValue: false, - NeedsClassClosure: false, - Varnames: []string{}, - Symbols: Symbols{ - "outer": Symbol{ - Flags: DefLocal, - Scope: ScopeLocal, - }, - }, - Children: Children{ - &SymTable{ - Type: FunctionBlock, - Name: "outer", - Lineno: 1, - Unoptimized: 0, - Nested: false, - Free: false, - ChildFree: true, - Generator: false, - Varargs: false, - Varkeywords: false, - ReturnsValue: false, - NeedsClassClosure: false, - Varnames: []string{}, - Symbols: Symbols{ - "inner": Symbol{ - Flags: DefLocal, - Scope: ScopeLocal, - }, - "x": Symbol{ - Flags: DefLocal, - Scope: ScopeCell, - }, - }, - Children: Children{ - &SymTable{ - Type: FunctionBlock, - Name: "inner", - Lineno: 3, - Unoptimized: 0, - Nested: true, - Free: true, - ChildFree: false, - Generator: false, - Varargs: false, - Varkeywords: false, - ReturnsValue: false, - NeedsClassClosure: false, - Varnames: []string{}, - Symbols: Symbols{ - "x": Symbol{ - Flags: DefLocal | DefNonlocal, - Scope: ScopeFree, - }, - }, - Children: Children{}, - }, - }, - }, - }, - }, nil, ""}, + {"def outer():\n x = 1\n def inner():\n print(x)\n nonlocal x\n", "exec", nil, py.SyntaxError, "name 'x' is used prior to nonlocal declaration"}, + {"def outer():\n x = 1\n def inner():\n x = 2\n nonlocal x", "exec", nil, py.SyntaxError, "name 'x' is assigned to before nonlocal declaration"}, {"def outer():\n x = 1\n def inner(x):\n nonlocal x", "exec", nil, py.SyntaxError, "name 'x' is parameter and nonlocal"}, {"def outer():\n x = 1\n def inner(x):\n global x", "exec", nil, py.SyntaxError, "name 'x' is parameter and global"}, {"def outer():\n def inner():\n global x\n nonlocal x\n ", "exec", nil, py.SyntaxError, "name 'x' is nonlocal and global"}, diff --git a/testdata/hello.py b/testdata/hello.py new file mode 100644 index 00000000..e94acca7 --- /dev/null +++ b/testdata/hello.py @@ -0,0 +1 @@ +print("hello, world!") diff --git a/testdata/hello_golden.txt b/testdata/hello_golden.txt new file mode 100644 index 00000000..270c611e --- /dev/null +++ b/testdata/hello_golden.txt @@ -0,0 +1 @@ +hello, world! diff --git a/version.go b/version.go new file mode 100644 index 00000000..4a74e91b --- /dev/null +++ b/version.go @@ -0,0 +1,11 @@ +// Copyright 2018 The go-python Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +var ( + version = "dev" + commit = "none" + date = "unknown" +) diff --git a/vm/benchmarks/fib.py b/vm/benchmarks/fib.py new file mode 100644 index 00000000..18c69a5a --- /dev/null +++ b/vm/benchmarks/fib.py @@ -0,0 +1,15 @@ +# Copyright 2019 The go-python Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# Benchmark adapted from https://github.com/d5/tengobench/ +doc="fib recursion test" +def fib(n): + if n == 0: + return 0 + elif n == 1: + return 1 + return fib(n - 2) + fib(n - 1) + +fib(25) +doc="finished" diff --git a/vm/benchmarks/fibtc.py b/vm/benchmarks/fibtc.py new file mode 100644 index 00000000..1b8b5f73 --- /dev/null +++ b/vm/benchmarks/fibtc.py @@ -0,0 +1,15 @@ +# Copyright 2019 The go-python Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# Benchmark adapted from https://github.com/d5/tengobench/ +doc="fib tail call recursion test" +def fib(n, a, b): + if n == 0: + return a + elif n == 1: + return b + return fib(n-1, b, a+b) + +fib(35, 0, 1) +doc="finished" diff --git a/vm/builtin.go b/vm/builtin.go index 3fcdccac..6466b31f 100644 --- a/vm/builtin.go +++ b/vm/builtin.go @@ -12,13 +12,13 @@ import ( "github.com/go-python/gpython/py" ) -func builtinEvalOrExec(self py.Object, args py.Tuple, kwargs, currentLocals, currentGlobals, builtins py.StringDict, mode string) (py.Object, error) { +func builtinEvalOrExec(ctx py.Context, args py.Tuple, kwargs, currentLocals, currentGlobals, builtins py.StringDict, mode py.CompileMode) (py.Object, error) { var ( cmd py.Object globals py.Object = py.None locals py.Object = py.None ) - err := py.UnpackTuple(args, kwargs, mode, 1, 3, &cmd, &globals, &locals) + err := py.UnpackTuple(args, kwargs, string(mode), 1, 3, &cmd, &globals, &locals) if err != nil { return nil, err } @@ -60,24 +60,23 @@ func builtinEvalOrExec(self py.Object, args py.Tuple, kwargs, currentLocals, cur } if code == nil { codeStr = strings.TrimLeft(codeStr, " \t") - obj, err := py.Compile(codeStr, "", mode, 0, true) + code, err = py.Compile(codeStr, "", mode, 0, true) if err != nil { return nil, err } - code = obj.(*py.Code) } if code.GetNumFree() > 0 { return nil, py.ExceptionNewf(py.TypeError, "code passed to %s() may not contain free variables", mode) } - return EvalCode(code, globalsDict, localsDict) + return ctx.RunCode(code, globalsDict, localsDict, nil) } -func builtinEval(self py.Object, args py.Tuple, kwargs, currentLocals, currentGlobals, builtins py.StringDict) (py.Object, error) { - return builtinEvalOrExec(self, args, kwargs, currentLocals, currentGlobals, builtins, "eval") +func builtinEval(ctx py.Context, args py.Tuple, kwargs, currentLocals, currentGlobals, builtins py.StringDict) (py.Object, error) { + return builtinEvalOrExec(ctx, args, kwargs, currentLocals, currentGlobals, builtins, py.EvalMode) } -func builtinExec(self py.Object, args py.Tuple, kwargs, currentLocals, currentGlobals, builtins py.StringDict) (py.Object, error) { - _, err := builtinEvalOrExec(self, args, kwargs, currentLocals, currentGlobals, builtins, "exec") +func builtinExec(ctx py.Context, args py.Tuple, kwargs, currentLocals, currentGlobals, builtins py.StringDict) (py.Object, error) { + _, err := builtinEvalOrExec(ctx, args, kwargs, currentLocals, currentGlobals, builtins, py.ExecMode) if err != nil { return nil, err } diff --git a/vm/eval.go b/vm/eval.go index e98392a9..d32cf734 100644 --- a/vm/eval.go +++ b/vm/eval.go @@ -767,7 +767,7 @@ func do_END_FINALLY(vm *Vm, arg int32) error { // Loads the __build_class__ helper function to the stack which // creates a new class object. func do_LOAD_BUILD_CLASS(vm *Vm, arg int32) error { - vm.PUSH(py.Builtins.Globals["__build_class__"]) + vm.PUSH(vm.context.Store().Builtins.Globals["__build_class__"]) return nil } @@ -1087,9 +1087,9 @@ func do_IMPORT_NAME(vm *Vm, namei int32) error { } v := vm.POP() u := vm.TOP() - var locals py.Object = vm.frame.Locals - if locals == nil { - locals = py.None + var locals py.Object = py.None + if vm.frame.Locals != nil { + locals = vm.frame.Locals } var args py.Tuple if _, ok := u.(py.Int); ok { @@ -1435,7 +1435,7 @@ func _make_function(vm *Vm, argc int32, opcode OpCode) { num_annotations := (argc >> 16) & 0x7fff qualname := vm.POP() code := vm.POP() - function := py.NewFunction(code.(*py.Code), vm.frame.Globals, string(qualname.(py.String))) + function := py.NewFunction(vm.context, code.(*py.Code), vm.frame.Globals, string(qualname.(py.String))) if opcode == MAKE_CLOSURE { function.Closure = vm.POP().(py.Tuple) @@ -1579,7 +1579,7 @@ func EvalGetFuncDesc(fn py.Object) string { } } -// As py.Call but takes an intepreter Frame object +// As py.Call but takes an interpreter Frame object // // Used to implement some interpreter magic like locals(), globals() etc func callInternal(fn py.Object, args py.Tuple, kwargs py.StringDict, f *py.Frame) (py.Object, error) { @@ -1592,13 +1592,13 @@ func callInternal(fn py.Object, args py.Tuple, kwargs py.StringDict, f *py.Frame f.FastToLocals() return f.Locals, nil case py.InternalMethodImport: - return py.BuiltinImport(nil, args, kwargs, f.Globals) + return py.BuiltinImport(f.Context, nil, args, kwargs, f.Globals) case py.InternalMethodEval: f.FastToLocals() - return builtinEval(nil, args, kwargs, f.Locals, f.Globals, f.Builtins) + return builtinEval(f.Context, args, kwargs, f.Locals, f.Globals, f.Builtins) case py.InternalMethodExec: f.FastToLocals() - return builtinExec(nil, args, kwargs, f.Locals, f.Globals, f.Builtins) + return builtinExec(f.Context, args, kwargs, f.Locals, f.Globals, f.Builtins) default: return nil, py.ExceptionNewf(py.SystemError, "Internal method %v not found", x) } @@ -1609,7 +1609,7 @@ func callInternal(fn py.Object, args py.Tuple, kwargs py.StringDict, f *py.Frame // Implements a function call - see CALL_FUNCTION for a description of // how the arguments are arranged. // -// Optionally pass in args and kwargs +// # Optionally pass in args and kwargs // // The result is put on the stack func (vm *Vm) Call(argc int32, starArgs py.Object, starKwargs py.Object) error { @@ -1731,7 +1731,8 @@ func (vm *Vm) UnwindExceptHandler(frame *py.Frame, block *py.TryBlock) { // This is the equivalent of PyEval_EvalFrame func RunFrame(frame *py.Frame) (res py.Object, err error) { var vm = Vm{ - frame: frame, + frame: frame, + context: frame.Context, } // FIXME need to do this to save the old exeption when we @@ -2033,7 +2034,14 @@ func tooManyPositional(co *py.Code, given, defcount int, fastlocals []py.Object) chooseString(given == 1 && kwonly_given == 0, "was", "were")) } -func EvalCodeEx(co *py.Code, globals, locals py.StringDict, args []py.Object, kws py.StringDict, defs []py.Object, kwdefs py.StringDict, closure py.Tuple) (retval py.Object, err error) { +// EvalCode runs a new virtual machine on a Code object. +// +// # Any parameters are expected to have been decoded into locals +// +// Returns an Object and an error. The error will be a py.ExceptionInfo +// +// This is the equivalent of PyEval_EvalCode with closure support +func EvalCode(ctx py.Context, co *py.Code, globals, locals py.StringDict, args []py.Object, kws py.StringDict, defs []py.Object, kwdefs py.StringDict, closure py.Tuple) (retval py.Object, err error) { total_args := int(co.Argcount + co.Kwonlyargcount) n := len(args) var kwdict py.StringDict @@ -2045,7 +2053,7 @@ func EvalCodeEx(co *py.Code, globals, locals py.StringDict, args []py.Object, kw //assert(tstate != nil) //assert(globals != nil) // f = PyFrame_New(tstate, co, globals, locals) - f := py.NewFrame(globals, locals, co, closure) // FIXME extra closure parameter? + f := py.NewFrame(ctx, globals, locals, co, closure) // FIXME extra closure parameter? fastlocals := f.Localsplus freevars := f.CellAndFreeVars @@ -2162,34 +2170,8 @@ func EvalCodeEx(co *py.Code, globals, locals py.StringDict, args []py.Object, kw return RunFrame(f) } -func EvalCode(co *py.Code, globals, locals py.StringDict) (py.Object, error) { - return EvalCodeEx(co, - globals, locals, - nil, - nil, - nil, - nil, nil) -} - -// Run the virtual machine on a Code object -// -// Any parameters are expected to have been decoded into locals -// -// Returns an Object and an error. The error will be a py.ExceptionInfo -// -// This is the equivalent of PyEval_EvalCode with closure support -func Run(globals, locals py.StringDict, code *py.Code, closure py.Tuple) (res py.Object, err error) { - return EvalCodeEx(code, - globals, locals, - nil, - nil, - nil, - nil, closure) -} - // Write the py global to avoid circular import func init() { - py.VmRun = Run + py.VmEvalCode = EvalCode py.VmRunFrame = RunFrame - py.VmEvalCodeEx = EvalCodeEx } diff --git a/vm/tests/class.py b/vm/tests/class.py index 2c8fd70b..f9781cd7 100644 --- a/vm/tests/class.py +++ b/vm/tests/class.py @@ -47,17 +47,16 @@ def method1(self, x): c = x() assert c.method1(1) == 2 -# FIXME doesn't work -# doc="CLASS_DEREF2" -# def classderef2(x): -# class DeRefTest: -# VAR = x -# def method1(self, x): -# "method1" -# return self.VAR+x -# return DeRefTest -# x = classderef2(1) -# c = x() -# assert c.method1(1) == 2 +doc="CLASS_DEREF2" +def classderef2(x): + class DeRefTest: + VAR = x + def method1(self, x): + "method1" + return self.VAR+x + return DeRefTest +x = classderef2(1) +c = x() +assert c.method1(1) == 2 doc="finished" diff --git a/vm/tests/decorators.py b/vm/tests/decorators.py new file mode 100644 index 00000000..b7e2d703 --- /dev/null +++ b/vm/tests/decorators.py @@ -0,0 +1,327 @@ +# Copyright 2023 The go-python Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# Copied from Python-3.4.9\Lib\test\test_decorators.py + +import libtest as self + +def funcattrs(**kwds): + def decorate(func): + # FIXME func.__dict__.update(kwds) + for k, v in kwds.items(): + func.__dict__[k] = v + return func + return decorate + +class MiscDecorators (object): + @staticmethod + def author(name): + def decorate(func): + func.__dict__['author'] = name + return func + return decorate + +# ----------------------------------------------- + +class DbcheckError (Exception): + def __init__(self, exprstr, func, args, kwds): + # A real version of this would set attributes here + Exception.__init__(self, "dbcheck %r failed (func=%s args=%s kwds=%s)" % + (exprstr, func, args, kwds)) + + +def dbcheck(exprstr, globals=None, locals=None): + "Decorator to implement debugging assertions" + def decorate(func): + expr = compile(exprstr, "dbcheck-%s" % func.__name__, "eval") + def check(*args, **kwds): + if not eval(expr, globals, locals): + raise DbcheckError(exprstr, func, args, kwds) + return func(*args, **kwds) + return check + return decorate + +# ----------------------------------------------- + +def countcalls(counts): + "Decorator to count calls to a function" + def decorate(func): + func_name = func.__name__ + counts[func_name] = 0 + def call(*args, **kwds): + counts[func_name] += 1 + return func(*args, **kwds) + call.__name__ = func_name + return call + return decorate + +# ----------------------------------------------- + +# FIXME: dict can only have string keys +# def memoize(func): +# saved = {} +# def call(*args): +# try: +# return saved[args] +# except KeyError: +# res = func(*args) +# saved[args] = res +# return res +# except TypeError: +# # Unhashable argument +# return func(*args) +# call.__name__ = func.__name__ +# return call +def memoize(func): + saved = {} + def call(*args): + try: + if isinstance(args[0], list): + raise TypeError + return saved[str(args)] + except KeyError: + res = func(*args) + saved[str(args)] = res + return res + except TypeError: + # Unhashable argument + return func(*args) + call.__name__ = func.__name__ + return call + +# ----------------------------------------------- + +doc="test_single" +# FIXME staticmethod +# class C(object): +# @staticmethod +# def foo(): return 42 +# self.assertEqual(C.foo(), 42) +# self.assertEqual(C().foo(), 42) + +doc="test_staticmethod_function" +@staticmethod +def notamethod(x): + return x +self.assertRaises(TypeError, notamethod, 1) + +doc="test_dotted" +# FIXME class decorator +# decorators = MiscDecorators() +# @decorators.author('Cleese') +# def foo(): return 42 +# self.assertEqual(foo(), 42) +# self.assertEqual(foo.author, 'Cleese') + +doc="test_argforms" +def noteargs(*args, **kwds): + def decorate(func): + setattr(func, 'dbval', (args, kwds)) + return func + return decorate + +args = ( 'Now', 'is', 'the', 'time' ) +kwds = dict(one=1, two=2) +@noteargs(*args, **kwds) +def f1(): return 42 +self.assertEqual(f1(), 42) +self.assertEqual(f1.dbval, (args, kwds)) + +@noteargs('terry', 'gilliam', eric='idle', john='cleese') +def f2(): return 84 +self.assertEqual(f2(), 84) +self.assertEqual(f2.dbval, (('terry', 'gilliam'), + dict(eric='idle', john='cleese'))) + +@noteargs(1, 2,) +def f3(): pass +self.assertEqual(f3.dbval, ((1, 2), {})) + +doc="test_dbcheck" +# FIXME TypeError: "catching 'BaseException' that does not inherit from BaseException is not allowed" +# @dbcheck('args[1] is not None') +# def f(a, b): +# return a + b +# self.assertEqual(f(1, 2), 3) +# self.assertRaises(DbcheckError, f, 1, None) + +doc="test_memoize" +counts = {} + +@memoize +@countcalls(counts) +def double(x): + return x * 2 +self.assertEqual(double.__name__, 'double') + +self.assertEqual(counts, dict(double=0)) + +# Only the first call with a given argument bumps the call count: +# +# Only the first call with a given argument bumps the call count: +# +self.assertEqual(double(2), 4) +self.assertEqual(counts['double'], 1) +self.assertEqual(double(2), 4) +self.assertEqual(counts['double'], 1) +self.assertEqual(double(3), 6) +self.assertEqual(counts['double'], 2) + +# Unhashable arguments do not get memoized: +# +self.assertEqual(double([10]), [10, 10]) +self.assertEqual(counts['double'], 3) +self.assertEqual(double([10]), [10, 10]) +self.assertEqual(counts['double'], 4) + +doc="test_errors" +# Test syntax restrictions - these are all compile-time errors: +# +for expr in [ "1+2", "x[3]", "(1, 2)" ]: + # Sanity check: is expr is a valid expression by itself? + compile(expr, "testexpr", "exec") + + codestr = "@%s\ndef f(): pass" % expr + self.assertRaises(SyntaxError, compile, codestr, "test", "exec") + +# You can't put multiple decorators on a single line: +# +self.assertRaises(SyntaxError, compile, + "@f1 @f2\ndef f(): pass", "test", "exec") + +# Test runtime errors + +def unimp(func): + raise NotImplementedError +context = dict(nullval=None, unimp=unimp) + +for expr, exc in [ ("undef", NameError), + ("nullval", TypeError), + ("nullval.attr", NameError), # FIXME ("nullval.attr", AttributeError), + ("unimp", NotImplementedError)]: + codestr = "@%s\ndef f(): pass\nassert f() is None" % expr + code = compile(codestr, "test", "exec") + self.assertRaises(exc, eval, code, context) + +doc="test_double" +class C(object): + @funcattrs(abc=1, xyz="haha") + @funcattrs(booh=42) + def foo(self): return 42 +self.assertEqual(C().foo(), 42) +self.assertEqual(C.foo.abc, 1) +self.assertEqual(C.foo.xyz, "haha") +self.assertEqual(C.foo.booh, 42) + + +doc="test_order" +# Test that decorators are applied in the proper order to the function +# they are decorating. +def callnum(num): + """Decorator factory that returns a decorator that replaces the + passed-in function with one that returns the value of 'num'""" + def deco(func): + return lambda: num + return deco +@callnum(2) +@callnum(1) +def foo(): return 42 +self.assertEqual(foo(), 2, + "Application order of decorators is incorrect") + + +doc="test_eval_order" +# Evaluating a decorated function involves four steps for each +# decorator-maker (the function that returns a decorator): +# +# 1: Evaluate the decorator-maker name +# 2: Evaluate the decorator-maker arguments (if any) +# 3: Call the decorator-maker to make a decorator +# 4: Call the decorator +# +# When there are multiple decorators, these steps should be +# performed in the above order for each decorator, but we should +# iterate through the decorators in the reverse of the order they +# appear in the source. +# FIXME class decorator +# actions = [] +# +# def make_decorator(tag): +# actions.append('makedec' + tag) +# def decorate(func): +# actions.append('calldec' + tag) +# return func +# return decorate +# +# class NameLookupTracer (object): +# def __init__(self, index): +# self.index = index +# +# def __getattr__(self, fname): +# if fname == 'make_decorator': +# opname, res = ('evalname', make_decorator) +# elif fname == 'arg': +# opname, res = ('evalargs', str(self.index)) +# else: +# assert False, "Unknown attrname %s" % fname +# actions.append('%s%d' % (opname, self.index)) +# return res +# +# c1, c2, c3 = map(NameLookupTracer, [ 1, 2, 3 ]) +# +# expected_actions = [ 'evalname1', 'evalargs1', 'makedec1', +# 'evalname2', 'evalargs2', 'makedec2', +# 'evalname3', 'evalargs3', 'makedec3', +# 'calldec3', 'calldec2', 'calldec1' ] +# +# actions = [] +# @c1.make_decorator(c1.arg) +# @c2.make_decorator(c2.arg) +# @c3.make_decorator(c3.arg) +# def foo(): return 42 +# self.assertEqual(foo(), 42) +# +# self.assertEqual(actions, expected_actions) +# +# # Test the equivalence claim in chapter 7 of the reference manual. +# # +# actions = [] +# def bar(): return 42 +# bar = c1.make_decorator(c1.arg)(c2.make_decorator(c2.arg)(c3.make_decorator(c3.arg)(bar))) +# self.assertEqual(bar(), 42) +# self.assertEqual(actions, expected_actions) + +doc="test_simple" +def plain(x): + x.extra = 'Hello' + return x +@plain +class C(object): pass +self.assertEqual(C.extra, 'Hello') + +doc="test_double" +def ten(x): + x.extra = 10 + return x +def add_five(x): + x.extra += 5 + return x + +@add_five +@ten +class C(object): pass +self.assertEqual(C.extra, 15) + +doc="test_order" +def applied_first(x): + x.extra = 'first' + return x +def applied_second(x): + x.extra = 'second' + return x +@applied_second +@applied_first +class C(object): pass +self.assertEqual(C.extra, 'second') +doc="finished" diff --git a/vm/tests/functions.py b/vm/tests/functions.py index da4bf924..aab079f9 100644 --- a/vm/tests/functions.py +++ b/vm/tests/functions.py @@ -21,18 +21,32 @@ def fn2(x,y=1): assert fn2(1,y=4) == 5 # Closure +doc="closure1" +closure1 = lambda x: lambda y: x+y +cf1 = closure1(1) +assert cf1(1) == 2 +assert cf1(2) == 3 + +doc="closure2" +def closure2(*args, **kwargs): + def inc(): + kwargs['x'] += 1 + return kwargs['x'] + return inc +cf2 = closure2(x=1) +assert cf2() == 2 +assert cf2() == 3 -# FIXME something wrong with closures over function arguments... -# doc="counter3" -# def counter3(x): -# def inc(): -# nonlocal x -# x += 1 -# return x -# return inc -# fn3 = counter3(1) -# assert fn3() == 2 -# assert fn3() == 3 +doc="counter3" +def counter3(x): + def inc(): + nonlocal x + x += 1 + return x + return inc +fn3 = counter3(1) +assert fn3() == 2 +assert fn3() == 3 doc="counter4" def counter4(initial): @@ -238,6 +252,4 @@ def fn16_6(*,a,b,c): ck(fn16_5, "fn16_5() missing 2 required keyword-only arguments: 'a' and 'b'") ck(fn16_6, "fn16_6() missing 3 required keyword-only arguments: 'a', 'b', and 'c'") -#FIXME decorators - doc="finished" diff --git a/vm/tests/libtest.py b/vm/tests/libtest.py new file mode 100644 index 00000000..8038556d --- /dev/null +++ b/vm/tests/libtest.py @@ -0,0 +1,57 @@ +# Copyright 2023 The go-python Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# Imitate the calling method of unittest + +def assertRaises(expecting, fn, *args, **kwargs): + """Check the exception was raised - don't check the text""" + try: + fn(*args, **kwargs) + except expecting as e: + pass + else: + assert False, "%s not raised" % (expecting,) + +def assertEqual(first, second, msg=None): + if msg: + assert first == second, "%s not equal" % (msg,) + else: + assert first == second + +def assertIs(expr1, expr2, msg=None): + if msg: + assert expr1 is expr2, "%s is not None" % (msg,) + else: + assert expr1 is expr2 + +def assertIsNone(obj, msg=None): + if msg: + assert obj is None, "%s is not None" % (msg,) + else: + assert obj is None + +def assertTrue(obj, msg=None): + if msg: + assert obj, "%s is not True" % (msg,) + else: + assert obj + +def assertRaisesText(expecting, text, fn, *args, **kwargs): + """Check the exception with text in is raised""" + try: + fn(*args, **kwargs) + except expecting as e: + assert text in e.args[0], "'%s' not found in '%s'" % (text, e.args[0]) + else: + assert False, "%s not raised" % (expecting,) + +def assertTypedEqual(actual, expect, msg=None): + assertEqual(actual, expect, msg) + def recurse(actual, expect): + if isinstance(expect, (tuple, list)): + for x, y in zip(actual, expect): + recurse(x, y) + else: + assertIs(type(actual), type(expect)) + recurse(actual, expect) diff --git a/vm/vm.go b/vm/vm.go index 4eb3cb20..9abc45ea 100644 --- a/vm/vm.go +++ b/vm/vm.go @@ -41,4 +41,6 @@ type Vm struct { curexc py.ExceptionInfo // Previous exception type, value and traceback exc py.ExceptionInfo + // VM access to state / modules + context py.Context } diff --git a/vm/vm_test.go b/vm/vm_test.go index 51aa30e3..fd5fba24 100644 --- a/vm/vm_test.go +++ b/vm/vm_test.go @@ -5,11 +5,90 @@ package vm_test import ( + "fmt" + "strconv" + "strings" + "sync" "testing" + "github.com/go-python/gpython/py" "github.com/go-python/gpython/pytest" ) func TestVm(t *testing.T) { pytest.RunTests(t, "tests") } + +func BenchmarkVM(b *testing.B) { + pytest.RunBenchmarks(b, "benchmarks") +} + +var jobSrcTemplate = ` + +doc="multi py.Context text" +WORKER_ID = "{{WORKER_ID}}" +def fib(n): + if n == 0: + return 0 + elif n == 1: + return 1 + return fib(n - 2) + fib(n - 1) + +x = {{FIB_TO}} +fx = fib(x) +print("%s says fib(%d) is %d" % (WORKER_ID, x, fx)) +` + +type worker struct { + name string + ctx py.Context +} + +// run loads a specific worker with a specific work load +func (w *worker) run(b testing.TB, pySrc string, countUpto int) { + pySrc = strings.Replace(pySrc, "{{WORKER_ID}}", w.name, -1) + pySrc = strings.Replace(pySrc, "{{FIB_TO}}", strconv.Itoa(countUpto), -1) + + module, code := pytest.CompileSrc(b, w.ctx, pySrc, w.name) + _, err := w.ctx.RunCode(code, module.Globals, module.Globals, nil) + if err != nil { + b.Fatal(err) + } +} + +func BenchmarkContext(b *testing.B) { + numWorkers := 4 + workersRunning := sync.WaitGroup{} + + numJobs := 35 + fmt.Printf("Starting %d workers to process %d jobs...\n", numWorkers, numJobs) + + jobPipe := make(chan int) + go func() { + for i := 0; i < numJobs; i++ { + jobPipe <- i + 1 + } + close(jobPipe) + }() + + workers := make([]worker, numWorkers) + for i := 0; i < numWorkers; i++ { + + workers[i] = worker{ + name: fmt.Sprintf("Worker #%d", i+1), + ctx: py.NewContext(py.DefaultContextOpts()), + } + + workersRunning.Add(1) + w := workers[i] + go func() { + for jobID := range jobPipe { + w.run(b, jobSrcTemplate, jobID) + //fmt.Printf("### %s finished job %v ###\n", w.name, jobID) + } + workersRunning.Done() + }() + } + + workersRunning.Wait() +}