diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..7ccf9a5 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,19 @@ +name: Test +on: [push, pull_request] + +jobs: + test: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + python-version: [3.6, 3.7, 3.8] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Test + run: python -m unittest diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..58200d4 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +__pycache__/ diff --git a/README.md b/README.md index 942f76f..1df2030 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,64 @@ # Codewars Test Framework for Python +### Installation -## Example +```bash +pip install git+https://github.com/codewars/python-test-framework.git#egg=codewars_test +``` + +### Basic Example ```python -from solution import add import codewars_test as test +from solution import add @test.describe('Example Tests') def example_tests(): + @test.it('Example Test Case') def example_test_case(): test.assert_equals(add(1, 1), 2, 'Optional Message on Failure') ``` + + diff --git a/codewars_test/test_framework.py b/codewars_test/test_framework.py index 50b9f85..7b6fc6a 100644 --- a/codewars_test/test_framework.py +++ b/codewars_test/test_framework.py @@ -11,7 +11,8 @@ def format_message(message): def display(type, message, label="", mode=""): print("\n<{0}:{1}:{2}>{3}".format( - type.upper(), mode.upper(), label, format_message(message))) + type.upper(), mode.upper(), label, format_message(message)) + , flush=True) def expect(passed=None, message=None, allow_raise=False): @@ -51,8 +52,8 @@ def expect_error(message, function, exception=Exception): function() except exception: passed = True - except Exception: - pass + except Exception as e: + message = "{}: {} should be {}".format(message or "Unexpected exception", repr(e), repr(exception)) expect(passed, message) @@ -62,7 +63,7 @@ def expect_no_error(message, function, exception=BaseException): except exception as e: fail("{}: {}".format(message or "Unexpected exception", repr(e))) return - except Exception: + except: pass pass_() diff --git a/gen-fixture.sh b/gen-fixture.sh new file mode 100755 index 0000000..4093395 --- /dev/null +++ b/gen-fixture.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# ./gen-fixture.sh tests/fixtures/example.py +# ./gen-fixture.sh tests/fixtures/example.py expected +# ./gen-fixture.sh tests/fixtures/example.py sample +# for f in $(ls tests/fixtures/*.py); do ./gen-fixture.sh "$f"; done + +PYTHONPATH=./ python "$1" > "${1%.py}.${2:-expected}.txt" diff --git a/setup.py b/setup.py index dc916f3..fea4d4d 100644 --- a/setup.py +++ b/setup.py @@ -2,10 +2,10 @@ setup( name="codewars_test", - version="0.1.0", + version="0.2.1", packages=["codewars_test"], license="MIT", description="Codewars test framework for Python", install_requires=[], - url="https://github.com/Codewars/python-test-framework", + url="https://github.com/codewars/python-test-framework", ) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/fixtures/expect_error_sample.expected.txt b/tests/fixtures/expect_error_sample.expected.txt new file mode 100644 index 0000000..b0cb56d --- /dev/null +++ b/tests/fixtures/expect_error_sample.expected.txt @@ -0,0 +1,76 @@ + +expect_error, new version + +f0 raises nothing + +f0 did not raise any exception + +f0 did not raise Exception + +f0 did not raise ArithmeticError + +f0 did not raise ZeroDivisionError + +f0 did not raise LookupError + +f0 did not raise KeyError + +f0 did not raise OSError + +0.03 + +f1 raises Exception + +Test Passed + +Test Passed + +f1 did not raise ArithmeticError: Exception() should be + +f1 did not raise ZeroDivisionError: Exception() should be + +f1 did not raise LookupError: Exception() should be + +f1 did not raise KeyError: Exception() should be + +f1 did not raise OSError: Exception() should be + +0.02 + +f2 raises Exception >> ArithmeticError >> ZeroDivisionError + +Test Passed + +Test Passed + +Test Passed + +Test Passed + +f2 did not raise LookupError: ZeroDivisionError('integer division or modulo by zero') should be + +f2 did not raise KeyError: ZeroDivisionError('integer division or modulo by zero') should be + +f2 did not raise OSError: ZeroDivisionError('integer division or modulo by zero') should be + +0.02 + +f3 raises Exception >> LookupError >> KeyError + +Test Passed + +Test Passed + +f3 did not raise ArithmeticError: KeyError(1) should be + +f3 did not raise ZeroDivisionError: KeyError(1) should be + +Test Passed + +Test Passed + +f3 did not raise OSError: KeyError(1) should be + +0.02 + +0.11 \ No newline at end of file diff --git a/tests/fixtures/expect_error_sample.py b/tests/fixtures/expect_error_sample.py new file mode 100644 index 0000000..e76962f --- /dev/null +++ b/tests/fixtures/expect_error_sample.py @@ -0,0 +1,60 @@ +# https://www.codewars.com/kumite/5ab735bee7093b17b2000084?sel=5ab735bee7093b17b2000084 +import codewars_test as test + + +def f0(): + pass + + +# BaseException >> Exception +def f1(): + raise Exception() + + +# BaseException >> Exception >> ArithmeticError >> ZeroDivisionError +def f2(): + return 1 // 0 + + +# BaseException >> Exception >> LookupError >> KeyError +def f3(): + return {}[1] + + +excn = ( + "Exception", + "ArithmeticError", + "ZeroDivisionError", + "LookupError", + "KeyError", + "OSError", +) +exc = (Exception, ArithmeticError, ZeroDivisionError, LookupError, KeyError, OSError) + + +@test.describe("expect_error, new version") +def d2(): + @test.it("f0 raises nothing") + def i0(): + test.expect_error("f0 did not raise any exception", f0) + for i in range(6): + test.expect_error("f0 did not raise {}".format(excn[i]), f0, exc[i]) + + @test.it("f1 raises Exception") + def i1(): + test.expect_error("f1 did not raise Exception", f1) + for i in range(6): + test.expect_error("f1 did not raise {}".format(excn[i]), f1, exc[i]) + + @test.it("f2 raises Exception >> ArithmeticError >> ZeroDivisionError") + def i2(): + test.expect_error("f2 did not raise Exception", f2) + for i in range(6): + test.expect_error("f2 did not raise {}".format(excn[i]), f2, exc[i]) + + @test.it("f3 raises Exception >> LookupError >> KeyError") + def i3(): + test.expect_error("f3 did not raise Exception", f3) + for i in range(6): + test.expect_error("f3 did not raise {}".format(excn[i]), f3, exc[i]) + diff --git a/tests/fixtures/multiple_groups.expected.txt b/tests/fixtures/multiple_groups.expected.txt new file mode 100644 index 0000000..f9432cf --- /dev/null +++ b/tests/fixtures/multiple_groups.expected.txt @@ -0,0 +1,20 @@ + +group 1 + +test 1 + +1 should equal 2 + +0.01 + +0.02 + +group 2 + +test 1 + +1 should equal 2 + +0.00 + +0.01 diff --git a/tests/fixtures/multiple_groups.py b/tests/fixtures/multiple_groups.py new file mode 100644 index 0000000..0a24b59 --- /dev/null +++ b/tests/fixtures/multiple_groups.py @@ -0,0 +1,15 @@ +import codewars_test as test + + +@test.describe("group 1") +def group_1(): + @test.it("test 1") + def test_1(): + test.assert_equals(1, 2) + + +@test.describe("group 2") +def group_2(): + @test.it("test 1") + def test_1(): + test.assert_equals(1, 2) diff --git a/tests/fixtures/nested_groups.expected.txt b/tests/fixtures/nested_groups.expected.txt new file mode 100644 index 0000000..7445137 --- /dev/null +++ b/tests/fixtures/nested_groups.expected.txt @@ -0,0 +1,14 @@ + +group 1 + +group 1 1 + +test 1 + +1 should equal 2 + +0.01 + +0.02 + +0.02 diff --git a/tests/fixtures/nested_groups.py b/tests/fixtures/nested_groups.py new file mode 100644 index 0000000..b9e3636 --- /dev/null +++ b/tests/fixtures/nested_groups.py @@ -0,0 +1,10 @@ +import codewars_test as test + + +@test.describe("group 1") +def group_1(): + @test.describe("group 1 1") + def group_1_1(): + @test.it("test 1") + def test_1(): + test.assert_equals(1, 2) diff --git a/tests/fixtures/old_group.expected.txt b/tests/fixtures/old_group.expected.txt new file mode 100644 index 0000000..e450a9b --- /dev/null +++ b/tests/fixtures/old_group.expected.txt @@ -0,0 +1,6 @@ + +group 1 + +test 1 + +Test Passed diff --git a/tests/fixtures/old_group.py b/tests/fixtures/old_group.py new file mode 100644 index 0000000..db08a36 --- /dev/null +++ b/tests/fixtures/old_group.py @@ -0,0 +1,6 @@ +# Deprecated and should not be used +import codewars_test as test + +test.describe("group 1") +test.it("test 1") +test.assert_equals(1, 1) diff --git a/tests/fixtures/old_top_level_assertion_fail.expected.txt b/tests/fixtures/old_top_level_assertion_fail.expected.txt new file mode 100644 index 0000000..0475c84 --- /dev/null +++ b/tests/fixtures/old_top_level_assertion_fail.expected.txt @@ -0,0 +1,2 @@ + +1 should equal 2 diff --git a/tests/fixtures/old_top_level_assertion_fail.py b/tests/fixtures/old_top_level_assertion_fail.py new file mode 100644 index 0000000..c801dc5 --- /dev/null +++ b/tests/fixtures/old_top_level_assertion_fail.py @@ -0,0 +1,4 @@ +# Deprecated and should not be used +import codewars_test as test + +test.assert_equals(1, 2) diff --git a/tests/fixtures/old_top_level_assertion_fail_pass.expected.txt b/tests/fixtures/old_top_level_assertion_fail_pass.expected.txt new file mode 100644 index 0000000..d47bd4f --- /dev/null +++ b/tests/fixtures/old_top_level_assertion_fail_pass.expected.txt @@ -0,0 +1,4 @@ + +1 should equal 2 + +Test Passed diff --git a/tests/fixtures/old_top_level_assertion_fail_pass.py b/tests/fixtures/old_top_level_assertion_fail_pass.py new file mode 100644 index 0000000..690bfc5 --- /dev/null +++ b/tests/fixtures/old_top_level_assertion_fail_pass.py @@ -0,0 +1,4 @@ +import codewars_test as test + +test.assert_equals(1, 2) +test.assert_equals(1, 1) diff --git a/tests/fixtures/old_top_level_assertion_pass.expected.txt b/tests/fixtures/old_top_level_assertion_pass.expected.txt new file mode 100644 index 0000000..4bfed57 --- /dev/null +++ b/tests/fixtures/old_top_level_assertion_pass.expected.txt @@ -0,0 +1,2 @@ + +Test Passed diff --git a/tests/fixtures/old_top_level_assertion_pass.py b/tests/fixtures/old_top_level_assertion_pass.py new file mode 100644 index 0000000..3a814f8 --- /dev/null +++ b/tests/fixtures/old_top_level_assertion_pass.py @@ -0,0 +1,4 @@ +# Deprecated and should not be used +import codewars_test as test + +test.assert_equals(1, 1) diff --git a/tests/fixtures/passing_failing.expected.txt b/tests/fixtures/passing_failing.expected.txt new file mode 100644 index 0000000..edeea1f --- /dev/null +++ b/tests/fixtures/passing_failing.expected.txt @@ -0,0 +1,16 @@ + +group 1 + +test 1 + +Test Passed + +0.00 + +test 2 + +1 should equal 2 + +0.00 + +0.02 diff --git a/tests/fixtures/passing_failing.py b/tests/fixtures/passing_failing.py new file mode 100644 index 0000000..1d76fc4 --- /dev/null +++ b/tests/fixtures/passing_failing.py @@ -0,0 +1,12 @@ +import codewars_test as test + + +@test.describe("group 1") +def group_1(): + @test.it("test 1") + def test_1(): + test.assert_equals(1, 1) + + @test.it("test 2") + def test_2(): + test.assert_equals(1, 2) diff --git a/tests/fixtures/single_failing.expected.txt b/tests/fixtures/single_failing.expected.txt new file mode 100644 index 0000000..f81b2a3 --- /dev/null +++ b/tests/fixtures/single_failing.expected.txt @@ -0,0 +1,10 @@ + +group 1 + +test 1 + +1 should equal 2 + +0.00 + +0.01 diff --git a/tests/fixtures/single_failing.py b/tests/fixtures/single_failing.py new file mode 100644 index 0000000..8580ff8 --- /dev/null +++ b/tests/fixtures/single_failing.py @@ -0,0 +1,8 @@ +import codewars_test as test + + +@test.describe("group 1") +def group_1(): + @test.it("test 1") + def test_1(): + test.assert_equals(1, 2) diff --git a/tests/fixtures/single_passing.expected.txt b/tests/fixtures/single_passing.expected.txt new file mode 100644 index 0000000..b35ba0e --- /dev/null +++ b/tests/fixtures/single_passing.expected.txt @@ -0,0 +1,10 @@ + +group 1 + +test 1 + +Test Passed + +0.01 + +0.02 diff --git a/tests/fixtures/single_passing.py b/tests/fixtures/single_passing.py new file mode 100644 index 0000000..626fea7 --- /dev/null +++ b/tests/fixtures/single_passing.py @@ -0,0 +1,8 @@ +import codewars_test as test + + +@test.describe("group 1") +def group_1(): + @test.it("test 1") + def test_1(): + test.assert_equals(1, 1) diff --git a/tests/fixtures/timeout_failing.expected.txt b/tests/fixtures/timeout_failing.expected.txt new file mode 100644 index 0000000..cf478d6 --- /dev/null +++ b/tests/fixtures/timeout_failing.expected.txt @@ -0,0 +1,6 @@ + +group 1 + +Exceeded time limit of 0.010 seconds + +30.41 diff --git a/tests/fixtures/timeout_failing.py b/tests/fixtures/timeout_failing.py new file mode 100644 index 0000000..00c2d06 --- /dev/null +++ b/tests/fixtures/timeout_failing.py @@ -0,0 +1,11 @@ +import codewars_test as test + + +@test.describe("group 1") +def group_1(): + @test.timeout(0.01) + def test_1(): + x = 0 + while x < 10 ** 9: + x += 1 + test.pass_() diff --git a/tests/fixtures/timeout_passing.expected.txt b/tests/fixtures/timeout_passing.expected.txt new file mode 100644 index 0000000..a0894ac --- /dev/null +++ b/tests/fixtures/timeout_passing.expected.txt @@ -0,0 +1,8 @@ + +group 1 + +Test Passed + +Test Passed + +17.19 diff --git a/tests/fixtures/timeout_passing.py b/tests/fixtures/timeout_passing.py new file mode 100644 index 0000000..aa63d1c --- /dev/null +++ b/tests/fixtures/timeout_passing.py @@ -0,0 +1,10 @@ +import codewars_test as test + + +@test.describe("group 1") +def group_1(): + # This outputs 2 PASSED + @test.timeout(0.01) + def test_1(): + test.assert_equals(1, 1) + diff --git a/tests/test_outputs.py b/tests/test_outputs.py new file mode 100644 index 0000000..e35f8a2 --- /dev/null +++ b/tests/test_outputs.py @@ -0,0 +1,81 @@ +import unittest +import subprocess +import os +import re +from pathlib import Path + + +class TestOutputs(unittest.TestCase): + pass + + +def test_against_expected(test_file, expected_file, env): + def test(self): + # Using `stdout=PIPE, stderr=PIPE` for Python 3.6 compatibility instead of `capture_output=True` + result = subprocess.run( + ["python", test_file], + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + with open(expected_file, "r", encoding="utf-8") as r: + # Allow duration to change + expected = re.sub(r"([()])", r"\\\1", r.read()) + expected = re.sub( + r"(?<=)\d+(?:\.\d+)?", r"\\d+(?:\\.\\d+)?", expected + ) + + self.assertRegex(result.stdout.decode("utf-8"), expected) + + return test + + +def get_commands(output): + return re.findall(r"<(?:DESCRIBE|IT|PASSED|FAILED|ERROR|COMPLETEDIN)::>", output) + + +def test_against_sample(test_file, sample_file, env): + def test(self): + # Using `stdout=PIPE, stderr=PIPE` for Python 3.6 compatibility instead of `capture_output=True` + result = subprocess.run( + ["python", test_file], + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + with open(sample_file, "r", encoding="utf-8") as r: + # Ensure that it contains the same output structure + self.assertEqual( + get_commands(result.stdout.decode("utf-8")), get_commands(r.read()) + ) + + return test + + +def define_tests(): + fixtures_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "fixtures") + package_dir = Path(fixtures_dir).parent.parent + files = (f for f in os.listdir(fixtures_dir) if f.endswith(".py")) + for f in files: + expected_file = os.path.join(fixtures_dir, f.replace(".py", ".expected.txt")) + if os.path.exists(expected_file): + test_func = test_against_expected( + os.path.join(fixtures_dir, f), + expected_file, + {"PYTHONPATH": package_dir}, + ) + else: + # Use `.sample.txt` when testing against outputs with more variables. + # This version only checks for the basic structure. + test_func = test_against_sample( + os.path.join(fixtures_dir, f), + os.path.join(fixtures_dir, f.replace(".py", ".sample.txt")), + {"PYTHONPATH": package_dir}, + ) + setattr(TestOutputs, "test_{0}".format(f.replace(".py", "")), test_func) + + +define_tests() + +if __name__ == "__main__": + unittest.main()