From 8154a1d43e0d7ef19a5145bd076b8493fd6d019b Mon Sep 17 00:00:00 2001 From: kazk Date: Sun, 12 Jul 2020 16:42:28 -0700 Subject: [PATCH 1/2] Add tests to prevent regression --- .github/workflows/test.yml | 17 +++++ gen-fixture.sh | 7 ++ tests/__init__.py | 0 .../fixtures/expect_error_sample.expected.txt | 76 +++++++++++++++++++ tests/fixtures/expect_error_sample.py | 60 +++++++++++++++ tests/fixtures/multiple_groups.expected.txt | 20 +++++ tests/fixtures/multiple_groups.py | 15 ++++ tests/fixtures/nested_groups.expected.txt | 14 ++++ tests/fixtures/nested_groups.py | 10 +++ tests/fixtures/old_group.expected.txt | 6 ++ tests/fixtures/old_group.py | 6 ++ .../old_top_level_assertion_fail.expected.txt | 2 + .../fixtures/old_top_level_assertion_fail.py | 4 + ...top_level_assertion_fail_pass.expected.txt | 4 + .../old_top_level_assertion_fail_pass.py | 4 + .../old_top_level_assertion_pass.expected.txt | 2 + .../fixtures/old_top_level_assertion_pass.py | 4 + tests/fixtures/passing_failing.expected.txt | 16 ++++ tests/fixtures/passing_failing.py | 12 +++ tests/fixtures/single_failing.expected.txt | 10 +++ tests/fixtures/single_failing.py | 8 ++ tests/fixtures/single_passing.expected.txt | 10 +++ tests/fixtures/single_passing.py | 8 ++ tests/fixtures/timeout_failing.expected.txt | 6 ++ tests/fixtures/timeout_failing.py | 11 +++ tests/fixtures/timeout_passing.expected.txt | 8 ++ tests/fixtures/timeout_passing.py | 10 +++ tests/test_outputs.py | 67 ++++++++++++++++ 28 files changed, 417 insertions(+) create mode 100644 .github/workflows/test.yml create mode 100755 gen-fixture.sh create mode 100644 tests/__init__.py create mode 100644 tests/fixtures/expect_error_sample.expected.txt create mode 100644 tests/fixtures/expect_error_sample.py create mode 100644 tests/fixtures/multiple_groups.expected.txt create mode 100644 tests/fixtures/multiple_groups.py create mode 100644 tests/fixtures/nested_groups.expected.txt create mode 100644 tests/fixtures/nested_groups.py create mode 100644 tests/fixtures/old_group.expected.txt create mode 100644 tests/fixtures/old_group.py create mode 100644 tests/fixtures/old_top_level_assertion_fail.expected.txt create mode 100644 tests/fixtures/old_top_level_assertion_fail.py create mode 100644 tests/fixtures/old_top_level_assertion_fail_pass.expected.txt create mode 100644 tests/fixtures/old_top_level_assertion_fail_pass.py create mode 100644 tests/fixtures/old_top_level_assertion_pass.expected.txt create mode 100644 tests/fixtures/old_top_level_assertion_pass.py create mode 100644 tests/fixtures/passing_failing.expected.txt create mode 100644 tests/fixtures/passing_failing.py create mode 100644 tests/fixtures/single_failing.expected.txt create mode 100644 tests/fixtures/single_failing.py create mode 100644 tests/fixtures/single_passing.expected.txt create mode 100644 tests/fixtures/single_passing.py create mode 100644 tests/fixtures/timeout_failing.expected.txt create mode 100644 tests/fixtures/timeout_failing.py create mode 100644 tests/fixtures/timeout_passing.expected.txt create mode 100644 tests/fixtures/timeout_passing.py create mode 100644 tests/test_outputs.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..288e153 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,17 @@ +name: Test +on: [push, pull_request] + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.6, 3.7, 3.8] + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Test + run: python -m unittest diff --git a/gen-fixture.sh b/gen-fixture.sh new file mode 100755 index 0000000..4093395 --- /dev/null +++ b/gen-fixture.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# ./gen-fixture.sh tests/fixtures/example.py +# ./gen-fixture.sh tests/fixtures/example.py expected +# ./gen-fixture.sh tests/fixtures/example.py sample +# for f in $(ls tests/fixtures/*.py); do ./gen-fixture.sh "$f"; done + +PYTHONPATH=./ python "$1" > "${1%.py}.${2:-expected}.txt" diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/fixtures/expect_error_sample.expected.txt b/tests/fixtures/expect_error_sample.expected.txt new file mode 100644 index 0000000..b2c7f56 --- /dev/null +++ b/tests/fixtures/expect_error_sample.expected.txt @@ -0,0 +1,76 @@ + +expect_error, new version + +f0 raises nothing + +f0 did not raise any exception + +f0 did not raise Exception + +f0 did not raise ArithmeticError + +f0 did not raise ZeroDivisionError + +f0 did not raise LookupError + +f0 did not raise KeyError + +f0 did not raise OSError + +0.03 + +f1 raises Exception + +Test Passed + +Test Passed + +f1 did not raise ArithmeticError + +f1 did not raise ZeroDivisionError + +f1 did not raise LookupError + +f1 did not raise KeyError + +f1 did not raise OSError + +0.02 + +f2 raises Exception >> ArithmeticError >> ZeroDivisionError + +Test Passed + +Test Passed + +Test Passed + +Test Passed + +f2 did not raise LookupError + +f2 did not raise KeyError + +f2 did not raise OSError + +0.02 + +f3 raises Exception >> LookupError >> KeyError + +Test Passed + +Test Passed + +f3 did not raise ArithmeticError + +f3 did not raise ZeroDivisionError + +Test Passed + +Test Passed + +f3 did not raise OSError + +0.02 + +0.11 diff --git a/tests/fixtures/expect_error_sample.py b/tests/fixtures/expect_error_sample.py new file mode 100644 index 0000000..e76962f --- /dev/null +++ b/tests/fixtures/expect_error_sample.py @@ -0,0 +1,60 @@ +# https://www.codewars.com/kumite/5ab735bee7093b17b2000084?sel=5ab735bee7093b17b2000084 +import codewars_test as test + + +def f0(): + pass + + +# BaseException >> Exception +def f1(): + raise Exception() + + +# BaseException >> Exception >> ArithmeticError >> ZeroDivisionError +def f2(): + return 1 // 0 + + +# BaseException >> Exception >> LookupError >> KeyError +def f3(): + return {}[1] + + +excn = ( + "Exception", + "ArithmeticError", + "ZeroDivisionError", + "LookupError", + "KeyError", + "OSError", +) +exc = (Exception, ArithmeticError, ZeroDivisionError, LookupError, KeyError, OSError) + + +@test.describe("expect_error, new version") +def d2(): + @test.it("f0 raises nothing") + def i0(): + test.expect_error("f0 did not raise any exception", f0) + for i in range(6): + test.expect_error("f0 did not raise {}".format(excn[i]), f0, exc[i]) + + @test.it("f1 raises Exception") + def i1(): + test.expect_error("f1 did not raise Exception", f1) + for i in range(6): + test.expect_error("f1 did not raise {}".format(excn[i]), f1, exc[i]) + + @test.it("f2 raises Exception >> ArithmeticError >> ZeroDivisionError") + def i2(): + test.expect_error("f2 did not raise Exception", f2) + for i in range(6): + test.expect_error("f2 did not raise {}".format(excn[i]), f2, exc[i]) + + @test.it("f3 raises Exception >> LookupError >> KeyError") + def i3(): + test.expect_error("f3 did not raise Exception", f3) + for i in range(6): + test.expect_error("f3 did not raise {}".format(excn[i]), f3, exc[i]) + diff --git a/tests/fixtures/multiple_groups.expected.txt b/tests/fixtures/multiple_groups.expected.txt new file mode 100644 index 0000000..f9432cf --- /dev/null +++ b/tests/fixtures/multiple_groups.expected.txt @@ -0,0 +1,20 @@ + +group 1 + +test 1 + +1 should equal 2 + +0.01 + +0.02 + +group 2 + +test 1 + +1 should equal 2 + +0.00 + +0.01 diff --git a/tests/fixtures/multiple_groups.py b/tests/fixtures/multiple_groups.py new file mode 100644 index 0000000..0a24b59 --- /dev/null +++ b/tests/fixtures/multiple_groups.py @@ -0,0 +1,15 @@ +import codewars_test as test + + +@test.describe("group 1") +def group_1(): + @test.it("test 1") + def test_1(): + test.assert_equals(1, 2) + + +@test.describe("group 2") +def group_2(): + @test.it("test 1") + def test_1(): + test.assert_equals(1, 2) diff --git a/tests/fixtures/nested_groups.expected.txt b/tests/fixtures/nested_groups.expected.txt new file mode 100644 index 0000000..7445137 --- /dev/null +++ b/tests/fixtures/nested_groups.expected.txt @@ -0,0 +1,14 @@ + +group 1 + +group 1 1 + +test 1 + +1 should equal 2 + +0.01 + +0.02 + +0.02 diff --git a/tests/fixtures/nested_groups.py b/tests/fixtures/nested_groups.py new file mode 100644 index 0000000..b9e3636 --- /dev/null +++ b/tests/fixtures/nested_groups.py @@ -0,0 +1,10 @@ +import codewars_test as test + + +@test.describe("group 1") +def group_1(): + @test.describe("group 1 1") + def group_1_1(): + @test.it("test 1") + def test_1(): + test.assert_equals(1, 2) diff --git a/tests/fixtures/old_group.expected.txt b/tests/fixtures/old_group.expected.txt new file mode 100644 index 0000000..e450a9b --- /dev/null +++ b/tests/fixtures/old_group.expected.txt @@ -0,0 +1,6 @@ + +group 1 + +test 1 + +Test Passed diff --git a/tests/fixtures/old_group.py b/tests/fixtures/old_group.py new file mode 100644 index 0000000..db08a36 --- /dev/null +++ b/tests/fixtures/old_group.py @@ -0,0 +1,6 @@ +# Deprecated and should not be used +import codewars_test as test + +test.describe("group 1") +test.it("test 1") +test.assert_equals(1, 1) diff --git a/tests/fixtures/old_top_level_assertion_fail.expected.txt b/tests/fixtures/old_top_level_assertion_fail.expected.txt new file mode 100644 index 0000000..0475c84 --- /dev/null +++ b/tests/fixtures/old_top_level_assertion_fail.expected.txt @@ -0,0 +1,2 @@ + +1 should equal 2 diff --git a/tests/fixtures/old_top_level_assertion_fail.py b/tests/fixtures/old_top_level_assertion_fail.py new file mode 100644 index 0000000..c801dc5 --- /dev/null +++ b/tests/fixtures/old_top_level_assertion_fail.py @@ -0,0 +1,4 @@ +# Deprecated and should not be used +import codewars_test as test + +test.assert_equals(1, 2) diff --git a/tests/fixtures/old_top_level_assertion_fail_pass.expected.txt b/tests/fixtures/old_top_level_assertion_fail_pass.expected.txt new file mode 100644 index 0000000..d47bd4f --- /dev/null +++ b/tests/fixtures/old_top_level_assertion_fail_pass.expected.txt @@ -0,0 +1,4 @@ + +1 should equal 2 + +Test Passed diff --git a/tests/fixtures/old_top_level_assertion_fail_pass.py b/tests/fixtures/old_top_level_assertion_fail_pass.py new file mode 100644 index 0000000..690bfc5 --- /dev/null +++ b/tests/fixtures/old_top_level_assertion_fail_pass.py @@ -0,0 +1,4 @@ +import codewars_test as test + +test.assert_equals(1, 2) +test.assert_equals(1, 1) diff --git a/tests/fixtures/old_top_level_assertion_pass.expected.txt b/tests/fixtures/old_top_level_assertion_pass.expected.txt new file mode 100644 index 0000000..4bfed57 --- /dev/null +++ b/tests/fixtures/old_top_level_assertion_pass.expected.txt @@ -0,0 +1,2 @@ + +Test Passed diff --git a/tests/fixtures/old_top_level_assertion_pass.py b/tests/fixtures/old_top_level_assertion_pass.py new file mode 100644 index 0000000..3a814f8 --- /dev/null +++ b/tests/fixtures/old_top_level_assertion_pass.py @@ -0,0 +1,4 @@ +# Deprecated and should not be used +import codewars_test as test + +test.assert_equals(1, 1) diff --git a/tests/fixtures/passing_failing.expected.txt b/tests/fixtures/passing_failing.expected.txt new file mode 100644 index 0000000..edeea1f --- /dev/null +++ b/tests/fixtures/passing_failing.expected.txt @@ -0,0 +1,16 @@ + +group 1 + +test 1 + +Test Passed + +0.00 + +test 2 + +1 should equal 2 + +0.00 + +0.02 diff --git a/tests/fixtures/passing_failing.py b/tests/fixtures/passing_failing.py new file mode 100644 index 0000000..1d76fc4 --- /dev/null +++ b/tests/fixtures/passing_failing.py @@ -0,0 +1,12 @@ +import codewars_test as test + + +@test.describe("group 1") +def group_1(): + @test.it("test 1") + def test_1(): + test.assert_equals(1, 1) + + @test.it("test 2") + def test_2(): + test.assert_equals(1, 2) diff --git a/tests/fixtures/single_failing.expected.txt b/tests/fixtures/single_failing.expected.txt new file mode 100644 index 0000000..f81b2a3 --- /dev/null +++ b/tests/fixtures/single_failing.expected.txt @@ -0,0 +1,10 @@ + +group 1 + +test 1 + +1 should equal 2 + +0.00 + +0.01 diff --git a/tests/fixtures/single_failing.py b/tests/fixtures/single_failing.py new file mode 100644 index 0000000..8580ff8 --- /dev/null +++ b/tests/fixtures/single_failing.py @@ -0,0 +1,8 @@ +import codewars_test as test + + +@test.describe("group 1") +def group_1(): + @test.it("test 1") + def test_1(): + test.assert_equals(1, 2) diff --git a/tests/fixtures/single_passing.expected.txt b/tests/fixtures/single_passing.expected.txt new file mode 100644 index 0000000..b35ba0e --- /dev/null +++ b/tests/fixtures/single_passing.expected.txt @@ -0,0 +1,10 @@ + +group 1 + +test 1 + +Test Passed + +0.01 + +0.02 diff --git a/tests/fixtures/single_passing.py b/tests/fixtures/single_passing.py new file mode 100644 index 0000000..626fea7 --- /dev/null +++ b/tests/fixtures/single_passing.py @@ -0,0 +1,8 @@ +import codewars_test as test + + +@test.describe("group 1") +def group_1(): + @test.it("test 1") + def test_1(): + test.assert_equals(1, 1) diff --git a/tests/fixtures/timeout_failing.expected.txt b/tests/fixtures/timeout_failing.expected.txt new file mode 100644 index 0000000..cf478d6 --- /dev/null +++ b/tests/fixtures/timeout_failing.expected.txt @@ -0,0 +1,6 @@ + +group 1 + +Exceeded time limit of 0.010 seconds + +30.41 diff --git a/tests/fixtures/timeout_failing.py b/tests/fixtures/timeout_failing.py new file mode 100644 index 0000000..00c2d06 --- /dev/null +++ b/tests/fixtures/timeout_failing.py @@ -0,0 +1,11 @@ +import codewars_test as test + + +@test.describe("group 1") +def group_1(): + @test.timeout(0.01) + def test_1(): + x = 0 + while x < 10 ** 9: + x += 1 + test.pass_() diff --git a/tests/fixtures/timeout_passing.expected.txt b/tests/fixtures/timeout_passing.expected.txt new file mode 100644 index 0000000..a0894ac --- /dev/null +++ b/tests/fixtures/timeout_passing.expected.txt @@ -0,0 +1,8 @@ + +group 1 + +Test Passed + +Test Passed + +17.19 diff --git a/tests/fixtures/timeout_passing.py b/tests/fixtures/timeout_passing.py new file mode 100644 index 0000000..aa63d1c --- /dev/null +++ b/tests/fixtures/timeout_passing.py @@ -0,0 +1,10 @@ +import codewars_test as test + + +@test.describe("group 1") +def group_1(): + # This outputs 2 PASSED + @test.timeout(0.01) + def test_1(): + test.assert_equals(1, 1) + diff --git a/tests/test_outputs.py b/tests/test_outputs.py new file mode 100644 index 0000000..f94d331 --- /dev/null +++ b/tests/test_outputs.py @@ -0,0 +1,67 @@ +import unittest +import subprocess +import os +import re +from pathlib import Path + + +class TestOutputs(unittest.TestCase): + pass + + +def test_against_expected(test_file, expected_file, env): + def test(self): + result = subprocess.run(["python", test_file], env=env, capture_output=True) + with open(expected_file, "r", encoding="utf-8") as r: + # Allow duration to change + expected = re.sub( + r"(?<=)\d+(?:\.\d+)?", r"\\d+(?:\\.\\d+)?", r.read() + ) + self.assertRegex(result.stdout.decode("utf-8"), expected) + + return test + + +def get_commands(output): + return re.findall(r"<(?:DESCRIBE|IT|PASSED|FAILED|ERROR|COMPLETEDIN)::>", output) + + +def test_against_sample(test_file, sample_file, env): + def test(self): + result = subprocess.run(["python", test_file], env=env, capture_output=True) + with open(sample_file, "r", encoding="utf-8") as r: + # Ensure that it contains the same output structure + self.assertEqual( + get_commands(result.stdout.decode("utf-8")), get_commands(r.read()) + ) + + return test + + +def define_tests(): + fixtures_dir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "fixtures") + package_dir = Path(fixtures_dir).parent.parent + files = (f for f in os.listdir(fixtures_dir) if f.endswith(".py")) + for f in files: + expected_file = os.path.join(fixtures_dir, f.replace(".py", ".expected.txt")) + if os.path.exists(expected_file): + test_func = test_against_expected( + os.path.join(fixtures_dir, f), + expected_file, + {"PYTHONPATH": package_dir}, + ) + else: + # Use `.sample.txt` when testing against outputs with more variables. + # This version only checks for the basic structure. + test_func = test_against_sample( + os.path.join(fixtures_dir, f), + os.path.join(fixtures_dir, f.replace(".py", ".sample.txt")), + {"PYTHONPATH": package_dir}, + ) + setattr(TestOutputs, "test_{0}".format(f.replace(".py", "")), test_func) + + +define_tests() + +if __name__ == "__main__": + unittest.main() From ae20b2cecbbd02ec28e7cfc4a7855d120fdcdf99 Mon Sep 17 00:00:00 2001 From: kazk Date: Sun, 12 Jul 2020 16:53:06 -0700 Subject: [PATCH 2/2] Fix Python 3.6 compatibility --- tests/test_outputs.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/test_outputs.py b/tests/test_outputs.py index f94d331..bfe8396 100644 --- a/tests/test_outputs.py +++ b/tests/test_outputs.py @@ -11,7 +11,13 @@ class TestOutputs(unittest.TestCase): def test_against_expected(test_file, expected_file, env): def test(self): - result = subprocess.run(["python", test_file], env=env, capture_output=True) + # Using `stdout=PIPE, stderr=PIPE` for Python 3.6 compatibility instead of `capture_output=True` + result = subprocess.run( + ["python", test_file], + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) with open(expected_file, "r", encoding="utf-8") as r: # Allow duration to change expected = re.sub( @@ -28,7 +34,13 @@ def get_commands(output): def test_against_sample(test_file, sample_file, env): def test(self): - result = subprocess.run(["python", test_file], env=env, capture_output=True) + # Using `stdout=PIPE, stderr=PIPE` for Python 3.6 compatibility instead of `capture_output=True` + result = subprocess.run( + ["python", test_file], + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) with open(sample_file, "r", encoding="utf-8") as r: # Ensure that it contains the same output structure self.assertEqual(