check for case conflicts.................................................Passed
check for merge conflicts................................................Passed
debug statements (python)................................................Passed
isort....................................................................Passed
black....................................................................Passed
flake8...................................................................Passed
pylint...................................................................Passed
bandit...................................................................Passed
mypy.....................................................................Passed
- hook id: mypy
- duration: 0.25s

Success: no issues found in 62 source files

pyright..................................................................Passed

=========================================================================== test session starts ===========================================================================platform linux -- Python 3.12.3, pytest-8.3.5, pluggy-1.5.0
rootdir: /home/user/prompt-declaration-language
configfile: pyproject.toml
plugins: langsmith-0.3.27, anyio-4.9.0
collected 265 items                                                                                                                                                       

tests/test_array.py ...                                                                                                                                             [  1%]
tests/test_ast_utils.py .                                                                                                                                           [  1%]
tests/test_code.py ..............                                                                                                                                   [  6%]
tests/test_cond.py ........                                                                                                                                         [  9%]
tests/test_data.py .......                                                                                                                                          [ 12%]
tests/test_defaults.py .......                                                                                                                                      [ 15%]
tests/test_defs.py ...                                                                                                                                              [ 16%]
tests/test_dump.py .                                                                                                                                                [ 16%]
tests/test_errors.py ...                                                                                                                                            [ 17%]
tests/test_examples_parse.py .                                                                                                                                      [ 18%]
tests/test_examples_run.py F                                                                                                                                        [ 18%]
tests/test_expr.py ...............                                                                                                                                  [ 24%]
tests/test_fallback.py .......                                                                                                                                      [ 26%]
tests/test_for.py .........                                                                                                                                         [ 30%]
tests/test_function.py .......                                                                                                                                      [ 32%]
tests/test_include.py ..                                                                                                                                            [ 33%]
tests/test_input.py ...                                                                                                                                             [ 34%]
tests/test_lib_version.py .                                                                                                                                         [ 35%]
tests/test_line_table.py ............................                                                                                                               [ 45%]
tests/test_linter.py ...........................                                                                                                                    [ 55%]
tests/test_match.py ............................                                                                                                                    [ 66%]
tests/test_messages.py .....                                                                                                                                        [ 68%]
tests/test_model.py .......                                                                                                                                         [ 70%]
tests/test_object.py ..                                                                                                                                             [ 71%]
tests/test_parse.py .                                                                                                                                               [ 72%]
tests/test_parser.py ...........                                                                                                                                    [ 76%]
tests/test_repeat.py ......................                                                                                                                         [ 84%]
tests/test_role.py .                                                                                                                                                [ 84%]
tests/test_runtime_errors.py ..........                                                                                                                             [ 88%]
tests/test_schema.py .                                                                                                                                              [ 89%]
tests/test_type_checking.py .......................                                                                                                                 [ 97%]
tests/test_var.py ......                                                                                                                                            [100%]

================================================================================ FAILURES =================================================================================___________________________________________________________________________ test_valid_programs ___________________________________________________________________________
capsys = <_pytest.capture.CaptureFixture object at 0x78121ad12ff0>, monkeypatch = <_pytest.monkeypatch.MonkeyPatch object at 0x781223337620>

    def test_valid_programs(capsys: CaptureFixture[str], monkeypatch: MonkeyPatch) -> None:
        actual_parse_error: set[str] = set()
        actual_runtime_error: set[str] = set()
        wrong_results = {}
    
        files = pathlib.Path(".").glob("**/*.pdl")
    
        for pdl_file_name in files:
    
            scope: ScopeType = PdlDict({})
            if str(pdl_file_name) in TO_SKIP:
                continue
            if str(pdl_file_name) in TESTS_WITH_INPUT:
                inputs = TESTS_WITH_INPUT[str(pdl_file_name)]
                if inputs.stdin is not None:
                    monkeypatch.setattr(
                        "sys.stdin",
                        io.StringIO(inputs.stdin),
                    )
                if inputs.scope is not None:
                    scope = inputs.scope
            try:
                random.seed(11)
                output = pdl.exec_file(
                    pdl_file_name,
                    scope=scope,
                    output="all",
                    config=pdl.InterpreterConfig(batch=0),
                )
                actual_result = output["result"]
    
                block_to_dict(output["trace"], json_compatible=True)
                result_dir_name = (
                    pathlib.Path(".") / "tests" / "results" / pdl_file_name.parent
                )
    
                if not __find_and_compare_results(pdl_file_name, str(actual_result)):
    
                    if OLLAMA_GHACTIONS_RESULTS:
                        print(
                            f"Program {str(pdl_file_name)} requries updating its result on GitHub Actions"
                        )
                        print(f"Actual results: {str(actual_result)}")
                        result_file_name = f"{pdl_file_name.stem}.ollama_ghactions.result"
                        __write_to_results_file(
                            result_dir_name, result_file_name, str(actual_result)
                        )
    
                        # Evaluate the results again. If fails again, then consider this program as failing
                        if not __find_and_compare_results(
                            pdl_file_name, str(actual_result)
                        ):
                            print(
                                f"Program {str(pdl_file_name)} failed second time even after generating results from Github Actions. Consider this failing!"
                            )
                            wrong_results[str(pdl_file_name)] = {
                                "actual": str(actual_result),
                            }
                        # If evaluating results produces correct result, then this is considered passing
                        else:
                            continue
    
                    if UPDATE_RESULTS:
                        result_file_name = (
                            f"{pdl_file_name.stem}.{str(RESULTS_VERSION)}.result"
                        )
                        __write_to_results_file(
                            result_dir_name, result_file_name, str(actual_result)
                        )
    
                    wrong_results[str(pdl_file_name)] = {
                        "actual": str(actual_result),
                    }
            except PDLParseError:
                actual_parse_error |= {str(pdl_file_name)}
            except Exception as exc:
                if str(pdl_file_name) not in set(str(p) for p in EXPECTED_RUNTIME_ERROR):
                    print(f"{pdl_file_name}: {exc}")  # unexpected error: breakpoint
                actual_runtime_error |= {str(pdl_file_name)}
                print(exc)
    
        # Parse errors
        expected_parse_error = set(str(p) for p in EXPECTED_PARSE_ERROR)
        unexpected_parse_error = sorted(list(actual_parse_error - expected_parse_error))
        assert (
            len(unexpected_parse_error) == 0
        ), f"Unexpected parse error: {unexpected_parse_error}"
    
        # Runtime errors
        expected_runtime_error = set(str(p) for p in EXPECTED_RUNTIME_ERROR)
        unexpected_runtime_error = sorted(
            list(actual_runtime_error - expected_runtime_error)
        )
>       assert (
            len(unexpected_runtime_error) == 0
        ), f"Unexpected runtime error: {unexpected_runtime_error}"
E       AssertionError: Unexpected runtime error: ['examples/chatbot/chatbot.pdl', 'examples/code/code-eval.pdl', 'examples/code/code-json.pdl', 'examples/code/code.pdl', 'examples/demo/1-hello.pdl', 'examples/demo/10-sdg.pdl', 'examples/demo/2-model-chaining.pdl', 'examples/demo/3-def-use.pdl', 'examples/demo/4-function.pdl', 'examples/demo/5-code-eval.pdl', 'examples/demo/6-code-json.pdl', 'examples/demo/7-chatbot-roles.pdl', 'examples/demo/8-tools.pdl', 'examples/demo/9-react.pdl', 'examples/fibonacci/fib.pdl', 'examples/gsm8k/gsm8k-plan-few-shots.pdl', 'examples/rag/tfidf_rag.pdl', 'examples/react/demo.pdl', 'examples/sdk/hello.pdl', 'examples/teacher/teacher.pdl', 'examples/tools/calc.pdl', 'examples/tutorial/calling_llm.pdl', 'examples/tutorial/calling_llm_chaining.pdl', 'examples/tutorial/calling_llm_with_input.pdl', 'examples/tutorial/calling_llm_with_input_messages.pdl', 'examples/tutorial/calling_llm_with_input_messages_var.pdl', 'examples/tutorial/code_pdl.pdl', 'examples/tutorial/defs.pdl', 'examples/tutorial/function_definition.pdl', 'examples/tutorial/function_empty_context.pdl', 'examples/tutorial/muting_block_output.pdl', 'examples/tutorial/parser-regex.pdl', 'examples/tutorial/parser_regex_code.pdl', 'examples/tutorial/programs/chatbot.pdl', 'examples/tutorial/programs/code-json.pdl', 'examples/tutorial/programs/tfidf_rag.pdl', 'examples/tutorial/programs/weather.pdl', 'examples/tutorial/variable_def_use.pdl', 'pdl-live-react/src-tauri/tests/cli/model-input-array.pdl', 'pdl-live-react/src-tauri/tests/cli/model-input-string.pdl', 'pdl-live-react/src-tauri/tests/cli/read-stdin.pdl']
E       assert 41 == 0
E        +  where 41 = len(['examples/chatbot/chatbot.pdl', 'examples/code/code-eval.pdl', 'examples/code/code-json.pdl', 'examples/code/code.pdl', 'examples/demo/1-hello.pdl', 'examples/demo/10-sdg.pdl', ...])

tests/test_examples_run.py:259: AssertionError