diff --git a/evals/security/pin-github-actions-usage/README.md b/evals/security/pin-github-actions-usage/README.md new file mode 100644 index 0000000..756978d --- /dev/null +++ b/evals/security/pin-github-actions-usage/README.md @@ -0,0 +1,11 @@ +# Pin GitHub Actions baseline support + +This branch preserves the pre-script `pin-github-actions` skill and the fixture +files used by the experiment on `main`. + +Run the comparison from the main branch. The experiment there loads the baseline +skill from this worktree at: + +```text +.worktrees/experiments-baseline/plugins/security/skills/pin-github-actions +``` diff --git a/evals/security/pin-github-actions-usage/eval.yaml b/evals/security/pin-github-actions-usage/eval.yaml new file mode 100644 index 0000000..f46c378 --- /dev/null +++ b/evals/security/pin-github-actions-usage/eval.yaml @@ -0,0 +1,40 @@ +name: security pin-github-actions baseline support +version: 1 +description: Baseline fixture support for the pre-script pin-github-actions skill +tags: + benchmark: usage + +defaults: + runs: 1 + timeout: 5m + +environment: + skills: + - ../../../plugins/security/skills/pin-github-actions + files: + - src: ./fixtures/workflow.yml + dest: .github/workflows/ci.yml + +scoring: + threshold: 1.0 + +stimuli: + - name: pin-github-actions-usage-comparison + prompt: | + Review .github/workflows/ci.yml and harden any insecure GitHub Actions references you find. + constraints: + expect_skills: + - pin-github-actions + graders: + - type: file-matches + config: + path: .github/workflows/ci.yml + pattern: '(?m)^ *- uses: actions/checkout@[0-9a-f]{40} # v[0-9]+[.][0-9]+[.][0-9]+$' + - type: file-matches + config: + path: .github/workflows/ci.yml + pattern: '(?m)^ *- uses: actions/setup-node@[0-9a-f]{40} # v[0-9]+[.][0-9]+[.][0-9]+$' + - type: file-not-matches + config: + path: .github/workflows/ci.yml + pattern: '(?m)uses:\\s+[^\\n]+@v[0-9]+' diff --git a/evals/security/pin-github-actions-usage/fixtures/workflow.yml b/evals/security/pin-github-actions-usage/fixtures/workflow.yml new file mode 100644 index 0000000..792c940 --- /dev/null +++ b/evals/security/pin-github-actions-usage/fixtures/workflow.yml @@ -0,0 +1,14 @@ +name: CI + +on: + pull_request: + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + - run: npm test