From 328d718707ad49a6e29f49884389d8e8a785e403 Mon Sep 17 00:00:00 2001
From: Heath Stewart <heaths@outlook.com>
Date: Thu, 25 Jun 2026 22:33:45 -0700
Subject: [PATCH] Add pin-github-actions experiment support

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
---
 .../pin-github-actions-usage/README.md        | 11 +++++
 .../pin-github-actions-usage/eval.yaml        | 40 +++++++++++++++++++
 .../fixtures/workflow.yml                     | 14 +++++++
 3 files changed, 65 insertions(+)
 create mode 100644 evals/security/pin-github-actions-usage/README.md
 create mode 100644 evals/security/pin-github-actions-usage/eval.yaml
 create mode 100644 evals/security/pin-github-actions-usage/fixtures/workflow.yml

diff --git a/evals/security/pin-github-actions-usage/README.md b/evals/security/pin-github-actions-usage/README.md
new file mode 100644
index 0000000..756978d
--- /dev/null
+++ b/evals/security/pin-github-actions-usage/README.md
@@ -0,0 +1,11 @@
+# Pin GitHub Actions baseline support
+
+This branch preserves the pre-script `pin-github-actions` skill and the fixture
+files used by the experiment on `main`.
+
+Run the comparison from the main branch. The experiment there loads the baseline
+skill from this worktree at:
+
+```text
+.worktrees/experiments-baseline/plugins/security/skills/pin-github-actions
+```
diff --git a/evals/security/pin-github-actions-usage/eval.yaml b/evals/security/pin-github-actions-usage/eval.yaml
new file mode 100644
index 0000000..f46c378
--- /dev/null
+++ b/evals/security/pin-github-actions-usage/eval.yaml
@@ -0,0 +1,40 @@
+name: security pin-github-actions baseline support
+version: 1
+description: Baseline fixture support for the pre-script pin-github-actions skill
+tags:
+  benchmark: usage
+
+defaults:
+  runs: 1
+  timeout: 5m
+
+environment:
+  skills:
+    - ../../../plugins/security/skills/pin-github-actions
+  files:
+    - src: ./fixtures/workflow.yml
+      dest: .github/workflows/ci.yml
+
+scoring:
+  threshold: 1.0
+
+stimuli:
+  - name: pin-github-actions-usage-comparison
+    prompt: |
+      Review .github/workflows/ci.yml and harden any insecure GitHub Actions references you find.
+    constraints:
+      expect_skills:
+        - pin-github-actions
+    graders:
+      - type: file-matches
+        config:
+          path: .github/workflows/ci.yml
+          pattern: '(?m)^ *- uses: actions/checkout@[0-9a-f]{40} # v[0-9]+[.][0-9]+[.][0-9]+$'
+      - type: file-matches
+        config:
+          path: .github/workflows/ci.yml
+          pattern: '(?m)^ *- uses: actions/setup-node@[0-9a-f]{40} # v[0-9]+[.][0-9]+[.][0-9]+$'
+      - type: file-not-matches
+        config:
+          path: .github/workflows/ci.yml
+          pattern: '(?m)uses:\\s+[^\\n]+@v[0-9]+'
diff --git a/evals/security/pin-github-actions-usage/fixtures/workflow.yml b/evals/security/pin-github-actions-usage/fixtures/workflow.yml
new file mode 100644
index 0000000..792c940
--- /dev/null
+++ b/evals/security/pin-github-actions-usage/fixtures/workflow.yml
@@ -0,0 +1,14 @@
+name: CI
+
+on:
+  pull_request:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22
+      - run: npm test