From d873a70ca70824d2f5611f9bff6d09c99e46ff29 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Tue, 11 Apr 2023 14:29:30 +0000 Subject: [PATCH 1/7] ci: Add script for fetching past test stats from CI --- scripts/ci-report/README.md | 4 + scripts/ci-report/fetch_stats_from_ci.sh | 124 +++++++++++++++++++++++ 2 files changed, 128 insertions(+) create mode 100755 scripts/ci-report/fetch_stats_from_ci.sh diff --git a/scripts/ci-report/README.md b/scripts/ci-report/README.md index 875ef9922f230..1748180ab7a4b 100644 --- a/scripts/ci-report/README.md +++ b/scripts/ci-report/README.md @@ -5,3 +5,7 @@ This program generates a CI report from the `gotests.json` generated by `go test ## Limitations We won't generate any report/stats for tests that weren't run. To find all existing tests, we could use: `go test ./... -list=. -json`, but the time it takes is probably not worth it. Usually most tests will run, even if there are errors and we're using `-failfast`. + +## Misc + +The script `fetch_stats_from_ci.sh` can be used to fetch historical stats from CI, e.g. for development or analysis. diff --git a/scripts/ci-report/fetch_stats_from_ci.sh b/scripts/ci-report/fetch_stats_from_ci.sh new file mode 100755 index 0000000000000..8eb3782285f28 --- /dev/null +++ b/scripts/ci-report/fetch_stats_from_ci.sh @@ -0,0 +1,124 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Usage: ./fetch_stats_from_ci.sh +# +# This script is for fetching historic test stats from GitHub Actions CI. +# +# Requires gh with credentials. +# +# https://github.com/cli/cli/blob/trunk/pkg/cmd/run/view/view.go#L434 + +dir="$(dirname "$0")"/ci-stats +mkdir -p "${dir}" + +pushd "${dir}" >/dev/null + +# Stats step name, used for filtering log. +job_step_name="Print test stats" + +if [[ ! -f list-ci.yaml.json ]]; then + gh run list -w ci.yaml -L 1000 --json conclusion,createdAt,databaseId,displayTitle,event,headBranch,headSha,name,number,startedAt,status,updatedAt,url,workflowDatabaseId,workflowName \ + >list-ci.yaml.json || { + rm -f list-ci.yaml.json + exit 1 + } +fi + +runs="$( + jq -r '.[] | select(.status == "completed") | select(.conclusion == "success" or .conclusion == "failure") | [.databaseId, .event, .displayTitle, .headBranch, .headSha] | @tsv' \ + "${run_jobs_file}" || { + rm -f "${run_jobs_file}" + exit 1 + } + fi + + jobs="$( + jq -r '.jobs[] | select(.name | startswith("test-go")) | select(.status == "completed") | select(.conclusion == "success" or .conclusion == "failure") | [.databaseId, .startedAt, .completedAt, .name, .url] | @tsv' \ + <"${run_jobs_file}" + )" + + while read -r job; do + mapfile -d $'\t' -t parts <<<"${job}" + parts[-1]="${parts[-1]%$'\n'}" + + job_database_id="${parts[0]}" + job_started_at="${parts[1]}" + job_completed_at="${parts[2]}" + job_name="${parts[3]}" + job_url="${parts[4]}" + + job_log=run-"${database_id}"-job-"${job_database_id}"-"${job_name}".log + if [[ ! -f "${job_log}" ]]; then + echo "Fetching log for: ${job_name} (${job_database_id}, ${job_url})" + # Example log (partial). + # test-go (ubuntu-latest) Print test stats 2023-04-11T03:02:18.4063489Z ##[group]Run # Artifacts are not available after rerunning a job, + # test-go (ubuntu-latest) Print test stats 2023-04-11T03:02:18.4063872Z # Artifacts are not available after rerunning a job, + # test-go (ubuntu-latest) Print test stats 2023-04-11T03:02:18.4064188Z # so we need to print the test stats to the log. + # test-go (ubuntu-latest) Print test stats 2023-04-11T03:02:18.4064642Z go run ./scripts/ci-report/main.go gotests.json | tee gotests_stats.json + # test-go (ubuntu-latest) Print test stats 2023-04-11T03:02:18.4110112Z shell: /usr/bin/bash -e {0} + # test-go (ubuntu-latest) Print test stats 2023-04-11T03:02:18.4110364Z ##[endgroup] + # test-go (ubuntu-latest) Print test stats 2023-04-11T03:02:19.3440469Z { + # test-go (ubuntu-latest) Print test stats 2023-04-11T03:02:19.3441078Z "packages": [ + # test-go (ubuntu-latest) Print test stats 2023-04-11T03:02:19.3441448Z { + # test-go (ubuntu-latest) Print test stats 2023-04-11T03:02:19.3442927Z "name": "agent", + # test-go (ubuntu-latest) Print test stats 2023-04-11T03:02:19.3443311Z "time": 17.538 + # test-go (ubuntu-latest) Print test stats 2023-04-11T03:02:19.3444048Z }, + # ... + gh run view --job "${job_database_id}" --log >"${job_log}" || { + # Sometimes gh failes to extract ZIP, etc. :'( + rm -f "${job_log}" + echo "Failed to fetch log for: ${job_name} (${job_database_id}, ${job_url}), skipping..." + continue + } + log_lines="$(wc -l "${job_log}" | awk '{print $1}')" + if [[ ${log_lines} -lt 2 ]]; then + # Sometimes gh returns nothing and gives no error :'( + rm -f "${job_log}" + echo "Log is empty for: ${job_name} (${job_database_id}, ${job_url}), skipping..." + continue + fi + fi + + job_stats="$( + grep "${job_name}.*${job_step_name}" "${job_log}" \ + | sed -E 's/.*[0-9-]{10}T[0-9:]{8}\.[0-9]*Z //' \ + | grep -E "^[{}\ ].*" + )" + + if ! jq -e . >/dev/null 2>&1 <<<"${job_stats}"; then + # Sometimes actions logs are partial when fetched via CLI :'( + echo "Failed to parse stats for: ${job_name} (${job_database_id}, ${job_url}), skipping..." + continue + fi + + job_stats_file=run-"${database_id}"-job-"${job_database_id}"-"${job_name}"-stats.json + jq \ + --arg event "${event}" \ + --arg branch "${head_branch}" \ + --arg sha "${head_sha}" \ + --arg started_at "${job_started_at}" \ + --arg completed_at "${job_completed_at}" \ + --arg display_title "${display_title}" \ + --arg url "${job_url}" \ + '{event: $event, branch: $branch, sha: $sha, started_at: $started_at, completed_at: $completed_at, display_title: $display_title, url: $url, stats: .}' \ + <<<"${job_stats}" \ + >"${job_stats_file}" + done <<<"${jobs}" +done <<<"${runs}" From d4833928c8d48b1d263b9288005d19237054c17a Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Tue, 11 Apr 2023 15:01:13 +0000 Subject: [PATCH 2/7] Fix typo --- scripts/ci-report/fetch_stats_from_ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/ci-report/fetch_stats_from_ci.sh b/scripts/ci-report/fetch_stats_from_ci.sh index 8eb3782285f28..c23d274579251 100755 --- a/scripts/ci-report/fetch_stats_from_ci.sh +++ b/scripts/ci-report/fetch_stats_from_ci.sh @@ -82,7 +82,7 @@ while read -r run; do # test-go (ubuntu-latest) Print test stats 2023-04-11T03:02:19.3444048Z }, # ... gh run view --job "${job_database_id}" --log >"${job_log}" || { - # Sometimes gh failes to extract ZIP, etc. :'( + # Sometimes gh fails to extract ZIP, etc. :'( rm -f "${job_log}" echo "Failed to fetch log for: ${job_name} (${job_database_id}, ${job_url}), skipping..." continue From a24a72a12b112f809c55f598ebb4ddd727e74edd Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Tue, 11 Apr 2023 15:17:02 +0000 Subject: [PATCH 3/7] Fix fmt --- scripts/ci-report/fetch_stats_from_ci.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/ci-report/fetch_stats_from_ci.sh b/scripts/ci-report/fetch_stats_from_ci.sh index c23d274579251..f3f6e8a085f8c 100755 --- a/scripts/ci-report/fetch_stats_from_ci.sh +++ b/scripts/ci-report/fetch_stats_from_ci.sh @@ -97,9 +97,9 @@ while read -r run; do fi job_stats="$( - grep "${job_name}.*${job_step_name}" "${job_log}" \ - | sed -E 's/.*[0-9-]{10}T[0-9:]{8}\.[0-9]*Z //' \ - | grep -E "^[{}\ ].*" + grep "${job_name}.*${job_step_name}" "${job_log}" | + sed -E 's/.*[0-9-]{10}T[0-9:]{8}\.[0-9]*Z //' | + grep -E "^[{}\ ].*" )" if ! jq -e . >/dev/null 2>&1 <<<"${job_stats}"; then From 7175cc76a3f0db273d938a9ad0ee04a24619c411 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Tue, 11 Apr 2023 15:49:40 +0000 Subject: [PATCH 4/7] More error handling --- scripts/ci-report/fetch_stats_from_ci.sh | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/scripts/ci-report/fetch_stats_from_ci.sh b/scripts/ci-report/fetch_stats_from_ci.sh index f3f6e8a085f8c..8219b3a5bd542 100755 --- a/scripts/ci-report/fetch_stats_from_ci.sh +++ b/scripts/ci-report/fetch_stats_from_ci.sh @@ -96,11 +96,14 @@ while read -r run; do fi fi - job_stats="$( + if ! job_stats="$( grep "${job_name}.*${job_step_name}" "${job_log}" | sed -E 's/.*[0-9-]{10}T[0-9:]{8}\.[0-9]*Z //' | grep -E "^[{}\ ].*" - )" + )"; then + echo "Failed to find stats in job log: ${job_name} (${job_database_id}, ${job_url}), skipping..." + continue + fi if ! jq -e . >/dev/null 2>&1 <<<"${job_stats}"; then # Sometimes actions logs are partial when fetched via CLI :'( @@ -109,6 +112,9 @@ while read -r run; do fi job_stats_file=run-"${database_id}"-job-"${job_database_id}"-"${job_name}"-stats.json + if [[ -f "${job_stats_file}" ]]; then + continue + fi jq \ --arg event "${event}" \ --arg branch "${head_branch}" \ @@ -119,6 +125,10 @@ while read -r run; do --arg url "${job_url}" \ '{event: $event, branch: $branch, sha: $sha, started_at: $started_at, completed_at: $completed_at, display_title: $display_title, url: $url, stats: .}' \ <<<"${job_stats}" \ - >"${job_stats_file}" + >"${job_stats_file}" || { + echo "Failed to write stats for: ${job_name} (${job_database_id}, ${job_url}), skipping..." + rm -f "${job_stats_file}" + exit 1 + } done <<<"${jobs}" done <<<"${runs}" From ee36e0e4a4669235a61a8e7cdc81f5698b9070e0 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Tue, 11 Apr 2023 16:04:52 +0000 Subject: [PATCH 5/7] Skip runs that are too old --- scripts/ci-report/fetch_stats_from_ci.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/ci-report/fetch_stats_from_ci.sh b/scripts/ci-report/fetch_stats_from_ci.sh index 8219b3a5bd542..4d83bb069840e 100755 --- a/scripts/ci-report/fetch_stats_from_ci.sh +++ b/scripts/ci-report/fetch_stats_from_ci.sh @@ -40,6 +40,11 @@ while read -r run; do head_branch="${parts[3]}" head_sha="${parts[4]}" + if [[ ${database_id} -le 4595490577 ]]; then + echo "Skipping ${database_id} (${display_title}), too old..." + continue + fi + run_jobs_file=run-"${database_id}"-"${event}"-jobs.json if [[ ! -f "${run_jobs_file}" ]]; then echo "Fetching jobs for run: ${display_title} (${database_id}, ${event}, ${head_branch})" From 8b323fac5a1a07602e32d644841b65a254841195 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Wed, 12 Apr 2023 09:31:28 +0000 Subject: [PATCH 6/7] Add more comments --- scripts/ci-report/fetch_stats_from_ci.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scripts/ci-report/fetch_stats_from_ci.sh b/scripts/ci-report/fetch_stats_from_ci.sh index 4d83bb069840e..976427c80554a 100755 --- a/scripts/ci-report/fetch_stats_from_ci.sh +++ b/scripts/ci-report/fetch_stats_from_ci.sh @@ -40,6 +40,8 @@ while read -r run; do head_branch="${parts[3]}" head_sha="${parts[4]}" + # Check if this run predates the stats PR, if yes, skip it: + # https://github.com/coder/coder/issues/6676 if [[ ${database_id} -le 4595490577 ]]; then echo "Skipping ${database_id} (${display_title}), too old..." continue @@ -102,6 +104,12 @@ while read -r run; do fi if ! job_stats="$( + # Extract the stats job output (JSON) from the job log, + # discarding the timestamp and non-JSON header. + # + # Example variable values: + # job_name="test-go (ubuntu-latest)" + # job_step_name="Print test stats" grep "${job_name}.*${job_step_name}" "${job_log}" | sed -E 's/.*[0-9-]{10}T[0-9:]{8}\.[0-9]*Z //' | grep -E "^[{}\ ].*" From 424e26ac3859e0faf2ae61280b153080d3271db2 Mon Sep 17 00:00:00 2001 From: Mathias Fredriksson Date: Wed, 12 Apr 2023 09:55:10 +0000 Subject: [PATCH 7/7] Add more data to JSON --- scripts/ci-report/fetch_stats_from_ci.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/scripts/ci-report/fetch_stats_from_ci.sh b/scripts/ci-report/fetch_stats_from_ci.sh index 976427c80554a..2ca8f586b5ab1 100755 --- a/scripts/ci-report/fetch_stats_from_ci.sh +++ b/scripts/ci-report/fetch_stats_from_ci.sh @@ -26,7 +26,7 @@ if [[ ! -f list-ci.yaml.json ]]; then fi runs="$( - jq -r '.[] | select(.status == "completed") | select(.conclusion == "success" or .conclusion == "failure") | [.databaseId, .event, .displayTitle, .headBranch, .headSha] | @tsv' \ + jq -r '.[] | select(.status == "completed") | select(.conclusion == "success" or .conclusion == "failure") | [.databaseId, .event, .displayTitle, .headBranch, .headSha, .url] | @tsv' \ "${job_stats_file}" || { echo "Failed to write stats for: ${job_name} (${job_database_id}, ${job_url}), skipping..."