benchmarks: introduce --admin flag for admin benchmarks

ethomson · ethomson · commit 771ec303d4fa · 2025-03-02T20:46:38.000Z
Some benchmarks require administrative privileges, namely the ones that
blow up the disk cache. Don't run them by default, to avoid obnoxious
sudo password prompts, etc. Users can specify `--admin` to run them.
diff --git a/tests/benchmarks/benchmark.sh b/tests/benchmarks/benchmark.sh
@@ -6,7 +6,7 @@ set -eo pipefail
 # parse the command line
 #
 
-usage() { echo "usage: $(basename "$0") [--cli <path>] [--name <cli-name>] [--baseline-cli <path>] [--suite <suite>] [--json <path>] [--flamegraph] [--zip <path>] [--verbose] [--debug]"; }
+usage() { echo "usage: $(basename "$0") [--cli <path>] [--name <cli-name>] [--baseline-cli <path>] [--suite <suite>] [--admin] [--json <path>] [--flamegraph] [--zip <path>] [--verbose] [--debug]"; }
 
 TEST_CLI="git"
 TEST_CLI_NAME=
@@ -16,6 +16,7 @@ JSON_RESULT=
 FLAMEGRAPH=
 ZIP_RESULT=
 OUTPUT_DIR=
+ADMIN=
 VERBOSE=
 DEBUG=
 NEXT=
@@ -58,6 +59,8 @@ for a in "$@"; do
 		NEXT="suite"
 	elif [[ "${a}" == "-s"* ]]; then
 		SUITE="${a/-s/}"
+	elif [ "${a}" == "--admin" ]; then
+		ADMIN=1
 	elif [ "${a}" = "-v" ] || [ "${a}" == "--verbose" ]; then
 		VERBOSE=1
 	elif [ "${a}" == "--debug" ]; then
@@ -223,14 +226,18 @@ for TEST_PATH in "${BENCHMARK_DIR}"/*; do
 		SHOW_OUTPUT="--show-output"
 	fi
 
+	if [ "${ADMIN}" = "1" ]; then
+		ALLOW_ADMIN="--admin"
+	fi
+
 	OUTPUT_FILE="${OUTPUT_DIR}/${TEST_FILE}.out"
 	ERROR_FILE="${OUTPUT_DIR}/${TEST_FILE}.err"
 	JSON_FILE="${OUTPUT_DIR}/${TEST_FILE}.json"
 	FLAMEGRAPH_FILE="${OUTPUT_DIR}/${TEST_FILE}.svg"
 
 	FAILED=
 	{
-	  ${TEST_PATH} --cli "${TEST_CLI}" --baseline-cli "${BASELINE_CLI}" --json "${JSON_FILE}" ${SHOW_OUTPUT} >"${OUTPUT_FILE}" 2>"${ERROR_FILE}";
+	  ${TEST_PATH} --cli "${TEST_CLI}" --baseline-cli "${BASELINE_CLI}" --json "${JSON_FILE}" ${ALLOW_ADMIN} ${SHOW_OUTPUT} >"${OUTPUT_FILE}" 2>"${ERROR_FILE}";
 	  FAILED=$?
 	} || true
 
@@ -311,6 +318,8 @@ for TEST_PATH in "${BENCHMARK_DIR}"/*; do
 				ANY_FAILED=1
 			fi
 		fi
+	else
+		echo ""
 	fi
 done
 
diff --git a/tests/benchmarks/benchmark_helpers.sh b/tests/benchmarks/benchmark_helpers.sh
@@ -7,11 +7,12 @@ set -eo pipefail
 # command-line parsing
 #
 
-usage() { echo "usage: $(basename "$0") [--cli <path>] [--baseline-cli <path>] [--output-style <style>] [--json <path>] [--profile] [--flamegraph <path>]"; }
+usage() { echo "usage: $(basename "$0") [--cli <path>] [--baseline-cli <path>] [--admin] [--output-style <style>] [--json <path>] [--show-output] [--output-style <style>] [--profile] [--flamegraph <path>]"; }
 
 NEXT=
 BASELINE_CLI=
 TEST_CLI="git"
+ADMIN=
 SHOW_OUTPUT=
 JSON=
 PROFILE=
@@ -69,6 +70,8 @@ for a in "$@"; do
                 JSON="${a/-j/}"
 	elif [ "${a}" = "-p" ] || [ "${a}" = "--profile" ]; then
 		PROFILE=1
+	elif [ "${a}" = "--admin" ]; then
+		ADMIN=1
 	elif [ "${a}" = "-F" ] || [ "${a}" = "--flamegraph" ]; then
 		NEXT="flamegraph"
 	elif [[ "${a}" == "-F"* ]]; then
@@ -335,6 +338,9 @@ parse_arguments() {
 	# this test should run `n` warmups
 	WARMUP=0
 
+	# this test requires administrative privileges
+	REQUIRES_ADMIN=0
+
 	if [ "$*" = "" ]; then
 		gitbench_usage 1>&2
 		exit 1
@@ -356,6 +362,8 @@ parse_arguments() {
 			NEXT="prepare"
 		elif [ "${a}" = "--chdir" ]; then
 			NEXT="chdir"
+		elif [ "${a}" = "--requires-admin" ]; then
+			REQUIRES_ADMIN=1
 		elif [[ "${a}" == "--" ]]; then
 			shift
 			break
@@ -379,6 +387,7 @@ parse_arguments() {
 	echo "PREPARE=\"${PREPARE}\""
 	echo "CHDIR=\"${CHDIR}\""
 	echo "WARMUP=\"${WARMUP}\""
+	echo "REQUIRES_ADMIN=\"${REQUIRES_ADMIN}\""
 
 	echo -n "GIT_ARGUMENTS=("
 
@@ -388,7 +397,7 @@ parse_arguments() {
 	echo " )"
 }
 
-gitbench_usage() { echo "usage: gitbench command..."; }
+gitbench_usage() { echo "usage: gitbench [--requires-admin] [--warmup <count>] [--prepare <command>] [--chdir <directory>] command..."; }
 
 exec_profiler() {
 	if [ "${BASELINE_CLI}" != "" ]; then
@@ -490,7 +499,8 @@ exec_hyperfine() {
 # invocation of hyperfine.
 #
 gitbench() {
-	eval $(parse_arguments "$@")
+	argument_data=$(parse_arguments "$@")
+	eval ${argument_data}
 
 	# sanity check
 
@@ -515,6 +525,17 @@ gitbench() {
 #	rm -rf "${SANDBOX_DIR:?}"
 }
 
+# helper script to ensure that --admin is specified
+needs_admin() {
+	if [ "${ADMIN}" != "1" ]; then
+		echo "$0: skipping administrator test" 1>&2
+		echo "" 1>&2
+		echo "This benchmark needs administrative (root) privileges. To run it," 1>&2
+		echo "specify '--admin' to the benchmark." 1>&2
+		exit 2
+	fi
+}
+
 # helper script to give useful error messages about configuration
 needs_repo() {
 	REPO="${1}"
diff --git a/tests/benchmarks/hash-object__text_nocache_100kb b/tests/benchmarks/hash-object__text_nocache_100kb
@@ -2,6 +2,8 @@
 
 . "$(dirname "$0")/benchmark_helpers.sh"
 
+needs_admin
+
 gitbench --prepare "create_text_file text_100kb 102400 &&
                     flush_disk_cache" \
          hash-object "text_100kb"
diff --git a/tests/benchmarks/hash-object__text_nocache_10mb b/tests/benchmarks/hash-object__text_nocache_10mb
@@ -2,6 +2,8 @@
 
 . "$(dirname "$0")/benchmark_helpers.sh"
 
+needs_admin
+
 gitbench --prepare "create_text_file text_10mb 10485760 &&
                     flush_disk_cache" \
          hash-object "text_10mb"
diff --git a/tests/benchmarks/hash-object__text_nocache_1kb b/tests/benchmarks/hash-object__text_nocache_1kb
@@ -2,6 +2,8 @@
 
 . "$(dirname "$0")/benchmark_helpers.sh"
 
+needs_admin
+
 gitbench --prepare "create_text_file text_1kb 1024 &&
                     flush_disk_cache" \
          hash-object "text_1kb"
diff --git a/tests/benchmarks/hash-object__write_text_nocache_100kb b/tests/benchmarks/hash-object__write_text_nocache_100kb
@@ -2,6 +2,8 @@
 
 . "$(dirname "$0")/benchmark_helpers.sh"
 
+needs_admin
+
 gitbench --prepare "sandbox_repo empty_standard_repo &&
                     create_text_file text_100kb 102400 &&
                     flush_disk_cache" \
diff --git a/tests/benchmarks/hash-object__write_text_nocache_10mb b/tests/benchmarks/hash-object__write_text_nocache_10mb
@@ -2,6 +2,8 @@
 
 . "$(dirname "$0")/benchmark_helpers.sh"
 
+needs_admin
+
 gitbench --prepare "sandbox_repo empty_standard_repo &&
                     create_text_file text_10mb 10485760 &&
                     flush_disk_cache" \
diff --git a/tests/benchmarks/hash-object__write_text_nocache_1kb b/tests/benchmarks/hash-object__write_text_nocache_1kb
@@ -2,6 +2,8 @@
 
 . "$(dirname "$0")/benchmark_helpers.sh"
 
+needs_admin
+
 gitbench --prepare "sandbox_repo empty_standard_repo &&
                     create_text_file text_1kb 1024 &&
                     flush_disk_cache" \
