Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 5ca5aa9

Browse files
agneumagneum
committed
Merge branch 'bash-test-rules' into 'master'
chore: define rules to reduce the running time of integration tests See merge request postgres-ai/database-lab!414
2 parents 88ff36d + 3aa437e commit 5ca5aa9

File tree

5 files changed

+49
-30
lines changed

5 files changed

+49
-30
lines changed

‎.gitlab-ci.yml

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -338,10 +338,23 @@ build-image-swagger-latest:
338338

339339

340340
.bash-test: &bash_test
341-
<<: *only_feature
342341
stage: integration-test
343342
variables:
344343
IMAGE_TAG: "${CI_COMMIT_REF_SLUG}"
344+
only:
345+
refs:
346+
- branches
347+
variables:
348+
- $CI_COMMIT_REF_SLUG != "master"
349+
changes:
350+
- cmd/**/*
351+
- configs/**/*
352+
- internal/**/*
353+
- pkg/**/*
354+
- test/**/*
355+
- Dockerfile.*
356+
- go.*
357+
- .gitlab-ci.yml
345358
script:
346359
- bash test/1.synthetic.sh
347360
- bash test/2.logical_generic.sh

‎test/2.logical_generic.sh

Lines changed: 15 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ export SOURCE_PORT="${SOURCE_PORT:-7432}"
1212
export SOURCE_USERNAME="${SOURCE_USERNAME:-postgres}"
1313
export SOURCE_PASSWORD="${SOURCE_PASSWORD:-secretpassword}"
1414
export POSTGRES_VERSION="${POSTGRES_VERSION:-13}"
15-
export DLE_TEST_MOUNT_DIR="/var/lib/test/dblab"
15+
export DLE_TEST_MOUNT_DIR="/var/lib/test/dblab_mount"
1616
export DLE_TEST_POOL_NAME="test_dblab_pool"
1717
export DLE_SERVER_PORT=${DLE_SERVER_PORT:-12345}
1818
export DLE_PORT_POOL_FROM=${DLE_PORT_POOL_FROM:-9000}
@@ -23,13 +23,7 @@ DIR=${0%/*}
2323
if [[ "${SOURCE_HOST}" = "172.17.0.1" ]]; then
2424
### Step 0. Create source database
2525
TMP_DATA_DIR="/tmp/dle_test/logical_generic"
26-
cleanup_testdata_dir() {
27-
sudo rm -rf "${TMP_DATA_DIR}"/postgresql/"${POSTGRES_VERSION}"/test || true
28-
}
29-
30-
trap cleanup_testdata_dir EXIT
3126

32-
cleanup_testdata_dir
3327
sudo docker rm postgres"${POSTGRES_VERSION}" || true
3428

3529
sudo docker run \
@@ -59,12 +53,21 @@ if [[ "${SOURCE_HOST}" = "172.17.0.1" ]]; then
5953

6054
check_database_readiness || (echo "test database is not ready" && exit 1)
6155

62-
# Generate data in the test database using pgbench
63-
# 1,000,000 accounts, ~0.14 GiB of data.
64-
sudo docker exec postgres"${POSTGRES_VERSION}" pgbench -U postgres -i -s 10 "${SOURCE_DBNAME}"
56+
check_data_existence(){
57+
sudo docker exec postgres"${POSTGRES_VERSION}" psql -d "${SOURCE_DBNAME}" -U postgres --command 'select from pgbench_accounts' > /dev/null 2>&1
58+
return $?
59+
}
60+
61+
generate_data(){
62+
# Generate data in the test database using pgbench
63+
# 1,000,000 accounts, ~0.14 GiB of data.
64+
sudo docker exec postgres"${POSTGRES_VERSION}" pgbench -U postgres -i -s 10 "${SOURCE_DBNAME}"
65+
66+
# Database info
67+
sudo docker exec postgres"${POSTGRES_VERSION}" psql -U postgres -c "\l+ ${SOURCE_DBNAME}"
68+
}
6569

66-
# Database info
67-
sudo docker exec postgres"${POSTGRES_VERSION}" psql -U postgres -c "\l+ ${SOURCE_DBNAME}"
70+
check_data_existence || generate_data
6871
fi
6972

7073
### Step 1. Prepare a machine with disk, Docker, and ZFS

‎test/4.physical_basebackup.sh

Lines changed: 17 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ export SOURCE_PORT="${SOURCE_PORT:-7432}"
1111
export SOURCE_USERNAME="${SOURCE_USERNAME:-postgres}"
1212
export SOURCE_PASSWORD="${SOURCE_PASSWORD:-secretpassword}"
1313
export POSTGRES_VERSION="${POSTGRES_VERSION:-13}"
14-
export DLE_TEST_MOUNT_DIR="/var/lib/test/dblab"
14+
export DLE_TEST_MOUNT_DIR="/var/lib/test/dblab_mount"
1515
export DLE_TEST_POOL_NAME="test_dblab_pool"
1616
export DLE_SERVER_PORT=${DLE_SERVER_PORT:-12345}
1717
export DLE_PORT_POOL_FROM=${DLE_PORT_POOL_FROM:-9000}
@@ -22,13 +22,7 @@ DIR=${0%/*}
2222
if [[ "${SOURCE_HOST}" = "172.17.0.1" ]]; then
2323
### Step 0. Create source database
2424
TMP_DATA_DIR="/tmp/dle_test/physical_basebackup"
25-
cleanup_testdata_dir() {
26-
sudo rm -rf "${TMP_DATA_DIR}"/postgresql/"${POSTGRES_VERSION}"/test || true
27-
}
28-
29-
trap cleanup_testdata_dir EXIT
3025

31-
cleanup_testdata_dir
3226
sudo docker rm postgres"${POSTGRES_VERSION}" || true
3327

3428
sudo docker run \
@@ -74,12 +68,21 @@ if [[ "${SOURCE_HOST}" = "172.17.0.1" ]]; then
7468
done
7569
fi
7670

77-
# Generate data in the test database using pgbench
78-
# 1,000,000 accounts, ~0.14 GiB of data.
79-
sudo docker exec postgres"${POSTGRES_VERSION}" pgbench -U postgres -i -s 10 test
71+
check_data_existence(){
72+
sudo docker exec postgres"${POSTGRES_VERSION}" psql -d test -U postgres --command 'select from pgbench_accounts' > /dev/null 2>&1
73+
return $?
74+
}
75+
76+
generate_data(){
77+
# Generate data in the test database using pgbench
78+
# 1,000,000 accounts, ~0.14 GiB of data.
79+
sudo docker exec postgres"${POSTGRES_VERSION}" pgbench -U postgres -i -s 10 test
80+
81+
# Database info
82+
sudo docker exec postgres"${POSTGRES_VERSION}" psql -U postgres -c "\l+ test"
83+
}
8084

81-
# Database info
82-
sudo docker exec postgres"${POSTGRES_VERSION}" psql -U postgres -c "\l+ test"
85+
check_data_existence || generate_data
8386
fi
8487

8588
### Step 1. Prepare a machine with disk, Docker, and ZFS
@@ -158,8 +161,8 @@ check_dle_readiness(){
158161
return 1
159162
}
160163

161-
### Waiting for the Database Lab Engine initialization.
162-
for i in {1..30}; do
164+
### Waiting for the Database Lab Engine initialization (7 minutes).
165+
for i in {1..42}; do
163166
check_dle_readiness && break || echo "Database Lab Engine is not ready yet"
164167
sleep 10
165168
done

‎test/_cleanup.sh

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#!/bin/bash
22
set -euxo pipefail
33

4-
DLE_TEST_MOUNT_DIR="/var/lib/test/dblab"
4+
DLE_TEST_MOUNT_DIR="/var/lib/test/dblab_mount"
55
DLE_TEST_POOL_NAME="test_dblab_pool"
66
ZFS_FILE="$(pwd)/zfs_file"
77

@@ -10,7 +10,7 @@ sudo docker ps -aq --filter label="test_dblab_pool" | xargs --no-run-if-empty su
1010
sudo docker ps -aq --filter label="dblab_test" | xargs --no-run-if-empty sudo docker rm -f
1111

1212
# Remove unused Docker images
13-
sudo docker images -q | xargs --no-run-if-empty sudo docker rmi || true
13+
sudo docker images --filter=reference='registry.gitlab.com/postgres-ai/database-lab/dblab-server:*' -q | xargs --no-run-if-empty sudo docker rmi || true
1414

1515
# Clean up the data directory
1616
sudo rm -rf ${DLE_TEST_MOUNT_DIR}/${DLE_TEST_POOL_NAME}/data/*

‎test/_zfs.file.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#!/bin/bash
22
set -euxo pipefail
33

4-
DLE_TEST_MOUNT_DIR="/var/lib/test/dblab"
4+
DLE_TEST_MOUNT_DIR="/var/lib/test/dblab_mount"
55
DLE_TEST_POOL_NAME="test_dblab_pool"
66
ZFS_FILE="$(pwd)/zfs_file"
77

0 commit comments

Comments
 (0)