diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index b2b6c1008..000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,553 +0,0 @@ -# version: 2.1 - -# parameters: -# run_default_flow: -# default: true -# type: boolean -# run_nightly_twice_a_week_flow_label: -# default: false -# type: boolean - -# commands: -# early-returns: -# steps: -# - run: -# name: Early return if this is a docs build -# command: | -# if [[ $CIRCLE_BRANCH == *docs ]]; then -# echo "Identifies as documents PR, no testing required." -# circleci step halt -# fi -# - run: -# name: Early return if this branch should ignore CI -# command: | -# if [[ $CIRCLE_BRANCH == *noci ]]; then -# echo "Identifies as actively ignoring CI, no testing required." -# circleci step halt -# fi - -# early-return-for-forked-pull-requests: -# description: >- -# If this build is from a fork, stop executing the current job and return success. -# This is useful to avoid steps that will fail due to missing credentials. -# steps: -# - run: -# name: Early return if this build is from a forked PR -# command: | -# if [[ -n "$CIRCLE_PR_NUMBER" ]]; then -# echo "Nothing to do for forked PRs, so marking this step successful" -# circleci step halt -# fi - -# setup-executor: -# steps: -# - run: -# name: Setup executor -# command: | -# apt-get -qq update -# apt-get -q install -y git openssh-client curl ca-certificates make tar gzip -# bash <(curl -fsSL https://raw.githubusercontent.com/docker/docker-install/master/install.sh) -# - setup_remote_docker: -# version: 20.10.14 -# docker_layer_caching: true - -# checkout-all: -# steps: -# - checkout -# - run: -# name: Checkout submodules -# command: git submodule update --init --recursive - -# setup-automation: -# steps: -# - run: -# name: Setup automation -# command: | -# git submodule update --init deps/readies -# if [[ $(uname -s) == Darwin ]]; then rm -f /usr/local/bin/python3; fi -# ./deps/readies/bin/getpy3 -# - run: -# name: Setup automation (part 2) -# shell: /bin/bash -l -eo pipefail -# command: | -# export HOMEBREW_NO_AUTO_UPDATE=1 -# ./deps/readies/bin/getaws -# ls -l /usr/local/bin/python* || true -# echo "python3: $(command -v python3)" -# python3 --version -# python3 -m pip list -# python3 -m pip install --upgrade setuptools six pip - -# install-prerequisites: -# parameters: -# redis_version: -# type: string -# default: "7.2" -# getredis_params: -# type: string -# default: "" -# system_setup_params: -# type: string -# default: "" -# steps: -# - setup-automation -# - run: -# name: System setup -# shell: /bin/bash -l -eo pipefail -# command: | -# ./sbin/system-setup.py <> -# - run: -# name: Install Redis -# shell: /bin/bash -l -eo pipefail -# command: | -# export HOMEBREW_NO_AUTO_UPDATE=1 -# ./deps/readies/bin/getredis -v '<>' --force <> -# - run: -# name: System report -# shell: /bin/bash -l -eo pipefail -# command: | -# source $HOME/.cargo/env -# make info - -# save-tests-logs: -# steps: -# - run: -# name: Cleanup test log dir -# command: | -# rm -f tests/pytest/logs/*.{aof,rdb} -# when: always -# - store_artifacts: -# path: tests/pytest/logs - -# persist-artifacts: -# steps: -# - early-return-for-forked-pull-requests -# - run: -# name: List artifacts -# command: | -# cd bin/artifacts -# du -ah --apparent-size * -# - persist_to_workspace: -# root: bin/ -# paths: -# - artifacts/*.zip -# - artifacts/*.tgz -# - artifacts/*.tar -# - artifacts/snapshots/* - -# build-steps: -# parameters: -# build_params: -# type: string -# default: "" -# test_params: -# type: string -# default: "" -# redis_version: -# type: string -# default: "7" -# getredis_params: -# type: string -# default: "" -# steps: -# - early-returns -# - checkout-all -# - install-prerequisites: -# redis_version: <> -# getredis_params: <> -# - restore_cache: -# keys: -# - v3-dependencies-{{ arch }}-{{ checksum "Cargo.lock" }} -# - run: -# name: Check formatting -# shell: /bin/bash -l -eo pipefail -# command: make lint -# - run: -# name: Build debug -# shell: /bin/bash -l -eo pipefail -# command: make build DEBUG=1 <> -# - run: -# name: Build release -# shell: /bin/bash -l -eo pipefail -# command: make build pack <> -# - save_cache: -# key: v3-dependencies-{{ arch }}-{{ checksum "Cargo.lock" }} -# paths: -# - "~/.cargo" -# - "./target" - -# test-steps: -# steps: -# - run: -# name: Run tests -# shell: /bin/bash -l -eo pipefail -# command: | -# python3 -m RLTest --version -# make test -# timeout: 30m -# no_output_timeout: 30m -# - save_cache: -# key: v2-dependencies-{{ arch }}-{{ checksum "Cargo.lock" }} -# paths: -# - "~/.cargo" -# - "./target" -# - save-tests-logs - -# build-platforms-steps: -# parameters: -# platform: -# type: string -# steps: -# - early-returns -# - setup-executor -# - checkout-all -# - setup-automation -# - run: -# name: Build for platform -# shell: /bin/bash -l -eo pipefail -# command: | -# ROOT=$PWD -# cd build/docker -# make build OSNICK=<> VERSION=$CIRCLE_TAG BRANCH=$CIRCLE_BRANCH TEST=1 OFFICIAL=1 SHOW=1 -# cd $ROOT -# mkdir -p tests/pytest/logs -# tar -C tests/pytest/logs -xzf bin/artifacts/pytest-logs*.tgz -# timeout: 60m -# no_output_timeout: 30m -# - save-tests-logs -# - early-return-for-forked-pull-requests -# - run: -# name: Upload artifacts to S3 -# shell: /bin/bash -l -eo pipefail -# command: | -# if [[ -n $CIRCLE_BRANCH ]]; then -# make upload-artifacts OSNICK=<> SHOW=1 -# fi -# - run: -# name: Publish container -# shell: /bin/bash -l -eo pipefail -# command: | -# docker login -u redisfab -p $DOCKER_REDISFAB_PWD -# cd build/docker -# make publish OSNICK=<> VERSION=$CIRCLE_TAG BRANCH=$CIRCLE_BRANCH OFFICIAL=1 SHOW=1 -# - persist-artifacts - -# vm-build-platforms-steps: -# parameters: -# platform: -# type: string -# steps: -# - early-returns -# - checkout -# - setup-automation -# - run: -# name: Install Docker -# shell: /bin/bash -l -eo pipefail -# command: ./deps/readies/bin/getdocker -# - run: -# name: Build for platform -# command: | -# ROOT=$PWD -# cd build/docker -# make build OSNICK=<> VERSION=$CIRCLE_TAG BRANCH=$CIRCLE_BRANCH TEST=1 OFFICIAL=1 SHOW=1 -# cd $ROOT -# mkdir -p tests/pytest/logs -# tar -C tests/pytest/logs -xzf bin/artifacts/pytest-logs*.tgz -# no_output_timeout: 30m -# - save-tests-logs -# - early-return-for-forked-pull-requests -# - run: -# name: Upload artifacts to S3 -# command: | -# if [[ -n $CIRCLE_BRANCH ]]; then -# make upload-artifacts OSNICK=<> SHOW=1 -# fi -# - run: -# name: Publish container -# command: | -# docker login -u redisfab -p $DOCKER_REDISFAB_PWD -# cd build/docker -# make publish OSNICK=<> VERSION=$CIRCLE_TAG BRANCH=$CIRCLE_BRANCH OFFICIAL=1 SHOW=1 -# - persist-artifacts - -# #---------------------------------------------------------------------------------------------------------------------------------- - -# jobs: -# build-linux-debian: -# docker: -# - image: redisfab/rmbuilder:6.2.7-x64-bullseye -# parameters: -# redis_version: -# type: string -# default: "7" -# persist: -# type: string -# default: "yes" -# steps: -# - build-steps: -# redis_version: <> -# - test-steps -# - run: -# name: Persist artifacts? -# command: | -# if [[ "<>" != "yes" ]]; then -# circleci step halt -# fi -# - persist-artifacts - -# build-platforms: -# parameters: -# platform: -# type: string -# # docker: -# # - image: debian:bullseye -# machine: -# enabled: true -# image: ubuntu-2204:edge -# resource_class: large -# steps: -# - vm-build-platforms-steps: -# platform: <> - -# # build-arm-platforms: -# # parameters: -# # platform: -# # type: string -# # machine: -# # image: ubuntu-2004:202101-01 -# # resource_class: arm.medium -# # steps: -# # - vm-build-platforms-steps: -# # platform: <> - -# build-macos-m1: -# macos: -# xcode: 15.4.0 -# resource_class: macos.m1.large.gen1 -# parameters: -# upload: -# type: string -# default: "yes" -# steps: -# - early-returns -# - build-steps -# - test-steps -# - persist-artifacts -# - run: -# name: Upload artifacts to S3 -# command: | -# if [[ -n $CIRCLE_BRANCH ]]; then -# make upload-artifacts SHOW=1 VERBOSE=1 -# else -# make upload-release SHOW=1 -# fi - -# coverage: -# docker: -# - image: redisfab/rmbuilder:6.2.7-x64-focal -# steps: -# - early-returns -# - checkout-all -# - install-prerequisites -# - run: -# name: Build & Test -# shell: /bin/bash -l -eo pipefail -# command: | -# make clang-install -# make coverage SHOW=1 -# make upload-cov SHOW=1 -# no_output_timeout: 30m -# - save-tests-logs - -# sanitize: -# docker: -# - image: redisfab/clang:16-x64-focal -# parameters: -# san-type: -# type: string -# steps: -# - early-returns -# - checkout-all -# - install-prerequisites -# - run: -# name: Build & test -# shell: /bin/bash -l -eo pipefail -# command: make SAN=<> build test SHOW=1 -# no_output_timeout: 30m -# - save-tests-logs - -# upload-artifacts: -# parameters: -# staging-lab: -# type: string -# default: "0" -# docker: -# - image: redisfab/rmbuilder:6.2.7-x64-bullseye -# steps: -# - early-returns -# - early-return-for-forked-pull-requests -# - checkout -# - setup-automation -# - attach_workspace: -# at: ~/workspace -# - run: -# name: Upload artifacts to S3 -# command: | -# mkdir -p bin -# ln -s ~/workspace/artifacts bin/artifacts -# if [[ -n $CIRCLE_TAG ]]; then -# make upload-release SHOW=1 -# else -# make upload-artifacts SHOW=1 -# fi - -# #---------------------------------------------------------------------------------------------------------------------------------- - -# on-any-branch: &on-any-branch -# filters: -# branches: -# only: /.*/ -# tags: -# only: /.*/ - -# always: &always -# filters: -# branches: -# only: /.*/ -# tags: -# only: /.*/ - -# never: &never -# filters: -# branches: -# ignore: /.*/ -# tags: -# ignore: /.*/ - -# on-master: &on-master -# filters: -# branches: -# only: master -# tags: -# ignore: /.*/ - -# on-integ-branch: &on-integ-branch -# filters: -# branches: -# only: -# - master -# - /^\d+\.\d+.*$/ -# - /^feature.*$/ -# tags: -# ignore: /.*/ - -# on-integ-branch-cron: &on-integ-branch-cron -# filters: -# branches: -# only: -# - master -# - /^\d+\.\d+.*$/ -# - /^feature.*$/ - -# not-on-integ-branch: ¬-on-integ-branch -# filters: -# branches: -# ignore: -# - master -# - /^\d+\.\d+.*$/ -# - /^feature.*$/ -# tags: -# ignore: /.*/ - -# on-version-tags: &on-version-tags -# filters: -# branches: -# ignore: /.*/ -# tags: -# only: /^v[0-9].*/ - -# on-integ-and-version-tags: &on-integ-and-version-tags -# filters: -# branches: -# only: -# - master -# - /^\d+\.\d+.*$/ -# - /^feature.*$/ -# tags: -# only: /^v[0-9].*/ - -# #---------------------------------------------------------------------------------------------------------------------------------- - -# workflows: -# version: 2 -# default-flow: -# when: -# << pipeline.parameters.run_default_flow >> -# jobs: -# - build-linux-debian: -# name: build -# <<: *not-on-integ-branch -# - build-platforms: -# <<: *on-integ-and-version-tags -# context: common -# matrix: -# parameters: -# platform: [centos7, rocky8, rocky9, bullseye, amzn2] -# - build-macos-m1: -# context: common -# <<: *on-integ-and-version-tags -# - coverage: -# <<: *always -# - sanitize: -# name: sanitize-<< matrix.san-type >> -# <<: *always -# matrix: -# parameters: -# san-type: [address] -# - upload-artifacts: -# name: upload-artifacts-to-staging-lab -# <<: *on-integ-branch -# staging-lab: "1" -# context: common -# requires: -# - build-platforms -# # - build-arm-platforms -# - build-macos-m1 -# - upload-artifacts: -# name: upload-release-artifacts -# <<: *on-version-tags -# context: common -# requires: -# - build-platforms -# # - build-arm-platforms -# - build-macos-m1 - -# nightly: -# triggers: -# - schedule: -# cron: "07 20 * * *" -# <<: *on-integ-branch-cron -# jobs: -# - build-linux-debian: -# name: build-with-redis-<> -# matrix: -# parameters: -# redis_version: ["7", "unstable"] - -# nightly-twice-a-week-by-param: -# when: -# << pipeline.parameters.run_nightly_twice_a_week_flow_label >> -# jobs: -# - build-macos-m1: -# context: common -# upload: "yes" - -# nightly-twice-a-week: -# triggers: -# - schedule: -# cron: "20 17 * * 0,3" -# <<: *on-integ-branch-cron -# jobs: -# - build-macos-m1: -# context: common -# upload: "yes" \ No newline at end of file diff --git a/.github/actions/build-json-module-and-redis-with-cargo/action.yml b/.github/actions/build-json-module-and-redis-with-cargo/action.yml index 0bea4d0fa..18da16fdd 100644 --- a/.github/actions/build-json-module-and-redis-with-cargo/action.yml +++ b/.github/actions/build-json-module-and-redis-with-cargo/action.yml @@ -19,6 +19,7 @@ runs: - name: Build module shell: bash run: | - . "$HOME/.cargo/env" + . $HOME/.cargo/env + echo "source $HOME/.cargo/env" >> $HOME/.bash_profile cargo build --release cp $(realpath ./target/release)/librejson.so $(realpath ./target/release)/rejson.so diff --git a/.github/actions/build-json-module-and-redis/action.yml b/.github/actions/build-json-module-and-redis/action.yml index 5239223fb..f317c9c97 100644 --- a/.github/actions/build-json-module-and-redis/action.yml +++ b/.github/actions/build-json-module-and-redis/action.yml @@ -3,10 +3,6 @@ description: | Build JSON module and Redis Server inputs: - redis-ref: - description: 'Redis version to build' - required: true - default: '7.4' sanitizer: type: string diff --git a/.github/actions/make-pack/action.yml b/.github/actions/make-pack/action.yml new file mode 100644 index 000000000..27170a11b --- /dev/null +++ b/.github/actions/make-pack/action.yml @@ -0,0 +1,9 @@ +name: Run make pack module script + +runs: + using: composite + steps: + - name: Pack module + shell: bash + run: | + make pack BRANCH=$TAG_OR_BRANCH diff --git a/.github/actions/pack-module/action.yml b/.github/actions/pack-module/action.yml new file mode 100644 index 000000000..ceb84b30c --- /dev/null +++ b/.github/actions/pack-module/action.yml @@ -0,0 +1,17 @@ +name: Run pack module script + +runs: + using: composite + steps: + - name: Pack module + shell: bash + run: | + if command -v scl_source &> /dev/null + then + . scl_source enable devtoolset-11 || true + fi + . venv/bin/activate + git config --global --add safe.directory $GITHUB_WORKSPACE + export PATH="$GITHUB_WORKSPACE/redis/src:$PATH" + MODULE=$(realpath ./target/release/rejson.so) BRANCH=$TAG_OR_BRANCH \ + SHOW=1 OSNICK=${{ matrix.docker.nick }} ./sbin/pack.sh diff --git a/.github/actions/run-tests/action.yml b/.github/actions/run-tests/action.yml index 7ceeb5289..3cd82ea15 100644 --- a/.github/actions/run-tests/action.yml +++ b/.github/actions/run-tests/action.yml @@ -1,122 +1,29 @@ -name: Common Flow for Tests - -# Documentation: https://redislabs.atlassian.net/wiki/spaces/DX/pages/3967844669/RediSearch+CI+refactor +name: Run module tests inputs: - env: - default: "ubuntu-22.04" - type: string - container: - type: string - sanitizer: - type: string - test-config: - description: 'Test configuration environment variable (e.g. "CONFIG=tls" or "QUICK=1")' - required: true + run_valgrind: + description: 'Run valgrind on the tests' type: string + default: 0 runs: using: composite steps: - - name: Get Installation Mode - shell: bash - id: mode - run: | - [[ -z "${{ inputs.container }}" ]] && echo "mode=sudo" >> $GITHUB_OUTPUT || echo "mode=" >> $GITHUB_OUTPUT - - name: Check if node20 is Supported - id: node20 - uses: ./.github/actions/node20-supported - with: - container: ${{ inputs.container }} - - name: Install git - shell: bash - run: | - # TODO: must be changed to run a script based on the input env - echo ::group::Install git - ${{ steps.mode.outputs.mode }} apt-get update && apt-get install -y git - echo ::endgroup:: - - name: Setup specific - shell: bash - working-directory: .install - run: | - echo ::group::OS-Specific Setup - ./install_script.sh ${{ steps.mode.outputs.mode }} - echo ::endgroup:: - echo ::group::Get Rust - ./getrust.sh ${{ steps.mode.outputs.mode }} - echo ::endgroup:: - - - name: Full checkout (node20 supported) - if: steps.node20.outputs.supported == 'true' - uses: actions/checkout@v4 - with: - submodules: recursive - - name: Full checkout (node20 unsupported) - if: steps.node20.outputs.supported == 'false' - uses: actions/checkout@v3 - with: - submodules: recursive - - - name: Get Redis - uses: actions/checkout@v3 - with: - repository: redis/redis - ref: 'unstable' # todo change per version/tag - path: redis - submodules: 'recursive' - - name: Build - uses: ./.github/actions/build-json-module-and-redis - with: - sanitizer: ${{ inputs.san }} - - - name: Set Artifact Names - shell: bash - # Artifact names have to be unique, so we base them on the environment. - # We also remove invalid characters from the name. - id: artifact-names - run: | - # Invalid characters include: Double quote ", Colon :, Less than <, Greater than >, Vertical bar |, Asterisk *, Question mark ? - echo "name=$(echo "${{ inputs.container || inputs.env }} ${{ runner.arch }}, Redis ${{ inputs.get-redis || 'unstable' }}" | \ - sed -e 's/[":\/\\<>\|*?]/_/g' -e 's/__*/_/g' -e 's/^_//' -e 's/_$//')" >> $GITHUB_OUTPUT - - name: Run tests shell: bash - id: test run: | echo ::group::Activate virtual environment python3 -m venv venv echo "source $PWD/venv/bin/activate" >> ~/.bash_profile - source venv/bin/activate + . venv/bin/activate echo ::endgroup:: echo ::group::Install python dependencies ./.install/common_installations.sh echo ::endgroup:: echo ::group::Unit tests - . "$HOME/.cargo/env" - make cargo_test LOG=1 CLEAR_LOGS=0 SAN=${{ inputs.san }} + . $HOME/.cargo/env + cargo test echo ::endgroup:: echo ::group::Flow tests - make pytest LOG=1 CLEAR_LOGS=0 SAN=${{ inputs.san }} ${{ inputs.test-config }} + MODULE=$(realpath ./target/release/rejson.so) RLTEST_ARGS='--no-progress' ./tests/pytest/tests.sh VG=${{inputs.run_valgrind}} echo ::endgroup:: - env: - PIP_BREAK_SYSTEM_PACKAGES: 1 - - - name: Upload test artifacts (node20 supported) - if: steps.node20.outputs.supported == 'true' && steps.test.outcome == 'failure' - uses: actions/upload-artifact@v4 - with: - name: Test logs ${{ steps.artifact-names.outputs.name }} - path: tests/**/logs/*.log* - if-no-files-found: ignore - - name: Upload test artifacts (node20 unsupported) - if: steps.node20.outputs.supported == 'false' && steps.test.outcome == 'failure' - uses: actions/upload-artifact@v3 - with: - name: Test logs ${{ steps.artifact-names.outputs.name }} - path: tests/**/logs/*.log* - if-no-files-found: ignore - - - name: Fail flow if tests failed - shell: bash - if: steps.test.outcome == 'failure' - run: exit 1 diff --git a/.github/actions/san-run-tests/action.yml b/.github/actions/san-run-tests/action.yml new file mode 100644 index 000000000..4673fbce9 --- /dev/null +++ b/.github/actions/san-run-tests/action.yml @@ -0,0 +1,126 @@ +name: Common Flow for Tests + +# Documentation: https://redislabs.atlassian.net/wiki/spaces/DX/pages/3967844669/RediSearch+CI+refactor + +inputs: + env: + default: "ubuntu-22.04" + type: string + container: + type: string + sanitizer: + type: string + test-config: + description: 'Test configuration environment variable (e.g. "CONFIG=tls" or "QUICK=1")' + required: true + type: string + redis-ref: + description: 'Redis ref to checkout' + type: string + required: true + +runs: + using: composite + steps: + - name: Get Installation Mode + shell: bash + id: mode + run: | + [[ -z "${{ inputs.container }}" ]] && echo "mode=sudo" >> $GITHUB_OUTPUT || echo "mode=" >> $GITHUB_OUTPUT + - name: Check if node20 is Supported + id: node20 + uses: ./.github/actions/node20-supported + with: + container: ${{ inputs.container }} + - name: Install git + shell: bash + run: | + # TODO: must be changed to run a script based on the input env + echo ::group::Install git + ${{ steps.mode.outputs.mode }} apt-get update && apt-get install -y git + echo ::endgroup:: + - name: Setup specific + shell: bash + working-directory: .install + run: | + echo ::group::OS-Specific Setup + ./install_script.sh ${{ steps.mode.outputs.mode }} + echo ::endgroup:: + echo ::group::Get Rust + ./getrust.sh ${{ steps.mode.outputs.mode }} + echo ::endgroup:: + + - name: Full checkout (node20 supported) + if: steps.node20.outputs.supported == 'true' + uses: actions/checkout@v4 + with: + submodules: recursive + - name: Full checkout (node20 unsupported) + if: steps.node20.outputs.supported == 'false' + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Get Redis + uses: actions/checkout@v4 + with: + repository: redis/redis + ref: ${{ inputs.redis-ref }} + path: redis + submodules: 'recursive' + - name: Build + uses: ./.github/actions/build-json-module-and-redis + with: + sanitizer: ${{ inputs.san }} + + - name: Set Artifact Names + shell: bash + # Artifact names have to be unique, so we base them on the environment. + # We also remove invalid characters from the name. + id: artifact-names + run: | + # Invalid characters include: Double quote ", Colon :, Less than <, Greater than >, Vertical bar |, Asterisk *, Question mark ? + echo "name=$(echo "${{ inputs.container || inputs.env }} ${{ runner.arch }}, Redis ${{ inputs.redis-ref }}" | \ + sed -e 's/[":\/\\<>\|*?]/_/g' -e 's/__*/_/g' -e 's/^_//' -e 's/_$//')" >> $GITHUB_OUTPUT + + - name: Run tests + shell: bash + id: test + run: | + echo ::group::Activate virtual environment + python3 -m venv venv + echo "source $PWD/venv/bin/activate" >> ~/.bash_profile + source venv/bin/activate + echo ::endgroup:: + echo ::group::Install python dependencies + ./.install/common_installations.sh + echo ::endgroup:: + echo ::group::Unit tests + . "$HOME/.cargo/env" + make cargo_test LOG=1 CLEAR_LOGS=0 SAN=${{ inputs.san }} + echo ::endgroup:: + echo ::group::Flow tests + make pytest LOG=1 CLEAR_LOGS=0 SAN=${{ inputs.san }} ${{ inputs.test-config }} + echo ::endgroup:: + env: + PIP_BREAK_SYSTEM_PACKAGES: 1 + + - name: Upload test artifacts (node20 supported) + if: steps.node20.outputs.supported == 'true' && steps.test.outcome == 'failure' + uses: actions/upload-artifact@v4 + with: + name: Test logs ${{ steps.artifact-names.outputs.name }} + path: tests/**/logs/*.log* + if-no-files-found: ignore + - name: Upload test artifacts (node20 unsupported) + if: steps.node20.outputs.supported == 'false' && steps.test.outcome == 'failure' + uses: actions/upload-artifact@v4 + with: + name: Test logs ${{ steps.artifact-names.outputs.name }} + path: tests/**/logs/*.log* + if-no-files-found: ignore + + - name: Fail flow if tests failed + shell: bash + if: steps.test.outcome == 'failure' + run: exit 1 diff --git a/.github/actions/setup-env/action.yml b/.github/actions/setup-env/action.yml index c9b607ab4..a24cfba86 100644 --- a/.github/actions/setup-env/action.yml +++ b/.github/actions/setup-env/action.yml @@ -65,4 +65,4 @@ runs: id: set-redis-ref run: | export REDIS_REF="${{ inputs.redis-ref || '7.4'}}" - echo "REDIS_REF=${REDIS_REF}" >> $GITHUB_OUTPUT \ No newline at end of file + echo "REDIS_REF=${REDIS_REF}" >> $GITHUB_OUTPUT diff --git a/.github/actions/upload-artifacts-to-s3-without-make/action.yml b/.github/actions/upload-artifacts-to-s3-without-make/action.yml index 9a9a43833..b28937dcf 100644 --- a/.github/actions/upload-artifacts-to-s3-without-make/action.yml +++ b/.github/actions/upload-artifacts-to-s3-without-make/action.yml @@ -9,9 +9,6 @@ inputs: aws-secret-access-key: description: 'AWS Secret Access Key' required: true - github-ref: - description: 'GitHub ref' - required: true osnick: description: 'OS Nickname' required: false @@ -20,29 +17,40 @@ inputs: runs: using: composite steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v3 - with: # todo: use role instead of access key - aws-access-key-id: ${{ inputs.aws-access-key-id }} - aws-secret-access-key: ${{ inputs.aws-secret-access-key }} - aws-region: "us-east-1" - - name: Upload artifacts to S3 - staging + - name: Configure AWS credentials and upload artifcats # todo - use aws role instead shell: bash run: | - echo ::group::install aws cli + echo ::group::install aws cli python3 -m venv .aws-cli-venv && source .aws-cli-venv/bin/activate && pip3 install --upgrade pip && pip3 install --no-cache-dir awscli && rm -rf /var/cache/apk/* - echo ::endgroup:: - echo ::group::upload artifacts - SNAPSHOT=1 SHOW=1 VERBOSE=1 ./sbin/upload-artifacts - echo ::endgroup:: - echo ::group::upload staging release - RELEASE=1 SHOW=1 STAGING=1 VERBOSE=1 ./sbin/upload-artifacts - echo ::endgroup:: - - echo ::group::upload production release - # todo: trigger this manually instead - if [[ "${{ inputs.github-ref}}" != 'refs/heads/master' ]]; then - RELEASE=1 SHOW=1 VERBOSE=1 ./sbin/upload-artifacts + echo ::endgroup:: + + # Variables from the workflow + export AWS_ACCESS_KEY_ID="${{ inputs.aws-access-key-id }}" + export AWS_SECRET_ACCESS_KEY="${{ inputs.aws-secret-access-key }}" + export AWS_REGION="us-east-1" + # Check if the required environment variables are set + if [ -z "$AWS_ACCESS_KEY_ID" ] || [ -z "$AWS_SECRET_ACCESS_KEY" ] || [ -z "$AWS_REGION" ]; then + echo "Missing AWS credentials or region configuration." + exit 1 fi - echo ::endgroup:: \ No newline at end of file + # Configure AWS CLI with provided credentials and region + echo "Configuring AWS CLI with access keys..." + aws configure set aws_access_key_id "$AWS_ACCESS_KEY_ID" + aws configure set aws_secret_access_key "$AWS_SECRET_ACCESS_KEY" + aws configure set region "$AWS_REGION" + + echo ::group::upload artifacts + SNAPSHOT=1 SHOW=1 VERBOSE=1 ./sbin/upload-artifacts + echo ::endgroup:: + echo ::group::upload staging release + RELEASE=1 SHOW=1 STAGING=1 VERBOSE=1 ./sbin/upload-artifacts + echo ::endgroup:: + + echo ::group::upload production release + REF="${{ inputs.github-ref }}" + PATTERN="refs/tags/v[0-9]+.*" + if [[ $REF =~ $PATTERN ]]; then + RELEASE=1 SHOW=1 VERBOSE=1 ./sbin/upload-artifacts + fi + echo ::endgroup:: diff --git a/.github/actions/upload-artifacts-to-s3/action.yml b/.github/actions/upload-artifacts-to-s3/action.yml index b0276085e..dfed5d609 100644 --- a/.github/actions/upload-artifacts-to-s3/action.yml +++ b/.github/actions/upload-artifacts-to-s3/action.yml @@ -20,12 +20,6 @@ inputs: runs: using: composite steps: - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v3 - with: # todo: use role instead of access key - aws-access-key-id: ${{ inputs.aws-access-key-id }} - aws-secret-access-key: ${{ inputs.aws-secret-access-key }} - aws-region: "us-east-1" - name: Upload artifacts to S3 - staging shell: bash run: | @@ -33,6 +27,22 @@ runs: python3 -m venv .aws-cli-venv && source .aws-cli-venv/bin/activate && pip3 install --upgrade pip && pip3 install --no-cache-dir awscli && rm -rf /var/cache/apk/* echo ::endgroup:: + + # Variables from the workflow + export AWS_ACCESS_KEY_ID="${{ inputs.aws-access-key-id }}" + export AWS_SECRET_ACCESS_KEY="${{ inputs.aws-secret-access-key }}" + export AWS_REGION="us-east-1" + # Check if the required environment variables are set + if [ -z "$AWS_ACCESS_KEY_ID" ] || [ -z "$AWS_SECRET_ACCESS_KEY" ] || [ -z "$AWS_REGION" ]; then + echo "Missing AWS credentials or region configuration." + exit 1 + fi + # Configure AWS CLI with provided credentials and region + echo "Configuring AWS CLI with access keys..." + aws configure set aws_access_key_id "$AWS_ACCESS_KEY_ID" + aws configure set aws_secret_access_key "$AWS_SECRET_ACCESS_KEY" + aws configure set region "$AWS_REGION" + echo ::group::upload artifacts make upload-artifacts SHOW=1 VERBOSE=1 echo ::endgroup:: @@ -45,4 +55,4 @@ runs: if [[ "${{ inputs.github-ref}}" != 'refs/heads/master' ]]; then make upload-release SHOW=1 VERBOSE=1 fi - echo ::endgroup:: \ No newline at end of file + echo ::endgroup:: diff --git a/.github/actions/upload-artifacts/action.yml b/.github/actions/upload-artifacts/action.yml new file mode 100644 index 000000000..f3f9ca630 --- /dev/null +++ b/.github/actions/upload-artifacts/action.yml @@ -0,0 +1,26 @@ +name: Pack the module and upload it to S3 +description: Pack the module and upload it to S3 + +inputs: + image: # The Docker image to use for the build + description: 'The Docker image to use for the build' + required: true + +runs: + using: composite + steps: + - name: Set Artifact Names + # Artifact names have to be unique, so we base them on the environment. + # We also remove invalid characters from the name. + id: artifact-names + shell: bash + run: | # Invalid characters include: Double quote ", Colon :, Less than <, Greater than >, Vertical bar |, Asterisk *, Question mark ? + echo "name=$(echo "${{ inputs.image }} x86-64, Redis unstable" | \ + sed -e 's/[":\/\\<>\|*?]/_/g' -e 's/__*/_/g' -e 's/^_//' -e 's/_$//')" >> $GITHUB_OUTPUT + - name: Upload test artifacts + if: inputs.image != 'amazonlinux:2' && inputs.image != 'ubuntu:bionic' + uses: actions/upload-artifact@v4 + with: + name: Test logs ${{ steps.artifact-names.outputs.name }} + path: tests/**/logs/*.log* + if-no-files-found: ignore diff --git a/.github/workflows/alpine-arm.yml b/.github/workflows/alpine-arm.yml deleted file mode 100644 index 37a59e57d..000000000 --- a/.github/workflows/alpine-arm.yml +++ /dev/null @@ -1,121 +0,0 @@ -name: alpine ARM64 - -on: - push: - branches: - - master - tags: - - 'v[0-9]+.[0-9]+.[0-9]+' - pull_request: - types: - - opened - - reopened - - review_requested - workflow_dispatch: # Allows you to run this workflow manually from the Actions tab - inputs: - redis-ref: - description: 'Redis ref to checkout' - required: true - default: 'unstable' - workflow_call: # Allows to run this workflow from another workflow - inputs: - redis-ref: - description: 'Redis ref to checkout' - type: string - required: true - -permissions: - id-token: write # This is required for requesting the JWT - contents: read # This is required for actions/checkout - -jobs: - setup-environment: - runs-on: ubuntu-latest - outputs: - TAGGED: ${{ steps.set-env.outputs.TAGGED }} - TAG: ${{ steps.set-env.outputs.TAG }} - BRANCH: ${{ steps.set-env.outputs.BRANCH }} - TAG_OR_BRANCH: ${{ steps.set-env.outputs.TAG }}${{ steps.set-env.outputs.BRANCH }} - redis-ref: ${{ steps.set-env.outputs.redis-ref }} - steps: - - name: checkout - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: set env - id: set-env - uses: ./.github/actions/setup-env - with: - github-ref: ${{ github.ref }} - redis-ref: ${{ inputs.redis-ref }} - - alpine-arm64: - runs-on: ubuntu24-arm64-4-16 # ubuntu24-arm64-2-8 - needs: setup-environment - defaults: - run: - shell: bash - env: - TAGGED: ${{ needs.setup-environment.outputs.TAGGED }} - VERSION: ${{ needs.setup-environment.outputs.TAG }} - BRANCH: ${{ needs.setup-environment.outputs.BRANCH }} - TAG_OR_BRANCH: ${{ needs.setup-environment.outputs.TAG_OR_BRANCH}} - - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true - ACTIONS_RUNNER_FORCED_INTERNAL_NODE_VERSION: node16 - ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16 - container: - image: alpine:3 # required to run the job on the ARM instance - steps: - - name: Workaround alpine-arm64 GHA issues - shell: sh - run: | - cp /etc/os-release /etc/os-release.bak - sed -i 's/ID=alpine/ID=NotpineForGHA/g' /etc/os-release - - name: Install prerequisites - shell: sh - run: | - echo ::group::install packages - apk add bash make libtool tar cmake python3 python3-dev \ - py3-pip gcc git curl build-base autoconf automake py3-cryptography \ - linux-headers musl-dev libffi-dev openssl-dev openssh py-virtualenv \ - clang18-libclang gcompat libstdc++ libgcc g++ openblas-dev \ - xsimd git xz bsd-compat-headers clang18 cargo - echo ::endgroup:: - - name: Checkout the module - uses: actions/checkout@v3 - with: - submodules: 'recursive' - - name: Checkout Redis - uses: actions/checkout@v3 - with: - repository: 'redis/redis' - ref: ${{needs.setup-environment.outputs.redis-ref}} - path: 'redis' - - name: Install python dependencies - run: | - echo ::group::install requirements - pip install -q --upgrade setuptools - pip install -q --upgrade pip - pip install -q -r tests/pytest/requirements.txt - pip install -q -r .install/build_package_requirements.txt - echo ::endgroup:: - env: - PIP_BREAK_SYSTEM_PACKAGES: 1 - - name: build - uses: ./.github/actions/build-json-module-and-redis - - name: Test - run: | - make test - - name: Pack module - run: | - git config --global --add safe.directory $GITHUB_WORKSPACE # to avoid git error - mv /etc/os-release.bak /etc/os-release - make pack BRANCH=$TAG_OR_BRANCH SHOW=1 - sed -i 's/ID=alpine/ID=NotpineForGHA/g' /etc/os-release - - name: Upload artifacts to S3 - uses: ./.github/actions/upload-artifacts-to-s3 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - github-ref: ${{ github.ref }} \ No newline at end of file diff --git a/.github/workflows/alpine.yml b/.github/workflows/alpine.yml deleted file mode 100644 index 05c9266d7..000000000 --- a/.github/workflows/alpine.yml +++ /dev/null @@ -1,58 +0,0 @@ -name: alpine - -on: - push: - branches: - - master - tags: - - 'v[0-9]+.[0-9]+.[0-9]+' - -jobs: - build: - runs-on: ubuntu-latest - defaults: - run: - shell: bash - container: - image: alpine:3 - steps: - - name: Install prerequisites - shell: sh - run: | - apk add bash make tar cargo python3 python3-dev py3-pip gcc git curl build-base autoconf automake py3-cryptography linux-headers musl-dev libffi-dev openssl-dev openssh py-virtualenv clang18-libclang - - name: Checkout the module - uses: actions/checkout@v4 - with: - submodules: 'recursive' - - name: Install python dependencies - run: | - pip install -q --upgrade setuptools - pip install -q --upgrade pip - pip install -q -r tests/pytest/requirements.txt - pip install -q -r .install/build_package_requirements.txt - env: - PIP_BREAK_SYSTEM_PACKAGES: 1 - - name: Checkout Redis - uses: actions/checkout@v4 - with: - repository: redis/redis - ref: '7.4' - path: redis - - name: Build Redis - working-directory: redis - run: make install - - name: Build module - run: | - make build - - name: Test - run: | - make test - - name: Pack module - run: | - make pack BRANCH=${{ github.ref_name }} - - name: Upload artifacts to S3 - uses: ./.github/actions/upload-artifacts-to-s3 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - github-ref: ${{ github.ref }} diff --git a/.github/workflows/backport_pr.yml b/.github/workflows/backport_pr.yml index 8ac05f0b3..43156c1e4 100644 --- a/.github/workflows/backport_pr.yml +++ b/.github/workflows/backport_pr.yml @@ -26,7 +26,7 @@ jobs: contains(github.event.comment.body, '/backport') ) steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Create backport pull requests uses: korthout/backport-action@v1 with: diff --git a/.github/workflows/build-linux-platforms.yml b/.github/workflows/build-linux-platforms.yml deleted file mode 100644 index 59fd9aced..000000000 --- a/.github/workflows/build-linux-platforms.yml +++ /dev/null @@ -1,129 +0,0 @@ -name: Build all supported linux platforms - -on: - push: - branches: - - master - tags: - - 'v[0-9]+.[0-9]+.[0-9]+' - pull_request: - types: - - opened - - reopened - - review_requested - -jobs: - build-linux-matrix: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - docker_image: - - image: "ubuntu:bionic" - pre_req_install_cmd: | - # https://github.com/actions/checkout/issues/1809 - echo "ACTIONS_RUNNER_FORCED_INTERNAL_NODE_VERSION=node16" >> $GITHUB_ENV - echo "ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION=node16" >> $GITHUB_ENV - # https://github.blog/changelog/2024-03-07-github-actions-all-actions-will-run-on-node20-instead-of-node16-by-default/ - echo "ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION=true" >> $GITHUB_ENV - apt-get update && apt-get install -y software-properties-common - add-apt-repository ppa:git-core/ppa && apt-get update && apt-get install -y git - - image: "ubuntu:focal" - # https://github.com/actions/checkout/issues/1386 - sparse checkout does not work on got 2.25 which is the default in focal - pre_req_install_cmd: | - apt-get update && apt-get install -y software-properties-common - add-apt-repository ppa:git-core/ppa && apt-get update && apt-get install -y git - - image: "ubuntu:jammy" - pre_req_install_cmd: | - apt-get update && apt-get install -y git - - image: "rockylinux:8" - pre_req_install_cmd: | - dnf install -y git - - image: "rockylinux:9" - pre_req_install_cmd: | - dnf install -y git - - image: "debian:bullseye" - pre_req_install_cmd: | - apt-get update && apt-get install -y git - - image: "amazonlinux:2" - pre_req_install_cmd: | - # https://github.com/actions/checkout/issues/1809 - echo "ACTIONS_RUNNER_FORCED_INTERNAL_NODE_VERSION=node16" >> $GITHUB_ENV - echo "ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION=node16" >> $GITHUB_ENV - # https://github.blog/changelog/2024-03-07-github-actions-all-actions-will-run-on-node20-instead-of-node16-by-default/ - echo "ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION=true" >> $GITHUB_ENV - yum install -y git - - image: "mcr.microsoft.com/cbl-mariner/base/core:2.0" - pre_req_install_cmd: tdnf install --noplugins --skipsignature -y ca-certificates git - container: - image: ${{ matrix.docker_image.image }} - defaults: - run: - shell: bash -l -eo pipefail {0} - steps: - - name: Install git - run: ${{ matrix.docker_image.pre_req_install_cmd }} - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: 'recursive' - - name: Setup - working-directory: .install - run: | - echo ::group::Install dependencies - ./install_script.sh - echo ::endgroup:: - echo ::group::Install rust - ./getrust.sh - echo ::endgroup:: - - name: Get Redis - uses: actions/checkout@v3 - with: - repository: redis/redis - ref: 'unstable' # todo change per version/tag - path: redis - submodules: 'recursive' - - name: build - uses: ./.github/actions/build-json-module-and-redis-with-cargo - - name: Set Artifact Names - # Artifact names have to be unique, so we base them on the environment. - # We also remove invalid characters from the name. - id: artifact-names - run: | # Invalid characters include: Double quote ", Colon :, Less than <, Greater than >, Vertical bar |, Asterisk *, Question mark ? - echo "name=$(echo "${{ matrix.docker_image.image }} x86-64, Redis unstable" | \ - sed -e 's/[":\/\\<>\|*?]/_/g' -e 's/__*/_/g' -e 's/^_//' -e 's/_$//')" >> $GITHUB_OUTPUT - - name: Run tests - run: | - echo ::group::Activate virtual environment - python3 -m venv venv - echo "source $PWD/venv/bin/activate" >> ~/.bash_profile - source venv/bin/activate - echo ::endgroup:: - echo ::group::Install python dependencies - ./.install/common_installations.sh - echo ::endgroup:: - echo ::group::Unit tests - cargo test - echo ::endgroup:: - echo ::group::Flow tests - MODULE=$(realpath ./target/release/rejson.so) RLTEST_ARGS='--no-progress' ./tests/pytest/tests.sh - echo ::endgroup:: - env: - PIP_BREAK_SYSTEM_PACKAGES: 1 - - name: Upload test artifacts - if: failure() - uses: actions/upload-artifact@v3 - with: - name: Test logs ${{ steps.artifact-names.outputs.name }} - path: tests/**/logs/*.log* - if-no-files-found: ignore - - name: Pack module - run: | - git config --global --add safe.directory /__w/RedisJSON/RedisJSON # to avoid git error - MODULE=$(realpath ./target/release/rejson.so) ./sbin/pack.sh - - name: Upload artifacts to S3 - uses: ./.github/actions/upload-artifacts-to-s3-without-make - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - github-ref: ${{ github.ref }} \ No newline at end of file diff --git a/.github/workflows/ci-basic.yml b/.github/workflows/ci-basic.yml deleted file mode 100644 index e400a4e25..000000000 --- a/.github/workflows/ci-basic.yml +++ /dev/null @@ -1,89 +0,0 @@ -name: CI Basic - -on: - push: - paths-ignore: - - '.circleci/**' - - 'docs/**' - - '*.md' - branches-ignore: - - main - - master - - '[0-9]+.[0-9]+.[0-9]+' - - '[0-9]+.[0-9]+' - - 'feature-*' - tags-ignore: - - 'v[0-9]+.[0-9]+.[0-9]+-rc[0-9]+' - - 'v[0-9]+.[0-9]+.[0-9]+-m[0-9]+' - - 'v[0-9]+.[0-9]+.[0-9]+' - -jobs: - build-linux-jammy: - runs-on: "ubuntu-latest" - container: - image: "ubuntu:jammy" - defaults: - run: - shell: bash -l -eo pipefail {0} - steps: - - name: Install git - run: | - apt-get update && apt-get install -y git - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: 'recursive' - - name: Setup - working-directory: .install - run: | - echo ::group::Install dependencies - ./ubuntu_22.04.sh - echo ::endgroup:: - echo ::group::Install rust - ./getrust.sh - echo ::endgroup:: - - name: Get Redis - uses: actions/checkout@v3 - with: - repository: redis/redis - ref: 'unstable' # todo change per version/tag - path: redis - - name: Build Redis - working-directory: redis - run: | - make install - - name: Build module - run: | - . "$HOME/.cargo/env" - cargo --version - cargo build --release - - name: Set Artifact Names - # Artifact names have to be unique, so we base them on the environment. - # We also remove invalid characters from the name. - id: artifact-names - run: | # Invalid characters include: Double quote ", Colon :, Less than <, Greater than >, Vertical bar |, Asterisk *, Question mark ? - echo "name=$(echo "ubuntu22 x86-64, Redis unstable" | \ - sed -e 's/[":\/\\<>\|*?]/_/g' -e 's/__*/_/g' -e 's/^_//' -e 's/_$//')" >> $GITHUB_OUTPUT - - name: Run tests - run: | - echo ::group::Activate virtual environment - ./.install/activate_venv.sh - echo ::endgroup:: - echo ::group::Install python dependencies - ./.install/common_installations.sh - echo ::endgroup:: - echo ::group::Flow tests - MODULE=$(realpath ./target/release/librejson.so) RLTEST_ARGS='--no-progress' ./tests/pytest/tests.sh - echo ::endgroup:: - echo ::group::Unit tests - cargo test - echo ::endgroup:: - env: - PIP_BREAK_SYSTEM_PACKAGES: 1 - - name: Upload artifacts - if: failure() - uses: actions/upload-artifact@v3 - with: - name: Test logs ${{ steps.artifact-names.outputs.name }} - path: tests/**/logs/*.log* - if-no-files-found: ignore \ No newline at end of file diff --git a/.github/workflows/event-ci.yml b/.github/workflows/event-ci.yml new file mode 100644 index 000000000..a47cc12f7 --- /dev/null +++ b/.github/workflows/event-ci.yml @@ -0,0 +1,59 @@ +name: Event CI + +permissions: + id-token: write + contents: read + +on: + pull_request: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + docs-only: + uses: ./.github/workflows/task-check-docs.yml + prepare-values: + runs-on: ubuntu-latest + outputs: + redis-ref: ${{ steps.set-env.outputs.redis-ref }} + steps: + - name: set env + id: set-env + run: | + echo "redis-ref=unstable" >> $GITHUB_OUTPUT # todo change per version/tag + linux: + uses: ./.github/workflows/flow-linux-x86.yml + needs: [prepare-values] + with: + os: jammy rocky9 amazonlinux2 + redis-ref: ${{needs.prepare-values.outputs.redis-ref}} + secrets: inherit + linux-valgrind: + uses: ./.github/workflows/flow-linux-x86.yml + needs: [prepare-values] + with: + os: jammy + redis-ref: ${{needs.prepare-values.outputs.redis-ref}} + run_valgrind: true + secrets: inherit + linux-sanitizer: + needs: [prepare-values, docs-only] + if: ${{ needs.docs-only.outputs.only-docs-changed == 'false' && !github.event.pull_request.draft }} + uses: ./.github/workflows/flow-sanitizer.yml + with: + container: ubuntu:jammy + redis-ref: ${{needs.prepare-values.outputs.redis-ref}} + secrets: inherit + linux-coverage: + needs: [prepare-values, docs-only] + if: ${{ needs.docs-only.outputs.only-docs-changed == 'false' && !github.event.pull_request.draft }} + uses: ./.github/workflows/flow-coverage.yml + secrets: inherit + spellcheck: + uses: ./.github/workflows/flow-spellcheck.yml + secrets: inherit + linter: + uses: ./.github/workflows/flow-linter.yml + secrets: inherit diff --git a/.github/workflows/event-nightly.yml b/.github/workflows/event-nightly.yml new file mode 100644 index 000000000..efbcdc84d --- /dev/null +++ b/.github/workflows/event-nightly.yml @@ -0,0 +1,71 @@ +name: Event Nightly + +permissions: + id-token: write + contents: read + +on: + schedule: + - cron: '20 20 * * *' # 20:20 UTC every day + workflow_dispatch: + inputs: + redis-ref: + description: 'Redis ref to checkout' + required: true + default: 'unstable' +jobs: + prepare-values: + runs-on: ubuntu-latest + outputs: + redis-ref: ${{ steps.set-env.outputs.redis-ref }} + steps: + - name: set env + id: set-env + run: | + echo "redis-ref=unstable" >> $GITHUB_OUTPUT # todo change per version/tag + linux: + uses: ./.github/workflows/flow-linux-x86.yml + needs: [prepare-values] + with: + os: bionic focal jammy rocky8 rocky9 bullseye amazonlinux2 mariner2 + redis-ref: ${{needs.prepare-values.outputs.redis-ref}} + secrets: inherit + ubuntu-arm64: + uses: ./.github/workflows/flow-ubuntu-arm.yml + needs: [prepare-values] + with: + redis-ref: ${{needs.prepare-values.outputs.redis-ref}} + secrets: inherit + alpine: + uses: ./.github/workflows/flow-alpine.yml + needs: [prepare-values] + with: + redis-ref: ${{needs.prepare-values.outputs.redis-ref}} + secrets: inherit + macos: + uses: ./.github/workflows/flow-macos.yml + needs: [prepare-values] + with: + redis-ref: ${{needs.prepare-values.outputs.redis-ref}} + secrets: inherit + linux-valgrind: + uses: ./.github/workflows/flow-linux-x86.yml + needs: [prepare-values] + with: + os: jammy + redis-ref: ${{needs.prepare-values.outputs.redis-ref}} + run_valgrind: true + secrets: inherit + linux-sanitizer: + uses: ./.github/workflows/flow-sanitizer.yml + needs: [prepare-values] + with: + container: ubuntu:jammy + redis-ref: ${{needs.prepare-values.outputs.redis-ref}} + secrets: inherit + spellcheck: + uses: ./.github/workflows/flow-spellcheck.yml + secrets: inherit + linter: + uses: ./.github/workflows/flow-linter.yml + secrets: inherit diff --git a/.github/workflows/event-pull-request.yml b/.github/workflows/event-pull-request.yml deleted file mode 100644 index 1968c92fc..000000000 --- a/.github/workflows/event-pull-request.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: Pull Request Flow - -on: - pull_request: - types: [opened, synchronize, reopened, ready_for_review] # Defaults + ready_for_review - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - docs-only: # Check whether the PR is only modifying docs - uses: ./.github/workflows/task-check-docs.yml - - coverage: - needs: docs-only - if: ${{ needs.docs-only.outputs.only-docs-changed == 'false' && !github.event.pull_request.draft }} - uses: ./.github/workflows/flow-coverage.yml - secrets: inherit - - sanitize: - needs: docs-only - if: ${{ needs.docs-only.outputs.only-docs-changed == 'false' && !github.event.pull_request.draft }} - uses: ./.github/workflows/flow-sanitizer.yml - secrets: inherit diff --git a/.github/workflows/event-tag.yml b/.github/workflows/event-tag.yml new file mode 100644 index 000000000..68dd5ff46 --- /dev/null +++ b/.github/workflows/event-tag.yml @@ -0,0 +1,58 @@ +name: Event TAG + +permissions: + id-token: write + contents: read + +on: + push: + tags: + - 'v[0-9]+.[0-9]+.[0-9]+' + workflow_dispatch: + inputs: + redis-ref: + description: 'Redis ref to checkout' + required: true + default: 'unstable' + +jobs: + prepare-values: + runs-on: ubuntu-latest + outputs: + redis-ref: ${{ steps.set-env.outputs.redis-ref }} + steps: + - name: set env + id: set-env + run: | + echo "redis-ref=unstable" >> $GITHUB_OUTPUT # todo change per version/tag + linux: + uses: ./.github/workflows/flow-linux-x86.yml + needs: [prepare-values] + with: + os: bionic focal jammy rocky8 rocky9 bullseye amazonlinux2 mariner2 + redis-ref: ${{needs.prepare-values.outputs.redis-ref}} + secrets: inherit + ubuntu-arm64: + uses: ./.github/workflows/flow-ubuntu-arm.yml + needs: [prepare-values] + with: + redis-ref: ${{needs.prepare-values.outputs.redis-ref}} + secrets: inherit + alpine: + uses: ./.github/workflows/flow-alpine.yml + needs: [prepare-values] + with: + redis-ref: ${{needs.prepare-values.outputs.redis-ref}} + secrets: inherit + macos: + uses: ./.github/workflows/flow-macos.yml + needs: [prepare-values] + with: + redis-ref: ${{needs.prepare-values.outputs.redis-ref}} + secrets: inherit + spellcheck: + uses: ./.github/workflows/flow-spellcheck.yml + secrets: inherit + linter: + uses: ./.github/workflows/flow-linter.yml + secrets: inherit diff --git a/.github/workflows/event-weekly.yml b/.github/workflows/event-weekly.yml index e26cf60a0..8845fb00f 100644 --- a/.github/workflows/event-weekly.yml +++ b/.github/workflows/event-weekly.yml @@ -1,12 +1,73 @@ -name: Weekly Flow +name: Event Weekly on: schedule: - cron: "0 0 * * 0" +permissions: + id-token: write + contents: read + jobs: run-benchmarks: uses: ./.github/workflows/benchmark-runner.yml secrets: inherit with: extended: true + + prepare-values: + runs-on: ubuntu-latest + outputs: + redis-ref: ${{ steps.set-env.outputs.redis-ref }} + steps: + - name: set env + id: set-env + run: | + echo "redis-ref=unstable" >> $GITHUB_OUTPUT # todo change per version/tag + linux: + uses: ./.github/workflows/flow-linux-x86.yml + needs: [prepare-values] + with: + # os: jammy rocky9 amazonlinux2 + os: bionic focal jammy rocky8 rocky9 bullseye amazonlinux2 mariner2 + redis-ref: ${{needs.prepare-values.outputs.redis-ref}} + secrets: inherit + ubuntu-arm64: + uses: ./.github/workflows/flow-ubuntu-arm.yml + needs: [prepare-values] + with: + redis-ref: ${{needs.prepare-values.outputs.redis-ref}} + secrets: inherit + alpine: + uses: ./.github/workflows/flow-alpine.yml + needs: [prepare-values] + with: + redis-ref: ${{needs.prepare-values.outputs.redis-ref}} + secrets: inherit + macos: + uses: ./.github/workflows/flow-macos.yml + needs: [prepare-values] + with: + redis-ref: ${{needs.prepare-values.outputs.redis-ref}} + secrets: inherit + linux-valgrind: + uses: ./.github/workflows/flow-linux-x86.yml + needs: [prepare-values] + with: + os: jammy + redis-ref: ${{needs.prepare-values.outputs.redis-ref}} + run_valgrind: true + secrets: inherit + linux-sanitizer: + uses: ./.github/workflows/flow-sanitizer.yml + needs: [prepare-values] + with: + container: ubuntu:jammy + redis-ref: ${{needs.prepare-values.outputs.redis-ref}} + secrets: inherit + spellcheck: + uses: ./.github/workflows/flow-spellcheck.yml + secrets: inherit + linter: + uses: ./.github/workflows/flow-linter.yml + secrets: inherit diff --git a/.github/workflows/flow-alpine.yml b/.github/workflows/flow-alpine.yml new file mode 100644 index 000000000..fe4a5d684 --- /dev/null +++ b/.github/workflows/flow-alpine.yml @@ -0,0 +1,110 @@ +name: Flow alpine + +permissions: + id-token: write + contents: read + +on: + workflow_dispatch: # Allows you to run this workflow manually from the Actions tab + inputs: + redis-ref: + description: 'Redis ref to checkout' + required: true + default: 'unstable' + run-test: + type: boolean + default: true + workflow_call: # Allows to run this workflow from another workflow + inputs: + redis-ref: + description: 'Redis ref to checkout' + type: string + required: true + run-test: + type: boolean + default: true + +jobs: + setup-environment: + runs-on: ubuntu-latest + outputs: + redis-ref: ${{ steps.set-env.outputs.redis-ref }} + TAGGED: ${{ steps.set-env.outputs.TAGGED }} + TAG: ${{ steps.set-env.outputs.TAG }} + BRANCH: ${{ steps.set-env.outputs.BRANCH }} + TAG_OR_BRANCH: ${{ steps.set-env.outputs.TAG }}${{ steps.set-env.outputs.BRANCH }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: set env + id: set-env + uses: ./.github/actions/setup-env + with: + github-ref: ${{ github.ref }} + redis-ref: ${{ inputs.redis-ref }} + build: + runs-on: ${{matrix.runs_on}} + needs: setup-environment + defaults: + run: + shell: bash + container: + image: alpine:3 + env: + TAGGED: ${{ needs.setup-environment.outputs.TAGGED }} + VERSION: ${{ needs.setup-environment.outputs.TAG }} + BRANCH: ${{ needs.setup-environment.outputs.BRANCH }} + TAG_OR_BRANCH: ${{ needs.setup-environment.outputs.TAG_OR_BRANCH}} + PIP_BREAK_SYSTEM_PACKAGES: 1 + strategy: + matrix: + runs_on: [ubuntu24-arm64-4-16, ubuntu-latest] + steps: + - name: Install prerequisites + shell: sh + run: | + apk add bash make tar cargo python3 python3-dev py3-pip gcc git curl build-base autoconf automake py3-cryptography linux-headers musl-dev libffi-dev openssl-dev openssh py-virtualenv clang18-libclang + - name: git checkout + run: | + # Perform checkout + REPO_URL="https://github.com/${{ github.repository }}.git" + # Clone the repository to the current directory + git clone --recurse-submodules --depth=1 $REPO_URL . + git config --global --add safe.directory /__w/${{ github.repository }} + REF=${{github.sha}} + git fetch origin ${REF} + git checkout ${REF} + - name: Install python dependencies + run: | + pip install -q --upgrade setuptools + pip install -q --upgrade pip + pip install -q -r tests/pytest/requirements.txt + pip install -q -r .install/build_package_requirements.txt + env: + PIP_BREAK_SYSTEM_PACKAGES: 1 + - name: checkout redis + run: | + # Perform checkout + REPO_URL="https://github.com/redis/redis.git" + # Clone the repository to the current directory + git clone --recurse-submodules --depth=1 $REPO_URL redis + cd redis + git fetch origin ${{inputs.redis-ref}} + git checkout ${{inputs.redis-ref}} + - name: Build Redis + working-directory: redis + run: make install + - name: Build module + run: | + make build + - name: Run tests + if: ${{inputs.run-test}} + run: | + make test + - name: Pack module + uses: ./.github/actions/make-pack + - name: Upload artifacts to S3 + uses: ./.github/actions/upload-artifacts-to-s3-without-make + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/flow-linter.yml b/.github/workflows/flow-linter.yml new file mode 100644 index 000000000..0e0ee2114 --- /dev/null +++ b/.github/workflows/flow-linter.yml @@ -0,0 +1,15 @@ +name: Flow linter +on: + workflow_call: # Allows to run this workflow from another workflow + +jobs: + linter: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Install build dependencies + run: sudo apt-get update && sudo apt-get install -y build-essential + - name: Linter + run: make lint diff --git a/.github/workflows/flow-linux-x86.yml b/.github/workflows/flow-linux-x86.yml new file mode 100644 index 000000000..ab1f13f20 --- /dev/null +++ b/.github/workflows/flow-linux-x86.yml @@ -0,0 +1,185 @@ +name: Build all supported linux platforms + +on: + workflow_dispatch: # Allows to run this workflow from another workflow + inputs: + redis-ref: + description: 'Redis ref to checkout' # todo change per version/tag + type: string + required: true + run-test: + type: boolean + default: true + os: + description: 'OS to build on, bash array style' + type: string + required: true + run_valgrind: + description: 'Run valgrind on the tests' + type: boolean + default: false + workflow_call: # Allows to run this workflow from another workflow + inputs: + redis-ref: + description: 'Redis ref to checkout' # todo change per version/tag + type: string + required: true + run-test: + type: boolean + default: true + os: + description: 'OS to build on, bash array style' + type: string + required: true + run_valgrind: + description: 'Run valgrind on the tests' + type: boolean + default: false + +jobs: + setup-environment: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + redis-ref: ${{ steps.set-env.outputs.redis-ref }} + TAGGED: ${{ steps.set-env.outputs.TAGGED }} + TAG: ${{ steps.set-env.outputs.TAG }} + BRANCH: ${{ steps.set-env.outputs.BRANCH }} + TAG_OR_BRANCH: ${{ steps.set-env.outputs.TAG }}${{ steps.set-env.outputs.BRANCH }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: set env + id: set-env + uses: ./.github/actions/setup-env + with: + github-ref: ${{ github.ref }} + redis-ref: ${{ inputs.redis-ref }} + - name: Set matrix + id: set-matrix + run: | + OS="${{ inputs.os }}" + if [ -z "${OS}" ]; then + OS="bionic focal jammy rocky8 rocky9 bullseye amazonlinux2 mariner2" + fi + MATRIX="[" + for os in $OS; do + case $os in + bionic) + MATRIX="${MATRIX}{\"image\": \"ubuntu:bionic\", \"pre_req_install_cmd\": \"apt-get update && apt-get install -y software-properties-common && add-apt-repository ppa:git-core/ppa && apt-get update && apt-get install -y git\"}," + ;; + focal) + MATRIX="${MATRIX}{\"image\": \"ubuntu:focal\", \"pre_req_install_cmd\": \"apt-get update && apt-get install -y software-properties-common && add-apt-repository ppa:git-core/ppa && apt-get update && apt-get install -y git\"}," + ;; + jammy) + MATRIX="${MATRIX}{\"image\": \"ubuntu:jammy\", \"pre_req_install_cmd\": \"apt-get update && apt-get install -y git\"}," + ;; + rocky8) + MATRIX="${MATRIX}{\"image\": \"rockylinux:8\", \"pre_req_install_cmd\": \"dnf install -y git\"}," + ;; + rocky9) + MATRIX="${MATRIX}{\"image\": \"rockylinux:9\", \"pre_req_install_cmd\": \"dnf install -y git\"}," + ;; + bullseye) + MATRIX="${MATRIX}{\"image\": \"debian:bullseye\", \"pre_req_install_cmd\": \"apt-get update && apt-get install -y git\"}," + ;; + amazonlinux2) + MATRIX="${MATRIX}{\"image\": \"amazonlinux:2\", \"pre_req_install_cmd\": \"yum update -y && yum install -y git\"}," + ;; + mariner2) + MATRIX="${MATRIX}{\"image\": \"mcr.microsoft.com/cbl-mariner/base/core:2.0\", \"pre_req_install_cmd\": \"tdnf install --noplugins --skipsignature -y ca-certificates git\"}," + ;; + *) + echo "Unknown OS: $os" + exit 1 + ;; + esac + done + MATRIX="${MATRIX%?}]" + echo "${MATRIX}" + echo "matrix=${MATRIX}" >> $GITHUB_OUTPUT + + build-linux-matrix: + name: ${{matrix.docker_image.image}}, ${{needs.setup-environment.outputs.redis-ref}} + runs-on: ubuntu-latest + needs: setup-environment + strategy: + fail-fast: false + matrix: + docker_image: ${{fromJson(needs.setup-environment.outputs.matrix)}} + env: + TAGGED: ${{ needs.setup-environment.outputs.TAGGED }} + VERSION: ${{ needs.setup-environment.outputs.TAG }} + BRANCH: ${{ needs.setup-environment.outputs.BRANCH }} + TAG_OR_BRANCH: ${{ needs.setup-environment.outputs.TAG_OR_BRANCH}} + PIP_BREAK_SYSTEM_PACKAGES: 1 + container: + image: ${{ matrix.docker_image.image }} + defaults: + run: + shell: bash -l -eo pipefail {0} + steps: + - name: Install git + run: ${{ matrix.docker_image.pre_req_install_cmd }} + - name: Checkout + if: matrix.docker_image.image != 'amazonlinux:2' && matrix.docker_image.image != 'ubuntu:bionic' + uses: actions/checkout@v4 + with: + submodules: 'recursive' + - name: Checkout Redis + if: matrix.docker_image.image != 'amazonlinux:2' && matrix.docker_image.image != 'ubuntu:bionic' + uses: actions/checkout@v4 + with: + repository: redis/redis + ref: ${{needs.setup-environment.outputs.redis-ref}} + path: redis + submodules: 'recursive' + - name: git checkout + if: matrix.docker_image.image == 'amazonlinux:2' || matrix.docker_image.image == 'ubuntu:bionic' + run: | + # Perform checkout + REPO_URL="https://github.com/${{ github.repository }}.git" + # Clone the repository to the current directory + git config --global --add safe.directory /__w/${{ github.repository }} + git clone --recurse-submodules --depth=1 $REPO_URL . + REF=${{github.sha}} + git fetch origin ${REF} + git checkout ${REF} + + # Perform checkout + REPO_URL="https://github.com/redis/redis.git" + # Clone the repository to the current directory + git clone --recurse-submodules --depth=1 $REPO_URL redis + cd redis + git fetch origin ${{needs.setup-environment.outputs.redis-ref}} + git checkout ${{needs.setup-environment.outputs.redis-ref}} + - name: Setup + working-directory: .install + run: | + echo ::group::Install dependencies + ./install_script.sh + echo ::endgroup:: + echo ::group::Install rust + ./getrust.sh + echo ::endgroup:: + - name: build + uses: ./.github/actions/build-json-module-and-redis-with-cargo + - name: Run tests + if: ${{inputs.run-test}} + uses: ./.github/actions/run-tests + with: + run_valgrind: ${{inputs.run_valgrind && '1' || '0'}} + - name: Upload test artifacts + if: failure() + uses: ./.github/actions/upload-artifacts + with: + image: ${{ matrix.docker_image.image }} + - name: Pack module + if: ${{!inputs.run_valgrind}} + uses: ./.github/actions/pack-module + - name: Upload artifacts to S3 + if: ${{!inputs.run_valgrind}} + uses: ./.github/actions/upload-artifacts-to-s3-without-make + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} diff --git a/.github/workflows/macos-build-and-deploy.yml b/.github/workflows/flow-macos.yml similarity index 58% rename from .github/workflows/macos-build-and-deploy.yml rename to .github/workflows/flow-macos.yml index 7850c2f69..8bf300a22 100644 --- a/.github/workflows/macos-build-and-deploy.yml +++ b/.github/workflows/flow-macos.yml @@ -1,21 +1,52 @@ name: Build for macos on: - push: - branches: - - master - tags: - - 'v[0-9]+.[0-9]+.[0-9]+' workflow_dispatch: # Allows you to run this workflow manually from the Actions tab inputs: redis-ref: description: 'Redis ref to checkout' required: true default: 'unstable' + run-test: + type: boolean + default: true + workflow_call: # Allows you to run this workflow manually from the Actions tab + inputs: + redis-ref: + description: 'Redis ref to checkout' + type: string + default: 'unstable' + run-test: + type: boolean + default: true jobs: + setup-environment: + runs-on: ubuntu-latest + outputs: + redis-ref: ${{ steps.set-env.outputs.redis-ref }} + TAGGED: ${{ steps.set-env.outputs.TAGGED }} + TAG: ${{ steps.set-env.outputs.TAG }} + BRANCH: ${{ steps.set-env.outputs.BRANCH }} + TAG_OR_BRANCH: ${{ steps.set-env.outputs.TAG }}${{ steps.set-env.outputs.BRANCH }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: set env + id: set-env + uses: ./.github/actions/setup-env + with: + github-ref: ${{ github.ref }} + redis-ref: ${{ inputs.redis-ref }} build-macos-x64: runs-on: macos-13 + needs: setup-environment + env: + TAGGED: ${{ needs.setup-environment.outputs.TAGGED }} + VERSION: ${{ needs.setup-environment.outputs.TAG }} + BRANCH: ${{ needs.setup-environment.outputs.BRANCH }} + TAG_OR_BRANCH: ${{ needs.setup-environment.outputs.TAG_OR_BRANCH}} + PIP_BREAK_SYSTEM_PACKAGES: 1 defaults: run: shell: bash -l -eo pipefail {0} @@ -25,7 +56,7 @@ jobs: with: submodules: 'recursive' - name: Deps checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: path: setup sparse-checkout-cone-mode: false @@ -33,15 +64,14 @@ jobs: .install tests/pytest/requirements.* - name: Setup specific - working-directory: setup/.install + working-directory: .install run: ./install_script.sh - name: Full checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: recursive - name: Setup common run: .install/common_installations.sh - - name: Get Redis uses: actions/checkout@v4 with: @@ -55,27 +85,26 @@ jobs: run: | make build - name: Test + if: ${{inputs.run-test}} run: | make test - name: Pack module - run: | - make pack BRANCH=${{ github.ref_name }} - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 + run: make pack BRANCH=$TAG_OR_BRANCH + - name: Upload artifacts to S3 + uses: ./.github/actions/upload-artifacts-to-s3-without-make with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: "us-east-1" - - name: Upload artifacts to S3 - staging - run: | - make upload-artifacts SHOW=1 VERBOSE=1 - make upload-release SHOW=1 STAGING=1 VERBOSE=1 - - name: Upload artifacts to S3 - release # todo: trigger this manually instead - if: ${{ github.ref != 'refs/heads/master' }} - run: make upload-release SHOW=1 VERBOSE=1 build-macos-m1: runs-on: macos-latest-xlarge + needs: setup-environment + env: + TAGGED: ${{ needs.setup-environment.outputs.TAGGED }} + VERSION: ${{ needs.setup-environment.outputs.TAG }} + BRANCH: ${{ needs.setup-environment.outputs.BRANCH }} + TAG_OR_BRANCH: ${{ needs.setup-environment.outputs.TAG_OR_BRANCH}} + PIP_BREAK_SYSTEM_PACKAGES: 1 defaults: run: shell: bash -l -eo pipefail {0} @@ -85,7 +114,7 @@ jobs: with: submodules: 'recursive' - name: Deps checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: path: setup sparse-checkout-cone-mode: false @@ -96,7 +125,7 @@ jobs: working-directory: setup/.install run: ./install_script.sh - name: Full checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: recursive - name: Setup common @@ -105,12 +134,11 @@ jobs: python3 -m venv venv echo "source venv/bin/activate" >> ~/.bashrc echo "source venv/bin/activate" >> ~/.zshrc - source venv/bin/activate + . venv/bin/activate echo ::endgroup:: echo ::group::Install python dependencies ./.install/common_installations.sh echo ::endgroup:: - - name: Get Redis uses: actions/checkout@v4 with: @@ -124,21 +152,13 @@ jobs: run: | make build - name: Test + if: ${{inputs.run-test}} run: | make test - name: Pack module - run: | - make pack BRANCH=${{ github.ref_name }} - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 + run: make pack BRANCH=$TAG_OR_BRANCH + - name: Upload artifacts to S3 + uses: ./.github/actions/upload-artifacts-to-s3-without-make with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: "us-east-1" - - name: Upload artifacts to S3 - staging - run: | - make upload-artifacts SHOW=1 VERBOSE=1 - make upload-release SHOW=1 STAGING=1 VERBOSE=1 - - name: Upload artifacts to S3 - release # todo: trigger this manually instead - if: ${{ github.ref != 'refs/heads/master' }} - run: make upload-release SHOW=1 VERBOSE=1 \ No newline at end of file diff --git a/.github/workflows/flow-sanitizer.yml b/.github/workflows/flow-sanitizer.yml index bd47a3c6f..9c59d6eb0 100644 --- a/.github/workflows/flow-sanitizer.yml +++ b/.github/workflows/flow-sanitizer.yml @@ -8,6 +8,11 @@ on: container: default: "ubuntu:jammy" type: string + required: true + redis-ref: + description: 'Redis ref to checkout' + type: string + required: true jobs: clang-sanitizer: @@ -19,11 +24,12 @@ jobs: image: ${{ inputs.container }} steps: - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: ./.github/actions/run-tests + - uses: ./.github/actions/san-run-tests with: container: ${{ inputs.container }} test-config: QUICK=1 sanitizer: addr + redis-ref: ${{inputs.redis-ref}} diff --git a/.github/workflows/spellcheck.yml b/.github/workflows/flow-spellcheck.yml similarity index 55% rename from .github/workflows/spellcheck.yml rename to .github/workflows/flow-spellcheck.yml index 8b445ccb1..bd8a5ba0b 100644 --- a/.github/workflows/spellcheck.yml +++ b/.github/workflows/flow-spellcheck.yml @@ -1,16 +1,14 @@ name: Spellcheck on: - push: - branches: [master] - pull_request: - branches: [master] + workflow_call: # Allows to run this workflow from another workflow + jobs: spellcheck: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Spellcheck - uses: rojopolis/spellcheck-github-actions@0.30.0 + uses: rojopolis/spellcheck-github-actions@0.45.0 with: config_path: .github/spellcheck-settings.yml task_name: Markdown diff --git a/.github/workflows/ubuntu-arm.yml b/.github/workflows/flow-ubuntu-arm.yml similarity index 82% rename from .github/workflows/ubuntu-arm.yml rename to .github/workflows/flow-ubuntu-arm.yml index 2a0b5e36d..183dac731 100644 --- a/.github/workflows/ubuntu-arm.yml +++ b/.github/workflows/flow-ubuntu-arm.yml @@ -1,28 +1,24 @@ name: Build and Test ubuntu ARM instances on: - push: - branches: - - master - tags: - - 'v[0-9]+.[0-9]+.[0-9]+' - pull_request: - types: - - opened - - reopened - - review_requested workflow_dispatch: # Allows you to run this workflow manually from the Actions tab inputs: redis-ref: description: 'Redis ref to checkout' required: true default: 'unstable' + run-test: + type: boolean + default: true workflow_call: # Allows to run this workflow from another workflow inputs: redis-ref: description: 'Redis ref to checkout' type: string required: true + run-test: + type: boolean + default: true permissions: id-token: write # This is required for requesting the JWT @@ -40,7 +36,7 @@ jobs: redis-ref: ${{ steps.set-env.outputs.redis-ref }} steps: - name: checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 - name: set env @@ -101,18 +97,26 @@ jobs: VERSION: ${{ needs.setup-environment.outputs.TAG }} BRANCH: ${{ needs.setup-environment.outputs.BRANCH }} TAG_OR_BRANCH: ${{ needs.setup-environment.outputs.TAG_OR_BRANCH}} - - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true - ACTIONS_RUNNER_FORCED_INTERNAL_NODE_VERSION: node16 - ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16 container: image: ${{ matrix.docker.image }} steps: - name: Install git run: | ${{ matrix.docker.install_git }} + - name: git checkout + if: matrix.docker.image == 'ubuntu:bionic' + run: | + # Perform checkout + REPO_URL="https://github.com/${{ github.repository }}.git" + # Clone the repository to the current directory + git config --global --add safe.directory /__w/${{ github.repository }} + git clone --recurse-submodules --depth=1 $REPO_URL . + REF=${{github.sha}} + git fetch origin ${REF} + git checkout ${REF} - name: Checkout the module - uses: actions/checkout@v3 + if: matrix.docker.image != 'ubuntu:bionic' + uses: actions/checkout@v4 with: submodules: 'recursive' - name: Install dependencies @@ -121,11 +125,19 @@ jobs: env: DEBIAN_FRONTEND: noninteractive - name: Checkout Redis - uses: actions/checkout@v3 + if: matrix.docker.image != 'ubuntu:bionic' + uses: actions/checkout@v4 with: repository: 'redis/redis' ref: ${{ needs.setup-environment.outputs.redis-ref }} path: 'redis' + - name: Get Redis + if: matrix.docker.image == 'ubuntu:bionic' + run: | + # Perform checkout + REPO_URL="https://github.com/redis/redis.git" + # Clone the repository to the current directory + git clone --recurse-submodules --depth=1 $REPO_URL redis - name: Get Rust run: | curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y @@ -151,19 +163,14 @@ jobs: - name: build uses: ./.github/actions/build-json-module-and-redis-with-cargo - name: Test + if: ${{inputs.run-test}} run: | source venv/bin/activate MODULE=$(realpath ./target/release/rejson.so) RLTEST_ARGS='--no-progress' ./tests/pytest/tests.sh - name: Pack module - run: | - source venv/bin/activate - git config --global --add safe.directory /__w/RedisJSON/RedisJSON # to avoid git error - MODULE=$(realpath ./target/release/rejson.so) BRANCH=$TAG_OR_BRANCH \ - SHOW=1 OSNICK=${{ matrix.docker.nick }} ./sbin/pack.sh + uses: ./.github/actions/pack-module - name: Upload artifacts to S3 - uses: ./.github/actions/upload-artifacts-to-s3 + uses: ./.github/actions/upload-artifacts-to-s3-without-make with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - github-ref: ${{ github.ref }} - osnick: ${{ matrix.docker.nick }} \ No newline at end of file diff --git a/.github/workflows/freebsd.yml b/.github/workflows/freebsd.yml index 99ec6078a..fab2384be 100644 --- a/.github/workflows/freebsd.yml +++ b/.github/workflows/freebsd.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: build uses: vmactions/freebsd-vm@v1 with: diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml deleted file mode 100644 index da24a8143..000000000 --- a/.github/workflows/macos.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: macos - -on: - push: - branches: [ master ] - pull_request: - branches: [ master ] - -jobs: - build: - - runs-on: macos-latest - - steps: - - uses: actions/checkout@v2 - - name: Build - run: cargo build --all --all-targets --verbose - - name: Run tests - run: cargo test --all --all-targets --verbose diff --git a/.github/workflows/trigger-build.yml b/.github/workflows/trigger-build.yml deleted file mode 100644 index 8c5dc3574..000000000 --- a/.github/workflows/trigger-build.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: Trigger website deploy -on: - push: - branches: - - master - - '[0-9]+.[0-9]+' - paths: - - 'docs/**' - - 'commands.json' - -jobs: - trigger: - runs-on: ubuntu-latest - steps: - - run: | - echo "'$DATA'" | xargs \ - curl \ - -X POST https://api.netlify.com/build_hooks/${NETLIFY_BUILD_HOOK_ID} \ - -d - env: - NETLIFY_BUILD_HOOK_ID: ${{ secrets.NETLIFY_BUILD_HOOK_ID }} - DATA: '{"repository":"${{ github.repository }}", "sha":"${{ github.sha }}", "ref":"${{ github.ref }}"}}' diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml deleted file mode 100644 index d1753b202..000000000 --- a/.github/workflows/ubuntu.yml +++ /dev/null @@ -1,103 +0,0 @@ -name: Build for Ubuntu - -on: - push: - branches: - - master - tags: - - 'v[0-9]+.[0-9]+.[0-9]+' - paths-ignore: - - '.circleci/**' - - 'docs/**' - - '*.md' - -# https://github.com/actions/checkout/issues/1487 -# A workaround for the old runtime is to use these environment -# variables, install the latest version of git within the container, -# and use the actions/checkout@v3 instead of @v4, until it is fixed. -env: - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true - ACTIONS_RUNNER_FORCED_INTERNAL_NODE_VERSION: node16 - ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16 - DEBIAN_FRONTEND: noninteractive - -jobs: - build: - strategy: - matrix: - docker_image: ["ubuntu:bionic", "ubuntu:focal", "ubuntu:jammy"] - runs-on: "ubuntu-latest" - container: - image: ${{ matrix.docker_image }} - defaults: - run: - shell: bash -l -eo pipefail {0} - steps: - - name: Update git - run: | - apt update - apt install -y software-properties-common - add-apt-repository ppa:git-core/ppa -y - apt update - apt install -y git - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: 'recursive' - - name: Install python 3.12 - run: | - apt install -y wget build-essential libncursesw5-dev libssl-dev libsqlite3-dev tk-dev libgdbm-dev libc6-dev libbz2-dev libffi-dev zlib1g-dev - wget https://www.python.org/ftp/python/3.12.2/Python-3.12.2.tgz - tar xzf Python-3.12.2.tgz - cd Python-3.12.2 - ./configure --enable-optimizations - make -j - make altinstall - python3 --version - python3.12 --version - pip3.12 --version - update-alternatives --install /usr/bin/python3 python3 `which python3.12` 1 - python3 --version - - name: Install prerequisites - run: | - apt install -y make build-essential pkg-config python3-pip libclang-dev clang - ./deps/readies/bin/getaws - ./deps/readies/bin/getrust - python3 -m pip list - python3 -m pip install --upgrade setuptools six pip - pip install -q -r .install/build_package_requirements.txt - python3 -m pip install -r tests/pytest/requirements.txt - - name: Get Redis - uses: actions/checkout@v3 - with: - repository: redis/redis - ref: '7.4' - path: redis - - name: Build Redis - working-directory: redis - run: | - make -j - make install - - name: Build module - run: | - make build OSNICK=jammy VERSION=${{ github.ref_name }} TEST=1 OFFICIAL=1 SHOW=1 - - name: Test - run: | - make test - - name: Pack module - run: | - make pack BRANCH=${{ github.ref_name }} - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v3 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: "us-east-1" - - name: Upload artifacts to S3 - staging - run: | - make upload-artifacts SHOW=1 VERBOSE=1 - make upload-release SHOW=1 STAGING=1 VERBOSE=1 - - name: Upload artifacts to S3 - release # todo: trigger this manually instead - if: ${{ github.ref != 'refs/heads/master' }} - run: make upload-release SHOW=1 VERBOSE=1 - diff --git a/sbin/upload-artifacts b/sbin/upload-artifacts index 99818dd6d..2a4e681e9 100755 --- a/sbin/upload-artifacts +++ b/sbin/upload-artifacts @@ -89,8 +89,10 @@ else fi cd artifacts${MAYBE_SNAP} -# in some versions --apparent-size is not supported -[[ $VERBOSE == 1 ]] && du -ah --apparent-size * || du -ah -s * +if du --help | grep -q -- --apparent-size; then + DU_ARGS='--apparent-size' +fi +[[ $VERBOSE == 1 ]] && du -ah ${DU_ARGS} * #----------------------------------------------------------------------------------------------