.github/workflows/build-deploy-linux-arm64.yml #716
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| on: | |
| workflow_dispatch: | |
| inputs: | |
| buildThreads: | |
| description: 'Build threads for libnd4j. Used to control memory usage of builds.' | |
| required: true | |
| default: 2 | |
| deployToReleaseStaging: | |
| description: 'Whether to deploy to release staging or not.' | |
| required: false | |
| default: 0 | |
| releaseVersion: | |
| description: 'Release version target' | |
| required: false | |
| default: 1.0.0-M3 | |
| serverId: | |
| description: 'Server id to publish to' | |
| required: false | |
| default: central | |
| snapshotVersion: | |
| description: 'Snapshot version target' | |
| required: false | |
| default: 1.0.0-SNAPSHOT | |
| mvnFlags: | |
| description: "Extra maven flags (must escape input yourself if used)" | |
| required: false | |
| default: | |
| libnd4jUrl: | |
| description: 'Sets a libnd4j download url for this build. LIBND4J_HOME will automatically be set. Should be used when only needing to build other modules.' | |
| required: false | |
| default: | |
| runsOn: | |
| description: 'System to run on' | |
| required: false | |
| default: ubuntu-22.04-arm | |
| debug_enabled: | |
| description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)' | |
| required: false | |
| default: false | |
| swapSize: | |
| description: 'Swap file size in GB (e.g., 8 for 8GB)' | |
| required: false | |
| default: 8 | |
| jobs: | |
| linux-arm64: | |
| runs-on: ${{ github.event.inputs.runsOn }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| helper: [armcompute, ""] | |
| include: | |
| - mvn_ext: ${{ github.event.inputs.mvnFlags }} | |
| - debug_enabled: ${{ github.event.inputs.debug_enabled }} | |
| - deploy_to_release_staging: ${{ github.event.inputs.deployToReleaseStaging }} | |
| - release_version: ${{ github.event.inputs.releaseVersion }} | |
| - snapshot_version: ${{ github.event.inputs.snapshotVersion }} | |
| - server_id: ${{ github.event.inputs.serverId }} | |
| - build_threads: ${{ github.event.inputs.buildThreads }} | |
| - swap_size: ${{ github.event.inputs.swapSize }} | |
| steps: | |
| - uses: AutoModality/action-clean@v1 | |
| - name: Cancel Previous Runs | |
| uses: styfle/[email protected] | |
| with: | |
| access_token: ${{ github.token }} | |
| - uses: actions/checkout@v2 | |
| - name: Free Disk Space | |
| uses: jlumbroso/free-disk-space@main | |
| with: | |
| tool-cache: false | |
| android: true | |
| dotnet: true | |
| haskell: true | |
| large-packages: true | |
| docker-images: true | |
| swap-storage: true | |
| - name: Configure swap space | |
| shell: bash | |
| run: | | |
| sudo fallocate -l 16G /swapfile | |
| sudo chmod 600 /swapfile | |
| sudo mkswap /swapfile | |
| sudo swapon /swapfile | |
| sudo sysctl vm.swappiness=60 | |
| free -h | |
| - name: Cache protobuf install | |
| uses: actions/cache@v4 | |
| id: cache-protobuf | |
| with: | |
| path: /opt/protobuf | |
| key: ${{ runner.os }}-protobuf | |
| - uses: ./.github/actions/install-protobuf-linux | |
| if: steps.cache-protobuf.outputs.cache-hit != 'true' | |
| - name: Set mvn build command based on matrix | |
| shell: bash | |
| run: | | |
| LIBND4J_CLASSIFIER="linux-arm64" | |
| echo "LIBND4J_CLASSIFIER=${LIBND4J_CLASSIFIER}" >> $GITHUB_ENV | |
| echo "OPENBLAS_JAR=openblas-0.3.28-1.5.11-linux-arm64.jar" >> $GITHUB_ENV | |
| echo "ARMCOMPUTE_TARGET=arm64-v8a" >> $GITHUB_ENV | |
| if [ "${{ github.event.inputs.libnd4jUrl }}" != '' ]; then | |
| modules=':nd4j-native,:nd4j-native-preset' | |
| else | |
| modules=':nd4j-native,:nd4j-native-preset,:libnd4j' | |
| fi | |
| command="mvn ${{ matrix.mvn_ext }} -Posx-aarch64-protoc -Dlibnd4j.generate.flatc=ON --no-transfer-progress -pl $modules -Pcpu -Dlibnd4j.buildthreads=${{ github.event.inputs.buildThreads }} -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.count=3 -Possrh -DskipTestResourceEnforcement=true -Dmaven.javadoc.failOnError=false -Djavacpp.platform=${LIBND4J_CLASSIFIER} -Pcpu --also-make --batch-mode deploy -DskipTests" | |
| if [ "${{ matrix.helper }}" != '' ]; then | |
| mvn_ext=" -Dlibnd4j.helper=${{ matrix.helper }} -Djavacpp.platform.extension=-${{ matrix.helper }} -Dlibnd4j.classifier=${LIBND4J_CLASSIFIER}-${{ matrix.helper }}" | |
| else | |
| mvn_ext="" | |
| fi | |
| command="${command} ${mvn_ext}" | |
| echo "COMMAND=${command}" >> $GITHUB_ENV | |
| - name: Set up Java for publishing to OSSRH | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: 11 | |
| distribution: 'temurin' | |
| server-id: ${{ github.event.inputs.serverId }} | |
| server-username: MAVEN_USERNAME | |
| server-password: MAVEN_PASSWORD | |
| gpg-private-key: ${{ secrets.SONATYPE_GPG_KEY }} | |
| gpg-passphrase: MAVEN_GPG_PASSPHRASE | |
| cache: 'maven' | |
| - name: Setup Path | |
| shell: bash | |
| run: | | |
| echo "/opt/protobuf/bin" >> $GITHUB_PATH | |
| echo "/opt/cmake/bin" >> $GITHUB_PATH | |
| - name: Setup OpenBLAS and ARM Compute Library | |
| shell: bash | |
| run: | | |
| # Setup OpenBLAS | |
| mkdir -p "${GITHUB_WORKSPACE}/openblas_home" | |
| cd "${GITHUB_WORKSPACE}/openblas_home" | |
| wget https://repo1.maven.org/maven2/org/bytedeco/openblas/0.3.28-1.5.11/${OPENBLAS_JAR} | |
| unzip ${OPENBLAS_JAR} | |
| OPENBLAS_DIR="${GITHUB_WORKSPACE}/openblas_home/org/bytedeco/openblas/linux-arm64" | |
| echo "OPENBLAS_PATH=${OPENBLAS_DIR}" >> "$GITHUB_ENV" | |
| # Ensure the symlink exists for consistency | |
| ln -sf "${OPENBLAS_DIR}/libopenblas.so.0" "${OPENBLAS_DIR}/libopenblas.so" | |
| # Setup ARM Compute Library if selected | |
| if [ "${{ matrix.helper }}" == "armcompute" ]; then | |
| ARMCOMPUTE_HOME="${GITHUB_WORKSPACE}/armcompute_home" | |
| mkdir -p "${ARMCOMPUTE_HOME}" | |
| cd "${ARMCOMPUTE_HOME}" | |
| wget https://github.com/ARM-software/ComputeLibrary/releases/download/v25.04/arm_compute-v25.04-linux-aarch64-cpu-bin.tar.gz | |
| tar -xzf arm_compute-v25.04-linux-aarch64-cpu-bin.tar.gz | |
| # Set ARMCOMPUTE_ROOT to the explicit path of the extracted contents | |
| EXTRACTED_DIR="${ARMCOMPUTE_HOME}/arm_compute-v25.04-linux-aarch64-cpu-bin" | |
| echo "ARMCOMPUTE_ROOT=${EXTRACTED_DIR}" >> "$GITHUB_ENV" | |
| echo "LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${EXTRACTED_DIR}/lib/armv8a-neon" >> "$GITHUB_ENV" | |
| fi | |
| - name: Build with Maven | |
| shell: bash | |
| env: | |
| MAVEN_GPG_KEY: ${{ secrets.SONATYPE_GPG_KEY }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| PERFORM_RELEASE: ${{ github.event.inputs.deployToReleaseStaging }} | |
| RELEASE_VERSION: ${{ github.event.inputs.releaseVersion }} | |
| SNAPSHOT_VERSION: ${{ github.event.inputs.snapshotVersion }} | |
| RELEASE_REPO_ID: ${{ github.event.inputs.releaseRepoId }} | |
| MAVEN_USERNAME: ${{ secrets.CENTRAL_SONATYPE_TOKEN_USERNAME }} | |
| MAVEN_PASSWORD: ${{ secrets.CENTRAL_SONATYPE_TOKEN_PASSWORD }} | |
| MAVEN_GPG_PASSPHRASE: ${{ secrets.PACKAGES_GPG_PASS }} | |
| MAVEN_OPTS: -Xmx2g | |
| PROTO_EXEC: /opt/protobuf/bin/protoc | |
| run: | | |
| # Pre-build native libraries with CMake if not using a pre-packaged libnd4j | |
| if [ "${{ github.event.inputs.libnd4jUrl }}" == '' ]; then | |
| echo "Starting manual pre-build of libnd4j..." | |
| # Define CMake arguments for building libnd4j | |
| cmake_args="-DOPENBLAS_PATH=${OPENBLAS_PATH}" | |
| # Add armcompute helper flags if enabled | |
| if [ "${{ matrix.helper }}" == "armcompute" ]; then | |
| echo "Enabling armcompute helper for native build." | |
| cmake_args="${cmake_args} -DHELPERS_armcompute=true -DARMCOMPUTE_ROOT=${ARMCOMPUTE_ROOT}" | |
| fi | |
| # Navigate to libnd4j build directory | |
| cd "${GITHUB_WORKSPACE}/libnd4j" | |
| mkdir -p blasbuild/linux-arm64 | |
| cd blasbuild/linux-arm64 | |
| # Configure with CMake | |
| echo "Configuring with CMake args: ${cmake_args}" | |
| cmake ${cmake_args} ../.. | |
| # Build with specified number of threads | |
| echo "Building with make -j${{ github.event.inputs.buildThreads }}" | |
| make -j${{ github.event.inputs.buildThreads }} | |
| cd ${GITHUB_WORKSPACE} | |
| echo "Manual pre-build of libnd4j finished." | |
| fi | |
| # Run the main Maven build | |
| echo "Running Maven command: ${COMMAND}" | |
| if [ "$PERFORM_RELEASE" == 1 ]; then | |
| bash ${GITHUB_WORKSPACE}/release-specified-component.sh "${RELEASE_VERSION}" "${SNAPSHOT_VERSION}" "${RELEASE_REPO_ID}" "${COMMAND}" | |
| else | |
| eval "${COMMAND}" | |
| fi | |
| - name: Cleanup swap | |
| if: always() | |
| shell: bash | |
| run: | | |
| sudo swapoff /swapfile || true | |
| sudo rm -f /swapfile || true |