Merge branch 'master' into pr_2_3_cmake_rendering_platform #210
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| on: | ||
|
Check failure on line 1 in .github/workflows/build-deploy-windows-cuda-12.6.yml
|
||
| workflow_dispatch: | ||
| inputs: | ||
| buildThreads: | ||
| description: 'Build threads for libnd4j. Used to control memory usage of builds.' | ||
| required: true | ||
| default: 4 | ||
| deployToReleaseStaging: | ||
| description: 'Whether to deploy to release staging or not.' | ||
| required: false | ||
| default: 0 | ||
| releaseVersion: | ||
| description: 'Release version target' | ||
| required: false | ||
| default: 1.0.0-M3 | ||
| snapshotVersion: | ||
| description: 'Snapshot version target' | ||
| required: false | ||
| default: 1.0.0-SNAPSHOT | ||
| releaseRepoId: | ||
| description: 'Release repository id' | ||
| required: false | ||
| default: | ||
| serverId: | ||
| description: 'Server id to publish to' | ||
| required: false | ||
| default: central | ||
| mvnFlags: | ||
| description: "Extra maven flags (must escape input yourself if used)" | ||
| required: false | ||
| default: | ||
| libnd4jUrl: | ||
| description: 'Whether to download libnd4j using https://github.com/KonduitAI/gh-actions-libnd4j-urls/ for this build. LIBND4J_HOME will automatically be set. Should be used when only needing to build other modules.' | ||
| required: false | ||
| default: | ||
| runsOn: | ||
| description: 'System to run on' | ||
| required: false | ||
| default: windows-2022 | ||
| debug_enabled: | ||
| description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)' | ||
| required: false | ||
| default: false | ||
| jobs: | ||
| windows-x86_64-cuda-12-6: | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| helper: [ cudnn,"" ] | ||
| extension: [ "" ] | ||
| include: | ||
| - mvn_ext: ${{ github.event.inputs.mvnFlags }} | ||
| experimental: true | ||
| name: Extra maven flags | ||
| - debug_enabled: ${{ github.event.inputs.debug_enabled }} | ||
| experimental: true | ||
| name: Debug enabled | ||
| - runs_on: ${{ github.event.inputs.runsOn }} | ||
| experimental: true | ||
| name: OS to run on | ||
| - libnd4j_file_download: ${{ github.event.inputs.libnd4jUrl }} | ||
| experimental: true | ||
| name: OS to run on | ||
| - deploy_to_release_staging: ${{ github.event.inputs.deployToReleaseStaging }} | ||
| experimental: true | ||
| name: Whether to deploy to release staging or not | ||
| - release_version: ${{ github.event.inputs.releaseVersion }} | ||
| experimental: true | ||
| name: Release version | ||
| - snapshot_version: ${{ github.event.inputs.snapshotVersion }} | ||
| experimental: true | ||
| name: Snapshot version | ||
| - server_id: ${{ github.event.inputs.serverId }} | ||
| experimental: true | ||
| name: Server id | ||
| - release_repo_id: ${{ github.event.inputs.releaseRepoId }} | ||
| experimental: true | ||
| name: The release repository to run on | ||
| - mvn_flags: ${{ github.event.inputs.mvnFlags }} | ||
| experimental: true | ||
| name: Extra maven flags to use as part of the build | ||
| - build_threads: ${{ github.event.inputs.buildThreads }} | ||
| experimental: true | ||
| name: The number of threads to build libnd4j with | ||
| runs-on: ${{ matrix.runs_on }} | ||
| steps: | ||
| - name: Cancel Previous Runs | ||
| uses: styfle/[email protected] | ||
| with: | ||
| access_token: ${{ github.token }} | ||
| - uses: actions/checkout@v4 | ||
| - name: Free Disk Space (Windows) | ||
| shell: powershell | ||
| run: | | ||
| # Show initial disk space | ||
| Write-Host "Initial disk space:" | ||
| Get-PSDrive C | Select-Object Used,Free | ||
| # Remove Windows Defender scan history | ||
| Remove-Item -Path "$env:ProgramData\Microsoft\Windows Defender\Scans\History\*" -Recurse -Force -ErrorAction SilentlyContinue | ||
| Write-Host "Removed Windows Defender scan history" | ||
| # Clear Windows temp folders | ||
| Remove-Item -Path "$env:SystemRoot\Temp\*" -Recurse -Force -ErrorAction SilentlyContinue | ||
| Remove-Item -Path "$env:TEMP\*" -Recurse -Force -ErrorAction SilentlyContinue | ||
| Write-Host "Cleared Windows temp folders" | ||
| # Clear Windows Update cache safely (without stopping/starting service) | ||
| try { | ||
| Remove-Item -Path "$env:SystemRoot\SoftwareDistribution\Download\*" -Recurse -Force -ErrorAction SilentlyContinue | ||
| Write-Host "Cleared Windows Update download cache" | ||
| } | ||
| catch { | ||
| Write-Host "Could not clear Windows Update cache. Continuing..." | ||
| } | ||
| # Clean package manager caches | ||
| if (Test-Path -Path "C:\npm\cache") { | ||
| Remove-Item -Path "C:\npm\cache\*" -Recurse -Force -ErrorAction SilentlyContinue | ||
| Write-Host "Cleared NPM cache" | ||
| } | ||
| choco cache remove -y -ErrorAction SilentlyContinue | ||
| Write-Host "Cleared Chocolatey cache" | ||
| # Remove Docker images if Docker is installed | ||
| try { | ||
| if (Get-Command "docker" -ErrorAction SilentlyContinue) { | ||
| docker image prune -a -f | ||
| docker container prune -f | ||
| docker volume prune -f | ||
| Write-Host "Pruned Docker resources" | ||
| } | ||
| } | ||
| catch { | ||
| Write-Host "Failed to prune Docker resources. Continuing..." | ||
| } | ||
| # Remove .NET SDK/Runtime backup folders | ||
| if (Test-Path -Path "$env:ProgramData\Microsoft\.NET\*.backup") { | ||
| Remove-Item -Path "$env:ProgramData\Microsoft\.NET\*.backup" -Recurse -Force -ErrorAction SilentlyContinue | ||
| Write-Host "Removed .NET backup folders" | ||
| } | ||
| # Clear Azure artifacts cache | ||
| if (Test-Path -Path "$env:LOCALAPPDATA\Microsoft\Azure\*") { | ||
| Remove-Item -Path "$env:LOCALAPPDATA\Microsoft\Azure\*" -Recurse -Force -ErrorAction SilentlyContinue | ||
| Write-Host "Cleared Azure artifacts cache" | ||
| } | ||
| # Optimize Windows Component Store | ||
| try { | ||
| Start-Process -FilePath "dism.exe" -ArgumentList "/online /Cleanup-Image /StartComponentCleanup" -NoNewWindow -Wait | ||
| Write-Host "Optimized Windows Component Store" | ||
| } | ||
| catch { | ||
| Write-Host "Failed to optimize Windows Component Store. Continuing..." | ||
| } | ||
| # Show final disk space | ||
| Write-Host "Final disk space:" | ||
| Get-PSDrive C | Select-Object Used,Free | ||
| - name: Set mvn build command based on matrix | ||
| shell: powershell | ||
| run: | | ||
| if ( "${{ matrix.libnd4j_file_download }}" -ne "" ) { | ||
| $modules=" :nd4j-cuda-12.6-preset,:nd4j-cuda-12.6" | ||
| } elseif ( "${{ matrix.helper }}" -ne "" ) { | ||
| $modules=":nd4j-cuda-12.6-preset,:nd4j-cuda-12.6,libnd4j" | ||
| } elseif ( "${{ matrix.helper }}" -eq "" ) { | ||
| $modules=":nd4j-cuda-12.6-preset,:nd4j-cuda-12.6,libnd4j,:nd4j-cuda-12.6-platform" | ||
| } | ||
| $command="mvn ${{ matrix.mvn_ext }} -Pcuda -Dlibnd4j.cuda.compile.skip=false -Dlibnd4j.chip=cuda -Pcuda --also-make -pl $modules -Dlibnd4j.compute=`"8.6 9.0`" -Dlibnd4j.cpu.compile.skip=true -Dlibnd4j.buildthreads=${{ matrix.build_threads }} -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.count=3 -Possrh -Dlibnd4j.buildthreads=${{ github.event.inputs.buildThreads }} -Djavacpp.platform=windows-x86_64 -Dlibnd4j.platform=windows-x86_64 -Pcuda -Dlibnd4j.chip=cuda deploy -DskipTests" | ||
| if ( "${{ matrix.helper }}" -ne "" -And "${{ matrix.extension }}" -ne "" ) { | ||
| $mvn_ext=" -Dlibnd4j.chip=cuda -Dlibnd4j.classifier=windows-x86_64-cuda-12.6-${{ matrix.helper }}-${{matrix.extension}} -Dlibnd4j.extension=${{ matrix.extension }} -Djavacpp.platform.extension=-${{ matrix.helper }}-${{ matrix.extension }} -Dlibnd4j.helper=${{ matrix.helper }} -Pcuda -Dlibnd4j.chip=cuda deploy -DskipTests" | ||
| $libnd4j_download_file_url="windows-cuda-12.6-${{ matrix.extension }}-${{ matrix.helper }}" | ||
| } elseif ( "${{ matrix.helper }}" -ne "" ) { | ||
| $mvn_ext=" -Dlibnd4j.classifier=windows-x86_64-cuda-12.6-${{ matrix.helper }} -Dlibnd4j.extension=${{ matrix.helper }} -Djavacpp.platform.extension=-${{ matrix.helper }} -Djavacpp.platform=windows-x86_64 -Dlibnd4j.helper=${{ matrix.helper }} -Dlibnd4j.platform=windows-x86_64 -Pcuda -Dlibnd4j.chip=cuda deploy -DskipTests" | ||
| $libnd4j_download_file_url="windows-cuda-12.6-${{ matrix.helper }}" | ||
| } else { | ||
| $libnd4j_download_file_url="windows-cuda-12.6" | ||
| $mvn_ext=" -Dlibnd4j.classifier=windows-x86_64-cuda-12.6" | ||
| } | ||
| $command2= -join("$($command)","$($mvn_ext)"); | ||
| $to_write = -join("COMMAND=","$($command2)"); | ||
| if ( "${{ matrix.libnd4j_file_download }}" -ne "") { | ||
| Write-Host "Adding libnd4j download URL to GITHUB_ENV" | ||
| $libnd4j_url_to_write = -join("LIBND4J_FILE_NAME=","$(${{ matrix.libnd4j_file_download }}/$libnd4j_download_file_url)"); | ||
| echo $libnd4j_url_to_write | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append | ||
| } | ||
| Write-Host "Setting command for helper ${{ matrix.helper }} and extension ${{ matrix.extension }} to $($command2) and the libnd4j bootstrap file name to $($libnd4j_download_file_url)" | ||
| echo $command2 | Out-File -FilePath "$env:GITHUB_WORKSPACE/mvn-command.bat" -Encoding utf8 | ||
| echo $to_write | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append | ||
| echo "CUDNN_ROOT_DIR=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.6" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append | ||
| - name: Install MSYS2 and dependencies | ||
| uses: msys2/setup-msys2@v2 | ||
| with: | ||
| msystem: MINGW64 | ||
| update: true | ||
| install: >- | ||
| base-devel git tar pkg-config unzip p7zip zip autoconf autoconf-archive automake patch | ||
| mingw-w64-x86_64-make mingw-w64-x86_64-gnupg mingw-w64-x86_64-cmake mingw-w64-x86_64-nasm | ||
| mingw-w64-x86_64-toolchain mingw-w64-x86_64-libtool mingw-w64-x86_64-gcc mingw-w64-x86_64-gcc-fortran | ||
| mingw-w64-x86_64-libwinpthread-git mingw-w64-x86_64-SDL mingw-w64-x86_64-ragel | ||
| pacboy: --noconfirm | ||
| - name: Cache cuda install | ||
| uses: actions/cache@v4 | ||
| id: cache-cuda-126 | ||
| with: | ||
| path: C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.6 | ||
| key: ${{ matrix.runs_on }}-cuda-12.6-${{ matrix.helper }}-${{ hashFiles('**/pom.xml') }} | ||
| restore-keys: | | ||
| ${{ matrix.runs_on }}-cuda-12.6-${{ matrix.helper }}- | ||
| - name: Install CUDA 12.6 (if not cached) | ||
| uses: konduitai/cuda-install/.github/actions/install-cuda-windows@master | ||
| env: | ||
| cuda: "12.6" | ||
| if: steps.cache-cuda-126.outputs.cache-hit != 'true' | ||
| - name: Set up Java and Maven cache/auth | ||
| uses: konduitai/setup-java@main | ||
| with: | ||
| java-version: 11 | ||
| distribution: 'temurin' | ||
| - name: Setup Windows PATH | ||
| shell: powershell | ||
| run: | | ||
| $cudaPath = "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.6" | ||
| $msysPath = "C:\msys64\mingw64\bin;C:\msys64\usr\bin" | ||
| $newPath = "$($msysPath);$($cudaPath)\bin;$($cudaPath)\libnvvp;" | ||
| Write-Host "Adding to PATH: $newPath" | ||
| echo $newPath | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append | ||
| echo "CUDA_PATH=$cudaPath" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append | ||
| - name: Import GPG Key | ||
| if: ${{ secrets.GPG_PRIVATE_KEY != '' && secrets.MAVEN_GPG_PASSPHRASE != '' }} | ||
| uses: crazy-max/ghaction-import-gpg@v6 | ||
| with: | ||
| gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} | ||
| passphrase: ${{ secrets.MAVEN_GPG_PASSPHRASE }} | ||
| - name: Run cuda build | ||
| shell: cmd | ||
| env: | ||
| MAVEN_USERNAME: ${{ secrets.CENTRAL_SONATYPE_TOKEN_USERNAME }} | ||
| MAVEN_PASSWORD: ${{ secrets.CENTRAL_SONATYPE_TOKEN_PASSWORD }} | ||
| MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }} | ||
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | ||
| PERFORM_RELEASE: ${{ matrix.deploy_to_release_staging }} | ||
| RELEASE_VERSION: ${{ matrix.release_version }} | ||
| SNAPSHOT_VERSION: ${{ matrix.snapshot_version }} | ||
| RELEASE_REPO_ID: ${{ matrix.release_repo_id }} | ||
| PUBLISH_TO: central | ||
| LIBND4J_HOME_SUFFIX: cuda | ||
| HELPER: ${{ matrix.helper }} | ||
| EXTENSION: ${{ matrix.extension }} | ||
| MAVEN_OPTS: "-Xmx2g" | ||
| run: | | ||
| echo "Libnd4j build threads: ${{ matrix.build_threads }}" | ||
| echo "Deploy to release staging: %PERFORM_RELEASE%" | ||
| echo "Release version: %RELEASE_VERSION%" | ||
| echo "Snapshot version: %SNAPSHOT_VERSION%" | ||
| echo "Debug enabled: ${{ matrix.debug_enabled }}" | ||
| echo "Libnd4j URL input: %LIBND4J_FILE_NAME%" | ||
| echo "Maven flags input: ${{ matrix.mvn_flags }}" | ||
| echo "Server id: ${{ matrix.server_id }}" | ||
| echo "Release repo id: %RELEASE_REPO_ID%" | ||
| echo "CUDA Path from env: %CUDA_PATH%" | ||
| echo "CUDNN Root Dir from env: %CUDNN_ROOT_DIR%" | ||
| call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat" | ||
| set MSYSTEM=MINGW64 | ||
| echo "Running cuda build..." | ||
| echo "Maven command from file:" | ||
| type "%GITHUB_WORKSPACE%\mvn-command.bat" | ||
| bash ./change-cuda-versions.sh 12.6 | ||
| Rem Ensure CUDNN_ROOT_DIR is set if needed by cmake; often CUDA_PATH is sufficient | ||
| if not defined CUDNN_ROOT_DIR set CUDNN_ROOT_DIR=%CUDA_PATH% | ||
| if "%PERFORM_RELEASE%"=="1" ( | ||
| echo "Running release build..." | ||
| bash "%GITHUB_WORKSPACE%/bootstrap-libnd4j-from-url.sh" windows cuda 12.6 "%HELPER%" "%EXTENSION%" | ||
| bash "./release-specified-component.sh" "%RELEASE_VERSION%" "%SNAPSHOT_VERSION%" "%RELEASE_REPO_ID%" | ||
| ) else ( | ||
| echo "Running snapshot build..." | ||
| bash "%GITHUB_WORKSPACE%/bootstrap-libnd4j-from-url.sh" windows cuda 12.6 "%HELPER%" "%EXTENSION%" | ||
| call "%GITHUB_WORKSPACE%\mvn-command.bat" | ||
| ) | ||