Thanks to visit codestin.com
Credit goes to github.com

Skip to content

CI: migrate from Jenkins to GitHub Actions #630

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: madlib2-master
Choose a base branch
from
Draft

CI: migrate from Jenkins to GitHub Actions #630

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
150 changes: 150 additions & 0 deletions .github/workflows/build-test.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,150 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

name: MADlib CI Checks

on:
push:
branches: [ madlib2-master, github-actions ]
pull_request:
branches: [ madlib2-master, github-actions ]
# Allow manual triggers for testing
workflow_dispatch:

permissions:
contents: read
actions: read
checks: write

jobs:
build-and-test:
name: Build and Test MADlib
runs-on: ubuntu-latest

container:
image: madlib/postgres_15:jenkins
options: >-
--ulimit core=-1
--privileged

steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
persist-credentials: false
set-safe-directory: true

- name: Configure PostgreSQL
run: |
cp tool/pg_hba.conf.postgres /etc/postgresql/15/main/pg_hba.conf
echo " * soft nproc unlimited" > /etc/security/limits.d/postgres-limits.conf
service postgresql start
sleep 5
# Verify PostgreSQL started successfully and show version
echo "PostgreSQL version:" && su -s /bin/bash - postgres -c 'psql -c "SELECT version()"' || exit 1

- name: Install Dependencies
run: |
# Fix PostgreSQL APT repository issue (focal-pgdg is no longer available)
rm -f /etc/apt/sources.list.d/pgdg.list || true

apt-get update
apt-get install -y python3-pip openjdk-11-jre-headless
# Install Python packages (pypmml requires Java to be installed first)
pip install mock pandas numpy xgboost scikit-learn pyyaml pyxb-x pypmml

- name: Build MADlib
run: |
mkdir -p logs
rm -rf build && mkdir -p build
cd build
echo "---------- CMake Configuration -----------"
cmake .. 2>&1 | tee ../logs/madlib_compile.log
echo "---------- Make Clean -----------"
make clean 2>&1 | tee -a ../logs/madlib_compile.log
echo "---------- Make Build (first pass) -----------"
make -j$(nproc) 2>&1 | tee -a ../logs/madlib_compile.log
echo "---------- Make Build (second pass) -----------"
make -j$(nproc) 2>&1 | tee -a ../logs/madlib_compile.log
echo "---------- Make Install -----------"
make install 2>&1 | tee -a ../logs/madlib_compile.log
echo "---------- Make Package -----------"
make package 2>&1 | tee -a ../logs/madlib_compile.log
chown -R postgres:postgres . || true

- name: Run Tests
run: |
export PATH=$PATH:/usr/lib/postgresql/15/bin/
WORKSPACE=$(pwd)

echo "---------- Installing MADlib -----------"
# Install MADlib as postgres user
# Note: Use absolute path because 'su -' resets working directory
su -s /bin/bash - postgres -c "export PATH=\$PATH:/usr/lib/postgresql/15/bin/; cd $WORKSPACE/build; $WORKSPACE/build/src/bin/madpack -s mad -p postgres -c postgres/postgres@localhost:5432/postgres install" 2>&1 | tee $WORKSPACE/logs/madlib_install.log

mkdir -p /tmp

echo "---------- Removing known problematic test files -----------"
# Remove known problematic test files from BUILD directory (as done in jenkins_build.sh)
# These tests are known to fail in the Docker environment
rm -rf $WORKSPACE/build/src/ports/postgres/modules/deep_learning/test || true
rm -rf $WORKSPACE/build/src/ports/postgres/15/modules/deep_learning/test || true
rm -rf $WORKSPACE/build/src/ports/postgres/modules/linalg/test/linalg.sql_in || true
rm -rf $WORKSPACE/build/src/ports/postgres/modules/prob/test/prob.sql_in || true
rm -rf $WORKSPACE/build/src/ports/postgres/modules/stats/test/cox_prop_hazards.sql_in || true
rm -rf $WORKSPACE/build/src/ports/postgres/modules/utilities/test/path.sql_in || true
rm -rf $WORKSPACE/build/src/ports/postgres/modules/mxgboost/test/madlib_xgboost.sql_in || true
rm -rf $WORKSPACE/build/src/ports/postgres/modules/kmeans/test/kmeans.sql_in || true

echo "---------- Running dev-check -----------"
# Run dev-check and unit tests as postgres user
# Note: These commands will fail the workflow if tests fail (no || true)
su -s /bin/bash - postgres -c "cd $WORKSPACE/build; $WORKSPACE/build/src/bin/madpack -s mad -p postgres -c postgres/postgres@localhost:5432/postgres -d /tmp dev-check" 2>&1 | tee $WORKSPACE/logs/madlib_dev_check.log

echo "---------- Running unit tests -----------"
su -s /bin/bash - postgres -c "cd $WORKSPACE/build; $WORKSPACE/build/src/bin/madpack -s mad -p postgres -c postgres/postgres@localhost:5432/postgres -d /tmp unit-test" 2>&1 | tee -a $WORKSPACE/logs/madlib_dev_check.log

- name: Process Test Results
if: always()
run: |
echo "---------- Converting test results to JUnit format -----------"
WORKSPACE=$(pwd)
python3 tool/jenkins/junit_export.py . $WORKSPACE/logs/madlib_dev_check.log $WORKSPACE/logs/madlib_dev_check.xml

- name: Publish Test Results
uses: EnricoMi/publish-unit-test-result-action@v2
if: always()
with:
files: '${{ github.workspace }}/logs/madlib_dev_check.xml'
check_name: '🧪 MADlib Test Results'
comment_mode: 'always'

- name: Upload Build Artifacts
if: always()
uses: actions/upload-artifact@v4
with:
name: madlib-packages
path: build/*.deb
if-no-files-found: warn

- name: Upload Build Logs
if: always()
uses: actions/upload-artifact@v4
with:
name: madlib-build-logs
path: logs/
if-no-files-found: warn
117 changes: 117 additions & 0 deletions .github/workflows/license-check.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

name: License Check

on:
push:
branches: [ madlib2-master, github-actions ]
pull_request:
branches: [ madlib2-master, github-actions ]
# Allow manual triggers for compliance verification
workflow_dispatch:

jobs:
rat-check:
name: Apache RAT License Check
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v4
with:
fetch-depth: 1

- name: Set up JDK and Maven
uses: actions/setup-java@v3
with:
java-version: '11'
distribution: 'temurin'
cache: 'maven'

- name: Run Apache RAT Check
run: |
echo "Running Apache Rat license check..."
mvn apache-rat:check | tee rat-output.log

if grep -q "\[INFO\] BUILD FAILURE" rat-output.log; then
echo "::error::Apache Rat check failed - build failure detected"
exit 1
fi

- name: Run NOTICE Year and binary file checks
run: |
set -o pipefail
chmod +x ./tool/jenkins/rat_check.sh
./tool/jenkins/rat_check.sh 2>&1 | tee license_checks.txt

- name: Upload Rat check results
if: always()
uses: actions/upload-artifact@v4
with:
name: rat-check-results
path: |
rat-output.log
license_checks.txt
retention-days: 7

- name: Generate Summary
if: always()
run: |
# Check RAT results
if [[ -f rat-output.log ]] && grep -q "\[INFO\] BUILD SUCCESS" rat-output.log; then
echo "### ✅ License Check Passed" >> "$GITHUB_STEP_SUMMARY"
echo "All files comply with Apache License requirements." >> "$GITHUB_STEP_SUMMARY"
elif [[ -f rat-output.log ]]; then
echo "### ❌ License Check Failed" >> "$GITHUB_STEP_SUMMARY"
echo "" >> "$GITHUB_STEP_SUMMARY"

if grep -q "Files with unapproved licenses:" rat-output.log; then
echo "**Files with unapproved licenses:**" >> "$GITHUB_STEP_SUMMARY"
echo "" >> "$GITHUB_STEP_SUMMARY"
sed -n '/Files with unapproved licenses:/,/\[INFO\] ------------------------------------------------------------------------/p' rat-output.log | \
grep -v "\[INFO\] ------------------------------------------------------------------------" | \
grep -v "^$" | \
head -20 | \
sed 's/^/- /' >> "$GITHUB_STEP_SUMMARY"
echo "" >> "$GITHUB_STEP_SUMMARY"
fi

if grep -q "Rat check: Summary over all files" rat-output.log; then
echo "**Summary:**" >> "$GITHUB_STEP_SUMMARY"
grep "Rat check: Summary over all files" rat-output.log | sed 's/\[INFO\] //' >> "$GITHUB_STEP_SUMMARY"
fi
else
echo "### ⚠️ No RAT Output Log Found" >> "$GITHUB_STEP_SUMMARY"
fi

# Check additional checks results
echo "" >> "$GITHUB_STEP_SUMMARY"
if [[ -f license_checks.txt ]]; then
if grep -q "Error" license_checks.txt || grep -q "FAILED" license_checks.txt; then
echo "### ❌ Additional Checks Failed" >> "$GITHUB_STEP_SUMMARY"
echo "" >> "$GITHUB_STEP_SUMMARY"
echo "**Check output:**" >> "$GITHUB_STEP_SUMMARY"
echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY"
cat license_checks.txt >> "$GITHUB_STEP_SUMMARY"
echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY"
else
echo "### ✅ Additional Checks Passed" >> "$GITHUB_STEP_SUMMARY"
echo "- NOTICE file year is current" >> "$GITHUB_STEP_SUMMARY"
echo "- Version numbers match" >> "$GITHUB_STEP_SUMMARY"
echo "- No unexpected binary files found" >> "$GITHUB_STEP_SUMMARY"
fi
fi
0 Jenkinsfile → Jenkinsfile.deprecated
View file
Open in desktop
File renamed without changes.
4 changes: 2 additions & 2 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<version>0.11</version>
<version>0.16.1</version>
<configuration>
<excludes>
<!-- The following comes from BSD licensed snowballstem.org project -->
Expand Down Expand Up @@ -692,7 +692,7 @@
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<version>0.11</version>
<version>0.16.1</version>
<executions>
<execution>
<phase>verify</phase>
Expand Down