Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit 2e0c098

Browse files
nealrichardsonkou
nealrichardson
and
kou
committed
ARROW-7675: [R][CI] Move Windows CI from Appveyor to GHA
This also includes https://issues.apache.org/jira/browse/ARROW-7202 Closes apache#6581 from nealrichardson/r-windows-gha Lead-authored-by: Neal Richardson <[email protected]> Co-authored-by: Sutou Kouhei <[email protected]> Signed-off-by: Sutou Kouhei <[email protected]>
1 parent 95af148 commit 2e0c098

9 files changed

Lines changed: 166 additions & 227 deletions

File tree

‎.github/workflows/r.yml‎

Lines changed: 53 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,21 @@ on:
2121
push:
2222
paths:
2323
- '.github/workflows/r.yml'
24-
- 'ci/**'
24+
- 'ci/scripts/r_*.sh'
25+
- 'ci/scripts/cpp_*.sh'
26+
- 'ci/scripts/PKGBUILD'
27+
- 'ci/etc/rprofile'
28+
- 'ci/docker/**'
2529
- 'cpp/**'
2630
- 'r/**'
2731
pull_request:
2832
paths:
2933
- '.github/workflows/r.yml'
30-
- 'ci/**'
34+
- 'ci/scripts/r_*.sh'
35+
- 'ci/scripts/cpp_*.sh'
36+
- 'ci/scripts/PKGBUILD'
37+
- 'ci/etc/rprofile'
38+
- 'ci/docker/**'
3139
- 'cpp/**'
3240
- 'r/**'
3341

@@ -110,3 +118,46 @@ jobs:
110118
docker login -u ${{ secrets.DOCKERHUB_USER }} \
111119
-p ${{ secrets.DOCKERHUB_TOKEN }}
112120
docker-compose push r
121+
122+
windows:
123+
name: AMD64 Windows RTools ${{ matrix.rtools }}
124+
runs-on: windows-latest
125+
strategy:
126+
fail-fast: false
127+
matrix:
128+
# TODO: test on rtools40
129+
rtools: [35]
130+
env:
131+
TEST_R_WITH_ARROW: "TRUE"
132+
ARROW_R_CXXFLAGS: '-Werror'
133+
_R_CHECK_TESTS_NLINES_: 0
134+
steps:
135+
- run: git config --global core.autocrlf false
136+
- uses: numworks/setup-msys2@v1
137+
with:
138+
update: true
139+
- uses: actions/checkout@v2
140+
- uses: r-lib/actions/setup-r@master
141+
with:
142+
rtools-version: ${{ matrix.rtools }}
143+
- name: Build Arrow C++
144+
run: msys2do ci/scripts/r_windows_build.sh
145+
- name: Install R package dependencies
146+
shell: Rscript {0}
147+
run: |
148+
options(pkgType="win.binary")
149+
install.packages(c("remotes", "rcmdcheck"))
150+
remotes::install_deps("r", dependencies = TRUE)
151+
- name: Check
152+
shell: Rscript {0}
153+
run: |
154+
Sys.setenv(
155+
RWINLIB_LOCAL = file.path(Sys.getenv("GITHUB_WORKSPACE"), "libarrow.zip"),
156+
MAKEFLAGS = paste0("-j", parallel::detectCores())
157+
)
158+
rcmdcheck::rcmdcheck("r",
159+
build_args = '--no-build-vignettes',
160+
args = c('--no-manual', '--as-cran', '--ignore-vignettes', '--run-donttest'),
161+
error_on = 'warning',
162+
check_dir = 'check'
163+
)

‎LICENSE.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1765,7 +1765,7 @@ preceding version 3 has the following license:
17651765

17661766
This project includes code from the rtools-backports project.
17671767

1768-
* ci/PKGBUILD and ci/appveyor-build-r.sh are based on code
1768+
* ci/scripts/PKGBUILD and ci/scripts/r_windows_build.sh are based on code
17691769
from the rtools-backports project.
17701770

17711771
Copyright: Copyright (c) 2013 - 2019, Алексей and Jeroen Ooms.

‎appveyor.yml‎

Lines changed: 4 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -19,14 +19,15 @@
1919
os: Visual Studio 2015
2020

2121
only_commits:
22-
# Skip commits not related to Python, C++, R, or Ruby
22+
# Skip commits not related to Python or C++
2323
files:
2424
- appveyor.yml
25-
- ci/
25+
- ci/appveyor*
26+
- ci/*.bat
27+
- ci/conda*
2628
- cpp/
2729
- format/
2830
- python/
29-
- r/
3031

3132
cache:
3233
- C:\Users\Appveyor\clcache1
@@ -73,13 +74,6 @@ environment:
7374
# NOTE: Since ARROW-5403 we have disabled the static CRT build
7475
# - JOB: "Static_Crt_Build"
7576
# GENERATOR: Ninja
76-
- JOB: "R with libarrow"
77-
USE_CLCACHE: false
78-
TEST_R_WITH_ARROW: "TRUE"
79-
RWINLIB_LOCAL: '%APPVEYOR_BUILD_FOLDER%\libarrow.zip'
80-
ARROW_R_CXXFLAGS: '-Werror'
81-
_R_CHECK_TESTS_NLINES_: 0
82-
WARNINGS_ARE_ERRORS: 1
8377

8478
MSVC_DEFAULT_OPTIONS: ON
8579
APPVEYOR_SAVE_CACHE_ON_ERROR: true
@@ -97,63 +91,6 @@ before_build:
9791
build_script:
9892
- call ci\appveyor-build.bat
9993

100-
for:
101-
-
102-
matrix:
103-
only:
104-
- JOB: "R with libarrow"
105-
init:
106-
ps: |
107-
$ErrorActionPreference = "Stop"
108-
Invoke-WebRequest http://raw.githubusercontent.com/krlmlr/r-appveyor/master/scripts/appveyor-tool.ps1 -OutFile "..\appveyor-tool.ps1"
109-
Import-Module '..\appveyor-tool.ps1'
110-
111-
before_build:
112-
- cd r
113-
- ps: Bootstrap
114-
- cd %APPVEYOR_BUILD_FOLDER%
115-
116-
build_script:
117-
- rmdir /s /Q C:\OpenSSL-Win32 C:\OpenSSL-Win64
118-
- C:\msys64\usr\bin\pacman --noconfirm --ask 20 --sync --refresh --refresh --sysupgrade --sysupgrade
119-
- C:\msys64\usr\bin\bash --login -c "$(cygpath ${APPVEYOR_BUILD_FOLDER})/ci/appveyor-build-r.sh"
120-
- pushd r
121-
- travis-tool.sh install_deps
122-
123-
test_script:
124-
- travis-tool.sh run_tests
125-
126-
on_failure:
127-
- travis-tool.sh dump_logs
128-
129-
environment:
130-
NOT_CRAN: true
131-
132-
artifacts:
133-
- path: '*.pkg.tar.xz'
134-
name: binaries
135-
136-
- path: 'build\*.zip'
137-
name: binaries
138-
139-
- path: 'r\*.Rcheck\**\*.log'
140-
name: Logs
141-
142-
- path: 'r\*.Rcheck\**\*.out'
143-
name: Logs
144-
145-
- path: 'r\*.Rcheck\**\*.fail'
146-
name: Logs
147-
148-
- path: 'r\*.Rcheck\**\*.Rout'
149-
name: Logs
150-
151-
- path: 'r\*_*.tar.gz'
152-
name: Bits
153-
154-
- path: 'r\*_*.zip'
155-
name: Bits
156-
15794
# Disable test discovery
15895
test: off
15996

‎ci/appveyor-build-r.sh‎

Lines changed: 0 additions & 51 deletions
This file was deleted.

ci/PKGBUILD renamed to ci/scripts/PKGBUILD

Lines changed: 13 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ options=("staticlibs" "strip" "!buildflags")
3636

3737
# For installing from a local checkout, set source_dir to . and don't include
3838
# a "source" param below
39-
source_dir="$APPVEYOR_BUILD_FOLDER"
39+
source_dir="$GITHUB_WORKSPACE"
4040
# else
4141
# source_dir=apache-${_realname}-${pkgver}
4242

@@ -49,7 +49,6 @@ source_dir="$APPVEYOR_BUILD_FOLDER"
4949
# sha256sums=("SKIP")
5050
# source_dir="${APPVEYOR_BUILD_FOLDER}/${source_dir}"
5151

52-
cmake_build_type=release
5352
cpp_build_dir=build-${CARCH}-cpp
5453

5554
pkgver() {
@@ -76,28 +75,28 @@ build() {
7675
${MINGW_PREFIX}/bin/cmake.exe \
7776
${ARROW_CPP_DIR} \
7877
-G "MSYS Makefiles" \
79-
-DCMAKE_INSTALL_PREFIX=${MINGW_PREFIX} \
80-
-DCMAKE_BUILD_TYPE=${cmake_build_type} \
81-
-DARROW_BUILD_STATIC=ON \
78+
-DARROW_BOOST_USE_SHARED=OFF \
8279
-DARROW_BUILD_SHARED=OFF \
80+
-DARROW_BUILD_STATIC=ON \
81+
-DARROW_BUILD_UTILITIES=OFF \
8382
-DARROW_COMPUTE=ON \
8483
-DARROW_CSV=ON \
8584
-DARROW_DATASET=ON \
8685
-DARROW_FILESYSTEM=ON \
87-
-DARROW_JSON=ON \
88-
-DARROW_PARQUET=ON \
8986
-DARROW_HDFS=OFF \
90-
-DARROW_BOOST_USE_SHARED=OFF \
9187
-DARROW_JEMALLOC=OFF \
92-
-DARROW_WITH_ZLIB=ON \
93-
-DARROW_WITH_SNAPPY=ON \
88+
-DARROW_JSON=ON \
89+
-DARROW_PACKAGE_PREFIX="${MINGW_PREFIX}" \
90+
-DARROW_PARQUET=ON \
91+
-DARROW_USE_GLOG=OFF \
9492
-DARROW_WITH_LZ4=ON \
93+
-DARROW_WITH_SNAPPY=ON \
94+
-DARROW_WITH_ZLIB=ON \
9595
-DARROW_WITH_ZSTD=ON \
96-
-DARROW_USE_GLOG=OFF \
97-
-DARROW_BUILD_UTILITIES=OFF \
98-
-DThrift_ROOT="${MINGW_PREFIX}"
96+
-DCMAKE_BUILD_TYPE="release" \
97+
-DCMAKE_INSTALL_PREFIX=${MINGW_PREFIX}
9998

100-
make
99+
make -j2
101100
popd
102101
}
103102

‎ci/scripts/r_windows_build.sh‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
#!/bin/bash
2+
3+
# Licensed to the Apache Software Foundation (ASF) under one
4+
# or more contributor license agreements. See the NOTICE file
5+
# distributed with this work for additional information
6+
# regarding copyright ownership. The ASF licenses this file
7+
# to you under the Apache License, Version 2.0 (the
8+
# "License"); you may not use this file except in compliance
9+
# with the License. You may obtain a copy of the License at
10+
#
11+
# http://www.apache.org/licenses/LICENSE-2.0
12+
#
13+
# Unless required by applicable law or agreed to in writing,
14+
# software distributed under the License is distributed on an
15+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
16+
# KIND, either express or implied. See the License for the
17+
# specific language governing permissions and limitations
18+
# under the License.
19+
20+
set -x
21+
22+
pacman --sync --noconfirm ccache
23+
24+
wget https://raw.githubusercontent.com/r-windows/rtools-backports/master/pacman.conf
25+
cp -f pacman.conf /etc/pacman.conf
26+
27+
pacman --noconfirm -Scc
28+
pacman --noconfirm -Syyu
29+
pacman --noconfirm --needed -S git base-devel binutils zip
30+
31+
# Install core build stuff
32+
pacman --noconfirm --needed -S mingw-w64-{i686,x86_64}-{toolchain,crt,winpthreads,gcc,libtre,pkg-config,xz}
33+
34+
# Force static linking
35+
rm -f /mingw32/lib/*.dll.a
36+
rm -f /mingw64/lib/*.dll.a
37+
export PKG_CONFIG="/${MINGW_PREFIX}/bin/pkg-config --static"
38+
39+
cp ci/scripts/PKGBUILD .
40+
export PKGEXT='.pkg.tar.xz' # pacman default changed to .zst in 2020, but keep the old ext for compat
41+
unset BOOST_ROOT
42+
printenv
43+
makepkg-mingw --noconfirm --noprogressbar --skippgpcheck --nocheck --syncdeps --cleanbuild
44+
45+
# Collect the build artifacts and make the shape of zip file that rwinlib expects
46+
ls
47+
mkdir build
48+
cp mingw* build
49+
cd build
50+
51+
# This may vary by system/CI provider
52+
MSYS_LIB_DIR="D:/a/_temp/msys/msys64"
53+
54+
ls $MSYS_LIB_DIR/mingw64/lib/
55+
ls $MSYS_LIB_DIR/mingw32/lib/
56+
57+
VERSION=$(grep Version ../r/DESCRIPTION | cut -d " " -f 2)
58+
DST_DIR="arrow-$VERSION"
59+
60+
# Untar the two builds we made
61+
ls | xargs -n 1 tar -xJf
62+
mkdir $DST_DIR
63+
# Grab the headers from one, either one is fine
64+
mv mingw64/include $DST_DIR
65+
66+
# Make the rest of the directory structure
67+
# lib-4.9.3 is for libraries compiled with gcc 4.9 (Rtools 3.5)
68+
mkdir -p $DST_DIR/lib-4.9.3/x64
69+
mkdir -p $DST_DIR/lib-4.9.3/i386
70+
# lib is for the new gcc 8 toolchain (Rtools 4.0)
71+
mkdir -p $DST_DIR/lib/x64
72+
mkdir -p $DST_DIR/lib/i386
73+
74+
# Move the 64-bit versions of libarrow into the expected location
75+
mv mingw64/lib/*.a $DST_DIR/lib-4.9.3/x64
76+
# Same for the 32-bit versions
77+
mv mingw32/lib/*.a $DST_DIR/lib-4.9.3/i386
78+
79+
# These are from https://dl.bintray.com/rtools/backports/
80+
cp $MSYS_LIB_DIR/mingw64/lib/lib{thrift,snappy,boost*}.a $DST_DIR/lib-4.9.3/x64
81+
cp $MSYS_LIB_DIR/mingw32/lib/lib{thrift,snappy,boost*}.a $DST_DIR/lib-4.9.3/i386
82+
83+
# These are from https://dl.bintray.com/rtools/mingw{32,64}/
84+
cp $MSYS_LIB_DIR/mingw64/lib/lib{zstd,lz4,crypto}.a $DST_DIR/lib/x64
85+
cp $MSYS_LIB_DIR/mingw32/lib/lib{zstd,lz4,crypto}.a $DST_DIR/lib/i386
86+
87+
# Create build artifact
88+
zip -r ${DST_DIR}.zip $DST_DIR
89+
90+
# Copy that to a file name/path that does not vary by version number so we
91+
# can easily find it in the R package tests on CI
92+
cp ${DST_DIR}.zip ../libarrow.zip

0 commit comments

Comments
 (0)