Thanks to visit codestin.com
Credit goes to github.com

Skip to content

Commit ac48f57

Browse files
kouraulcd
kou
authored and
raulcd
committed
GH-38295: [CI][R] Free up disk space for Azure Pipelines jobs (#38302)
### Rationale for this change test-r-rhub-ubuntu-gcc-release-latest doesn't have enough disk space. ### What changes are included in this PR? Remove pre-installed files on Azure Pipelines too. ### Are these changes tested? Yes. ### Are there any user-facing changes? No. * Closes: #38295 Authored-by: Sutou Kouhei <[email protected]> Signed-off-by: Raúl Cumplido <[email protected]>
1 parent f208e7f commit ac48f57

File tree

3 files changed

+130
-132
lines changed

3 files changed

+130
-132
lines changed

‎ci/scripts/util_free_space.sh‎

Lines changed: 76 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -19,81 +19,79 @@
1919

2020
set -eux
2121

22-
if [ "${GITHUB_ACTIONS}" = "true" ]; then
23-
df -h
24-
echo "::group::/usr/local/*"
25-
du -hsc /usr/local/*
26-
echo "::endgroup::"
27-
# ~1GB
28-
sudo rm -rf \
29-
/usr/local/aws-cli \
30-
/usr/local/aws-sam-cil \
31-
/usr/local/julia* || :
32-
echo "::group::/usr/local/bin/*"
33-
du -hsc /usr/local/bin/*
34-
echo "::endgroup::"
35-
# ~1GB (From 1.2GB to 214MB)
36-
sudo rm -rf \
37-
/usr/local/bin/aliyun \
38-
/usr/local/bin/aws \
39-
/usr/local/bin/aws_completer \
40-
/usr/local/bin/azcopy \
41-
/usr/local/bin/bicep \
42-
/usr/local/bin/cmake-gui \
43-
/usr/local/bin/cpack \
44-
/usr/local/bin/helm \
45-
/usr/local/bin/hub \
46-
/usr/local/bin/kubectl \
47-
/usr/local/bin/minikube \
48-
/usr/local/bin/node \
49-
/usr/local/bin/packer \
50-
/usr/local/bin/pulumi* \
51-
/usr/local/bin/sam \
52-
/usr/local/bin/stack \
53-
/usr/local/bin/terraform || :
54-
# 142M
55-
sudo rm -rf /usr/local/bin/oc || : \
56-
echo "::group::/usr/local/share/*"
57-
du -hsc /usr/local/share/*
58-
echo "::endgroup::"
59-
# 506MB
60-
sudo rm -rf /usr/local/share/chromium || :
61-
# 1.3GB
62-
sudo rm -rf /usr/local/share/powershell || :
63-
echo "::group::/usr/local/lib/*"
64-
du -hsc /usr/local/lib/*
65-
echo "::endgroup::"
66-
# 15GB
67-
sudo rm -rf /usr/local/lib/android || :
68-
# 341MB
69-
sudo rm -rf /usr/local/lib/heroku || :
70-
# 1.2GB
71-
sudo rm -rf /usr/local/lib/node_modules || :
72-
echo "::group::/opt/*"
73-
du -hsc /opt/*
74-
echo "::endgroup::"
75-
# 679MB
76-
sudo rm -rf /opt/az || :
77-
echo "::group::/opt/microsoft/*"
78-
du -hsc /opt/microsoft/*
79-
echo "::endgroup::"
80-
# 197MB
81-
sudo rm -rf /opt/microsoft/powershell || :
82-
echo "::group::/opt/hostedtoolcache/*"
83-
du -hsc /opt/hostedtoolcache/*
84-
echo "::endgroup::"
85-
# 5.3GB
86-
sudo rm -rf /opt/hostedtoolcache/CodeQL || :
87-
# 1.4GB
88-
sudo rm -rf /opt/hostedtoolcache/go || :
89-
# 489MB
90-
sudo rm -rf /opt/hostedtoolcache/PyPy || :
91-
# 376MB
92-
sudo rm -rf /opt/hostedtoolcache/node || :
93-
# Remove Web browser packages
94-
sudo apt purge -y \
95-
firefox \
96-
google-chrome-stable \
97-
microsoft-edge-stable
98-
df -h
99-
fi
22+
df -h
23+
echo "::group::/usr/local/*"
24+
du -hsc /usr/local/*
25+
echo "::endgroup::"
26+
# ~1GB
27+
sudo rm -rf \
28+
/usr/local/aws-cli \
29+
/usr/local/aws-sam-cil \
30+
/usr/local/julia* || :
31+
echo "::group::/usr/local/bin/*"
32+
du -hsc /usr/local/bin/*
33+
echo "::endgroup::"
34+
# ~1GB (From 1.2GB to 214MB)
35+
sudo rm -rf \
36+
/usr/local/bin/aliyun \
37+
/usr/local/bin/aws \
38+
/usr/local/bin/aws_completer \
39+
/usr/local/bin/azcopy \
40+
/usr/local/bin/bicep \
41+
/usr/local/bin/cmake-gui \
42+
/usr/local/bin/cpack \
43+
/usr/local/bin/helm \
44+
/usr/local/bin/hub \
45+
/usr/local/bin/kubectl \
46+
/usr/local/bin/minikube \
47+
/usr/local/bin/node \
48+
/usr/local/bin/packer \
49+
/usr/local/bin/pulumi* \
50+
/usr/local/bin/sam \
51+
/usr/local/bin/stack \
52+
/usr/local/bin/terraform || :
53+
# 142M
54+
sudo rm -rf /usr/local/bin/oc || : \
55+
echo "::group::/usr/local/share/*"
56+
du -hsc /usr/local/share/*
57+
echo "::endgroup::"
58+
# 506MB
59+
sudo rm -rf /usr/local/share/chromium || :
60+
# 1.3GB
61+
sudo rm -rf /usr/local/share/powershell || :
62+
echo "::group::/usr/local/lib/*"
63+
du -hsc /usr/local/lib/*
64+
echo "::endgroup::"
65+
# 15GB
66+
sudo rm -rf /usr/local/lib/android || :
67+
# 341MB
68+
sudo rm -rf /usr/local/lib/heroku || :
69+
# 1.2GB
70+
sudo rm -rf /usr/local/lib/node_modules || :
71+
echo "::group::/opt/*"
72+
du -hsc /opt/*
73+
echo "::endgroup::"
74+
# 679MB
75+
sudo rm -rf /opt/az || :
76+
echo "::group::/opt/microsoft/*"
77+
du -hsc /opt/microsoft/*
78+
echo "::endgroup::"
79+
# 197MB
80+
sudo rm -rf /opt/microsoft/powershell || :
81+
echo "::group::/opt/hostedtoolcache/*"
82+
du -hsc /opt/hostedtoolcache/*
83+
echo "::endgroup::"
84+
# 5.3GB
85+
sudo rm -rf /opt/hostedtoolcache/CodeQL || :
86+
# 1.4GB
87+
sudo rm -rf /opt/hostedtoolcache/go || :
88+
# 489MB
89+
sudo rm -rf /opt/hostedtoolcache/PyPy || :
90+
# 376MB
91+
sudo rm -rf /opt/hostedtoolcache/node || :
92+
# Remove Web browser packages
93+
sudo apt purge -y \
94+
firefox \
95+
google-chrome-stable \
96+
microsoft-edge-stable
97+
df -h

‎dev/tasks/macros.jinja‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -182,6 +182,11 @@ on:
182182
displayName: Clone arrow
183183
{% endmacro %}
184184

185+
{%- macro azure_free_space() -%}
186+
- script: arrow/ci/scripts/util_free_space.sh
187+
displayName: Free up disk space
188+
{% endmacro %}
189+
185190
{%- macro azure_upload_releases(pattern) -%}
186191
- task: UsePythonVersion@0
187192
inputs:

‎dev/tasks/r/azure.linux.yml‎

Lines changed: 49 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -16,62 +16,57 @@
1616
{% import 'macros.jinja' as macros with context %}
1717

1818
jobs:
19-
- job: linux
20-
pool:
21-
vmImage: ubuntu-latest
22-
timeoutInMinutes: 360
23-
{% if env is defined %}
24-
variables:
19+
- job: linux
20+
pool:
21+
vmImage: ubuntu-latest
22+
timeoutInMinutes: 360
23+
{% if env is defined %}
24+
variables:
2525
{% for key, value in env.items() %}
26-
{{ key }}: {{ value }}
26+
{{ key }}: {{ value }}
2727
{% endfor %}
28-
{% endif %}
29-
steps:
30-
- script: |
31-
set -ex
32-
git clone --no-checkout {{ arrow.remote }} arrow
33-
git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
34-
git -C arrow checkout FETCH_HEAD
35-
git -C arrow submodule update --init --recursive
36-
displayName: Clone arrow
28+
{% endif %}
29+
steps:
30+
{{ macros.azure_checkout_arrow() }}
31+
{{ macros.azure_free_space() }}
3732

38-
- script: |
39-
set -ex
40-
docker -v
41-
docker-compose -v
42-
cd arrow
43-
export R_ORG={{ r_org }}
44-
export R_IMAGE={{ r_image }}
45-
export R_TAG={{ r_tag }}
46-
export DEVTOOLSET_VERSION={{ devtoolset_version|default("") }}
47-
export R_CUSTOM_CCACHE={{ r_custom_ccache|default("false") }}
48-
docker-compose pull --ignore-pull-failures r
49-
docker-compose build r
50-
displayName: Docker build
51-
env:
52-
{{ macros.azure_set_sccache_envvars()|indent(8) }}
33+
- script: |
34+
set -ex
35+
docker -v
36+
docker-compose -v
37+
cd arrow
38+
export R_ORG={{ r_org }}
39+
export R_IMAGE={{ r_image }}
40+
export R_TAG={{ r_tag }}
41+
export DEVTOOLSET_VERSION={{ devtoolset_version|default("") }}
42+
export R_CUSTOM_CCACHE={{ r_custom_ccache|default("false") }}
43+
docker-compose pull --ignore-pull-failures r
44+
docker-compose build r
45+
displayName: Docker build
46+
env:
47+
{{ macros.azure_set_sccache_envvars()|indent(4) }}
5348
54-
- script: |
55-
set -ex
56-
cd arrow
57-
export R_ORG={{ r_org }}
58-
export R_IMAGE={{ r_image }}
59-
export R_TAG={{ r_tag }}
60-
export ARROW_R_DEV={{ not_cran|default("TRUE") }}
61-
# Note that by default, ci/scripts/r_test.sh sets NOT_CRAN=true
62-
# if ARROW_R_DEV=TRUE. Pass `-e NOT_CRAN=false` to turn that off.
63-
docker-compose run {{ flags|default("") }} r
64-
displayName: Docker run
65-
env:
66-
{{ macros.azure_set_sccache_envvars()|indent(8) }}
49+
- script: |
50+
set -ex
51+
cd arrow
52+
export R_ORG={{ r_org }}
53+
export R_IMAGE={{ r_image }}
54+
export R_TAG={{ r_tag }}
55+
export ARROW_R_DEV={{ not_cran|default("TRUE") }}
56+
# Note that by default, ci/scripts/r_test.sh sets NOT_CRAN=true
57+
# if ARROW_R_DEV=TRUE. Pass `-e NOT_CRAN=false` to turn that off.
58+
docker-compose run {{ flags|default("") }} r
59+
displayName: Docker run
60+
env:
61+
{{ macros.azure_set_sccache_envvars()|indent(4) }}
6762
68-
- script: |
69-
set -ex
70-
cat arrow/r/check/arrow.Rcheck/00install.out
71-
displayName: Dump install logs
72-
condition: succeededOrFailed()
73-
- script: |
74-
set -ex
75-
cat arrow/r/check/arrow.Rcheck/tests/testthat.Rout*
76-
displayName: Dump test logs
77-
condition: succeededOrFailed()
63+
- script: |
64+
set -ex
65+
cat arrow/r/check/arrow.Rcheck/00install.out
66+
displayName: Dump install logs
67+
condition: succeededOrFailed()
68+
- script: |
69+
set -ex
70+
cat arrow/r/check/arrow.Rcheck/tests/testthat.Rout*
71+
displayName: Dump test logs
72+
condition: succeededOrFailed()

0 commit comments

Comments
 (0)