From f99f05bb54c992b1a6cc0494b0d6be077f140ea1 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Sat, 14 Jul 2018 12:22:16 +0100 Subject: [PATCH 01/39] ci: Windows PowerShell build script --- ci/build.ps1 | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 ci/build.ps1 diff --git a/ci/build.ps1 b/ci/build.ps1 new file mode 100644 index 00000000000..d01b142e80b --- /dev/null +++ b/ci/build.ps1 @@ -0,0 +1,42 @@ +Set-StrictMode -Version Latest + +$ErrorActionPreference = "Stop" +$PSDefaultParameterValues['*:ErrorAction'] = 'Stop' + +if ($Env:SOURCE_DIR) { $SourceDirectory = $Env:SOURCE_DIR } else { $SourceDirectory = Split-Path (Split-Path $MyInvocation.MyCommand.Path -Parent) -Parent } +$BuildDirectory = $(Get-Location).Path + +Write-Host "Source directory: ${SourceDirectory}" +Write-Host "Build directory: ${BuildDirectory}" +Write-Host "" +Write-Host "Operating system version:" +Get-CimInstance Win32_OperatingSystem | Select-Object Caption, Version, ServicePackMajorVersion, BuildNumber, OSArchitecture | Format-List +Write-Host "PATH:" +Write-Host "${Env:PATH}" +Write-Host "" + +Write-Host "##############################################################################" +Write-Host "## Configuring build environment" +Write-Host "##############################################################################" + +cmake $SourceDirectory -DBUILD_EXAMPLES=ON ${Env:CMAKE_OPTIONS} +if ($LastExitCode -ne 0) { [Environment]::Exit($LastExitCode) } + +Write-Host "" +Write-Host "##############################################################################" +Write-Host "## Building libgit2" +Write-Host "##############################################################################" + +cmake --build . +if ($LastExitCode -ne 0) { [Environment]::Exit($LastExitCode) } + +if ($Env:SKIP_TESTS) { exit } + +Write-Host "" +Write-Host "#######################################################################" +Write-Host "## Running default tests" +Write-Host "#######################################################################" + +ctest -V -R libgit2_clar +if ($LastExitCode -ne 0) { [Environment]::Exit($LastExitCode) } + From 8e28c09354cf709542e1b1161b511890c44d702b Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Sat, 8 Sep 2018 19:17:31 +0100 Subject: [PATCH 02/39] ci: move tests into citest.ps1 Add citest.ps1 PowerShell script to run the tests. --- ci/build.ps1 | 11 ----------- ci/test.ps1 | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 11 deletions(-) create mode 100644 ci/test.ps1 diff --git a/ci/build.ps1 b/ci/build.ps1 index d01b142e80b..c5c7c870a60 100644 --- a/ci/build.ps1 +++ b/ci/build.ps1 @@ -29,14 +29,3 @@ Write-Host "#################################################################### cmake --build . if ($LastExitCode -ne 0) { [Environment]::Exit($LastExitCode) } - -if ($Env:SKIP_TESTS) { exit } - -Write-Host "" -Write-Host "#######################################################################" -Write-Host "## Running default tests" -Write-Host "#######################################################################" - -ctest -V -R libgit2_clar -if ($LastExitCode -ne 0) { [Environment]::Exit($LastExitCode) } - diff --git a/ci/test.ps1 b/ci/test.ps1 new file mode 100644 index 00000000000..91b78269934 --- /dev/null +++ b/ci/test.ps1 @@ -0,0 +1,38 @@ +Set-StrictMode -Version Latest + +$ErrorActionPreference = "Stop" +$PSDefaultParameterValues['*:ErrorAction'] = 'Stop' + +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + +if ($Env:SKIP_TESTS) { exit } + +Write-Host "##############################################################################" +Write-Host "## Configuring test environment" +Write-Host "##############################################################################" + +Write-Host "" +Write-Host "Starting HTTP proxy..." +Invoke-WebRequest -Method GET -Uri https://github.com/ethomson/poxyproxy/releases/download/v0.1.0/poxyproxy-0.1.0.jar -OutFile poxyproxy.jar +javaw -jar poxyproxy.jar -d --port 8080 --credentials foo:bar + +Write-Host "" +Write-Host "##############################################################################" +Write-Host "## Running default tests" +Write-Host "##############################################################################" + +ctest -V -R libgit2_clar +if ($LastExitCode -ne 0) { [Environment]::Exit($LastExitCode) } + +Write-Host "" +Write-Host "##############################################################################" +Write-Host "## Running proxy tests" +Write-Host "##############################################################################" + +$Env:GITTEST_REMOTE_PROXY_URL="localhost:8080" +$Env:GITTEST_REMOTE_PROXY_USER="foo" +$Env:GITTEST_REMOTE_PROXY_PASS="bar" +ctest -V -R libgit2_clar-proxy_credentials +if ($LastExitCode -ne 0) { [Environment]::Exit($LastExitCode) } + +taskkill /F /IM javaw.exe From 2c2d2fa7f2e17ac2fa3b2300866c8f4f3b7aae62 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Sat, 8 Sep 2018 19:20:36 +0100 Subject: [PATCH 03/39] ci: refactor unix ci build/test scripts --- ci/build.sh | 37 ++++++++++++++++++++ ci/test.sh | 99 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 136 insertions(+) create mode 100755 ci/build.sh create mode 100644 ci/test.sh diff --git a/ci/build.sh b/ci/build.sh new file mode 100755 index 00000000000..39d35f1b527 --- /dev/null +++ b/ci/build.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# +# Environment variables: +# +# SOURCE_DIR: Set to the directory of the libgit2 source (optional) +# If not set, it will be derived relative to this script. + +set -e + +SOURCE_DIR=${SOURCE_DIR:-$( cd "$( dirname "${BASH_SOURCE[0]}" )" && dirname $( pwd ) )} +BUILD_DIR=$(pwd) + +indent() { sed "s/^/ /"; } + +echo "Source directory: ${SOURCE_DIR}" +echo "Build directory: ${BUILD_DIR}" +echo "" +echo "Operating system version:" +uname -a 2>&1 | indent +echo "CMake version:" +cmake --version 2>&1 | indent +echo "Compiler version:" +gcc --version 2>&1 | indent +echo "" + +echo "##############################################################################" +echo "## Configuring build environment" +echo "##############################################################################" + +cmake ${SOURCE_DIR} -DBUILD_EXAMPLES=ON ${CMAKE_OPTIONS} + +echo "" +echo "##############################################################################" +echo "## Building libgit2" +echo "##############################################################################" + +cmake --build . diff --git a/ci/test.sh b/ci/test.sh new file mode 100644 index 00000000000..7245428b2dc --- /dev/null +++ b/ci/test.sh @@ -0,0 +1,99 @@ +#!/bin/sh + +set -e + +if [ -n "$SKIP_TESTS" ]; then + exit $? +fi + +# Configure the test environment; run them early so that we're certain +# that they're started by the time we need them. + +echo "################################################################################" +echo "## Configuring test environment" +echo "################################################################################" + +echo "Starting HTTP proxy..." +curl -L https://github.com/ethomson/poxyproxy/releases/download/v0.1.0/poxyproxy-0.1.0.jar >poxyproxy.jar +java -jar poxyproxy.jar -d --port 8080 --credentials foo:bar >/dev/null 2>&1 & + +echo "Starting git daemon..." +GITDAEMON_DIR=`mktemp -d ${TMPDIR}/gitdaemon.XXXXXXXX` +git init --bare "${GITDAEMON_DIR}/test.git" +git daemon --listen=localhost --export-all --enable=receive-pack --base-path="${GITDAEMON_DIR}" "${GITDAEMON_DIR}" 2>/dev/null & + +echo "Starting ssh daemon..." +HOME=`mktemp -d ${TMPDIR}/home.XXXXXXXX` +SSH_DIR="${HOME}/.ssh" +SSHD_DIR=`mktemp -d ${TMPDIR}/sshd.XXXXXXXX` +mkdir ${SSH_DIR} +cat >"${SSHD_DIR}/sshd_config" <<-EOF + Port 2222 + ListenAddress 0.0.0.0 + Protocol 2 + HostKey ${SSHD_DIR}/id_rsa + PidFile ${SSHD_DIR}/pid + RSAAuthentication yes + PasswordAuthentication yes + PubkeyAuthentication yes + ChallengeResponseAuthentication no + # Required here as sshd will simply close connection otherwise + UsePAM no +EOF +ssh-keygen -t rsa -f "${SSHD_DIR}/id_rsa" -N "" -q +/usr/sbin/sshd -f "${SSHD_DIR}/sshd_config" + +# Set up keys +ssh-keygen -t rsa -f "${SSH_DIR}/id_rsa" -N "" -q +cat "${SSH_DIR}/id_rsa.pub" >>"${SSH_DIR}/authorized_keys" +while read algorithm key comment; do + echo "[localhost]:2222 $algorithm $key" >>"${SSH_DIR}/known_hosts" +done <"${SSHD_DIR}/id_rsa.pub" + +# Get the fingerprint for localhost and remove the colons so we can parse it as +# a hex number. The Mac version is newer so it has a different output format. +if [ "$TRAVIS_OS_NAME" = "osx" ]; then + export GITTEST_REMOTE_SSH_FINGERPRINT=$(ssh-keygen -E md5 -F '[localhost]:2222' -l | tail -n 1 | cut -d ' ' -f 3 | cut -d : -f2- | tr -d :) +else + export GITTEST_REMOTE_SSH_FINGERPRINT=$(ssh-keygen -F '[localhost]:2222' -l | tail -n 1 | cut -d ' ' -f 2 | tr -d ':') +fi + +# Run the tests that do not require network connectivity. + +echo "" +echo "################################################################################" +echo "## Running default tests ##" +echo "################################################################################" + +ctest -V -R libgit2_clar + +# Run the various online tests. The "online" test suite only includes the +# default online tests that do not require additional configuration. The +# "proxy" and "ssh" test suites require further setup. + +echo "" +echo "################################################################################" +echo "## Running (online) tests ##" +echo "################################################################################" + +echo "" +echo "Running proxy tests" +echo "" + +export GITTEST_REMOTE_PROXY_URL="localhost:8080" +export GITTEST_REMOTE_PROXY_USER="foo" +export GITTEST_REMOTE_PROXY_PASS="bar" +ctest -V -R libgit2_clar-proxy_credentials + +echo "" +echo "Running ssh tests" +echo "" + +export GITTEST_REMOTE_URL="ssh://localhost:2222/$HOME/_temp/test.git" +export GITTEST_REMOTE_USER=$USER +export GITTEST_REMOTE_SSH_KEY="${SSH_DIR}/id_rsa" +export GITTEST_REMOTE_SSH_PUBKEY="${SSH_DIR}/id_rsa.pub" +export GITTEST_REMOTE_SSH_PASSPHRASE="" +ctest -V -R libgit2_clar-ssh + +kill $(cat "${SSHD_DIR}/pid") From a0d5e07c0a6ad87b61a5d72a3dc7564ee7ef8529 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Sat, 8 Sep 2018 19:21:25 +0100 Subject: [PATCH 04/39] fixup! ci: move tests into citest.ps1 --- ci/test.ps1 | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ci/test.ps1 b/ci/test.ps1 index 91b78269934..21e8bd7644f 100644 --- a/ci/test.ps1 +++ b/ci/test.ps1 @@ -26,9 +26,13 @@ if ($LastExitCode -ne 0) { [Environment]::Exit($LastExitCode) } Write-Host "" Write-Host "##############################################################################" -Write-Host "## Running proxy tests" +Write-Host "## Running online tests" Write-Host "##############################################################################" +Write-Host "" +Write-Host "Running proxy tests" +Write-Host "" + $Env:GITTEST_REMOTE_PROXY_URL="localhost:8080" $Env:GITTEST_REMOTE_PROXY_USER="foo" $Env:GITTEST_REMOTE_PROXY_PASS="bar" From ce0cdd17a419cb5d04d5dae616310a90bb2532cc Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Sat, 8 Sep 2018 19:26:22 +0100 Subject: [PATCH 05/39] ci: improved flexibility for citest.sh Refactor citest.sh to enable local testing by developers. --- ci/test.sh | 177 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 113 insertions(+), 64 deletions(-) mode change 100644 => 100755 ci/test.sh diff --git a/ci/test.sh b/ci/test.sh old mode 100644 new mode 100755 index 7245428b2dc..e8fd55f75ca --- a/ci/test.sh +++ b/ci/test.sh @@ -1,99 +1,148 @@ -#!/bin/sh +#!/usr/bin/env bash set -e if [ -n "$SKIP_TESTS" ]; then - exit $? + exit 0 fi +cleanup() { + echo "Cleaning up..." + + if [ ! -z "$GITDAEMON_DIR" -a -f "${GITDAEMON_DIR}/pid" ]; then + kill $(cat "${GITDAEMON_DIR}/pid") + fi + + if [ ! -z "$SSHD_DIR" -a -f "${SSHD_DIR}/pid" ]; then + kill $(cat "${SSHD_DIR}/pid") + fi +} + +die() { + cleanup + exit $1 +} + +TMPDIR=${TMPDIR:-/tmp} +USER=${USER:-$(whoami)} + # Configure the test environment; run them early so that we're certain # that they're started by the time we need them. -echo "################################################################################" +echo "##############################################################################" echo "## Configuring test environment" -echo "################################################################################" - -echo "Starting HTTP proxy..." -curl -L https://github.com/ethomson/poxyproxy/releases/download/v0.1.0/poxyproxy-0.1.0.jar >poxyproxy.jar -java -jar poxyproxy.jar -d --port 8080 --credentials foo:bar >/dev/null 2>&1 & - -echo "Starting git daemon..." -GITDAEMON_DIR=`mktemp -d ${TMPDIR}/gitdaemon.XXXXXXXX` -git init --bare "${GITDAEMON_DIR}/test.git" -git daemon --listen=localhost --export-all --enable=receive-pack --base-path="${GITDAEMON_DIR}" "${GITDAEMON_DIR}" 2>/dev/null & - -echo "Starting ssh daemon..." -HOME=`mktemp -d ${TMPDIR}/home.XXXXXXXX` -SSH_DIR="${HOME}/.ssh" -SSHD_DIR=`mktemp -d ${TMPDIR}/sshd.XXXXXXXX` -mkdir ${SSH_DIR} -cat >"${SSHD_DIR}/sshd_config" <<-EOF +echo "##############################################################################" + +if [ -z "$SKIP_GITDAEMON_TESTS" ]; then + echo "Starting git daemon..." + GITDAEMON_DIR=`mktemp -d ${TMPDIR}/gitdaemon.XXXXXXXX` + git init --bare "${GITDAEMON_DIR}/test.git" + git daemon --listen=localhost --export-all --enable=receive-pack --pid-file="${GITDAEMON_DIR}/pid" --base-path="${GITDAEMON_DIR}" "${GITDAEMON_DIR}" 2>/dev/null & +fi + +if [ -z "$SKIP_PROXY_TESTS" ]; then + echo "Starting HTTP proxy..." + curl -L https://github.com/ethomson/poxyproxy/releases/download/v0.1.0/poxyproxy-0.1.0.jar >poxyproxy.jar + java -jar poxyproxy.jar -d --port 8080 --credentials foo:bar >/dev/null 2>&1 & +fi + +if [ -z "$SKIP_SSH_TESTS" ]; then + echo "Starting ssh daemon..." + HOME=`mktemp -d ${TMPDIR}/home.XXXXXXXX` + SSHD_DIR=`mktemp -d ${TMPDIR}/sshd.XXXXXXXX` + git init --bare "${SSHD_DIR}/test.git" + cat >"${SSHD_DIR}/sshd_config" <<-EOF Port 2222 ListenAddress 0.0.0.0 Protocol 2 HostKey ${SSHD_DIR}/id_rsa PidFile ${SSHD_DIR}/pid + AuthorizedKeysFile ${HOME}/.ssh/authorized_keys + LogLevel DEBUG RSAAuthentication yes PasswordAuthentication yes PubkeyAuthentication yes ChallengeResponseAuthentication no + StrictModes no # Required here as sshd will simply close connection otherwise UsePAM no -EOF -ssh-keygen -t rsa -f "${SSHD_DIR}/id_rsa" -N "" -q -/usr/sbin/sshd -f "${SSHD_DIR}/sshd_config" - -# Set up keys -ssh-keygen -t rsa -f "${SSH_DIR}/id_rsa" -N "" -q -cat "${SSH_DIR}/id_rsa.pub" >>"${SSH_DIR}/authorized_keys" -while read algorithm key comment; do - echo "[localhost]:2222 $algorithm $key" >>"${SSH_DIR}/known_hosts" -done <"${SSHD_DIR}/id_rsa.pub" - -# Get the fingerprint for localhost and remove the colons so we can parse it as -# a hex number. The Mac version is newer so it has a different output format. -if [ "$TRAVIS_OS_NAME" = "osx" ]; then - export GITTEST_REMOTE_SSH_FINGERPRINT=$(ssh-keygen -E md5 -F '[localhost]:2222' -l | tail -n 1 | cut -d ' ' -f 3 | cut -d : -f2- | tr -d :) -else - export GITTEST_REMOTE_SSH_FINGERPRINT=$(ssh-keygen -F '[localhost]:2222' -l | tail -n 1 | cut -d ' ' -f 2 | tr -d ':') + EOF + ssh-keygen -t rsa -f "${SSHD_DIR}/id_rsa" -N "" -q + /usr/sbin/sshd -f "${SSHD_DIR}/sshd_config" -E "${SSHD_DIR}/log" + + # Set up keys + mkdir "${HOME}/.ssh" + ssh-keygen -t rsa -f "${HOME}/.ssh/id_rsa" -N "" -q + cat "${HOME}/.ssh/id_rsa.pub" >>"${HOME}/.ssh/authorized_keys" + while read algorithm key comment; do + echo "[localhost]:2222 $algorithm $key" >>"${HOME}/.ssh/known_hosts" + done <"${SSHD_DIR}/id_rsa.pub" + + # Get the fingerprint for localhost and remove the colons so we can + # parse it as a hex number. Older versions have a different output + # format. + if [[ $(ssh -V 2>&1) == OpenSSH_6* ]]; then + SSH_FINGERPRINT=$(ssh-keygen -F '[localhost]:2222' -f "${HOME}/.ssh/known_hosts" -l | tail -n 1 | cut -d ' ' -f 2 | tr -d ':') + else + SSH_FINGERPRINT=$(ssh-keygen -E md5 -F '[localhost]:2222' -f "${HOME}/.ssh/known_hosts" -l | tail -n 1 | cut -d ' ' -f 3 | cut -d : -f2- | tr -d :) + fi fi # Run the tests that do not require network connectivity. +if [ -z "$SKIP_GITDAEMON_TESTS" ]; then + export GITTEST_REMOTE_URL="git://localhost/test.git" +fi + echo "" -echo "################################################################################" -echo "## Running default tests ##" -echo "################################################################################" +echo "##############################################################################" +echo "## Running (offline) tests" +echo "##############################################################################" -ctest -V -R libgit2_clar +ctest -V -R libgit2_clar || die $? +unset GITTEST_REMOTE_URL # Run the various online tests. The "online" test suite only includes the # default online tests that do not require additional configuration. The # "proxy" and "ssh" test suites require further setup. echo "" -echo "################################################################################" -echo "## Running (online) tests ##" -echo "################################################################################" - -echo "" -echo "Running proxy tests" -echo "" - -export GITTEST_REMOTE_PROXY_URL="localhost:8080" -export GITTEST_REMOTE_PROXY_USER="foo" -export GITTEST_REMOTE_PROXY_PASS="bar" -ctest -V -R libgit2_clar-proxy_credentials - -echo "" -echo "Running ssh tests" -echo "" +echo "##############################################################################" +echo "## Running (online) tests" +echo "##############################################################################" + +if [ -z "$SKIP_PROXY_TESTS" ]; then + echo "" + echo "Running proxy tests" + echo "" + + export GITTEST_REMOTE_PROXY_URL="localhost:8080" + export GITTEST_REMOTE_PROXY_USER="foo" + export GITTEST_REMOTE_PROXY_PASS="bar" + ctest -V -R libgit2_clar-proxy_credentials || die $? + unset GITTEST_REMOTE_PROXY_URL + unset GITTEST_REMOTE_PROXY_USER + unset GITTEST_REMOTE_PROXY_PASS +fi -export GITTEST_REMOTE_URL="ssh://localhost:2222/$HOME/_temp/test.git" -export GITTEST_REMOTE_USER=$USER -export GITTEST_REMOTE_SSH_KEY="${SSH_DIR}/id_rsa" -export GITTEST_REMOTE_SSH_PUBKEY="${SSH_DIR}/id_rsa.pub" -export GITTEST_REMOTE_SSH_PASSPHRASE="" -ctest -V -R libgit2_clar-ssh +if [ -z "$SKIP_SSH_TESTS" ]; then + echo "" + echo "Running ssh tests" + echo "" + + export GITTEST_REMOTE_URL="ssh://localhost:2222/$SSHD_DIR/test.git" + export GITTEST_REMOTE_USER=$USER + export GITTEST_REMOTE_SSH_KEY="${HOME}/.ssh/id_rsa" + export GITTEST_REMOTE_SSH_PUBKEY="${HOME}/.ssh/id_rsa.pub" + export GITTEST_REMOTE_SSH_PASSPHRASE="" + export GITTEST_REMOTE_SSH_FINGERPRINT="${SSH_FINGERPRINT}" + ctest -V -R libgit2_clar-ssh || die $? + unset GITTEST_REMOTE_URL + unset GITTEST_REMOTE_USER + unset GITTEST_REMOTE_SSH_KEY + unset GITTEST_REMOTE_SSH_PUBKEY + unset GITTEST_REMOTE_SSH_PASSPHRASE + unset GITTEST_REMOTE_SSH_FINGERPRINT +fi -kill $(cat "${SSHD_DIR}/pid") +cleanup From 35377b25778633a2c2088502480f0fc3365c64ae Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Sat, 14 Jul 2018 12:34:05 +0100 Subject: [PATCH 06/39] ci: setup a linux host Sets up a linux host to prepare for a build. --- ci/setup-linux.sh | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100755 ci/setup-linux.sh diff --git a/ci/setup-linux.sh b/ci/setup-linux.sh new file mode 100755 index 00000000000..5de286668a5 --- /dev/null +++ b/ci/setup-linux.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +set -x + +apt-get update +apt-get -y install build-essential pkg-config cmake openssl libssl-dev libssh2-1-dev libcurl4-gnutls-dev openssh-server + +mkdir -p /var/run/sshd From fe31cf0224cdae2479f385b717fbaff5e7158e60 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Sat, 14 Jul 2018 12:35:02 +0100 Subject: [PATCH 07/39] ci: set up a macos host Script to set up dependencies on a macOS build system. --- ci/setup-macos.sh | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100755 ci/setup-macos.sh diff --git a/ci/setup-macos.sh b/ci/setup-macos.sh new file mode 100755 index 00000000000..a9066918590 --- /dev/null +++ b/ci/setup-macos.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +set -x + +brew update +brew install pkgconfig zlib curl openssl libssh2 From 56462cd4ebe7160b36083f0b99e67cd385aa97e1 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Sat, 14 Jul 2018 13:03:16 +0100 Subject: [PATCH 08/39] ci: scripts to setup mingw build environment --- ci/setup-mingw-amd64.ps1 | 20 ++++++++++++++++++++ ci/setup-mingw-x86.ps1 | 20 ++++++++++++++++++++ 2 files changed, 40 insertions(+) create mode 100644 ci/setup-mingw-amd64.ps1 create mode 100644 ci/setup-mingw-x86.ps1 diff --git a/ci/setup-mingw-amd64.ps1 b/ci/setup-mingw-amd64.ps1 new file mode 100644 index 00000000000..eaa670968eb --- /dev/null +++ b/ci/setup-mingw-amd64.ps1 @@ -0,0 +1,20 @@ +Set-StrictMode -Version Latest + +$ErrorActionPreference = "Stop" +$PSDefaultParameterValues['*:ErrorAction'] = 'Stop' + +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + +[Reflection.Assembly]::LoadWithPartialName("System.IO.Compression.FileSystem"); + +Write-Host "##############################################################################" +Write-Host "## Downloading mingw" +Write-Host "##############################################################################" + +$mingw_uri = "https://bintray.com/libgit2/build-dependencies/download_file?file_path=mingw-w64-x86_64-8.1.0-release-win32-seh-rt_v6-rev0.zip" +$platform = "x86_64" + +$wc = New-Object net.webclient +$wc.Downloadfile($mingw_uri, "${Env:TEMP}/mingw-${platform}.zip") + +[System.IO.Compression.ZipFile]::ExtractToDirectory("${Env:TEMP}/mingw-${platform}.zip", $Env:TEMP) diff --git a/ci/setup-mingw-x86.ps1 b/ci/setup-mingw-x86.ps1 new file mode 100644 index 00000000000..832c0f53799 --- /dev/null +++ b/ci/setup-mingw-x86.ps1 @@ -0,0 +1,20 @@ +Set-StrictMode -Version Latest + +$ErrorActionPreference = "Stop" +$PSDefaultParameterValues['*:ErrorAction'] = 'Stop' + +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 + +[Reflection.Assembly]::LoadWithPartialName("System.IO.Compression.FileSystem"); + +Write-Host "##############################################################################" +Write-Host "## Downloading mingw" +Write-Host "##############################################################################" + +$mingw_uri = "https://bintray.com/libgit2/build-dependencies/download_file?file_path=mingw-w64-i686-8.1.0-release-win32-sjlj-rt_v6-rev0.zip" +$platform = "x86" + +$wc = New-Object net.webclient +$wc.Downloadfile($mingw_uri, "${Env:TEMP}/mingw-${platform}.zip") + +[System.IO.Compression.ZipFile]::ExtractToDirectory("${Env:TEMP}/mingw-${platform}.zip", $Env:TEMP) From bce0713bccbb8fdd45ccfef2c1b7550ac3c6802d Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Sat, 14 Jul 2018 12:42:50 +0100 Subject: [PATCH 09/39] ci: introduce vsts builds --- .vsts-ci.yml | 134 +++++++++++++++++++++++++++++++++++++++++++++++++++ ci/build.sh | 1 + 2 files changed, 135 insertions(+) create mode 100644 .vsts-ci.yml diff --git a/.vsts-ci.yml b/.vsts-ci.yml new file mode 100644 index 00000000000..b461d2ec10b --- /dev/null +++ b/.vsts-ci.yml @@ -0,0 +1,134 @@ +resources: +- repo: self + +phases: +- phase: linux_trusty_openssl + displayName: 'Linux (Trusty; OpenSSL)' + queue: + name: 'Hosted Linux Preview' + steps: + - task: Docker@0 + displayName: Build + inputs: + action: 'Run an image' + imageName: 'ethomson/libgit2-trusty-openssl:latest' + volumes: | + $(Build.SourcesDirectory):/src + $(Build.BinariesDirectory):/build + workDir: '/build' + containerCommand: '/src/ci/build.sh' + detached: false + - task: Docker@0 + displayName: Test + inputs: + action: 'Run an image' + imageName: 'ethomson/libgit2-trusty-openssl:latest' + volumes: | + $(Build.SourcesDirectory):/src + $(Build.BinariesDirectory):/build + workDir: '/build' + containerCommand: '/src/ci/test.sh' + detached: false + +- phase: linux_trusty_mbedtls + displayName: 'Linux (Trusty; mbedTLS)' + queue: + name: 'Hosted Linux Preview' + steps: + - task: Docker@0 + displayName: Build + inputs: + action: 'Run an image' + imageName: 'ethomson/libgit2-trusty-mbedtls:latest' + volumes: | + $(Build.SourcesDirectory):/src + $(Build.BinariesDirectory):/build + envVars: 'CMAKE_OPTIONS=-DUSE_HTTPS=mbedTLS -DSHA1_BACKEND=mbedTLS' + workDir: '/build' + containerCommand: '/src/ci/build.sh' + detached: false + - task: Docker@0 + displayName: Test + inputs: + action: 'Run an image' + imageName: 'ethomson/libgit2-trusty-mbedtls:latest' + volumes: | + $(Build.SourcesDirectory):/src + $(Build.BinariesDirectory):/build + workDir: '/build' + containerCommand: '/src/ci/test.sh' + detached: false + +- phase: macos + displayName: 'macOS' + queue: + name: 'Hosted macOS Preview' + steps: + - bash: . '$(Build.SourcesDirectory)/ci/setup-macos.sh' + displayName: Setup + - bash: . '$(Build.SourcesDirectory)/ci/build.sh' + displayName: Build + env: + PKG_CONFIG_PATH: /usr/local/opt/openssl/lib/pkgconfig + - bash: . '$(Build.SourcesDirectory)/ci/test.sh' + displayName: Test + env: + TMPDIR: $(Agent.TempDirectory) + +- phase: windows_vs_amd64 + displayName: 'Windows (Visual Studio; amd64)' + queue: + name: Hosted + steps: + - powershell: . '$(Build.SourcesDirectory)\ci\build.ps1' + displayName: Build + env: + CMAKE_OPTIONS: -G"Visual Studio 12 2013 Win64" + - powershell: . '$(Build.SourcesDirectory)\ci\test.ps1' + displayName: Test + +- phase: windows_vs_x86 + displayName: 'Windows (Visual Studio; x86)' + queue: + name: Hosted + steps: + - powershell: . '$(Build.SourcesDirectory)\ci\build.ps1' + displayName: Build + env: + CMAKE_OPTIONS: -G"Visual Studio 12 2013" + - powershell: . '$(Build.SourcesDirectory)\ci\test.ps1' + displayName: Test + +- phase: windows_mingw_amd64 + displayName: 'Windows (MinGW; amd64)' + queue: + name: Hosted + steps: + - powershell: . '$(Build.SourcesDirectory)\ci\setup-mingw-amd64.ps1' + displayName: Setup + env: + TEMP: $(Agent.TempDirectory) + - powershell: . '$(Build.SourcesDirectory)\ci\build.ps1' + displayName: Build + env: + CMAKE_OPTIONS: -G"MinGW Makefiles" + PATH: $(Agent.TempDirectory)\mingw64\bin;C:\ProgramData\Oracle\Java\javapath;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Windows\System32\WindowsPowerShell\v1.0\;C:\Program Files (x86)\CMake\bin + - powershell: . '$(Build.SourcesDirectory)\ci\test.ps1' + displayName: Test + +- phase: windows_mingw_x86 + displayName: 'Windows (MinGW; x86)' + queue: + name: Hosted + steps: + - powershell: . '$(Build.SourcesDirectory)\ci\setup-mingw-x86.ps1' + displayName: Setup + env: + TEMP: $(Agent.TempDirectory) + - powershell: . '$(Build.SourcesDirectory)\ci\build.ps1' + displayName: Build + env: + CMAKE_OPTIONS: -G"MinGW Makefiles" + PATH: $(Agent.TempDirectory)\mingw32\bin;C:\ProgramData\Oracle\Java\javapath;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Windows\System32\WindowsPowerShell\v1.0\;C:\Program Files (x86)\CMake\bin + - powershell: . '$(Build.SourcesDirectory)\ci\test.ps1' + displayName: Test diff --git a/ci/build.sh b/ci/build.sh index 39d35f1b527..45f0748101c 100755 --- a/ci/build.sh +++ b/ci/build.sh @@ -27,6 +27,7 @@ echo "########################################################################## echo "## Configuring build environment" echo "##############################################################################" +echo cmake ${SOURCE_DIR} -DBUILD_EXAMPLES=ON ${CMAKE_OPTIONS} cmake ${SOURCE_DIR} -DBUILD_EXAMPLES=ON ${CMAKE_OPTIONS} echo "" From 5646008268dcd6ddafd8ff4ba1f84aef84d42a70 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Fri, 20 Jul 2018 19:01:55 +0100 Subject: [PATCH 10/39] ci: valgrind leak-checking --- .vsts-ci.yml | 2 ++ ci/test.sh | 30 ++++++++++++++++++++++++------ 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index b461d2ec10b..6f43ccecdc1 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -26,6 +26,7 @@ phases: volumes: | $(Build.SourcesDirectory):/src $(Build.BinariesDirectory):/build + envVars: 'LEAK_CHECK=valgrind' workDir: '/build' containerCommand: '/src/ci/test.sh' detached: false @@ -55,6 +56,7 @@ phases: volumes: | $(Build.SourcesDirectory):/src $(Build.BinariesDirectory):/build + envVars: 'LEAK_CHECK=valgrind' workDir: '/build' containerCommand: '/src/ci/test.sh' detached: false diff --git a/ci/test.sh b/ci/test.sh index e8fd55f75ca..96adc3c3491 100755 --- a/ci/test.sh +++ b/ci/test.sh @@ -6,6 +6,13 @@ if [ -n "$SKIP_TESTS" ]; then exit 0 fi +SOURCE_DIR=${SOURCE_DIR:-$( cd "$( dirname "${BASH_SOURCE[0]}" )" && dirname $( pwd ) )} +BUILD_DIR=$(pwd) +TMPDIR=${TMPDIR:-/tmp} +USER=${USER:-$(whoami)} + +VALGRIND="valgrind --leak-check=full --show-reachable=yes --error-exitcode=125 --num-callers=50 --suppressions=\"$SOURCE_DIR/libgit2_clar.supp\"" + cleanup() { echo "Cleaning up..." @@ -23,8 +30,20 @@ die() { exit $1 } -TMPDIR=${TMPDIR:-/tmp} -USER=${USER:-$(whoami)} +# Ask ctest what it would run if we were to invoke it directly. This lets us manage the +# test configuration in a single place (tests/CMakeLists.txt) instead of running clar +# here as well. But it allows us to wrap our test harness with a leak checker like valgrind. +run_test() { + TEST_CMD=$(ctest -N -V -R $1 | sed -n 's/^[0-9]*: Test command: //p') + + if [ "$LEAK_CHECK" = "valgrind" ]; then + RUNNER="$VALGRIND $TEST_CMD" + else + RUNNER="$TEST_CMD" + fi + + eval $RUNNER || die $? +} # Configure the test environment; run them early so that we're certain # that they're started by the time we need them. @@ -99,8 +118,7 @@ echo "########################################################################## echo "## Running (offline) tests" echo "##############################################################################" -ctest -V -R libgit2_clar || die $? -unset GITTEST_REMOTE_URL +run_test libgit2_clar # Run the various online tests. The "online" test suite only includes the # default online tests that do not require additional configuration. The @@ -119,7 +137,7 @@ if [ -z "$SKIP_PROXY_TESTS" ]; then export GITTEST_REMOTE_PROXY_URL="localhost:8080" export GITTEST_REMOTE_PROXY_USER="foo" export GITTEST_REMOTE_PROXY_PASS="bar" - ctest -V -R libgit2_clar-proxy_credentials || die $? + run_test libgit2_clar-proxy_credentials unset GITTEST_REMOTE_PROXY_URL unset GITTEST_REMOTE_PROXY_USER unset GITTEST_REMOTE_PROXY_PASS @@ -136,7 +154,7 @@ if [ -z "$SKIP_SSH_TESTS" ]; then export GITTEST_REMOTE_SSH_PUBKEY="${HOME}/.ssh/id_rsa.pub" export GITTEST_REMOTE_SSH_PASSPHRASE="" export GITTEST_REMOTE_SSH_FINGERPRINT="${SSH_FINGERPRINT}" - ctest -V -R libgit2_clar-ssh || die $? + run_test libgit2_clar-ssh unset GITTEST_REMOTE_URL unset GITTEST_REMOTE_USER unset GITTEST_REMOTE_SSH_KEY From 7514b2dd00c6b6e4e2086229117f7d23e278e4a8 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Fri, 20 Jul 2018 14:14:16 -0700 Subject: [PATCH 11/39] buf tests: allocate a smaller size for the oom On Linux (where we run valgrind) allocate a smaller buffer, but still an insanely large size. This will cause malloc to fail but will not cause valgrind to report a likely error with a negative-sized malloc. Keep the original buffer size on non-Linux platforms: this is well-tested on them and changing it may be problematic. On macOS, for example, using the new size causes `malloc` to print a warning to stderr. --- tests/buf/oom.c | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/tests/buf/oom.c b/tests/buf/oom.c index b9fd29cbb0a..16a03cc1a3a 100644 --- a/tests/buf/oom.c +++ b/tests/buf/oom.c @@ -1,10 +1,22 @@ #include "clar_libgit2.h" #include "buffer.h" -#if defined(GIT_ARCH_64) -#define TOOBIG 0xffffffffffffff00 +/* + * We want to use some ridiculous size that `malloc` will fail with + * but that does not otherwise interfere with testing. On Linux, choose + * a number that is large enough to fail immediately but small enough + * that valgrind doesn't believe it to erroneously be a negative number. + * On macOS, choose a number that is large enough to fail immediately + * without having libc print warnings to stderr. + */ +#if defined(GIT_ARCH_64) && defined(__linux__) +# define TOOBIG 0x0fffffffffffffff +#elif defined(__linux__) +# define TOOBIG 0x0fffffff +#elif defined(GIT_ARCH_64) +# define TOOBIG 0xffffffffffffff00 #else -#define TOOBIG 0xffffff00 +# define TOOBIG 0xffffff00 #endif /** From f44e9101e68533ff1c498fde757c07e57656bc8d Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Fri, 20 Jul 2018 17:20:15 -0700 Subject: [PATCH 12/39] ci: xcode leaks leak-checking --- ci/test.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ci/test.sh b/ci/test.sh index 96adc3c3491..ac7d385319d 100755 --- a/ci/test.sh +++ b/ci/test.sh @@ -12,6 +12,7 @@ TMPDIR=${TMPDIR:-/tmp} USER=${USER:-$(whoami)} VALGRIND="valgrind --leak-check=full --show-reachable=yes --error-exitcode=125 --num-callers=50 --suppressions=\"$SOURCE_DIR/libgit2_clar.supp\"" +LEAKS="MallocStackLogging=1 MallocScribble=1 leaks -quiet -atExit --" cleanup() { echo "Cleaning up..." @@ -38,6 +39,9 @@ run_test() { if [ "$LEAK_CHECK" = "valgrind" ]; then RUNNER="$VALGRIND $TEST_CMD" + elif [ "$LEAK_CHECK" = "leaks" ]; then + RUNNER="$LEAKS $TEST_CMD" + echo $RUNNER else RUNNER="$TEST_CMD" fi From b5f20a28db2d11cd8374b42efb345a3ab8da8d0d Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Fri, 20 Jul 2018 18:09:38 -0700 Subject: [PATCH 13/39] ci: msvc leak-checking --- .vsts-ci.yml | 4 ++-- ci/build.ps1 | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 6f43ccecdc1..57e56b717e3 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -85,7 +85,7 @@ phases: - powershell: . '$(Build.SourcesDirectory)\ci\build.ps1' displayName: Build env: - CMAKE_OPTIONS: -G"Visual Studio 12 2013 Win64" + CMAKE_OPTIONS: -DMSVC_CRTDBG=ON -G"Visual Studio 12 2013 Win64" - powershell: . '$(Build.SourcesDirectory)\ci\test.ps1' displayName: Test @@ -97,7 +97,7 @@ phases: - powershell: . '$(Build.SourcesDirectory)\ci\build.ps1' displayName: Build env: - CMAKE_OPTIONS: -G"Visual Studio 12 2013" + CMAKE_OPTIONS: -DMSVC_CRTDBG=ON -G"Visual Studio 12 2013" - powershell: . '$(Build.SourcesDirectory)\ci\test.ps1' displayName: Test diff --git a/ci/build.ps1 b/ci/build.ps1 index c5c7c870a60..159c1dd1b92 100644 --- a/ci/build.ps1 +++ b/ci/build.ps1 @@ -11,15 +11,14 @@ Write-Host "Build directory: ${BuildDirectory}" Write-Host "" Write-Host "Operating system version:" Get-CimInstance Win32_OperatingSystem | Select-Object Caption, Version, ServicePackMajorVersion, BuildNumber, OSArchitecture | Format-List -Write-Host "PATH:" -Write-Host "${Env:PATH}" +Write-Host "PATH: ${Env:PATH}" Write-Host "" Write-Host "##############################################################################" Write-Host "## Configuring build environment" Write-Host "##############################################################################" -cmake $SourceDirectory -DBUILD_EXAMPLES=ON ${Env:CMAKE_OPTIONS} +Invoke-Expression "cmake ${SourceDirectory} -DBUILD_EXAMPLES=ON ${Env:CMAKE_OPTIONS}" if ($LastExitCode -ne 0) { [Environment]::Exit($LastExitCode) } Write-Host "" From e70dc2cd735e846768a048824c7aa214fff641ae Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Fri, 20 Jul 2018 19:47:40 -0700 Subject: [PATCH 14/39] ci: enable leak checking on osx --- .vsts-ci.yml | 1 + ci/setup-macos.sh | 2 ++ script/install-deps-osx.sh | 2 ++ 3 files changed, 5 insertions(+) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 57e56b717e3..ace22f4efc0 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -76,6 +76,7 @@ phases: displayName: Test env: TMPDIR: $(Agent.TempDirectory) + LEAK_CHECK: leaks - phase: windows_vs_amd64 displayName: 'Windows (Visual Studio; amd64)' diff --git a/ci/setup-macos.sh b/ci/setup-macos.sh index a9066918590..564910e4158 100755 --- a/ci/setup-macos.sh +++ b/ci/setup-macos.sh @@ -4,3 +4,5 @@ set -x brew update brew install pkgconfig zlib curl openssl libssh2 + +ln -s /Applications/Xcode.app/Contents/Developer/usr/lib/libLeaksAtExit.dylib /usr/local/lib diff --git a/script/install-deps-osx.sh b/script/install-deps-osx.sh index 94314dbaac6..8b88f84713c 100755 --- a/script/install-deps-osx.sh +++ b/script/install-deps-osx.sh @@ -7,3 +7,5 @@ brew install zlib brew install curl brew install openssl brew install libssh2 + +ln -s /Applications/Xcode.app/Contents/Developer/usr/lib/libLeaksAtExit.dylib /usr/local/lib From cbcc9597cc112fcc142d883ac06539970825c6fa Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Sat, 21 Jul 2018 10:49:23 -0700 Subject: [PATCH 15/39] ci: some additional debugging --- ci/test.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ci/test.sh b/ci/test.sh index ac7d385319d..f07346b6f04 100755 --- a/ci/test.sh +++ b/ci/test.sh @@ -18,15 +18,21 @@ cleanup() { echo "Cleaning up..." if [ ! -z "$GITDAEMON_DIR" -a -f "${GITDAEMON_DIR}/pid" ]; then + echo "Stopping git daemon..." kill $(cat "${GITDAEMON_DIR}/pid") fi if [ ! -z "$SSHD_DIR" -a -f "${SSHD_DIR}/pid" ]; then + echo "Stopping SSH..." kill $(cat "${SSHD_DIR}/pid") fi + + echo "Done." } die() { + echo "Test exited with code: $1" + cleanup exit $1 } @@ -167,4 +173,6 @@ if [ -z "$SKIP_SSH_TESTS" ]; then unset GITTEST_REMOTE_SSH_FINGERPRINT fi +echo "Success." cleanup +exit 0 From 6f38f245a06f680691e3dfbe232521ecb8c392c9 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Wed, 25 Jul 2018 01:04:55 +0100 Subject: [PATCH 16/39] ci: dissociate test from leaks process The leaks process is not good about handling children. Ensure that its child is `nohup`ed so that the grandparent shell won't wait for it to exit. --- ci/test.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ci/test.sh b/ci/test.sh index f07346b6f04..13f19df98fb 100755 --- a/ci/test.sh +++ b/ci/test.sh @@ -12,7 +12,7 @@ TMPDIR=${TMPDIR:-/tmp} USER=${USER:-$(whoami)} VALGRIND="valgrind --leak-check=full --show-reachable=yes --error-exitcode=125 --num-callers=50 --suppressions=\"$SOURCE_DIR/libgit2_clar.supp\"" -LEAKS="MallocStackLogging=1 MallocScribble=1 leaks -quiet -atExit --" +LEAKS="MallocStackLogging=1 MallocScribble=1 leaks -quiet -atExit -- nohup" cleanup() { echo "Cleaning up..." @@ -47,7 +47,6 @@ run_test() { RUNNER="$VALGRIND $TEST_CMD" elif [ "$LEAK_CHECK" = "leaks" ]; then RUNNER="$LEAKS $TEST_CMD" - echo $RUNNER else RUNNER="$TEST_CMD" fi From bbfd6611ac98c9cb9dda1be0309066be991462c2 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Thu, 26 Jul 2018 14:04:19 +0100 Subject: [PATCH 17/39] ci: perform clang builds on Linux --- .vsts-ci.yml | 21 ++++++++++++--------- ci/build.sh | 3 ++- ci/setup-linux.sh | 2 +- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index ace22f4efc0..0bdb98d12cf 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -2,8 +2,8 @@ resources: - repo: self phases: -- phase: linux_trusty_openssl - displayName: 'Linux (Trusty; OpenSSL)' +- phase: linux_trusty_gcc_openssl + displayName: 'Linux (Trusty; GCC; OpenSSL)' queue: name: 'Hosted Linux Preview' steps: @@ -26,13 +26,15 @@ phases: volumes: | $(Build.SourcesDirectory):/src $(Build.BinariesDirectory):/build - envVars: 'LEAK_CHECK=valgrind' + envVars: | + CC=gcc + LEAK_CHECK=valgrind workDir: '/build' containerCommand: '/src/ci/test.sh' detached: false -- phase: linux_trusty_mbedtls - displayName: 'Linux (Trusty; mbedTLS)' +- phase: linux_trusty_clang_openssl + displayName: 'Linux (Trusty; Clang; OpenSSL)' queue: name: 'Hosted Linux Preview' steps: @@ -40,11 +42,10 @@ phases: displayName: Build inputs: action: 'Run an image' - imageName: 'ethomson/libgit2-trusty-mbedtls:latest' + imageName: 'ethomson/libgit2-trusty-openssl:latest' volumes: | $(Build.SourcesDirectory):/src $(Build.BinariesDirectory):/build - envVars: 'CMAKE_OPTIONS=-DUSE_HTTPS=mbedTLS -DSHA1_BACKEND=mbedTLS' workDir: '/build' containerCommand: '/src/ci/build.sh' detached: false @@ -52,11 +53,13 @@ phases: displayName: Test inputs: action: 'Run an image' - imageName: 'ethomson/libgit2-trusty-mbedtls:latest' + imageName: 'ethomson/libgit2-trusty-openssl:latest' volumes: | $(Build.SourcesDirectory):/src $(Build.BinariesDirectory):/build - envVars: 'LEAK_CHECK=valgrind' + envVars: | + CC=clang + LEAK_CHECK=valgrind workDir: '/build' containerCommand: '/src/ci/test.sh' detached: false diff --git a/ci/build.sh b/ci/build.sh index 45f0748101c..a1deab3f2d3 100755 --- a/ci/build.sh +++ b/ci/build.sh @@ -9,6 +9,7 @@ set -e SOURCE_DIR=${SOURCE_DIR:-$( cd "$( dirname "${BASH_SOURCE[0]}" )" && dirname $( pwd ) )} BUILD_DIR=$(pwd) +CC=${CC:-cc} indent() { sed "s/^/ /"; } @@ -20,7 +21,7 @@ uname -a 2>&1 | indent echo "CMake version:" cmake --version 2>&1 | indent echo "Compiler version:" -gcc --version 2>&1 | indent +$CC --version 2>&1 | indent echo "" echo "##############################################################################" diff --git a/ci/setup-linux.sh b/ci/setup-linux.sh index 5de286668a5..03e4a1d2f9d 100755 --- a/ci/setup-linux.sh +++ b/ci/setup-linux.sh @@ -3,6 +3,6 @@ set -x apt-get update -apt-get -y install build-essential pkg-config cmake openssl libssl-dev libssh2-1-dev libcurl4-gnutls-dev openssh-server +apt-get -y install build-essential pkg-config clang cmake openssl libssl-dev libssh2-1-dev libcurl4-gnutls-dev openssh-server mkdir -p /var/run/sshd From 2d3ba70503c1f3d7b15a16136f994c9477a1fbbe Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Thu, 26 Jul 2018 14:29:41 +0100 Subject: [PATCH 18/39] ci: use docker containers from libgit2 account --- .vsts-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 0bdb98d12cf..d5c55683902 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -11,7 +11,7 @@ phases: displayName: Build inputs: action: 'Run an image' - imageName: 'ethomson/libgit2-trusty-openssl:latest' + imageName: 'libgit2/trusty-openssl:latest' volumes: | $(Build.SourcesDirectory):/src $(Build.BinariesDirectory):/build @@ -22,7 +22,7 @@ phases: displayName: Test inputs: action: 'Run an image' - imageName: 'ethomson/libgit2-trusty-openssl:latest' + imageName: 'libgit2/trusty-openssl:latest' volumes: | $(Build.SourcesDirectory):/src $(Build.BinariesDirectory):/build @@ -42,7 +42,7 @@ phases: displayName: Build inputs: action: 'Run an image' - imageName: 'ethomson/libgit2-trusty-openssl:latest' + imageName: 'libgit2/trusty-openssl:latest' volumes: | $(Build.SourcesDirectory):/src $(Build.BinariesDirectory):/build @@ -53,7 +53,7 @@ phases: displayName: Test inputs: action: 'Run an image' - imageName: 'ethomson/libgit2-trusty-openssl:latest' + imageName: 'libgit2/trusty-openssl:latest' volumes: | $(Build.SourcesDirectory):/src $(Build.BinariesDirectory):/build From 8aaf715fe8e4c540af600e8db20bfd37c579f067 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Thu, 26 Jul 2018 15:06:01 +0100 Subject: [PATCH 19/39] ci: use a single setup script for mingw --- .vsts-ci.yml | 6 ++++-- ci/setup-mingw-x86.ps1 | 20 ------------------- ci/{setup-mingw-amd64.ps1 => setup-mingw.ps1} | 13 ++++++++---- 3 files changed, 13 insertions(+), 26 deletions(-) delete mode 100644 ci/setup-mingw-x86.ps1 rename ci/{setup-mingw-amd64.ps1 => setup-mingw.ps1} (57%) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index d5c55683902..633390654b7 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -110,10 +110,11 @@ phases: queue: name: Hosted steps: - - powershell: . '$(Build.SourcesDirectory)\ci\setup-mingw-amd64.ps1' + - powershell: . '$(Build.SourcesDirectory)\ci\setup-mingw.ps1' displayName: Setup env: TEMP: $(Agent.TempDirectory) + ARCH: amd64 - powershell: . '$(Build.SourcesDirectory)\ci\build.ps1' displayName: Build env: @@ -127,10 +128,11 @@ phases: queue: name: Hosted steps: - - powershell: . '$(Build.SourcesDirectory)\ci\setup-mingw-x86.ps1' + - powershell: . '$(Build.SourcesDirectory)\ci\setup-mingw.ps1' displayName: Setup env: TEMP: $(Agent.TempDirectory) + ARCH: x86 - powershell: . '$(Build.SourcesDirectory)\ci\build.ps1' displayName: Build env: diff --git a/ci/setup-mingw-x86.ps1 b/ci/setup-mingw-x86.ps1 deleted file mode 100644 index 832c0f53799..00000000000 --- a/ci/setup-mingw-x86.ps1 +++ /dev/null @@ -1,20 +0,0 @@ -Set-StrictMode -Version Latest - -$ErrorActionPreference = "Stop" -$PSDefaultParameterValues['*:ErrorAction'] = 'Stop' - -[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 - -[Reflection.Assembly]::LoadWithPartialName("System.IO.Compression.FileSystem"); - -Write-Host "##############################################################################" -Write-Host "## Downloading mingw" -Write-Host "##############################################################################" - -$mingw_uri = "https://bintray.com/libgit2/build-dependencies/download_file?file_path=mingw-w64-i686-8.1.0-release-win32-sjlj-rt_v6-rev0.zip" -$platform = "x86" - -$wc = New-Object net.webclient -$wc.Downloadfile($mingw_uri, "${Env:TEMP}/mingw-${platform}.zip") - -[System.IO.Compression.ZipFile]::ExtractToDirectory("${Env:TEMP}/mingw-${platform}.zip", $Env:TEMP) diff --git a/ci/setup-mingw-amd64.ps1 b/ci/setup-mingw.ps1 similarity index 57% rename from ci/setup-mingw-amd64.ps1 rename to ci/setup-mingw.ps1 index eaa670968eb..76ecd3987ad 100644 --- a/ci/setup-mingw-amd64.ps1 +++ b/ci/setup-mingw.ps1 @@ -11,10 +11,15 @@ Write-Host "#################################################################### Write-Host "## Downloading mingw" Write-Host "##############################################################################" -$mingw_uri = "https://bintray.com/libgit2/build-dependencies/download_file?file_path=mingw-w64-x86_64-8.1.0-release-win32-seh-rt_v6-rev0.zip" -$platform = "x86_64" +if ($env:ARCH -eq "amd64") { + $mingw_uri = "https://bintray.com/libgit2/build-dependencies/download_file?file_path=mingw-w64-x86_64-8.1.0-release-win32-seh-rt_v6-rev0.zip" + $platform = "x86_64" +} else { + $mingw_uri = "https://bintray.com/libgit2/build-dependencies/download_file?file_path=mingw-w64-i686-8.1.0-release-win32-sjlj-rt_v6-rev0.zip" + $platform = "x86" +} $wc = New-Object net.webclient -$wc.Downloadfile($mingw_uri, "${Env:TEMP}/mingw-${platform}.zip") +$wc.Downloadfile($mingw_uri, "${Env:TEMP}/mingw-${Env:ARCH}.zip") -[System.IO.Compression.ZipFile]::ExtractToDirectory("${Env:TEMP}/mingw-${platform}.zip", $Env:TEMP) +[System.IO.Compression.ZipFile]::ExtractToDirectory("${Env:TEMP}/mingw-${Env:ARCH}.zip", $Env:TEMP) From 1570b727c8b9b096e746dba106a3beda58ac00bf Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Thu, 26 Jul 2018 15:14:37 +0100 Subject: [PATCH 20/39] ci: move appveyor to new scripts --- appveyor.yml | 58 +++++++++++++++------------------------- script/appveyor-mingw.sh | 23 ---------------- 2 files changed, 21 insertions(+), 60 deletions(-) delete mode 100755 script/appveyor-mingw.sh diff --git a/appveyor.yml b/appveyor.yml index f76830cb417..d89e3fb6c90 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -10,51 +10,35 @@ environment: matrix: - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2013 - GENERATOR: "Visual Studio 10 2010" - ARCH: 32 + CMAKE_OPTIONS: -G"Visual Studio 10 2010" + ARCH: x86 - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2013 - GENERATOR: "Visual Studio 10 2010 Win64" - ARCH: 64 + CMAKE_OPTIONS: -G"Visual Studio 10 2010 Win64" + ARCH: amd64 - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015 - GENERATOR: "Visual Studio 14 2015" - ARCH: 32 + CMAKE_OPTIONS: -G"Visual Studio 14 2015" + ARCH: x86 - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015 - GENERATOR: "Visual Studio 14 2015 Win64" - ARCH: 64 + CMAKE_OPTIONS: -G"Visual Studio 14 2015 Win64" + ARCH: amd64 - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015 - GENERATOR: "MSYS Makefiles" - ARCH: i686 # this is for 32-bit MinGW-w64 + CMAKE_OPTIONS: -G"MinGW Makefiles" + ARCH: x86 - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015 - GENERATOR: "MSYS Makefiles" - ARCH: 64 -cache: -- i686-4.9.2-release-win32-sjlj-rt_v3-rev1.7z -- x86_64-4.9.2-release-win32-seh-rt_v3-rev1.7z + CMAKE_OPTIONS: -G"MinGW Makefiles" + ARCH: amd64 -build_script: +install: +- set PATH=%TEMP%\mingw64\bin;%TEMP%\mingw32\bin;C:\ProgramData\Oracle\Java\javapath;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Windows\System32\WindowsPowerShell\v1.0\;C:\Program Files (x86)\CMake\bin - ps: | mkdir build cd build - if ($env:GENERATOR -ne "MSYS Makefiles") { - cmake -D ENABLE_TRACE=ON -D BUILD_CLAR=ON -D BUILD_EXAMPLES=ON -D MSVC_CRTDBG=ON .. -G"$env:GENERATOR" - cmake --build . --config Debug + if ($env:CMAKE_OPTIONS -eq '-G"MinGW Makefiles"') { + ../ci/setup-mingw.ps1 } -- cmd: | - if "%GENERATOR%"=="MSYS Makefiles" (C:\MinGW\msys\1.0\bin\sh --login /c/projects/libgit2/script/appveyor-mingw.sh) + +build_script: +- cmd: powershell ../ci/build.ps1 + test_script: -- ps: | - # Disable DHE key exchange to fix intermittent build failures ("A buffer - # provided was too small") due to SChannel bug. See e.g. - # - https://github.com/aws/aws-sdk-cpp/issues/671 - # - https://github.com/dotnet/corefx/issues/7812 - New-Item HKLM:\SYSTEM\CurrentControlSet\Control\SecurityProviders\SCHANNEL\KeyExchangeAlgorithms\Diffie-Hellman -Force | New-ItemProperty -Name Enabled -Value 0 -Force - $ErrorActionPreference="Stop" - Start-FileDownload https://github.com/ethomson/poxyproxy/releases/download/v0.1.0/poxyproxy-0.1.0.jar -FileName poxyproxy.jar - # Run this early so we know it's ready by the time we need it - $proxyJob = Start-Job { java -jar $Env:APPVEYOR_BUILD_FOLDER\build\poxyproxy.jar -d --port 8080 --credentials foo:bar } - ctest -V -R libgit2_clar - Receive-Job -Job $proxyJob - $env:GITTEST_REMOTE_PROXY_URL = "localhost:8080" - $env:GITTEST_REMOTE_PROXY_USER = "foo" - $env:GITTEST_REMOTE_PROXY_PASS = "bar" - ctest -V -R libgit2_clar-proxy_credentials +- cmd: powershell ../ci/test.ps1 diff --git a/script/appveyor-mingw.sh b/script/appveyor-mingw.sh deleted file mode 100755 index 6b2a9425eda..00000000000 --- a/script/appveyor-mingw.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/sh -set -e -cd `dirname "$0"`/.. -if [ "$ARCH" = "i686" ]; then - f=i686-4.9.2-release-win32-sjlj-rt_v3-rev1.7z - if ! [ -e $f ]; then - curl -LsSO http://sourceforge.net/projects/mingw-w64/files/Toolchains%20targetting%20Win32/Personal%20Builds/mingw-builds/4.9.2/threads-win32/sjlj/$f - fi - 7z x $f > /dev/null - export PATH=`pwd`/mingw32/bin:$PATH -else - f=x86_64-4.9.2-release-win32-seh-rt_v3-rev1.7z - if ! [ -e $f ]; then - curl -LsSO http://sourceforge.net/projects/mingw-w64/files/Toolchains%20targetting%20Win64/Personal%20Builds/mingw-builds/4.9.2/threads-win32/seh/$f - fi - 7z x $f > /dev/null - export PATH=`pwd`/mingw64/bin:$PATH -fi -cd build -gcc --version -cmake --version -cmake -D ENABLE_TRACE=ON -D BUILD_CLAR=ON -D BUILD_EXAMPLES=ON .. -G"$GENERATOR" -cmake --build . --config RelWithDebInfo From cebffc24a5965d89d5170699a628d7b55d62bc37 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Fri, 27 Jul 2018 12:31:32 +0100 Subject: [PATCH 21/39] ci: move travis to the new scripts --- .travis.yml | 22 ++++++++++++---------- .vsts-ci.yml | 2 +- {script => ci}/coverity.sh | 5 +---- ci/setup-linux.sh | 23 +++++++++++++++++++++-- ci/{setup-macos.sh => setup-osx.sh} | 0 5 files changed, 35 insertions(+), 17 deletions(-) rename {script => ci}/coverity.sh (94%) rename ci/{setup-macos.sh => setup-osx.sh} (100%) diff --git a/.travis.yml b/.travis.yml index 3a55e86b040..31a282cf434 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,9 +16,12 @@ env: global: - secure: "YnhS+8n6B+uoyaYfaJ3Lei7cSJqHDPiKJCKFIF2c87YDfmCvAJke8QtE7IzjYDs7UFkTCM4ox+ph2bERUrxZbSCyEkHdjIZpKuMJfYWja/jgMqTMxdyOH9y8JLFbZsSXDIXDwqBlC6vVyl1fP90M35wuWcNTs6tctfVWVofEFbs=" - GITTEST_INVASIVE_FS_SIZE=1 + - MBEDTLS_DIR=/tmp/mbedtls + - SKIP_APT=1 + - SKIP_MBEDTLS_INSTALL=1 matrix: - - OPTIONS="-DTHREADSAFE=ON -DENABLE_TRACE=ON -DCMAKE_BUILD_TYPE=Release -DENABLE_WERROR=ON" - - OPTIONS="-DTHREADSAFE=OFF -DBUILD_EXAMPLES=ON -DENABLE_WERROR=ON" + - CMAKE_OPTIONS="-DTHREADSAFE=ON -DENABLE_TRACE=ON -DCMAKE_BUILD_TYPE=Release" + - CMAKE_OPTIONS="-DTHREADSAFE=OFF -DBUILD_EXAMPLES=ON" dist: trusty osx_image: xcode8.3 @@ -52,23 +55,22 @@ matrix: dist: trusty - compiler: gcc env: - - VALGRIND=1 - OPTIONS="-DBUILD_CLAR=ON -DBUILD_EXAMPLES=OFF -DDEBUG_POOL=ON -DCMAKE_BUILD_TYPE=Debug" + - LEAK_CHECK=valgrind + CMAKE_OPTIONS="-DBUILD_CLAR=ON -DBUILD_EXAMPLES=OFF -DDEBUG_POOL=ON -DCMAKE_BUILD_TYPE=Debug" os: linux dist: trusty allow_failures: - env: COVERITY=1 install: - - if [ -f ./script/install-deps-${TRAVIS_OS_NAME}.sh ]; then ./script/install-deps-${TRAVIS_OS_NAME}.sh; fi + - if [ -f ./ci/setup-${TRAVIS_OS_NAME}.sh ]; then ./ci/setup-${TRAVIS_OS_NAME}.sh; fi # Run the Build script and tests script: - - script/cibuild.sh - -# Run Tests -after_success: - - if [ "$TRAVIS_OS_NAME" = "linux" -a -n "$VALGRIND" ]; then valgrind --leak-check=full --show-reachable=yes --suppressions=./libgit2_clar.supp _build/libgit2_clar -ionline; fi + - mkdir build + - cd build + - if [ "$COVERITY" ]; then ../ci/coverity.sh; fi + - if [ -z "$COVERITY" ]; then ../ci/build.sh && ../ci/test.sh; fi # Only watch the development and master branches branches: diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 633390654b7..4879aeab85e 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -69,7 +69,7 @@ phases: queue: name: 'Hosted macOS Preview' steps: - - bash: . '$(Build.SourcesDirectory)/ci/setup-macos.sh' + - bash: . '$(Build.SourcesDirectory)/ci/setup-osx.sh' displayName: Setup - bash: . '$(Build.SourcesDirectory)/ci/build.sh' displayName: Build diff --git a/script/coverity.sh b/ci/coverity.sh similarity index 94% rename from script/coverity.sh rename to ci/coverity.sh index 5fe16c0311d..57f4111748f 100755 --- a/script/coverity.sh +++ b/ci/coverity.sh @@ -32,14 +32,11 @@ if [ ! -d "$TOOL_BASE" ]; then ln -s "$TOOL_DIR" "$TOOL_BASE"/cov-analysis fi -cp script/user_nodefs.h "$TOOL_BASE"/cov-analysis/config/user_nodefs.h +cp ../script/user_nodefs.h "$TOOL_BASE"/cov-analysis/config/user_nodefs.h COV_BUILD="$TOOL_BASE/cov-analysis/bin/cov-build" # Configure and build -rm -rf _build -mkdir _build -cd _build cmake .. -DTHREADSAFE=ON COVERITY_UNSUPPORTED=1 \ $COV_BUILD --dir cov-int \ diff --git a/ci/setup-linux.sh b/ci/setup-linux.sh index 03e4a1d2f9d..c5ecb550b88 100755 --- a/ci/setup-linux.sh +++ b/ci/setup-linux.sh @@ -1,8 +1,27 @@ #!/bin/sh +set -e set -x -apt-get update -apt-get -y install build-essential pkg-config clang cmake openssl libssl-dev libssh2-1-dev libcurl4-gnutls-dev openssh-server +TMPDIR=${TMPDIR:-/tmp} + +if [ -z "$SKIP_APT" ]; then + apt-get update + apt-get -y install build-essential pkg-config clang cmake openssl libssl-dev libssh2-1-dev libcurl4-gnutls-dev openssh-server +fi mkdir -p /var/run/sshd + +if [ "$MBEDTLS" ]; then + MBEDTLS_DIR=${MBEDTLS_DIR:-$(mktemp -d ${TMPDIR}/mbedtls.XXXXXXXX)} + + git clone --depth 10 --single-branch --branch mbedtls-2.6.1 https://github.com/ARMmbed/mbedtls.git ${MBEDTLS_DIR} + cd ${MBEDTLS_DIR} + + CFLAGS=-fPIC cmake -DENABLE_PROGRAMS=OFF -DENABLE_TESTING=OFF -DUSE_SHARED_MBEDTLS_LIBRARY=OFF -DUSE_STATIC_MBEDTLS_LIBRARY=ON . + cmake --build . + + if [ -z "$SKIP_MBEDTLS_INSTALL" ]; then + make install + fi +fi diff --git a/ci/setup-macos.sh b/ci/setup-osx.sh similarity index 100% rename from ci/setup-macos.sh rename to ci/setup-osx.sh From b82abc3d2f344317e034682a6497293223aa1e08 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Fri, 27 Jul 2018 16:40:44 +0100 Subject: [PATCH 22/39] ci: remove unused old ci scripts --- script/cibuild.sh | 102 ------------------------------------- script/install-deps-osx.sh | 11 ---- 2 files changed, 113 deletions(-) delete mode 100755 script/cibuild.sh delete mode 100755 script/install-deps-osx.sh diff --git a/script/cibuild.sh b/script/cibuild.sh deleted file mode 100755 index 5d70e750640..00000000000 --- a/script/cibuild.sh +++ /dev/null @@ -1,102 +0,0 @@ -#!/bin/sh - -set -x - -if [ -n "$COVERITY" ]; -then - ./script/coverity.sh; - exit $?; -fi - -if [ "$TRAVIS_OS_NAME" = "osx" ]; then - export PKG_CONFIG_PATH=$(ls -d /usr/local/Cellar/{curl,zlib}/*/lib/pkgconfig | paste -s -d':' -) - - # Set up a ramdisk for us to put our test data on to speed up tests on macOS - export CLAR_TMP="$HOME"/_clar_tmp - mkdir -p $CLAR_TMP - - # 5*2M sectors aka ~5GB of space - device=$(hdiutil attach -nomount ram://$((5 * 2 * 1024 * 1024))) - newfs_hfs $device - mount -t hfs $device $CLAR_TMP -fi - -# Should we ask Travis to cache this file? -curl -L https://github.com/ethomson/poxyproxy/releases/download/v0.1.0/poxyproxy-0.1.0.jar >poxyproxy.jar || exit $? -# Run this early so we know it's ready by the time we need it -java -jar poxyproxy.jar -d --port 8080 --credentials foo:bar & - -mkdir _build -cd _build -# shellcheck disable=SC2086 -cmake .. -DBUILD_EXAMPLES=ON -DCMAKE_INSTALL_PREFIX=../_install $OPTIONS || exit $? -make -j2 install || exit $? - -# If this platform doesn't support test execution, bail out now -if [ -n "$SKIP_TESTS" ]; -then - exit $?; -fi - -# Create a test repo which we can use for the online::push tests -mkdir "$HOME"/_temp -git init --bare "$HOME"/_temp/test.git -git daemon --listen=localhost --export-all --enable=receive-pack --base-path="$HOME"/_temp "$HOME"/_temp 2>/dev/null & -export GITTEST_REMOTE_URL="git://localhost/test.git" - -# Run the test suite -ctest -V -R libgit2_clar || exit $? - -# Now that we've tested the raw git protocol, let's set up ssh to we -# can do the push tests over it - -killall git-daemon - -# Set up sshd -mkdir ~/sshd/ -cat >~/sshd/sshd_config<<-EOF - Port 2222 - ListenAddress 0.0.0.0 - Protocol 2 - HostKey ${HOME}/sshd/id_rsa - PidFile ${HOME}/sshd/pid - RSAAuthentication yes - PasswordAuthentication yes - PubkeyAuthentication yes - ChallengeResponseAuthentication no - # Required here as sshd will simply close connection otherwise - UsePAM no -EOF -ssh-keygen -t rsa -f ~/sshd/id_rsa -N "" -q -/usr/sbin/sshd -f ~/sshd/sshd_config - -# Set up keys -ssh-keygen -t rsa -f ~/.ssh/id_rsa -N "" -q -cat ~/.ssh/id_rsa.pub >>~/.ssh/authorized_keys -while read algorithm key comment; do - echo "[localhost]:2222 $algorithm $key" >>~/.ssh/known_hosts -done <~/sshd/id_rsa.pub - -# Get the fingerprint for localhost and remove the colons so we can parse it as -# a hex number. The Mac version is newer so it has a different output format. -if [ "$TRAVIS_OS_NAME" = "osx" ]; then - export GITTEST_REMOTE_SSH_FINGERPRINT=$(ssh-keygen -E md5 -F '[localhost]:2222' -l | tail -n 1 | cut -d ' ' -f 3 | cut -d : -f2- | tr -d :) -else - export GITTEST_REMOTE_SSH_FINGERPRINT=$(ssh-keygen -F '[localhost]:2222' -l | tail -n 1 | cut -d ' ' -f 2 | tr -d ':') -fi - -# Use the SSH server -export GITTEST_REMOTE_URL="ssh://localhost:2222/$HOME/_temp/test.git" -export GITTEST_REMOTE_USER=$USER -export GITTEST_REMOTE_SSH_KEY="$HOME/.ssh/id_rsa" -export GITTEST_REMOTE_SSH_PUBKEY="$HOME/.ssh/id_rsa.pub" -export GITTEST_REMOTE_SSH_PASSPHRASE="" -ctest -V -R libgit2_clar-ssh || exit $? - -# Use the proxy we started at the beginning -export GITTEST_REMOTE_PROXY_URL="localhost:8080" -export GITTEST_REMOTE_PROXY_USER="foo" -export GITTEST_REMOTE_PROXY_PASS="bar" -ctest -V -R libgit2_clar-proxy_credentials || exit $? - -kill $(cat "$HOME/sshd/pid") diff --git a/script/install-deps-osx.sh b/script/install-deps-osx.sh deleted file mode 100755 index 8b88f84713c..00000000000 --- a/script/install-deps-osx.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh - -set -x - -brew update -brew install zlib -brew install curl -brew install openssl -brew install libssh2 - -ln -s /Applications/Xcode.app/Contents/Developer/usr/lib/libLeaksAtExit.dylib /usr/local/lib From 651a229e7b5dca407a38362059f98b59523fef37 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Fri, 20 Jul 2018 08:20:48 -0700 Subject: [PATCH 23/39] winhttp: retry erroneously failing requests Early Windows TLS 1.2 implementations have an issue during key exchange with OpenSSL implementations that cause negotiation to fail with the error "the buffer supplied to a function was too small." This is a transient error on the connection, so when that error is received, retry up to 5 times to create a connection to the remote server before actually giving up. --- src/transports/winhttp.c | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/src/transports/winhttp.c b/src/transports/winhttp.c index e52d54b6d70..fbb69b3872d 100644 --- a/src/transports/winhttp.c +++ b/src/transports/winhttp.c @@ -845,23 +845,27 @@ static int winhttp_connect( static int do_send_request(winhttp_stream *s, size_t len, int ignore_length) { - if (ignore_length) { - if (!WinHttpSendRequest(s->request, - WINHTTP_NO_ADDITIONAL_HEADERS, 0, - WINHTTP_NO_REQUEST_DATA, 0, - WINHTTP_IGNORE_REQUEST_TOTAL_LENGTH, 0)) { - return -1; - } - } else { - if (!WinHttpSendRequest(s->request, - WINHTTP_NO_ADDITIONAL_HEADERS, 0, - WINHTTP_NO_REQUEST_DATA, 0, - len, 0)) { - return -1; + int attempts; + bool success; + + for (attempts = 0; attempts < 5; attempts++) { + if (ignore_length) { + success = WinHttpSendRequest(s->request, + WINHTTP_NO_ADDITIONAL_HEADERS, 0, + WINHTTP_NO_REQUEST_DATA, 0, + WINHTTP_IGNORE_REQUEST_TOTAL_LENGTH, 0); + } else { + success = WinHttpSendRequest(s->request, + WINHTTP_NO_ADDITIONAL_HEADERS, 0, + WINHTTP_NO_REQUEST_DATA, 0, + len, 0); } + + if (success || GetLastError() != SEC_E_BUFFER_TOO_SMALL) + break; } - return 0; + return success ? 0 : -1; } static int send_request(winhttp_stream *s, size_t len, int ignore_length) From 319d677a843bb09418e5a6d7e952994e32139e7b Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Sat, 28 Jul 2018 22:29:53 +0100 Subject: [PATCH 24/39] ci: run coverity from travis's cron Instead of trying to run coverity builds during the regular PR process, run them during a regularly scheduled cron process. These only need to run nightly, so it makes sense to bring them out of the PR process. --- .travis.yml | 8 ++------ ci/coverity.sh | 25 +++++++++++-------------- 2 files changed, 13 insertions(+), 20 deletions(-) diff --git a/.travis.yml b/.travis.yml index 31a282cf434..9477914a46d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -49,10 +49,6 @@ matrix: - os: osx compiler: gcc include: - - compiler: gcc - env: COVERITY=1 - os: linux - dist: trusty - compiler: gcc env: - LEAK_CHECK=valgrind @@ -69,8 +65,8 @@ install: script: - mkdir build - cd build - - if [ "$COVERITY" ]; then ../ci/coverity.sh; fi - - if [ -z "$COVERITY" ]; then ../ci/build.sh && ../ci/test.sh; fi + - if [ "$TRAVIS_EVENT_TYPE" = "cron" ]; then ../ci/coverity.sh; fi + - if [ "$TRAVIS_EVENT_TYPE" != "cron" ]; then ../ci/build.sh && ../ci/test.sh; fi # Only watch the development and master branches branches: diff --git a/ci/coverity.sh b/ci/coverity.sh index 57f4111748f..ae6d46ef4c9 100755 --- a/ci/coverity.sh +++ b/ci/coverity.sh @@ -1,17 +1,13 @@ #!/bin/bash -set -e -# Only run this on our branches -echo "Branch: $TRAVIS_BRANCH | Pull request: $TRAVIS_PULL_REQUEST | Slug: $TRAVIS_REPO_SLUG" -if [ "$TRAVIS_BRANCH" != "master" -o "$TRAVIS_PULL_REQUEST" != "false" -o "$TRAVIS_REPO_SLUG" != "libgit2/libgit2" ]; -then - echo "Only analyzing the 'master' brach of the main repository." - exit 0 -fi +set -e # Environment check [ -z "$COVERITY_TOKEN" ] && echo "Need to set a coverity token" && exit 1 +SOURCE_DIR=${SOURCE_DIR:-$( cd "$( dirname "${BASH_SOURCE[0]}" )" && dirname $( pwd ) )} +BUILD_DIR=$(pwd) + case $(uname -m) in i?86) BITS=32 ;; amd64|x86_64) BITS=64 ;; @@ -32,28 +28,29 @@ if [ ! -d "$TOOL_BASE" ]; then ln -s "$TOOL_DIR" "$TOOL_BASE"/cov-analysis fi -cp ../script/user_nodefs.h "$TOOL_BASE"/cov-analysis/config/user_nodefs.h +cp "${SOURCE_DIR}/script/user_nodefs.h" "$TOOL_BASE"/cov-analysis/config/user_nodefs.h COV_BUILD="$TOOL_BASE/cov-analysis/bin/cov-build" # Configure and build -cmake .. -DTHREADSAFE=ON +cmake ${SOURCE_DIR} + COVERITY_UNSUPPORTED=1 \ $COV_BUILD --dir cov-int \ cmake --build . # Upload results tar czf libgit2.tgz cov-int -SHA=$(git rev-parse --short HEAD) +SHA=$(cd ${SOURCE_DIR} && git rev-parse --short HEAD) HTML="$(curl \ --silent \ --write-out "\n%{http_code}" \ --form token="$COVERITY_TOKEN" \ - --form email=bs@github.com \ + --form email=libgit2@gmail.com \ --form file=@libgit2.tgz \ --form version="$SHA" \ - --form description="Travis build" \ + --form description="libgit2 build" \ https://scan.coverity.com/builds?project=libgit2)" # Body is everything up to the last line BODY="$(echo "$HTML" | head -n-1)" @@ -62,7 +59,7 @@ STATUS_CODE="$(echo "$HTML" | tail -n1)" echo "${BODY}" -if [ "${STATUS_CODE}" != "201" ]; then +if [ "${STATUS_CODE}" != "200" -o "${STATUS_CODE}" != "201" ]; then echo "Received error code ${STATUS_CODE} from Coverity" exit 1 fi From b4e580f2b15ffa06e6bae18a4d1b448d74213d49 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Sun, 29 Jul 2018 17:26:44 +0100 Subject: [PATCH 25/39] ci: run coverity from a nightly VSTS build --- .vsts-nightly.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .vsts-nightly.yml diff --git a/.vsts-nightly.yml b/.vsts-nightly.yml new file mode 100644 index 00000000000..6c5d0ebb4b1 --- /dev/null +++ b/.vsts-nightly.yml @@ -0,0 +1,22 @@ +resources: +- repo: self + +phases: +- phase: coverity + displayName: 'Coverity' + queue: + name: 'Hosted Linux Preview' + steps: + - task: Docker@0 + displayName: Build + inputs: + action: 'Run an image' + imageName: 'libgit2/trusty-openssl:latest' + volumes: | + $(Build.SourcesDirectory):/src + $(Build.BinariesDirectory):/build + envVars: | + COVERITY_TOKEN=$(COVERITY_TOKEN) + workDir: '/build' + containerCommand: '/src/ci/coverity.sh' + detached: false From 6f45c88240248ed4ee1905f2ad2001468d3f89d4 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Thu, 2 Aug 2018 20:43:21 +0100 Subject: [PATCH 26/39] ci: run VSTS builds on master and maint branches --- .vsts-ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 4879aeab85e..36ce777602c 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -1,6 +1,10 @@ resources: - repo: self +trigger: +- master +- maint/* + phases: - phase: linux_trusty_gcc_openssl displayName: 'Linux (Trusty; GCC; OpenSSL)' From 50f57abd283df4ac184873394ae98b150f627f73 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Mon, 6 Aug 2018 16:33:15 -0500 Subject: [PATCH 27/39] ci: remove appveyor --- appveyor.yml | 44 -------------------------------------------- 1 file changed, 44 deletions(-) delete mode 100644 appveyor.yml diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index d89e3fb6c90..00000000000 --- a/appveyor.yml +++ /dev/null @@ -1,44 +0,0 @@ -version: '{build}' -branches: - only: - - master - - appveyor - - /^maint.*/ -environment: - GITTEST_INVASIVE_FS_STRUCTURE: 1 - GITTEST_INVASIVE_FS_SIZE: 1 - - matrix: - - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2013 - CMAKE_OPTIONS: -G"Visual Studio 10 2010" - ARCH: x86 - - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2013 - CMAKE_OPTIONS: -G"Visual Studio 10 2010 Win64" - ARCH: amd64 - - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015 - CMAKE_OPTIONS: -G"Visual Studio 14 2015" - ARCH: x86 - - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015 - CMAKE_OPTIONS: -G"Visual Studio 14 2015 Win64" - ARCH: amd64 - - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015 - CMAKE_OPTIONS: -G"MinGW Makefiles" - ARCH: x86 - - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015 - CMAKE_OPTIONS: -G"MinGW Makefiles" - ARCH: amd64 - -install: -- set PATH=%TEMP%\mingw64\bin;%TEMP%\mingw32\bin;C:\ProgramData\Oracle\Java\javapath;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Windows\System32\WindowsPowerShell\v1.0\;C:\Program Files (x86)\CMake\bin -- ps: | - mkdir build - cd build - if ($env:CMAKE_OPTIONS -eq '-G"MinGW Makefiles"') { - ../ci/setup-mingw.ps1 - } - -build_script: -- cmd: powershell ../ci/build.ps1 - -test_script: -- cmd: powershell ../ci/test.ps1 From 416080832f26e2c7da87a459b50b2ba27edff372 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Thu, 9 Aug 2018 09:39:39 -0500 Subject: [PATCH 28/39] readme: remove appveyor build badge --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9fe99d8d568..ce6baaaa4dd 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ libgit2 - the Git linkable library ================================== [![Travis Build Status](https://secure.travis-ci.org/libgit2/libgit2.svg?branch=master)](http://travis-ci.org/libgit2/libgit2) -[![AppVeyor Build Status](https://ci.appveyor.com/api/projects/status/xvof5b4t5480a2q3/branch/master?svg=true)](https://ci.appveyor.com/project/libgit2/libgit2/branch/master) +[![VSTS Build Status](https://libgit2.visualstudio.com/libgit2/_apis/build/status/libgit2)](https://libgit2.visualstudio.com/libgit2/_build/latest?definitionId=7) [![Coverity Scan Build Status](https://scan.coverity.com/projects/639/badge.svg)](https://scan.coverity.com/projects/639) `libgit2` is a portable, pure C implementation of the Git core methods @@ -64,7 +64,7 @@ slack channel once you've registered. If you have questions about the library, please be sure to check out the [API documentation](http://libgit2.github.com/libgit2/). If you still have -questions, reach out to us on Slack or post a question on +questions, reach out to us on Slack or post a question on [StackOverflow](http://stackoverflow.com/questions/tagged/libgit2) (with the `libgit2` tag). **Reporting Bugs** From 470f5c04486b66bd11bdbf9cf832ee752888cf00 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Thu, 30 Aug 2018 21:53:58 +0100 Subject: [PATCH 29/39] ci: remove travis --- .travis.yml | 90 ----------------------------------------------------- 1 file changed, 90 deletions(-) delete mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 9477914a46d..00000000000 --- a/.travis.yml +++ /dev/null @@ -1,90 +0,0 @@ -# Travis-CI Build for libgit2 -# see travis-ci.org for details - -language: c - -os: - - linux - - osx - -compiler: - - gcc - - clang - -# Settings to try -env: - global: - - secure: "YnhS+8n6B+uoyaYfaJ3Lei7cSJqHDPiKJCKFIF2c87YDfmCvAJke8QtE7IzjYDs7UFkTCM4ox+ph2bERUrxZbSCyEkHdjIZpKuMJfYWja/jgMqTMxdyOH9y8JLFbZsSXDIXDwqBlC6vVyl1fP90M35wuWcNTs6tctfVWVofEFbs=" - - GITTEST_INVASIVE_FS_SIZE=1 - - MBEDTLS_DIR=/tmp/mbedtls - - SKIP_APT=1 - - SKIP_MBEDTLS_INSTALL=1 - matrix: - - CMAKE_OPTIONS="-DTHREADSAFE=ON -DENABLE_TRACE=ON -DCMAKE_BUILD_TYPE=Release" - - CMAKE_OPTIONS="-DTHREADSAFE=OFF -DBUILD_EXAMPLES=ON" - -dist: trusty -osx_image: xcode8.3 -sudo: false - -addons: - apt: - sources: - - sourceline: 'deb https://dl.bintray.com/libgit2/ci-dependencies trusty libgit2deps' - key_url: 'https://bintray.com/user/downloadSubjectPublicKey?username=bintray' - packages: - cmake - curl - libcurl3 - libcurl3-gnutls - libcurl4-gnutls-dev - libssh2-1-dev - openssh-client - openssh-server - valgrind - -matrix: - fast_finish: true - exclude: - - os: osx - compiler: gcc - include: - - compiler: gcc - env: - - LEAK_CHECK=valgrind - CMAKE_OPTIONS="-DBUILD_CLAR=ON -DBUILD_EXAMPLES=OFF -DDEBUG_POOL=ON -DCMAKE_BUILD_TYPE=Debug" - os: linux - dist: trusty - allow_failures: - - env: COVERITY=1 - -install: - - if [ -f ./ci/setup-${TRAVIS_OS_NAME}.sh ]; then ./ci/setup-${TRAVIS_OS_NAME}.sh; fi - -# Run the Build script and tests -script: - - mkdir build - - cd build - - if [ "$TRAVIS_EVENT_TYPE" = "cron" ]; then ../ci/coverity.sh; fi - - if [ "$TRAVIS_EVENT_TYPE" != "cron" ]; then ../ci/build.sh && ../ci/test.sh; fi - -# Only watch the development and master branches -branches: - only: - - master - - /^maint.*/ - -# Notify development list when needed -notifications: - irc: - channels: - - irc.freenode.net#libgit2 - on_success: change - on_failure: always - use_notice: true - skip_join: true - campfire: - on_success: always - on_failure: always - rooms: - - secure: "sH0dpPWMirbEe7AvLddZ2yOp8rzHalGmv0bYL/LIhVw3JDI589HCYckeLMSB\n3e/FeXw4bn0EqXWEXijVa4ijbilVY6d8oprdqMdWHEodng4KvY5vID3iZSGT\nxylhahO1XHmRynKQLOAvxlc93IlpVW38vQfby8giIY1nkpspb2w=" From 66ea45797f16cbf499b8b7e4e4bdc2d5800a6312 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Mon, 3 Sep 2018 19:27:30 +0100 Subject: [PATCH 30/39] README: remove travis --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index ce6baaaa4dd..d297aaa1062 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,6 @@ libgit2 - the Git linkable library ================================== -[![Travis Build Status](https://secure.travis-ci.org/libgit2/libgit2.svg?branch=master)](http://travis-ci.org/libgit2/libgit2) [![VSTS Build Status](https://libgit2.visualstudio.com/libgit2/_apis/build/status/libgit2)](https://libgit2.visualstudio.com/libgit2/_build/latest?definitionId=7) [![Coverity Scan Build Status](https://scan.coverity.com/projects/639/badge.svg)](https://scan.coverity.com/projects/639) From 6ffdebb1fbb8321bad7dcde7c0d0a42171965f3a Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Wed, 5 Sep 2018 23:34:04 +0100 Subject: [PATCH 31/39] ci: use newer valgrind suppression --- libgit2_clar.supp | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/libgit2_clar.supp b/libgit2_clar.supp index bd22ada460f..0cc89b57f4c 100644 --- a/libgit2_clar.supp +++ b/libgit2_clar.supp @@ -47,3 +47,28 @@ ... fun:__check_pf } + +{ + ignore-curl-global-init + Memcheck:Leak + ... + fun:curl_global_init +} + +{ + ignore-libssh2-gcrypt-leak + Memcheck:Leak + ... + fun:gcry_control + obj:*libssh2.so* +} + +{ + ignore-noai6ai_cached-double-free + Memcheck:Free + fun:free + fun:__libc_freeres + ... + fun:exit + ... +} From ff41a8767ddc0d442895fb3a0050edc0a86560f2 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Fri, 20 Jul 2018 21:51:36 +0100 Subject: [PATCH 32/39] push tests: deeply free the push status Don't just free the push status structure, actually free the strings that were strdup'd into the struct as well. --- tests/online/push.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/online/push.c b/tests/online/push.c index f72b4f8cb22..cdfb5a64e46 100644 --- a/tests/online/push.c +++ b/tests/online/push.c @@ -152,8 +152,12 @@ static void do_verify_push_status(record_callbacks_data *data, const push_status git_buf_free(&msg); } - git_vector_foreach(actual, i, iter) - git__free(iter); + git_vector_foreach(actual, i, iter) { + push_status *s = (push_status *)iter; + git__free(s->ref); + git__free(s->msg); + git__free(s); + } git_vector_free(actual); } From ea9fe923bec45ee36df6903e2bf1d0037a712410 Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Fri, 20 Jul 2018 21:52:24 +0100 Subject: [PATCH 33/39] push tests: deeply free the specs Don't just free the spec vector, also free the specs themselves. --- tests/online/push.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/online/push.c b/tests/online/push.c index cdfb5a64e46..3b98278e054 100644 --- a/tests/online/push.c +++ b/tests/online/push.c @@ -397,7 +397,7 @@ void test_online_push__initialize(void) } git_remote_disconnect(_remote); - git_vector_free(&delete_specs); + git_vector_free_deep(&delete_specs); /* Now that we've deleted everything, fetch from the remote */ memcpy(&fetch_opts.callbacks, &_record_cbs, sizeof(git_remote_callbacks)); From 6353e6a94b082ad15a06a7210d45aecbec685afb Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Fri, 20 Jul 2018 21:50:58 +0100 Subject: [PATCH 34/39] smart subtransport: free url when resetting stream Free the url field when resetting the stream to avoid leaking it. --- src/transports/smart.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/transports/smart.c b/src/transports/smart.c index 6d5d95fbff5..5fedd394e7b 100644 --- a/src/transports/smart.c +++ b/src/transports/smart.c @@ -45,6 +45,11 @@ GIT_INLINE(int) git_smart__reset_stream(transport_smart *t, bool close_subtransp t->current_stream = NULL; } + if (t->url) { + git__free(t->url); + t->url = NULL; + } + if (close_subtransport && t->wrapped->close(t->wrapped) < 0) return -1; From 89024c7e6a1a38071a3b08504b8b1b247cebe37c Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Mon, 10 Sep 2018 12:10:29 +0100 Subject: [PATCH 35/39] ci: only run the exact named test Our CI test system invokes ctest with the name of the given tests it wishes to invoke. ctest (with the `-R` flag) treats this name as a regular expression. Provide anchors in the regular expression to avoid matching additional tests in this search. --- ci/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/test.sh b/ci/test.sh index 13f19df98fb..e9736461fb0 100755 --- a/ci/test.sh +++ b/ci/test.sh @@ -41,7 +41,7 @@ die() { # test configuration in a single place (tests/CMakeLists.txt) instead of running clar # here as well. But it allows us to wrap our test harness with a leak checker like valgrind. run_test() { - TEST_CMD=$(ctest -N -V -R $1 | sed -n 's/^[0-9]*: Test command: //p') + TEST_CMD=$(ctest -N -V -R "^${1}$" | sed -n 's/^[0-9]*: Test command: //p') if [ "$LEAK_CHECK" = "valgrind" ]; then RUNNER="$VALGRIND $TEST_CMD" From 7ae554fd7bf3c8edd97ee7478682cc64d5d17f2a Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Thu, 20 Sep 2018 20:11:36 +1000 Subject: [PATCH 36/39] online::clone: free url and username before resetting Before resetting the url and username, ensure that we free them in case they were set by environment variables. --- tests/online/clone.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/online/clone.c b/tests/online/clone.c index 27b7b96610d..4d409cb77dd 100644 --- a/tests/online/clone.c +++ b/tests/online/clone.c @@ -263,6 +263,9 @@ static int cred_failure_cb( void test_online_clone__cred_callback_failure_return_code_is_tunnelled(void) { + git__free(_remote_url); _remote_url = NULL; + git__free(_remote_user); _remote_user = NULL; + _remote_url = git__strdup("https://github.com/libgit2/non-existent"); _remote_user = git__strdup("libgit2test"); @@ -293,6 +296,9 @@ void test_online_clone__cred_callback_called_again_on_auth_failure(void) { size_t counter = 0; + git__free(_remote_url); _remote_url = NULL; + git__free(_remote_user); _remote_user = NULL; + _remote_url = git__strdup("https://github.com/libgit2/non-existent"); _remote_user = git__strdup("libgit2test"); From 9b122bfbf51d21683e2bc8042487818e715a707c Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Sun, 26 Aug 2018 17:27:54 +0100 Subject: [PATCH 37/39] ci: explicitly run in the build directory Explicitly run from the build directory, not the source. (I was mistaken about the default working directory for VSTS agents.) --- .vsts-ci.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 36ce777602c..2098deda76d 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -75,12 +75,15 @@ phases: steps: - bash: . '$(Build.SourcesDirectory)/ci/setup-osx.sh' displayName: Setup + workingDirectory: '$(Build.BinariesDirectory)' - bash: . '$(Build.SourcesDirectory)/ci/build.sh' displayName: Build + workingDirectory: '$(Build.BinariesDirectory)' env: PKG_CONFIG_PATH: /usr/local/opt/openssl/lib/pkgconfig - bash: . '$(Build.SourcesDirectory)/ci/test.sh' displayName: Test + workingDirectory: '$(Build.BinariesDirectory)' env: TMPDIR: $(Agent.TempDirectory) LEAK_CHECK: leaks @@ -92,10 +95,12 @@ phases: steps: - powershell: . '$(Build.SourcesDirectory)\ci\build.ps1' displayName: Build + workingDirectory: '$(Build.BinariesDirectory)' env: CMAKE_OPTIONS: -DMSVC_CRTDBG=ON -G"Visual Studio 12 2013 Win64" - powershell: . '$(Build.SourcesDirectory)\ci\test.ps1' displayName: Test + workingDirectory: '$(Build.BinariesDirectory)' - phase: windows_vs_x86 displayName: 'Windows (Visual Studio; x86)' @@ -104,10 +109,12 @@ phases: steps: - powershell: . '$(Build.SourcesDirectory)\ci\build.ps1' displayName: Build + workingDirectory: '$(Build.BinariesDirectory)' env: CMAKE_OPTIONS: -DMSVC_CRTDBG=ON -G"Visual Studio 12 2013" - powershell: . '$(Build.SourcesDirectory)\ci\test.ps1' displayName: Test + workingDirectory: '$(Build.BinariesDirectory)' - phase: windows_mingw_amd64 displayName: 'Windows (MinGW; amd64)' @@ -116,16 +123,19 @@ phases: steps: - powershell: . '$(Build.SourcesDirectory)\ci\setup-mingw.ps1' displayName: Setup + workingDirectory: '$(Build.BinariesDirectory)' env: TEMP: $(Agent.TempDirectory) ARCH: amd64 - powershell: . '$(Build.SourcesDirectory)\ci\build.ps1' displayName: Build + workingDirectory: '$(Build.BinariesDirectory)' env: CMAKE_OPTIONS: -G"MinGW Makefiles" PATH: $(Agent.TempDirectory)\mingw64\bin;C:\ProgramData\Oracle\Java\javapath;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Windows\System32\WindowsPowerShell\v1.0\;C:\Program Files (x86)\CMake\bin - powershell: . '$(Build.SourcesDirectory)\ci\test.ps1' displayName: Test + workingDirectory: '$(Build.BinariesDirectory)' - phase: windows_mingw_x86 displayName: 'Windows (MinGW; x86)' @@ -134,13 +144,16 @@ phases: steps: - powershell: . '$(Build.SourcesDirectory)\ci\setup-mingw.ps1' displayName: Setup + workingDirectory: '$(Build.BinariesDirectory)' env: TEMP: $(Agent.TempDirectory) ARCH: x86 - powershell: . '$(Build.SourcesDirectory)\ci\build.ps1' displayName: Build + workingDirectory: '$(Build.BinariesDirectory)' env: CMAKE_OPTIONS: -G"MinGW Makefiles" PATH: $(Agent.TempDirectory)\mingw32\bin;C:\ProgramData\Oracle\Java\javapath;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Windows\System32\WindowsPowerShell\v1.0\;C:\Program Files (x86)\CMake\bin - powershell: . '$(Build.SourcesDirectory)\ci\test.ps1' displayName: Test + workingDirectory: '$(Build.BinariesDirectory)' From 8ce01d817e87e6f8cc890f2017a319547b06877a Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Sun, 26 Aug 2018 18:05:31 +0100 Subject: [PATCH 38/39] ci: use templates for VSTS builds Our build YAML is becoming unweildly and full of copy-pasta. Simplify with templates. --- .vsts-ci.yml | 118 +++++++++++------------------------------ ci/vsts-bash.yml | 17 ++++++ ci/vsts-docker.yml | 33 ++++++++++++ ci/vsts-powershell.yml | 17 ++++++ 4 files changed, 97 insertions(+), 88 deletions(-) create mode 100644 ci/vsts-bash.yml create mode 100644 ci/vsts-docker.yml create mode 100644 ci/vsts-powershell.yml diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 2098deda76d..6788e0d8f6c 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -11,57 +11,22 @@ phases: queue: name: 'Hosted Linux Preview' steps: - - task: Docker@0 - displayName: Build - inputs: - action: 'Run an image' + - template: ci/vsts-docker.yml + parameters: imageName: 'libgit2/trusty-openssl:latest' - volumes: | - $(Build.SourcesDirectory):/src - $(Build.BinariesDirectory):/build - workDir: '/build' - containerCommand: '/src/ci/build.sh' - detached: false - - task: Docker@0 - displayName: Test - inputs: - action: 'Run an image' - imageName: 'libgit2/trusty-openssl:latest' - volumes: | - $(Build.SourcesDirectory):/src - $(Build.BinariesDirectory):/build - envVars: | + environmentVariables: | CC=gcc LEAK_CHECK=valgrind - workDir: '/build' - containerCommand: '/src/ci/test.sh' - detached: false - phase: linux_trusty_clang_openssl displayName: 'Linux (Trusty; Clang; OpenSSL)' queue: name: 'Hosted Linux Preview' steps: - - task: Docker@0 - displayName: Build - inputs: - action: 'Run an image' - imageName: 'libgit2/trusty-openssl:latest' - volumes: | - $(Build.SourcesDirectory):/src - $(Build.BinariesDirectory):/build - workDir: '/build' - containerCommand: '/src/ci/build.sh' - detached: false - - task: Docker@0 - displayName: Test - inputs: - action: 'Run an image' + - template: ci/vsts-docker.yml + parameters: imageName: 'libgit2/trusty-openssl:latest' - volumes: | - $(Build.SourcesDirectory):/src - $(Build.BinariesDirectory):/build - envVars: | + environmentVariables: | CC=clang LEAK_CHECK=valgrind workDir: '/build' @@ -75,46 +40,32 @@ phases: steps: - bash: . '$(Build.SourcesDirectory)/ci/setup-osx.sh' displayName: Setup - workingDirectory: '$(Build.BinariesDirectory)' - - bash: . '$(Build.SourcesDirectory)/ci/build.sh' - displayName: Build - workingDirectory: '$(Build.BinariesDirectory)' - env: - PKG_CONFIG_PATH: /usr/local/opt/openssl/lib/pkgconfig - - bash: . '$(Build.SourcesDirectory)/ci/test.sh' - displayName: Test - workingDirectory: '$(Build.BinariesDirectory)' - env: - TMPDIR: $(Agent.TempDirectory) - LEAK_CHECK: leaks + - template: ci/vsts-bash.yml + parameters: + environmentVariables: + TMPDIR: $(Agent.TempDirectory) + PKG_CONFIG_PATH: /usr/local/opt/openssl/lib/pkgconfig + LEAK_CHECK: leaks - phase: windows_vs_amd64 displayName: 'Windows (Visual Studio; amd64)' queue: name: Hosted steps: - - powershell: . '$(Build.SourcesDirectory)\ci\build.ps1' - displayName: Build - workingDirectory: '$(Build.BinariesDirectory)' - env: - CMAKE_OPTIONS: -DMSVC_CRTDBG=ON -G"Visual Studio 12 2013 Win64" - - powershell: . '$(Build.SourcesDirectory)\ci\test.ps1' - displayName: Test - workingDirectory: '$(Build.BinariesDirectory)' + - template: ci/vsts-powershell.yml + parameters: + environmentVariables: + CMAKE_OPTIONS: -DMSVC_CRTDBG=ON -G"Visual Studio 12 2013 Win64" - phase: windows_vs_x86 displayName: 'Windows (Visual Studio; x86)' queue: name: Hosted steps: - - powershell: . '$(Build.SourcesDirectory)\ci\build.ps1' - displayName: Build - workingDirectory: '$(Build.BinariesDirectory)' - env: - CMAKE_OPTIONS: -DMSVC_CRTDBG=ON -G"Visual Studio 12 2013" - - powershell: . '$(Build.SourcesDirectory)\ci\test.ps1' - displayName: Test - workingDirectory: '$(Build.BinariesDirectory)' + - template: ci/vsts-powershell.yml + parameters: + environmentVariables: + CMAKE_OPTIONS: -DMSVC_CRTDBG=ON -G"Visual Studio 12 2013" - phase: windows_mingw_amd64 displayName: 'Windows (MinGW; amd64)' @@ -123,19 +74,14 @@ phases: steps: - powershell: . '$(Build.SourcesDirectory)\ci\setup-mingw.ps1' displayName: Setup - workingDirectory: '$(Build.BinariesDirectory)' env: TEMP: $(Agent.TempDirectory) ARCH: amd64 - - powershell: . '$(Build.SourcesDirectory)\ci\build.ps1' - displayName: Build - workingDirectory: '$(Build.BinariesDirectory)' - env: - CMAKE_OPTIONS: -G"MinGW Makefiles" - PATH: $(Agent.TempDirectory)\mingw64\bin;C:\ProgramData\Oracle\Java\javapath;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Windows\System32\WindowsPowerShell\v1.0\;C:\Program Files (x86)\CMake\bin - - powershell: . '$(Build.SourcesDirectory)\ci\test.ps1' - displayName: Test - workingDirectory: '$(Build.BinariesDirectory)' + - template: ci/vsts-powershell.yml + parameters: + environmentVariables: + CMAKE_OPTIONS: -G"MinGW Makefiles" + PATH: $(Agent.TempDirectory)\mingw64\bin;C:\ProgramData\Oracle\Java\javapath;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Windows\System32\WindowsPowerShell\v1.0\;C:\Program Files (x86)\CMake\bin - phase: windows_mingw_x86 displayName: 'Windows (MinGW; x86)' @@ -148,12 +94,8 @@ phases: env: TEMP: $(Agent.TempDirectory) ARCH: x86 - - powershell: . '$(Build.SourcesDirectory)\ci\build.ps1' - displayName: Build - workingDirectory: '$(Build.BinariesDirectory)' - env: - CMAKE_OPTIONS: -G"MinGW Makefiles" - PATH: $(Agent.TempDirectory)\mingw32\bin;C:\ProgramData\Oracle\Java\javapath;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Windows\System32\WindowsPowerShell\v1.0\;C:\Program Files (x86)\CMake\bin - - powershell: . '$(Build.SourcesDirectory)\ci\test.ps1' - displayName: Test - workingDirectory: '$(Build.BinariesDirectory)' + - template: ci/vsts-powershell.yml + parameters: + environmentVariables: + CMAKE_OPTIONS: -G"MinGW Makefiles" + PATH: $(Agent.TempDirectory)\mingw32\bin;C:\ProgramData\Oracle\Java\javapath;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Windows\System32\WindowsPowerShell\v1.0\;C:\Program Files (x86)\CMake\bin diff --git a/ci/vsts-bash.yml b/ci/vsts-bash.yml new file mode 100644 index 00000000000..d776a364996 --- /dev/null +++ b/ci/vsts-bash.yml @@ -0,0 +1,17 @@ +# These are the steps used for building on machines with bash. +steps: +- bash: . '$(Build.SourcesDirectory)/ci/build.sh' + displayName: Build + workingDirectory: '$(Build.BinariesDirectory)' + env: ${{ parameters.environmentVariables }} +- bash: . '$(Build.SourcesDirectory)/ci/test.sh' + displayName: Test + workingDirectory: '$(Build.BinariesDirectory)' + env: ${{ parameters.environmentVariables }} +- task: PublishTestResults@2 + displayName: Publish Test Results + condition: succeededOrFailed() + inputs: + testResultsFiles: 'results_*.xml' + searchFolder: '$(Build.BinariesDirectory)' + mergeTestResults: true diff --git a/ci/vsts-docker.yml b/ci/vsts-docker.yml new file mode 100644 index 00000000000..e9251047832 --- /dev/null +++ b/ci/vsts-docker.yml @@ -0,0 +1,33 @@ +# These are the steps used in a container-based build in VSTS. +steps: +- task: docker@0 + displayName: Build + inputs: + action: 'Run an image' + imageName: ${{ parameters.imageName }} + volumes: | + $(Build.SourcesDirectory):/src + $(Build.BinariesDirectory):/build + envVars: ${{ parameters.environmentVariables }} + workDir: '/build' + containerCommand: '/src/ci/build.sh' + detached: false +- task: docker@0 + displayName: Test + inputs: + action: 'Run an image' + imageName: ${{ parameters.imageName }} + volumes: | + $(Build.SourcesDirectory):/src + $(Build.BinariesDirectory):/build + envVars: ${{ parameters.environmentVariables }} + workDir: '/build' + containerCommand: '/src/ci/test.sh' + detached: false +- task: publishtestresults@2 + displayName: Publish Test Results + condition: succeededOrFailed() + inputs: + testResultsFiles: 'results_*.xml' + searchFolder: '$(Build.BinariesDirectory)' + mergeTestResults: true diff --git a/ci/vsts-powershell.yml b/ci/vsts-powershell.yml new file mode 100644 index 00000000000..a2eb175d571 --- /dev/null +++ b/ci/vsts-powershell.yml @@ -0,0 +1,17 @@ +# These are the steps used for building on machines with PowerShell. +steps: +- powershell: . '$(Build.SourcesDirectory)\ci\build.ps1' + displayName: Build + workingDirectory: '$(Build.BinariesDirectory)' + env: ${{ parameters.environmentVariables }} +- powershell: . '$(Build.SourcesDirectory)\ci\test.ps1' + displayName: Test + workingDirectory: '$(Build.BinariesDirectory)' + env: ${{ parameters.environmentVariables }} +- task: PublishTestResults@2 + displayName: Publish Test Results + condition: succeededOrFailed() + inputs: + testResultsFiles: 'results_*.xml' + searchFolder: '$(Build.BinariesDirectory)' + mergeTestResults: true From 0897cf37e7a98b8b68d735722c7141460caaecaa Mon Sep 17 00:00:00 2001 From: Edward Thomson Date: Tue, 18 Sep 2018 13:35:25 +1000 Subject: [PATCH 39/39] ci: rename vsts to azure-pipelines --- .vsts-ci.yml => azure-pipelines.yml | 14 +++++++------- ci/{vsts-bash.yml => bash.yml} | 0 ci/{vsts-docker.yml => docker.yml} | 0 .vsts-nightly.yml => ci/nightly.yml | 0 ci/{vsts-powershell.yml => powershell.yml} | 0 5 files changed, 7 insertions(+), 7 deletions(-) rename .vsts-ci.yml => azure-pipelines.yml (91%) rename ci/{vsts-bash.yml => bash.yml} (100%) rename ci/{vsts-docker.yml => docker.yml} (100%) rename .vsts-nightly.yml => ci/nightly.yml (100%) rename ci/{vsts-powershell.yml => powershell.yml} (100%) diff --git a/.vsts-ci.yml b/azure-pipelines.yml similarity index 91% rename from .vsts-ci.yml rename to azure-pipelines.yml index 6788e0d8f6c..5844270c215 100644 --- a/.vsts-ci.yml +++ b/azure-pipelines.yml @@ -11,7 +11,7 @@ phases: queue: name: 'Hosted Linux Preview' steps: - - template: ci/vsts-docker.yml + - template: ci/docker.yml parameters: imageName: 'libgit2/trusty-openssl:latest' environmentVariables: | @@ -23,7 +23,7 @@ phases: queue: name: 'Hosted Linux Preview' steps: - - template: ci/vsts-docker.yml + - template: ci/docker.yml parameters: imageName: 'libgit2/trusty-openssl:latest' environmentVariables: | @@ -40,7 +40,7 @@ phases: steps: - bash: . '$(Build.SourcesDirectory)/ci/setup-osx.sh' displayName: Setup - - template: ci/vsts-bash.yml + - template: ci/bash.yml parameters: environmentVariables: TMPDIR: $(Agent.TempDirectory) @@ -52,7 +52,7 @@ phases: queue: name: Hosted steps: - - template: ci/vsts-powershell.yml + - template: ci/powershell.yml parameters: environmentVariables: CMAKE_OPTIONS: -DMSVC_CRTDBG=ON -G"Visual Studio 12 2013 Win64" @@ -62,7 +62,7 @@ phases: queue: name: Hosted steps: - - template: ci/vsts-powershell.yml + - template: ci/powershell.yml parameters: environmentVariables: CMAKE_OPTIONS: -DMSVC_CRTDBG=ON -G"Visual Studio 12 2013" @@ -77,7 +77,7 @@ phases: env: TEMP: $(Agent.TempDirectory) ARCH: amd64 - - template: ci/vsts-powershell.yml + - template: ci/powershell.yml parameters: environmentVariables: CMAKE_OPTIONS: -G"MinGW Makefiles" @@ -94,7 +94,7 @@ phases: env: TEMP: $(Agent.TempDirectory) ARCH: x86 - - template: ci/vsts-powershell.yml + - template: ci/powershell.yml parameters: environmentVariables: CMAKE_OPTIONS: -G"MinGW Makefiles" diff --git a/ci/vsts-bash.yml b/ci/bash.yml similarity index 100% rename from ci/vsts-bash.yml rename to ci/bash.yml diff --git a/ci/vsts-docker.yml b/ci/docker.yml similarity index 100% rename from ci/vsts-docker.yml rename to ci/docker.yml diff --git a/.vsts-nightly.yml b/ci/nightly.yml similarity index 100% rename from .vsts-nightly.yml rename to ci/nightly.yml diff --git a/ci/vsts-powershell.yml b/ci/powershell.yml similarity index 100% rename from ci/vsts-powershell.yml rename to ci/powershell.yml