From df33cad1717d6690ebe315be8c3ec8cd614f9ca0 Mon Sep 17 00:00:00 2001 From: Matthew Brett Date: Thu, 19 Feb 2015 18:35:12 -0800 Subject: [PATCH] TST: trigger travis OSX tests if Linux tests pass Use travis_after_all to wait for all Linux tests to pass, before triggering the OSX tests at MacPython/matplotlib-wheels. Uses: * https://github.com/dmakhno/travis_after_all * https://github.com/ajdm/travis-build-children following model at: https://github.com/python-pillow/Pillow/pull/1067 Authorization token to allow trigger of OSX build comes from: travis login travis token # My (Matthew Brett's) github token travis encrypt AUTH_TOKEN= The output becomes: env: global: - secure: "dfjNqGKzQG5bu3FnDNwLG8H/C4QoieFo4PfFmZPdM2RY7WIzukwKFNT6kiDfOrpwt+2bR7FhzjOGlDECGtlGOtYPN8XuXGjhcP4a4IfakdbDfF+D3NPIpf5VlE6776k0VpvcZBTMYJKNFIMc7QPkOwjvNJ2aXyfe3hBuGlKJzQU=" in .travis.yml --- .travis.yml | 52 ++++++++++++++----- .travis/build_children.sh | 9 ++++ .travis/travis_after_all.py | 100 ++++++++++++++++++++++++++++++++++++ 3 files changed, 148 insertions(+), 13 deletions(-) create mode 100755 .travis/build_children.sh create mode 100644 .travis/travis_after_all.py diff --git a/.travis.yml b/.travis.yml index e8c93af5e6a5..c0f6f1a49b8f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,7 @@ env: - ARTIFACTS_S3_BUCKET=matplotlib-test-results - secure: RgJI7BBL8aX5FTOQe7xiXqWHMxWokd6GNUWp1NUV2mRLXPb9dI0RXqZt3UJwKTAzf1z/OtlHDmEkBoTVK81E9iUxK5npwyyjhJ8yTJmwfQtQF2n51Q1Ww9p+XSLORrOzZc7kAo6Kw6FIXN1pfctgYq2bQkrwJPRx/oPR8f6hcbY= - secure: E7OCdqhZ+PlwJcn+Hd6ns9TDJgEUXiUNEI0wu7xjxB2vBRRIKtZMbuaZjd+iKDqCKuVOJKu0ClBUYxmgmpLicTwi34CfTUYt6D4uhrU+8hBBOn1iiK51cl/aBvlUUrqaRLVhukNEBGZcyqAjXSA/Qsnp2iELEmAfOUa92ZYo1sk= + - secure: "dfjNqGKzQG5bu3FnDNwLG8H/C4QoieFo4PfFmZPdM2RY7WIzukwKFNT6kiDfOrpwt+2bR7FhzjOGlDECGtlGOtYPN8XuXGjhcP4a4IfakdbDfF+D3NPIpf5VlE6776k0VpvcZBTMYJKNFIMc7QPkOwjvNJ2aXyfe3hBuGlKJzQU=" - BUILD_DOCS=false - TEST_ARGS=--no-pep8 - NUMPY=numpy @@ -51,26 +52,35 @@ script: # multiple processes - gcc --version - python -c "from matplotlib import font_manager" - - if [[ $BUILD_DOCS == false ]]; then export MPL_REPO_DIR=$PWD; fi # pep8-conformance test of the examples - - if [[ $BUILD_DOCS == false ]]; then mkdir ../tmp_test_dir; fi - - if [[ $BUILD_DOCS == false ]]; then cd ../tmp_test_dir; fi - - if [[ $BUILD_DOCS == false ]]; then gdb -return-child-result -batch -ex r -ex bt --args python ../matplotlib/tests.py -sv --processes=8 --process-timeout=300 $TEST_ARGS; fi - - if [[ $BUILD_DOCS == true ]]; then cd doc; python make.py html --small --warningsaserrors; fi - # We don't build the LaTeX docs here, so linkchecker will complain - - if [[ $BUILD_DOCS == true ]]; then touch build/html/Matplotlib.pdf; fi - - if [[ $BUILD_DOCS == true ]]; then linkchecker build/html/index.html; fi + - | + if [[ $BUILD_DOCS == false ]]; then + export MPL_REPO_DIR=$PWD # pep8-conformance test of the examples + mkdir ../tmp_test_dir + cd ../tmp_test_dir + gdb -return-child-result -batch -ex r -ex bt --args python ../matplotlib/tests.py -sv --processes=8 --process-timeout=300 $TEST_ARGS + else + cd doc + python make.py html --small --warningsaserrors + # We don't build the LaTeX docs here, so linkchecker will complain + touch build/html/Matplotlib.pdf + linkchecker build/html/index.html + fi after_failure: - - tar cjf result_images.tar.bz2 result_images - - if [[ $TRAVIS_PULL_REQUEST == false ]]; then gem install travis-artifacts; fi - - if [[ $TRAVIS_PULL_REQUEST == false ]]; then travis-artifacts upload --path result_images.tar.bz2; fi - - if [[ $TRAVIS_PULL_REQUEST != false ]]; then echo "The result images will only be uploaded if they are on the matplotlib/matplotlib repo - this is for security reasons to prevent arbitrary PRs echoing security details." else echo https://s3.amazonaws.com/matplotlib-test-results/artifacts/${TRAVIS_BUILD_NUMBER}/${TRAVIS_JOB_NUMBER}/result_images.tar.bz2; fi + | + if [[ $BUILD_DOCS == false && $TRAVIS_PULL_REQUEST == false && $TRAVIS_REPO_SLUG == 'matplotlib/matplotlib' ]]; then + gem install travis-artifacts + cd $MPL_REPO_DIR/../tmp_test_dir + tar cjf result_images.tar.bz2 result_images + travis-artifacts upload --path result_images.tar.bz2 + echo "The result images will only be uploaded if they are on the matplotlib/matplotlib repo - this is for security reasons to prevent arbitrary PRs echoing security details." else echo https://s3.amazonaws.com/matplotlib-test-results/artifacts/${TRAVIS_BUILD_NUMBER}/${TRAVIS_JOB_NUMBER}/result_images.tar.bz2 + fi after_success: | if [[ $TRAVIS_PULL_REQUEST == false && $BUILD_DOCS == true && $TRAVIS_BRANCH == 'master' ]]; then + cd $MPL_REPO_DIR echo "Uploading documentation" - cd .. openssl aes-256-cbc -K $encrypted_cc802e084cd0_key -iv $encrypted_cc802e084cd0_iv -in .travis/matplotlibDeployKey.enc -out .travis/matplotlibDeployKey -d eval `ssh-agent -s` chmod 600 .travis/matplotlibDeployKey @@ -90,3 +100,19 @@ after_success: else echo "Will only deploy docs build from matplotlib master branch" fi + if [[ $TRAVIS_PULL_REQUEST == false ]] && \ + [[ $TRAVIS_REPO_SLUG == 'matplotlib/matplotlib' ]] && \ + [[ $TRAVIS_BRANCH == 'master' ]]; then + cd $MPL_REPO_DIR + python .travis/travis_after_all.py + export $(cat .to_export_back) + if [ "$BUILD_LEADER" = "YES" ]; then + if [ "$BUILD_AGGREGATE_STATUS" = "others_succeeded" ]; then + echo "All Succeeded! Triggering OSX build..." + ./.travis/build_children.sh + else + echo "Some Failed; no OSX build" + fi + fi + fi + fi diff --git a/.travis/build_children.sh b/.travis/build_children.sh new file mode 100755 index 000000000000..5f26c4890cc1 --- /dev/null +++ b/.travis/build_children.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# Modified from: +# https://github.com/ajdm/travis-build-children/blob/master/build_children.sh + +# Get last child project build number from branch named "latest" +BUILD_NUM=$(curl -s 'https://api.travis-ci.org/repos/MacPython/matplotlib-wheels/branches/latest' | grep -o '^{"branch":{"id":[0-9]*,' | grep -o '[0-9]' | tr -d '\n') + +# Restart last child project build +curl -X POST https://api.travis-ci.org/builds/$BUILD_NUM/restart --header "Authorization: token "$AUTH_TOKEN diff --git a/.travis/travis_after_all.py b/.travis/travis_after_all.py new file mode 100644 index 000000000000..4e35b39ab9a5 --- /dev/null +++ b/.travis/travis_after_all.py @@ -0,0 +1,100 @@ +# Copied from: +# https://github.com/dmakhno/travis_after_all +# commit b7172bca9 +import os +import json +import time +import logging + +try: + import urllib.request as urllib2 +except ImportError: + import urllib2 + +log = logging.getLogger("travis.leader") +log.addHandler(logging.StreamHandler()) +log.setLevel(logging.INFO) + +TRAVIS_JOB_NUMBER = 'TRAVIS_JOB_NUMBER' +TRAVIS_BUILD_ID = 'TRAVIS_BUILD_ID' +POLLING_INTERVAL = 'LEADER_POLLING_INTERVAL' + +build_id = os.getenv(TRAVIS_BUILD_ID) +polling_interval = int(os.getenv(POLLING_INTERVAL, '5')) + +#assume, first job is the leader +is_leader = lambda job_number: job_number.endswith('.1') + +if not os.getenv(TRAVIS_JOB_NUMBER): + # seems even for builds with only one job, this won't get here + log.fatal("Don't use defining leader for build without matrix") + exit(1) +elif is_leader(os.getenv(TRAVIS_JOB_NUMBER)): + log.info("This is a leader") +else: + #since python is subprocess, env variables are exported back via file + with open(".to_export_back", "w") as export_var: + export_var.write("BUILD_MINION=YES") + log.info("This is a minion") + exit(0) + + +class MatrixElement(object): + def __init__(self, json_raw): + self.is_finished = json_raw['finished_at'] is not None + self.is_succeeded = json_raw['result'] == 0 + self.number = json_raw['number'] + self.is_leader = is_leader(self.number) + + +def matrix_snapshot(): + """ + :return: Matrix List + """ + response = urllib2.build_opener().open("https://api.travis-ci.org/builds/{0}".format(build_id)).read() + raw_json = json.loads(response) + matrix_without_leader = [MatrixElement(element) for element in raw_json["matrix"]] + return matrix_without_leader + + +def wait_others_to_finish(): + def others_finished(): + """ + Dumps others to finish + Leader cannot finish, it is working now + :return: tuple(True or False, List of not finished jobs) + """ + snapshot = matrix_snapshot() + finished = [el.is_finished for el in snapshot if not el.is_leader] + return reduce(lambda a, b: a and b, finished), [el.number for el in snapshot if + not el.is_leader and not el.is_finished] + + while True: + finished, waiting_list = others_finished() + if finished: break + log.info("Leader waits for minions {0}...".format(waiting_list)) # just in case do not get "silence timeout" + time.sleep(polling_interval) + + +try: + wait_others_to_finish() + + final_snapshot = matrix_snapshot() + log.info("Final Results: {0}".format([(e.number, e.is_succeeded) for e in final_snapshot])) + + BUILD_AGGREGATE_STATUS = 'BUILD_AGGREGATE_STATUS' + others_snapshot = [el for el in final_snapshot if not el.is_leader] + if reduce(lambda a, b: a and b, [e.is_succeeded for e in others_snapshot]): + os.environ[BUILD_AGGREGATE_STATUS] = "others_succeeded" + elif reduce(lambda a, b: a and b, [not e.is_succeeded for e in others_snapshot]): + log.error("Others Failed") + os.environ[BUILD_AGGREGATE_STATUS] = "others_failed" + else: + log.warn("Others Unknown") + os.environ[BUILD_AGGREGATE_STATUS] = "unknown" + #since python is subprocess, env variables are exported back via file + with open(".to_export_back", "w") as export_var: + export_var.write("BUILD_LEADER=YES {0}={1}".format(BUILD_AGGREGATE_STATUS, os.environ[BUILD_AGGREGATE_STATUS])) + +except Exception as e: + log.fatal(e)