From 0e8dafb2d63d5b4466a3b95e0f164795652e962b Mon Sep 17 00:00:00 2001 From: Wolfgang Kerzendorf Date: Thu, 30 Aug 2018 16:58:25 +0200 Subject: [PATCH] Add Travis Caching and separate scripts for CI (#866) * add caching * remove cached miniconda * swap coveralls for codecov * adding new ci helper * caching tardis env * add refdata to caching * fetch reference data * ref data avail * remove the fetch reference data caching - to be done in next PR * update ref_data_home * update comments from unoebauer * update refdata path * add single quotes * remove git lfs from separate install * update clone refdata * remove space after equal * update travis yaml --- .travis.yml | 64 ++++++++++++------------------ ci-helpers/fetch_reference_data.sh | 17 ++++++++ ci-helpers/install_miniconda.sh | 18 +++++++++ ci-helpers/install_tardis_env.sh | 17 ++++++++ tardis_env27.yml | 5 ++- 5 files changed, 80 insertions(+), 41 deletions(-) create mode 100644 ci-helpers/fetch_reference_data.sh create mode 100644 ci-helpers/install_miniconda.sh create mode 100644 ci-helpers/install_tardis_env.sh diff --git a/.travis.yml b/.travis.yml index 3eed3c16a05..9266d788cc0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,8 +8,11 @@ python: virtualenv: system_site_packages: true -cache: - apt: true +addons: + apt: + packages: + - clang + - gdb sudo: true @@ -20,23 +23,25 @@ env: - ASTROPY_USE_SYSTEM_PYTEST=1 - SETUP_CMD='test' - TEST_MODE='normal' - - TARDIS_REF_DATA_URL='https://github.com/tardis-sn/tardis-refdata.git' + - REF_DATA_HOME=$HOME/tardis-refdata + - REF_DATA_GITHUBURL='https://github.com/tardis-sn/tardis-refdata.git' - MINICONDA_URL='http://repo.continuum.io/miniconda/Miniconda2-latest-Linux-x86_64.sh' - SAVE_COVERAGE=false + - GIT_LFS_SKIP_SMUDGE=1 matrix: include: - python: 2.7 env: - COMPILER=gcc - - SETUP_CMD='test --coverage --args="--tardis-refdata=$HOME/tardis-refdata/"' + - SETUP_CMD='test --coverage --args="--tardis-refdata=$REF_DATA_HOME"' - TEST_MODE='spectrum' - SAVE_COVERAGE=true - python: 2.7 env: - COMPILER=clang - - SETUP_CMD='test --args="--tardis-refdata=$HOME/tardis-refdata/"' + - SETUP_CMD='test --args="--tardis-refdata=$REF_DATA_HOME"' - TEST_MODE='spectrum' #trouble with osx building due to segfault at cython (https://github.com/cython/cython/issues/2199) @@ -44,37 +49,26 @@ matrix: language: generic env: - COMPILER=clang - - SETUP_CMD='test --args="--tardis-refdata=$HOME/tardis-refdata/"' + - SETUP_CMD='test --args="--tardis-refdata=$REF_DATA_HOME"' - TEST_MODE='spectrum' - MINICONDA_URL='http://repo.continuum.io/miniconda/Miniconda2-latest-MacOSX-x86_64.sh' - - - python: 2.7 - env: SETUP_CMD='test' - -addons: - apt: - packages: - - clang - - gdb - - +cache: + apt: true + directories: + - $HOME/miniconda + #- $REF_DATA_HOME before_install: # We do this to make sure we get the dependencies so pip works below - - export PYTHONIOENCODING=UTF8 - - wget $MINICONDA_URL -O miniconda.sh - - chmod +x miniconda.sh - - bash miniconda.sh -b -p $HOME/miniconda - - export PATH=$HOME/miniconda/bin:$PATH - - hash -r - - conda update --yes conda - - if [[ $TEST_MODE == 'spectrum' ]]; then conda install -c conda-forge git-lfs=2.2.1 -y; fi + - source ci-helpers/install_miniconda.sh + #- if [[ $TEST_MODE == 'spectrum' ]]; then conda install -c conda-forge git-lfs=2.2.1 -y; fi - if [[ $TEST_MODE == 'spectrum' ]]; then git lfs install --skip-smudge; fi - - if [[ $TEST_MODE == 'spectrum' ]]; then git clone $TARDIS_REF_DATA_URL $HOME/tardis-refdata; fi - - if [[ $TEST_MODE == 'spectrum' ]]; then cd $HOME/tardis-refdata; fi - # Use the following to get the ref-data from the master; + - echo git clone $REF_DATA_GITHUBURL $REF_DATA_HOME + - if [[ $TEST_MODE == 'spectrum' ]]; then git clone $REF_DATA_GITHUBURL $REF_DATA_HOME; fi + - if [[ $TEST_MODE == 'spectrum' ]]; then cd $REF_DATA_HOME; fi +# Use the following to get the ref-data from the master; - if [[ $TEST_MODE == 'spectrum' ]]; then git fetch origin; fi - if [[ $TEST_MODE == 'spectrum' ]]; then git checkout origin/master; fi # Use the following to get the ref-data from a specific pull request; @@ -86,19 +80,11 @@ before_install: - if [[ $TEST_MODE == 'spectrum' ]]; then echo MD5 `md5sum unit_test_data.h5`; fi - if [[ $TEST_MODE == 'spectrum' ]]; then cd $TRAVIS_BUILD_DIR; fi +#- source ci-helpers/fetch_reference_data.sh + install: - - cd $TRAVIS_BUILD_DIR - - conda env create -f tardis_env27.yml - - source activate tardis - - conda install -y -c conda-forge codecov - #trouble with building due to segfault at cython (https://github.com/cython/cython/issues/2199) - #remove if we can get normal cython through conda - - git clone https://github.com/cython/cython - - cd cython - - git checkout c485b1b77264c3c75d090a3c526de24966830d42 - - CFLAGS="$CFLAGS -D CYTHON_CLINE_IN_TRACEBACK=0" python setup.py install - - cd .. + - source ci-helpers/install_tardis_env.sh script: - echo CC=$COMPILER python setup.py $SETUP_CMD diff --git a/ci-helpers/fetch_reference_data.sh b/ci-helpers/fetch_reference_data.sh new file mode 100644 index 00000000000..9002286ca2b --- /dev/null +++ b/ci-helpers/fetch_reference_data.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +git lfs install +if test -e $REF_DATA_HOME; then + echo "Ref data available" +else + git clone https://github.com/tardis-sn/tardis-refdata.git $REF_DATA_HOME +cd $REF_DATA_HOME +# Use the following to get the ref-data from the master; +git fetch origin +git checkout origin/master +# Use the following to get the ref-data from a specific pull request +#git fetch origin pull/11/head:thomson-ref +git lfs pull --include="atom_data/kurucz_cd23_chianti_H_He.h5" origin/master +git lfs pull --include="atom_data/chianti_He.h5" origin/master +git lfs pull --include="plasma_reference/" origin/master +git lfs pull --include="unit_test_data.h5" origin/master +cd $TRAVIS_BUILD_DIR diff --git a/ci-helpers/install_miniconda.sh b/ci-helpers/install_miniconda.sh new file mode 100644 index 00000000000..399323e61ad --- /dev/null +++ b/ci-helpers/install_miniconda.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +export PYTHONIOENCODING=UTF8 +if test -e $HOME/miniconda/bin; then + echo "miniconda already installed."; + export PATH=$HOME/miniconda/bin:$PATH + hash -r + #conda update --yes conda + +else + wget $MINICONDA_URL -O miniconda.sh + chmod +x miniconda.sh + rm -r $HOME/miniconda + bash miniconda.sh -b -p $HOME/miniconda + + export PATH=$HOME/miniconda/bin:$PATH + hash -r + conda update --yes conda +fi diff --git a/ci-helpers/install_tardis_env.sh b/ci-helpers/install_tardis_env.sh new file mode 100644 index 00000000000..3851d30b218 --- /dev/null +++ b/ci-helpers/install_tardis_env.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +cd $TRAVIS_BUILD_DIR +if test -e $HOME/miniconda/envs/tardis; then + echo "TARDIS env already installed."; + # Also check for tardis_env27.yml change +else + conda env create -f tardis_env27.yml + #trouble with building due to segfault at cython (https://github.com/cython/cython/issues/2199) + #remove if we can get normal cython through conda + git clone https://github.com/cython/cython + cd cython + git checkout c485b1b77264c3c75d090a3c526de24966830d42 + CFLAGS="$CFLAGS -D CYTHON_CLINE_IN_TRACEBACK=0" python setup.py install + cd .. +fi + +source activate tardis \ No newline at end of file diff --git a/tardis_env27.yml b/tardis_env27.yml index 5a6b337f99d..de11427d97e 100644 --- a/tardis_env27.yml +++ b/tardis_env27.yml @@ -34,12 +34,13 @@ dependencies: - requests=2.9.1 - docopt=0.6.2 - pytest-cov=2.2.1 - +- codecov +#testing requirements +- git-lfs=2.4 - pip: - sphinx_bootstrap_theme - sphinxcontrib-bibtex - sphinxcontrib-tikz - - coveralls - pytest-html==1.10.0 - dokuwiki - dot2tex