From 7e995287a4024e4e72b19d48f556b0802e232f6c Mon Sep 17 00:00:00 2001 From: Florian Pagnoux Date: Tue, 15 May 2018 13:41:19 -0400 Subject: [PATCH 01/14] Publish py3 versions in CI & switch to Circle v2 --- .circleci/config.yml | 158 +++++++++++++++++++++++++++++++++++++++++++ circle.yml | 26 ------- 2 files changed, 158 insertions(+), 26 deletions(-) create mode 100644 .circleci/config.yml delete mode 100644 circle.yml diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000..bdb30f4b --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,158 @@ +version: 2 + +jobs: + checkout: &checkout + working_directory: ~/country-template + docker: + - image: python:2.7.14 + + steps: + - checkout + + - run: + name: Fetch remote refs + command: git fetch + + - save_cache: + key: v1-checkout-{{ .Environment.CIRCLE_SHA1 }} + paths: + - . + - ~/.ssh/known_hosts + + dependencies: &dependencies + working_directory: ~/country-template + docker: + - image: python:2.7.14 + environment: + CREATE_VENV: "virtualenv" + + steps: + - restore_cache: + keys: + - v1-checkout-{{ .Environment.CIRCLE_SHA1 }} + + - run: + name: Create virtualenv + command: | + mkdir -p /tmp/venv/country-template + ${CREATE_VENV} /tmp/venv/country-template + + # pip >= 8.0 needed to be compatible with "manylinux" wheels, used by numpy >= 1.11 + - run: + name: Install dependencies + command: | + . /tmp/venv/country-template/bin/activate + pip install --upgrade pip twine wheel + pip install .[test] --upgrade + + # Uncomment and adapt the next line to use a particular feature branch of OpenFisca-Core to run Circle CI tests + # - run: + # name: Install a particular feature branch of OpenFisca-Core + # command: | + # . /tmp/venv/country-template/bin/activate + # pip install --editable git+https://github.com/openfisca/openfisca-core.git@BRANCH_NAME#egg=OpenFisca-Core + + - save_cache: + # PY_VERSION is set in the context + key: v1-{{ .Environment.PY_VERSION }}-dependencies-{{ .Environment.CIRCLE_SHA1 }} + paths: + - /tmp/venv/country-template + + dependencies_python3: + <<: *dependencies + docker: + - image: python:3.6 + environment: + CREATE_VENV: "python -m venv" + + tests: &tests + working_directory: ~/country-template + docker: + - image: python:2.7.14 + + steps: + - restore_cache: + keys: + - v1-checkout-{{ .Environment.CIRCLE_SHA1 }} + + - restore_cache: + keys: + - v1-{{ .Environment.PY_VERSION }}-dependencies-{{ .Environment.CIRCLE_SHA1 }} + + - run: + name: Run tests + command: | + . /tmp/venv/country-template/bin/activate + make test + . check-version-bump.sh + + tests_python3: + <<: *tests + docker: + - image: python:3.6 + + deploy: &deploy + working_directory: ~/country-template + docker: + - image: python:2.7.14 + environment: + PYPI_USERNAME: openfisca-bot + # PYPI_PASSWORD: this value is set in CircleCI's web interface; do not set it here, it is a secret! + + steps: + - restore_cache: + keys: + - v1-checkout-{{ .Environment.CIRCLE_SHA1 }} + + - restore_cache: + keys: + - v1-{{ .Environment.PY_VERSION }}-dependencies-{{ .Environment.CIRCLE_SHA1 }} + + - run: + name: Deploy (if version bump) + command: | + . /tmp/venv/country-template/bin/activate + . deploy-if-version-bump.sh + + deploy_python3: + <<: *deploy + docker: + - image: python:3.6 + +workflows: + version: 2 + country-template: + jobs: + - checkout + - dependencies: + context: Py2 + requires: + - checkout + - tests: + context: Py2 + requires: + - dependencies + - dependencies_python3: + context: Py3 + requires: + - checkout + - tests_python3: + context: Py3 + requires: + - dependencies_python3 + - deploy: + context: Py2 + requires: + - tests + - tests_python3 + filters: + branches: + only: master + - deploy_python3: + context: Py3 + requires: + - tests + - tests_python3 + filters: + branches: + only: master diff --git a/circle.yml b/circle.yml deleted file mode 100644 index 1fec3785..00000000 --- a/circle.yml +++ /dev/null @@ -1,26 +0,0 @@ -machine: - python: - version: 2.7.12 # this specific version is preinstalled on the CI servers and allows faster builds - environment: - PYPI_USERNAME: openfisca-bot # set here the name of your Pypi account to automatically publish your package to Pypi - # PYPI_PASSWORD: this value is set in CircleCI's web interface; do not set it here, it is a secret! - -dependencies: - override: - - pip install --upgrade pip wheel # pip >= 8.0 needed to be compatible with "manylinux" wheels, used by numpy >= 1.11 - - pip install twine - # Uncomment and adapt the next line to use a particular feature branch of OpenFisca-Core to run Circle CI tests - # - pip install --editable git+https://github.com/openfisca/openfisca-core.git@BRANCH_NAME#egg=OpenFisca-Core - - pip install .[test] --upgrade -test: - pre: - - git fetch - override: - - make test - - ./check-version-bump.sh -deployment: - master: - owner: openfisca # update this to your GitHub user (or organisation) name, in order to prevent failed builds on forks - branch: master - commands: - - ./deploy-if-version-bump.sh From 9a1384163550084d518217b607baec8e8f3ce984 Mon Sep 17 00:00:00 2001 From: Florian Pagnoux Date: Tue, 15 May 2018 14:52:58 -0400 Subject: [PATCH 02/14] Don't try to publish the tag twice --- deploy-if-version-bump.sh | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/deploy-if-version-bump.sh b/deploy-if-version-bump.sh index f1eede6c..c19d8239 100755 --- a/deploy-if-version-bump.sh +++ b/deploy-if-version-bump.sh @@ -1,11 +1,20 @@ #!/bin/sh set -e -if ! git rev-parse `python setup.py --version` 2>/dev/null ; then - git tag `python setup.py --version` - git push --tags # update the repository version - python setup.py bdist_wheel # build this package in the dist directory - twine upload dist/* --username $PYPI_USERNAME --password $PYPI_PASSWORD # publish -else - echo "No deployment - Only non-functional elements were modified in this change" +if [ $PY_VERSION = "2" ]; then + if ! git rev-parse `python setup.py --version` 2>/dev/null ; then + git tag `python setup.py --version` + git push --tags # update the repository version + python setup.py bdist_wheel # build this package in the dist directory + twine upload dist/* --username $PYPI_USERNAME --password $PYPI_PASSWORD # publish + else + echo "No deployment - Only non-functional elements were modified in this change" + fi +fi + +if [ $PY_VERSION = "3" ]; then + python setup.py bdist_wheel # build this package in the dist directory + if ! twine upload dist/* --username $PYPI_USERNAME --password $PYPI_PASSWORD; then # publish + echo "Could not upload this package version on Pypi. This usually means that the version already exists, and that only non-functional elements were modified in this change. If this it not the case, check the error message above." + fi fi From f989ab38af41789c71ecbfaaf1fc919af0c8b857 Mon Sep 17 00:00:00 2001 From: Florian Pagnoux Date: Tue, 15 May 2018 15:04:34 -0400 Subject: [PATCH 03/14] Bump version number --- CHANGELOG.md | 4 ++++ setup.py | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e921ae4..84871fd3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +### 3.1.0 - [#41](https://github.com/openfisca/country-template/pull/41) + +* Make package compatible with Python 3 + ### 3.0.2 - [#37](https://github.com/openfisca/country-template/pull/37) * Declare package compatible with OpenFisca Core v23 diff --git a/setup.py b/setup.py index 77a17bf5..f60013b3 100755 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name='OpenFisca-Country-Template', - version='3.0.2', + version='3.1.0', author='OpenFisca Team', author_email='contact@openfisca.fr', description=u'OpenFisca tax and benefit system for Country-Template', @@ -16,7 +16,7 @@ url='https://github.com/openfisca/openfisca-country-template', include_package_data = True, # Will read MANIFEST.in install_requires=[ - 'OpenFisca-Core >= 23, < 24.0', + 'OpenFisca-Core >= 23.1, < 24.0', ], extras_require = { 'api': [ From e1e9f3d542cdddf8235f1f41fb7b860ecc891522 Mon Sep 17 00:00:00 2001 From: Florian Pagnoux Date: Wed, 16 May 2018 12:40:37 -0400 Subject: [PATCH 04/14] Merge conf with #40's suggestion --- .../check-version-bump.sh | 4 +- .circleci/config.yml | 177 ++++++++---------- .circleci/deploy-python-2.sh | 17 ++ .circleci/deploy-python-3.sh | 15 ++ deploy-if-version-bump.sh | 20 -- 5 files changed, 114 insertions(+), 119 deletions(-) rename check-version-bump.sh => .circleci/check-version-bump.sh (91%) create mode 100755 .circleci/deploy-python-2.sh create mode 100755 .circleci/deploy-python-3.sh delete mode 100755 deploy-if-version-bump.sh diff --git a/check-version-bump.sh b/.circleci/check-version-bump.sh similarity index 91% rename from check-version-bump.sh rename to .circleci/check-version-bump.sh index 6b56bbe3..a5ab03a5 100755 --- a/check-version-bump.sh +++ b/.circleci/check-version-bump.sh @@ -3,7 +3,9 @@ VERSION_CHANGE_TRIGGERS="setup.py MANIFEST.in openfisca_country_template" if git diff-index --quiet origin/master -- $VERSION_CHANGE_TRIGGERS ":(exclude)*.md" -then exit 0 # there are no changes at all, the version is correct +then + echo "No functional change. No need for a version update." + exit 0 fi current_version=`python setup.py --version` diff --git a/.circleci/config.yml b/.circleci/config.yml index bdb30f4b..ef74c3c1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,158 +1,139 @@ +# CircleCI 2.0 configuration file. See . version: 2 - jobs: - checkout: &checkout - working_directory: ~/country-template + build_python2: docker: - image: python:2.7.14 steps: - checkout + - restore_cache: + key: v1-py2-{{ checksum "setup.py" }} + - run: - name: Fetch remote refs - command: git fetch + name: Create a virtualenv + command: | + mkdir -p /tmp/venv/country_template + virtualenv /tmp/venv/country_template + echo "source /tmp/venv/country_template/bin/activate" >> $BASH_ENV + + - run: + name: Install dependencies + command: | + pip install --upgrade pip twine wheel + pip install --editable .[test] --upgrade + # pip install --editable git+https://github.com/openfisca/openfisca-core.git@BRANCH_NAME#egg=OpenFisca-Core # use a specific branch of OpenFisca-Core - save_cache: - key: v1-checkout-{{ .Environment.CIRCLE_SHA1 }} + key: v1-py2-{{ checksum "setup.py" }} paths: - - . - - ~/.ssh/known_hosts + - /tmp/venv/country_template + + - run: + name: Run tests + command: make test + + - run: + name: Check version number has been properly updated + command: | + git fetch + ./.circleci/check-version-bump.sh - dependencies: &dependencies - working_directory: ~/country-template + deploy_python2: docker: - image: python:2.7.14 environment: - CREATE_VENV: "virtualenv" + PYPI_USERNAME: openfisca-bot + # PYPI_PASSWORD: this value is set in CircleCI's web interface; do not set it here, it is a secret! steps: + - checkout + + - restore_cache: + key: v1-py2-{{ checksum "setup.py" }} + + - run: + name: Upload a Python 2 package to Pypi and publish a tag if functional changes were made + command: | + source /tmp/venv/country_template/bin/activate + ./.circleci/deploy-python-2.sh + + build_python3: + docker: + - image: python:3.6 + + steps: + - checkout + - restore_cache: - keys: - - v1-checkout-{{ .Environment.CIRCLE_SHA1 }} + key: v1-py3-{{ checksum "setup.py" }} - run: - name: Create virtualenv + name: Create a virtualenv command: | - mkdir -p /tmp/venv/country-template - ${CREATE_VENV} /tmp/venv/country-template + mkdir -p /tmp/venv/country_template + python -m venv /tmp/venv/country_template + echo "source /tmp/venv/country_template/bin/activate" >> $BASH_ENV - # pip >= 8.0 needed to be compatible with "manylinux" wheels, used by numpy >= 1.11 - run: name: Install dependencies command: | - . /tmp/venv/country-template/bin/activate pip install --upgrade pip twine wheel - pip install .[test] --upgrade - - # Uncomment and adapt the next line to use a particular feature branch of OpenFisca-Core to run Circle CI tests - # - run: - # name: Install a particular feature branch of OpenFisca-Core - # command: | - # . /tmp/venv/country-template/bin/activate - # pip install --editable git+https://github.com/openfisca/openfisca-core.git@BRANCH_NAME#egg=OpenFisca-Core + pip install --editable .[test] --upgrade + # pip install --editable git+https://github.com/openfisca/country-template.git@BRANCH_NAME#egg=OpenFisca-Country-Template # use a specific branch of OpenFisca-Country-Template - save_cache: - # PY_VERSION is set in the context - key: v1-{{ .Environment.PY_VERSION }}-dependencies-{{ .Environment.CIRCLE_SHA1 }} + key: v1-py3-{{ checksum "setup.py" }} paths: - - /tmp/venv/country-template - - dependencies_python3: - <<: *dependencies - docker: - - image: python:3.6 - environment: - CREATE_VENV: "python -m venv" - - tests: &tests - working_directory: ~/country-template - docker: - - image: python:2.7.14 - - steps: - - restore_cache: - keys: - - v1-checkout-{{ .Environment.CIRCLE_SHA1 }} - - - restore_cache: - keys: - - v1-{{ .Environment.PY_VERSION }}-dependencies-{{ .Environment.CIRCLE_SHA1 }} + - /tmp/venv/country_template - run: name: Run tests + command: make test + + - run: + name: Check version number has been properly updated command: | - . /tmp/venv/country-template/bin/activate - make test - . check-version-bump.sh + git fetch + ./.circleci/check-version-bump.sh - tests_python3: - <<: *tests + deploy_python3: docker: - image: python:3.6 - - deploy: &deploy - working_directory: ~/country-template - docker: - - image: python:2.7.14 environment: PYPI_USERNAME: openfisca-bot # PYPI_PASSWORD: this value is set in CircleCI's web interface; do not set it here, it is a secret! steps: - - restore_cache: - keys: - - v1-checkout-{{ .Environment.CIRCLE_SHA1 }} + - checkout - restore_cache: - keys: - - v1-{{ .Environment.PY_VERSION }}-dependencies-{{ .Environment.CIRCLE_SHA1 }} + key: v1-py3-{{ checksum "setup.py" }} - run: - name: Deploy (if version bump) + name: Upload a Python 3 package to Pypi command: | - . /tmp/venv/country-template/bin/activate - . deploy-if-version-bump.sh - - deploy_python3: - <<: *deploy - docker: - - image: python:3.6 + source /tmp/venv/country_template/bin/activate + ./.circleci/deploy-python-3.sh workflows: version: 2 - country-template: + build_and_deploy: jobs: - - checkout - - dependencies: - context: Py2 - requires: - - checkout - - tests: - context: Py2 - requires: - - dependencies - - dependencies_python3: - context: Py3 - requires: - - checkout - - tests_python3: - context: Py3 - requires: - - dependencies_python3 - - deploy: - context: Py2 + - build_python2 + - build_python3 + - deploy_python2: requires: - - tests - - tests_python3 + - build_python2 + - build_python3 filters: branches: only: master - deploy_python3: - context: Py3 requires: - - tests - - tests_python3 + - build_python2 + - build_python3 filters: branches: only: master diff --git a/.circleci/deploy-python-2.sh b/.circleci/deploy-python-2.sh new file mode 100755 index 00000000..ce7c8e28 --- /dev/null +++ b/.circleci/deploy-python-2.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +set -e + +VERSION_CHANGE_TRIGGERS="setup.py MANIFEST.in openfisca_core openfisca_web_api_preview" + +if git diff-index --quiet HEAD^ -- $VERSION_CHANGE_TRIGGERS ":(exclude)*.md" +then + echo "No deployment - Only non-functional elements were modified in this change" + exit 0 # there are no changes at all, the version is correct +fi + +git tag `python setup.py --version` +git push --tags # update the repository version +python setup.py bdist_wheel # build this package in the dist directory +twine upload dist/* --username $PYPI_USERNAME --password $PYPI_PASSWORD # publish +ssh -o StrictHostKeyChecking=no deploy-api@fr.openfisca.org # Deploy the OpenFisca-France public API diff --git a/.circleci/deploy-python-3.sh b/.circleci/deploy-python-3.sh new file mode 100755 index 00000000..31785d15 --- /dev/null +++ b/.circleci/deploy-python-3.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +set -e + +VERSION_CHANGE_TRIGGERS="setup.py MANIFEST.in openfisca_core openfisca_web_api_preview" + +if git diff-index --quiet HEAD^ -- $VERSION_CHANGE_TRIGGERS ":(exclude)*.md" +then + echo "No deployment - Only non-functional elements were modified in this change" + exit 0 # there are no changes at all, the version is correct +fi + + +python setup.py bdist_wheel # build this package in the dist directory +twine upload dist/* --username $PYPI_USERNAME --password $PYPI_PASSWORD # publish diff --git a/deploy-if-version-bump.sh b/deploy-if-version-bump.sh deleted file mode 100755 index c19d8239..00000000 --- a/deploy-if-version-bump.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/sh - -set -e -if [ $PY_VERSION = "2" ]; then - if ! git rev-parse `python setup.py --version` 2>/dev/null ; then - git tag `python setup.py --version` - git push --tags # update the repository version - python setup.py bdist_wheel # build this package in the dist directory - twine upload dist/* --username $PYPI_USERNAME --password $PYPI_PASSWORD # publish - else - echo "No deployment - Only non-functional elements were modified in this change" - fi -fi - -if [ $PY_VERSION = "3" ]; then - python setup.py bdist_wheel # build this package in the dist directory - if ! twine upload dist/* --username $PYPI_USERNAME --password $PYPI_PASSWORD; then # publish - echo "Could not upload this package version on Pypi. This usually means that the version already exists, and that only non-functional elements were modified in this change. If this it not the case, check the error message above." - fi -fi From e1e45980749cabe3159395de768ee4da020cbbc1 Mon Sep 17 00:00:00 2001 From: Matti Schneider Date: Thu, 17 May 2018 14:53:37 +1200 Subject: [PATCH 05/14] Remove duplication in functional changes detection --- .circleci/config.yml | 4 ++-- .circleci/deploy-python-2.sh | 17 +++++++---------- .circleci/deploy-python-3.sh | 12 ++++-------- .circleci/detect-functional-changes.sh | 8 ++++++++ ...-bump.sh => is-version-number-acceptable.sh} | 10 ++++------ 5 files changed, 25 insertions(+), 26 deletions(-) create mode 100755 .circleci/detect-functional-changes.sh rename .circleci/{check-version-bump.sh => is-version-number-acceptable.sh} (64%) diff --git a/.circleci/config.yml b/.circleci/config.yml index ef74c3c1..13f45a61 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -38,7 +38,7 @@ jobs: name: Check version number has been properly updated command: | git fetch - ./.circleci/check-version-bump.sh + .circleci/is-version-number-acceptable.sh deploy_python2: docker: @@ -96,7 +96,7 @@ jobs: name: Check version number has been properly updated command: | git fetch - ./.circleci/check-version-bump.sh + .circleci/is-version-number-acceptable.sh deploy_python3: docker: diff --git a/.circleci/deploy-python-2.sh b/.circleci/deploy-python-2.sh index ce7c8e28..2e9ab4b5 100755 --- a/.circleci/deploy-python-2.sh +++ b/.circleci/deploy-python-2.sh @@ -2,16 +2,13 @@ set -e -VERSION_CHANGE_TRIGGERS="setup.py MANIFEST.in openfisca_core openfisca_web_api_preview" - -if git diff-index --quiet HEAD^ -- $VERSION_CHANGE_TRIGGERS ":(exclude)*.md" +if ./detect-functional-changes.sh then + git tag `python setup.py --version` + git push --tags # update the repository version + python setup.py bdist_wheel # build this package in the dist directory + twine upload dist/* --username $PYPI_USERNAME --password $PYPI_PASSWORD # publish + ssh -o StrictHostKeyChecking=no deploy-api@fr.openfisca.org # Deploy the OpenFisca-France public API +else echo "No deployment - Only non-functional elements were modified in this change" - exit 0 # there are no changes at all, the version is correct fi - -git tag `python setup.py --version` -git push --tags # update the repository version -python setup.py bdist_wheel # build this package in the dist directory -twine upload dist/* --username $PYPI_USERNAME --password $PYPI_PASSWORD # publish -ssh -o StrictHostKeyChecking=no deploy-api@fr.openfisca.org # Deploy the OpenFisca-France public API diff --git a/.circleci/deploy-python-3.sh b/.circleci/deploy-python-3.sh index 31785d15..1047660f 100755 --- a/.circleci/deploy-python-3.sh +++ b/.circleci/deploy-python-3.sh @@ -2,14 +2,10 @@ set -e -VERSION_CHANGE_TRIGGERS="setup.py MANIFEST.in openfisca_core openfisca_web_api_preview" - -if git diff-index --quiet HEAD^ -- $VERSION_CHANGE_TRIGGERS ":(exclude)*.md" +if ./detect-functional-changes.sh then + python setup.py bdist_wheel # build this package in the dist directory + twine upload dist/* --username $PYPI_USERNAME --password $PYPI_PASSWORD # publish +else echo "No deployment - Only non-functional elements were modified in this change" - exit 0 # there are no changes at all, the version is correct fi - - -python setup.py bdist_wheel # build this package in the dist directory -twine upload dist/* --username $PYPI_USERNAME --password $PYPI_PASSWORD # publish diff --git a/.circleci/detect-functional-changes.sh b/.circleci/detect-functional-changes.sh new file mode 100755 index 00000000..24913635 --- /dev/null +++ b/.circleci/detect-functional-changes.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +VERSION_CHANGE_TRIGGERS="setup.py MANIFEST.in openfisca_country_template" + +if git diff-index --quiet origin/master -- $VERSION_CHANGE_TRIGGERS ":(exclude)*.md" +then echo "No functional changes detected." +else exit 1 +fi diff --git a/.circleci/check-version-bump.sh b/.circleci/is-version-number-acceptable.sh similarity index 64% rename from .circleci/check-version-bump.sh rename to .circleci/is-version-number-acceptable.sh index a5ab03a5..acec468a 100755 --- a/.circleci/check-version-bump.sh +++ b/.circleci/is-version-number-acceptable.sh @@ -1,10 +1,8 @@ #! /usr/bin/env bash -VERSION_CHANGE_TRIGGERS="setup.py MANIFEST.in openfisca_country_template" - -if git diff-index --quiet origin/master -- $VERSION_CHANGE_TRIGGERS ":(exclude)*.md" +if ./detect-functional-changes.sh then - echo "No functional change. No need for a version update." + echo "No need for a version update." exit 0 fi @@ -12,7 +10,7 @@ current_version=`python setup.py --version` if git rev-parse --verify --quiet $current_version then - echo "Version $current_version already exists:" + echo "Version $current_version already exists in commit:" git --no-pager log -1 $current_version echo echo "Update the version number in setup.py before merging this branch into master." @@ -22,7 +20,7 @@ fi if git diff-index --quiet origin/master CHANGELOG.md then - echo "CHANGELOG.md has not been modified, while the code has changed." + echo "CHANGELOG.md has not been modified, while functional changes were made." echo "Explain what you changed before merging this branch into master." echo "Look at the CONTRIBUTING.md file to learn how to write the changelog." exit 2 From 78c7ab416e6408efee490b97575359f3de9037aa Mon Sep 17 00:00:00 2001 From: Matti Schneider Date: Thu, 17 May 2018 14:53:51 +1200 Subject: [PATCH 06/14] Minor simplification of CircleCI syntax --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 13f45a61..df9ce4fa 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -57,7 +57,7 @@ jobs: name: Upload a Python 2 package to Pypi and publish a tag if functional changes were made command: | source /tmp/venv/country_template/bin/activate - ./.circleci/deploy-python-2.sh + .circleci/deploy-python-2.sh build_python3: docker: @@ -115,7 +115,7 @@ jobs: name: Upload a Python 3 package to Pypi command: | source /tmp/venv/country_template/bin/activate - ./.circleci/deploy-python-3.sh + .circleci/deploy-python-3.sh workflows: version: 2 From 006273db40d3734a31cc0fc8f7912a690d078aea Mon Sep 17 00:00:00 2001 From: Florian Pagnoux Date: Fri, 18 May 2018 12:39:30 -0400 Subject: [PATCH 07/14] Add comment about Pypi username --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index df9ce4fa..51349a22 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -44,7 +44,7 @@ jobs: docker: - image: python:2.7.14 environment: - PYPI_USERNAME: openfisca-bot + PYPI_USERNAME: openfisca-bot # Edit this value to replace it by your Pypi username # PYPI_PASSWORD: this value is set in CircleCI's web interface; do not set it here, it is a secret! steps: @@ -102,7 +102,7 @@ jobs: docker: - image: python:3.6 environment: - PYPI_USERNAME: openfisca-bot + PYPI_USERNAME: openfisca-bot # Edit this value to replace it by your Pypi username # PYPI_PASSWORD: this value is set in CircleCI's web interface; do not set it here, it is a secret! steps: From 5846f05aeabba4eeb3f3bffd2e52dea59dbdf90e Mon Sep 17 00:00:00 2001 From: Florian Pagnoux Date: Fri, 18 May 2018 13:03:16 -0400 Subject: [PATCH 08/14] Make script work in test and deploy context --- .circleci/detect-functional-changes.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.circleci/detect-functional-changes.sh b/.circleci/detect-functional-changes.sh index 24913635..ce034643 100755 --- a/.circleci/detect-functional-changes.sh +++ b/.circleci/detect-functional-changes.sh @@ -1,8 +1,14 @@ #!/bin/sh VERSION_CHANGE_TRIGGERS="setup.py MANIFEST.in openfisca_country_template" +CURRENT_BRANCH=`git symbolic-ref --short HEAD` -if git diff-index --quiet origin/master -- $VERSION_CHANGE_TRIGGERS ":(exclude)*.md" +if [[ "$CURRENT_BRANCH" == "master" ]] +then LAST_MASTER_VERSION="HEAD^" +else LAST_MASTER_VERSION="origin/master" +fi + +if git diff-index --quiet $LAST_MASTER_VERSION -- $VERSION_CHANGE_TRIGGERS ":(exclude)*.md" then echo "No functional changes detected." else exit 1 fi From 9c2153d7f9977e330e1d2136f0f58b3f114b7919 Mon Sep 17 00:00:00 2001 From: Florian Pagnoux Date: Fri, 18 May 2018 13:14:00 -0400 Subject: [PATCH 09/14] Refactor deployment --- .circleci/config.yml | 24 ++++++++++++++++++++---- .circleci/deploy-python-2.sh | 14 -------------- .circleci/deploy-python-3.sh | 11 ----------- .circleci/detect-functional-changes.sh | 2 +- .circleci/publish-git-tag.sh | 4 ++++ .circleci/publish-python-package.sh | 4 ++++ 6 files changed, 29 insertions(+), 30 deletions(-) delete mode 100755 .circleci/deploy-python-2.sh delete mode 100755 .circleci/deploy-python-3.sh create mode 100755 .circleci/publish-git-tag.sh create mode 100755 .circleci/publish-python-package.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 51349a22..5c3c7bd8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -54,10 +54,18 @@ jobs: key: v1-py2-{{ checksum "setup.py" }} - run: - name: Upload a Python 2 package to Pypi and publish a tag if functional changes were made + name: Check for functional changes + command: if .circleci/detect-functional-changes.sh ; then circleci step halt ; fi + + - run: + name: Upload a Python package to Pypi command: | source /tmp/venv/country_template/bin/activate - .circleci/deploy-python-2.sh + .circleci/publish-python-package.sh + + - run: + name: Publish a git tag + command: .circleci/publish-git-tag.sh build_python3: docker: @@ -112,10 +120,18 @@ jobs: key: v1-py3-{{ checksum "setup.py" }} - run: - name: Upload a Python 3 package to Pypi + name: Check for functional changes + command: if .circleci/detect-functional-changes.sh ; then circleci step halt ; fi + + - run: + name: Upload a Python package to Pypi command: | source /tmp/venv/country_template/bin/activate - .circleci/deploy-python-3.sh + .circleci/publish-python-package.sh + + - run: + name: Publish a git tag + command: .circleci/publish-git-tag.sh workflows: version: 2 diff --git a/.circleci/deploy-python-2.sh b/.circleci/deploy-python-2.sh deleted file mode 100755 index 2e9ab4b5..00000000 --- a/.circleci/deploy-python-2.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh - -set -e - -if ./detect-functional-changes.sh -then - git tag `python setup.py --version` - git push --tags # update the repository version - python setup.py bdist_wheel # build this package in the dist directory - twine upload dist/* --username $PYPI_USERNAME --password $PYPI_PASSWORD # publish - ssh -o StrictHostKeyChecking=no deploy-api@fr.openfisca.org # Deploy the OpenFisca-France public API -else - echo "No deployment - Only non-functional elements were modified in this change" -fi diff --git a/.circleci/deploy-python-3.sh b/.circleci/deploy-python-3.sh deleted file mode 100755 index 1047660f..00000000 --- a/.circleci/deploy-python-3.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/sh - -set -e - -if ./detect-functional-changes.sh -then - python setup.py bdist_wheel # build this package in the dist directory - twine upload dist/* --username $PYPI_USERNAME --password $PYPI_PASSWORD # publish -else - echo "No deployment - Only non-functional elements were modified in this change" -fi diff --git a/.circleci/detect-functional-changes.sh b/.circleci/detect-functional-changes.sh index ce034643..949500d8 100755 --- a/.circleci/detect-functional-changes.sh +++ b/.circleci/detect-functional-changes.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#! /usr/bin/env bash VERSION_CHANGE_TRIGGERS="setup.py MANIFEST.in openfisca_country_template" CURRENT_BRANCH=`git symbolic-ref --short HEAD` diff --git a/.circleci/publish-git-tag.sh b/.circleci/publish-git-tag.sh new file mode 100755 index 00000000..4450357c --- /dev/null +++ b/.circleci/publish-git-tag.sh @@ -0,0 +1,4 @@ +#! /usr/bin/env bash + +git tag `python setup.py --version` +git push --tags # update the repository version diff --git a/.circleci/publish-python-package.sh b/.circleci/publish-python-package.sh new file mode 100755 index 00000000..8d331bd9 --- /dev/null +++ b/.circleci/publish-python-package.sh @@ -0,0 +1,4 @@ +#! /usr/bin/env bash + +python setup.py bdist_wheel # build this package in the dist directory +twine upload dist/* --username $PYPI_USERNAME --password $PYPI_PASSWORD # publish From 96d7dae2c1848ccb60819dc742f6924bb909bc34 Mon Sep 17 00:00:00 2001 From: Florian Pagnoux Date: Fri, 18 May 2018 14:01:09 -0400 Subject: [PATCH 10/14] Bypass version check on master --- .circleci/detect-functional-changes.sh | 3 +-- .circleci/is-version-number-acceptable.sh | 6 ++++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.circleci/detect-functional-changes.sh b/.circleci/detect-functional-changes.sh index 949500d8..233a40c9 100755 --- a/.circleci/detect-functional-changes.sh +++ b/.circleci/detect-functional-changes.sh @@ -1,9 +1,8 @@ #! /usr/bin/env bash VERSION_CHANGE_TRIGGERS="setup.py MANIFEST.in openfisca_country_template" -CURRENT_BRANCH=`git symbolic-ref --short HEAD` -if [[ "$CURRENT_BRANCH" == "master" ]] +if [[ "$CIRCLE_BRANCH" == "master" ]] then LAST_MASTER_VERSION="HEAD^" else LAST_MASTER_VERSION="origin/master" fi diff --git a/.circleci/is-version-number-acceptable.sh b/.circleci/is-version-number-acceptable.sh index acec468a..82d247f3 100755 --- a/.circleci/is-version-number-acceptable.sh +++ b/.circleci/is-version-number-acceptable.sh @@ -1,5 +1,11 @@ #! /usr/bin/env bash +if [[ $CIRCLE_BRANCH == master ]] +then + echo "No need for a version check on master." + exit 0 +fi + if ./detect-functional-changes.sh then echo "No need for a version update." From e959284c9de7b4e0e7fedcccb1fc82a170580c1b Mon Sep 17 00:00:00 2001 From: Florian Pagnoux Date: Fri, 18 May 2018 13:54:32 -0400 Subject: [PATCH 11/14] Use relative path --- .circleci/is-version-number-acceptable.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/is-version-number-acceptable.sh b/.circleci/is-version-number-acceptable.sh index 82d247f3..ea14c2ea 100755 --- a/.circleci/is-version-number-acceptable.sh +++ b/.circleci/is-version-number-acceptable.sh @@ -6,7 +6,7 @@ then exit 0 fi -if ./detect-functional-changes.sh +if $(dirname "$BASH_SOURCE")/detect-functional-changes.sh then echo "No need for a version update." exit 0 From 9c09dd0e49c13dc4ac489dc79140330b9705e6c4 Mon Sep 17 00:00:00 2001 From: Florian Pagnoux Date: Fri, 18 May 2018 14:06:36 -0400 Subject: [PATCH 12/14] Don't publish a tag in Py3 --- .circleci/config.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5c3c7bd8..9f34bcad 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -129,10 +129,6 @@ jobs: source /tmp/venv/country_template/bin/activate .circleci/publish-python-package.sh - - run: - name: Publish a git tag - command: .circleci/publish-git-tag.sh - workflows: version: 2 build_and_deploy: From 266dd14bc7c9c308f17fdb3f8cad97fac44d95e4 Mon Sep 17 00:00:00 2001 From: Matti Schneider Date: Sat, 19 May 2018 16:57:47 +1200 Subject: [PATCH 13/14] Increase resilience of last tagged version search Do not rely on master merge commits to necessarily have been tagged and published: if there is an issue there, we would enter a cycle of failing deployments --- .circleci/detect-functional-changes.sh | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.circleci/detect-functional-changes.sh b/.circleci/detect-functional-changes.sh index 233a40c9..539375a7 100755 --- a/.circleci/detect-functional-changes.sh +++ b/.circleci/detect-functional-changes.sh @@ -2,12 +2,9 @@ VERSION_CHANGE_TRIGGERS="setup.py MANIFEST.in openfisca_country_template" -if [[ "$CIRCLE_BRANCH" == "master" ]] -then LAST_MASTER_VERSION="HEAD^" -else LAST_MASTER_VERSION="origin/master" -fi +last_tagged_commit=`git describe --tags --abbrev=0 --first-parent` # --first-parent ensures we don't follow tags not published in master through an unlikely intermediary merge commit -if git diff-index --quiet $LAST_MASTER_VERSION -- $VERSION_CHANGE_TRIGGERS ":(exclude)*.md" +if git diff-index --quiet $last_tagged_commit -- $VERSION_CHANGE_TRIGGERS ":(exclude)*.md" then echo "No functional changes detected." else exit 1 fi From 86886c41f16204f312c6f2c58cbdea9884d00a9b Mon Sep 17 00:00:00 2001 From: Matti Schneider Date: Sat, 19 May 2018 16:58:59 +1200 Subject: [PATCH 14/14] Give a short list of functional changes If none are detected, there will still be no output --- .circleci/detect-functional-changes.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/detect-functional-changes.sh b/.circleci/detect-functional-changes.sh index 539375a7..805e7869 100755 --- a/.circleci/detect-functional-changes.sh +++ b/.circleci/detect-functional-changes.sh @@ -4,7 +4,7 @@ VERSION_CHANGE_TRIGGERS="setup.py MANIFEST.in openfisca_country_template" last_tagged_commit=`git describe --tags --abbrev=0 --first-parent` # --first-parent ensures we don't follow tags not published in master through an unlikely intermediary merge commit -if git diff-index --quiet $last_tagged_commit -- $VERSION_CHANGE_TRIGGERS ":(exclude)*.md" +if git diff-index --shortstat $last_tagged_commit -- $VERSION_CHANGE_TRIGGERS ":(exclude)*.md" then echo "No functional changes detected." else exit 1 fi