From fa97c70431ab894b014c02a8285012e1be87b5cb Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Sun, 18 Dec 2016 03:08:06 -0500 Subject: [PATCH 1/7] Fix source channel order in CIs. The order the source channels were listed in `conda-forge.yml` was not being respected. In the case of Travis CI and AppVeyor, these were getting added in the wrong order so the priority would have been backwards (i.e. last channel in `conda-forge.yml` would have come first). In the CircleCI/docker build, the channels were being sorted, which should not occur. Thus order was not being respected at all in that case. Here we reverse source channel additions in the Travis CI and AppVeyor configuration files so that will get the right priority. In the docker build configuration, we drop the unneeded sorting. --- conda_smithy/templates/appveyor.yml.tmpl | 2 +- conda_smithy/templates/run_docker_build.tmpl | 2 +- conda_smithy/templates/travis.yml.tmpl | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/conda_smithy/templates/appveyor.yml.tmpl b/conda_smithy/templates/appveyor.yml.tmpl index 8caa5377d..12c69ebda 100644 --- a/conda_smithy/templates/appveyor.yml.tmpl +++ b/conda_smithy/templates/appveyor.yml.tmpl @@ -45,7 +45,7 @@ install: - cmd: set "OLDPATH=%PATH%" - cmd: set "PATH=%CONDA_INSTALL_LOCN%\\Scripts;%CONDA_INSTALL_LOCN%\\Library\\bin;%PATH%" - cmd: conda config --set show_channel_urls true - {%- for channel in channels.get('sources', []) %} + {%- for channel in channels.get('sources', [])|reverse %} - cmd: conda config --add channels {{ channel }}{% endfor %} # Add a hack to switch to `conda` version `4.1.12` before activating. diff --git a/conda_smithy/templates/run_docker_build.tmpl b/conda_smithy/templates/run_docker_build.tmpl index 8277df928..451b45282 100644 --- a/conda_smithy/templates/run_docker_build.tmpl +++ b/conda_smithy/templates/run_docker_build.tmpl @@ -13,7 +13,7 @@ docker info config=$(cat < Date: Sun, 18 Dec 2016 18:47:44 -0500 Subject: [PATCH 2/7] Clean up various AppVeyor hacks and kludge. Some of the changes included in here install a very particular version of `conda`. Since then newer version have been released that we use successfully. So it makes sense to remove the pinning. However, the update is still needed as AppVeyor's included `conda` is so old. Also we found ourselves in a situation where Python 2.7 jobs where running out of memory with particular configurations of the root environment when running `conda-forge-build-setup`. Some hacks were added to adjust the `root` environment to fix this. Though these appear to be no longer necessary. --- conda_smithy/templates/appveyor.yml.tmpl | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/conda_smithy/templates/appveyor.yml.tmpl b/conda_smithy/templates/appveyor.yml.tmpl index 12c69ebda..3ab455a61 100644 --- a/conda_smithy/templates/appveyor.yml.tmpl +++ b/conda_smithy/templates/appveyor.yml.tmpl @@ -41,17 +41,10 @@ install: # Cywing's git breaks conda-build. (See https://github.com/conda-forge/conda-smithy-feedstock/pull/2.) - cmd: rmdir C:\cygwin /s /q - # Add our channels. + # Add a hack to update conda as the included copy is too old. - cmd: set "OLDPATH=%PATH%" - cmd: set "PATH=%CONDA_INSTALL_LOCN%\\Scripts;%CONDA_INSTALL_LOCN%\\Library\\bin;%PATH%" - - cmd: conda config --set show_channel_urls true - {%- for channel in channels.get('sources', [])|reverse %} - - cmd: conda config --add channels {{ channel }}{% endfor %} - - # Add a hack to switch to `conda` version `4.1.12` before activating. - # This is required to handle a long path activation issue. - # Please see PR ( https://github.com/conda/conda/pull/3349 ). - - cmd: conda install --yes --quiet conda=4.1.12 + - cmd: conda update --yes --quiet conda - cmd: set "PATH=%OLDPATH%" - cmd: set "OLDPATH=" @@ -59,6 +52,12 @@ install: - cmd: call %CONDA_INSTALL_LOCN%\Scripts\activate.bat - cmd: set PYTHONUNBUFFERED=1 + # Add our channels. + - cmd: conda config --set show_channel_urls true + {%- for channel in channels.get('sources', [])|reverse %} + - cmd: conda config --add channels {{ channel }}{% endfor %} + + # Configure the VM. - cmd: conda install -n root --quiet --yes obvious-ci - cmd: conda install -n root --quiet --yes conda-forge-build-setup {% if build_setup -%} From abe3bbe8866924752cdd0548e9b67cc6f10d3bae Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Sun, 18 Dec 2016 03:17:05 -0500 Subject: [PATCH 3/7] Add `defaults` to the source channels explicitly. --- conda_smithy/configure_feedstock.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conda_smithy/configure_feedstock.py b/conda_smithy/configure_feedstock.py index ff0c1e0c2..da19b3437 100755 --- a/conda_smithy/configure_feedstock.py +++ b/conda_smithy/configure_feedstock.py @@ -529,7 +529,8 @@ def main(forge_file_directory): 'travis': {}, 'circle': {}, 'appveyor': {}, - 'channels': {'sources': ['conda-forge'], 'targets': [['conda-forge', 'main']]}, + 'channels': {'sources': ['conda-forge', 'defaults'], + 'targets': [['conda-forge', 'main']]}, 'github': {'user_or_org': 'conda-forge', 'repo_name': ''}, 'recipe_dir': recipe_dir} forge_dir = os.path.abspath(forge_file_directory) From 795712ec3f861a0beb6b6e86b15b38d4eb2ec4cc Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Sun, 18 Dec 2016 03:37:21 -0500 Subject: [PATCH 4/7] Explicitly remove the `defaults` channel. It is possible that a user may not want the `defaults` channel. This may be desired as they wish to build a clean stack from scratch. Though our main concern is that including `defaults` implicitly or hard-coding its inclusion can result in differences between re-rendering and build time that will cause a variety of subtle bugs. Given `defaults` is included if the source channels are not specified, there is no harm in us dropping it in this way. Here we drop the hard-coding of `defaults` from the docker build `.condarc`. Also we explicitly remove `defaults` from the `.condarc` on Travis CI and AppVeyor before proceeding to add channels. As a result, we can be assured that only the channels that we specify in `conda-forge.yml` or the default set included in `conda-smithy` will be the only channels used as sources and in the order we have specified. --- conda_smithy/templates/appveyor.yml.tmpl | 1 + conda_smithy/templates/run_docker_build.tmpl | 1 - conda_smithy/templates/travis.yml.tmpl | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conda_smithy/templates/appveyor.yml.tmpl b/conda_smithy/templates/appveyor.yml.tmpl index 3ab455a61..45406f51b 100644 --- a/conda_smithy/templates/appveyor.yml.tmpl +++ b/conda_smithy/templates/appveyor.yml.tmpl @@ -54,6 +54,7 @@ install: # Add our channels. - cmd: conda config --set show_channel_urls true + - cmd: conda config --remove channels defaults {%- for channel in channels.get('sources', [])|reverse %} - cmd: conda config --add channels {{ channel }}{% endfor %} diff --git a/conda_smithy/templates/run_docker_build.tmpl b/conda_smithy/templates/run_docker_build.tmpl index 451b45282..6b7a009b1 100644 --- a/conda_smithy/templates/run_docker_build.tmpl +++ b/conda_smithy/templates/run_docker_build.tmpl @@ -16,7 +16,6 @@ channels: {%- for channel in channels.get('sources', []) %} - {{ channel }} {%- endfor %} - - defaults # As we need conda-build conda-build: root-dir: /feedstock_root/build_artefacts diff --git a/conda_smithy/templates/travis.yml.tmpl b/conda_smithy/templates/travis.yml.tmpl index 36e8a6a3d..b1f30dab8 100644 --- a/conda_smithy/templates/travis.yml.tmpl +++ b/conda_smithy/templates/travis.yml.tmpl @@ -44,7 +44,7 @@ install: bash $MINICONDA_FILE -b source /Users/travis/miniconda3/bin/activate root - + conda config --remove channels defaults {%- for channel in channels.get('sources', [])|reverse %} conda config --add channels {{ channel }} {%- endfor %} From 2aa06adec9493dfa0d27387d6f5f69db2ec1b339 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Sun, 18 Dec 2016 03:55:29 -0500 Subject: [PATCH 5/7] Optionally provide channels when getting the index. The index will try pick up channels from `.condarc` normally. However, that may not be the ideal behavior. This allows us to override the channels to use when filling the index. As a result, we should get consistent behavior on different user's machines regardless of their own channel preferences. --- conda_smithy/configure_feedstock.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/conda_smithy/configure_feedstock.py b/conda_smithy/configure_feedstock.py index da19b3437..c9b26e817 100755 --- a/conda_smithy/configure_feedstock.py +++ b/conda_smithy/configure_feedstock.py @@ -500,8 +500,9 @@ def meta_of_feedstock(forge_dir, config=None): return meta -def compute_build_matrix(meta, existing_matrix=None): - index = conda.api.get_index() +def compute_build_matrix(meta, existing_matrix=None, channel_sources=tuple()): + channel_sources = tuple(channel_sources) + index = conda.api.get_index(channel_urls=channel_sources) mtx = special_case_version_matrix(meta, index) mtx = list(filter_cases(mtx, ['python >=2.7,<3|>=3.4', 'numpy >=1.10'])) if existing_matrix: From 2fe3c200ae5a4c74eb56003ae849ad7e13a33af2 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Sun, 18 Dec 2016 03:55:57 -0500 Subject: [PATCH 6/7] Override the definition of `defaults`. When getting the index, `conda` has a notion of `defaults` (at least in older versions) that is dependent on the platform it is run on. Unfortunately, this may result in using things like the `msys2` channel when re-rendering on Linux even though there are no Linux packages in this case. To fix this we add a hack that detects the `defaults` channel if listed and redefines it based on the platform we are masquerading as. This hopefully should allow Windows users to re-render natively. Also this should ensure that we do use `msys2` on Windows so that feedstocks requiring it will be re-rendered properly. --- conda_smithy/configure_feedstock.py | 44 +++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/conda_smithy/configure_feedstock.py b/conda_smithy/configure_feedstock.py index c9b26e817..4eab102ac 100755 --- a/conda_smithy/configure_feedstock.py +++ b/conda_smithy/configure_feedstock.py @@ -11,6 +11,33 @@ import conda.api import conda.config + +try: + # Try conda's API in newer 4.2.x and 4.3.x. + from conda.exports import ( + DEFAULT_CHANNELS_UNIX, + DEFAULT_CHANNELS_WIN, + ) +except ImportError: + try: + # Fallback for old versions of 4.2.x and 4.3.x. + from conda.base.constants import ( + DEFAULT_CHANNELS_UNIX, + DEFAULT_CHANNELS_WIN, + ) + except ImportError: + # Fallback for very old conda (e.g. 4.1.x). + DEFAULT_CHANNELS_UNIX = ( + 'https://repo.continuum.io/pkgs/free', + 'https://repo.continuum.io/pkgs/pro', + ) + + DEFAULT_CHANNELS_WIN = ( + 'https://repo.continuum.io/pkgs/free', + 'https://repo.continuum.io/pkgs/pro', + 'https://repo.continuum.io/pkgs/msys2', + ) + import conda_build.metadata try: import conda_build.api @@ -502,6 +529,23 @@ def meta_of_feedstock(forge_dir, config=None): def compute_build_matrix(meta, existing_matrix=None, channel_sources=tuple()): channel_sources = tuple(channel_sources) + + # Override what `defaults` means depending on the platform used. + try: + i = channel_sources.index("defaults") + + defaults = DEFAULT_CHANNELS_UNIX + if meta_config(meta).subdir.startswith("win"): + defaults = DEFAULT_CHANNELS_WIN + + channel_sources = ( + channel_sources[:i] + + defaults + + channel_sources[i+1:] + ) + except ValueError: + pass + index = conda.api.get_index(channel_urls=channel_sources) mtx = special_case_version_matrix(meta, index) mtx = list(filter_cases(mtx, ['python >=2.7,<3|>=3.4', 'numpy >=1.10'])) From 0a99bfb7d578a6b71c6ebf8df19d8013dff5c087 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Sun, 18 Dec 2016 03:56:53 -0500 Subject: [PATCH 7/7] Use `channels/sources` for our index. Instead of using whatever `conda` finds to generate our index (e.g. `.condarc`), this uses the `channels/sources` from the feedstock's `conda-forge.yml` to determine what should constitute the index. This matches more closely with what the CIs would do anyways as they get all dependencies from this list. --- conda_smithy/configure_feedstock.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/conda_smithy/configure_feedstock.py b/conda_smithy/configure_feedstock.py index 4eab102ac..55660df24 100755 --- a/conda_smithy/configure_feedstock.py +++ b/conda_smithy/configure_feedstock.py @@ -71,7 +71,11 @@ def render_run_docker_build(jinja_env, forge_config, forge_dir): meta = forge_config['package'] with fudge_subdir('linux-64', build_config=meta_config(meta)): meta.parse_again() - matrix = compute_build_matrix(meta, forge_config.get('matrix')) + matrix = compute_build_matrix( + meta, + forge_config.get('matrix'), + forge_config.get('channels', {}).get('sources', tuple()) + ) cases_not_skipped = [] for case in matrix: pkgs, vars = split_case(case) @@ -172,7 +176,11 @@ def render_circle(jinja_env, forge_config, forge_dir): meta = forge_config['package'] with fudge_subdir('linux-64', build_config=meta_config(meta)): meta.parse_again() - matrix = compute_build_matrix(meta, forge_config.get('matrix')) + matrix = compute_build_matrix( + meta, + forge_config.get('matrix'), + forge_config.get('channels', {}).get('sources', tuple()) + ) cases_not_skipped = [] for case in matrix: @@ -225,7 +233,11 @@ def render_travis(jinja_env, forge_config, forge_dir): meta = forge_config['package'] with fudge_subdir('osx-64', build_config=meta_config(meta)): meta.parse_again() - matrix = compute_build_matrix(meta, forge_config.get('matrix')) + matrix = compute_build_matrix( + meta, + forge_config.get('matrix'), + forge_config.get('channels', {}).get('sources', tuple()) + ) cases_not_skipped = [] for case in matrix: @@ -357,7 +369,11 @@ def render_appveyor(jinja_env, forge_config, forge_dir): for platform, arch in [['win-32', 'x86'], ['win-64', 'x64']]: with fudge_subdir(platform, build_config=meta_config(meta)): meta.parse_again() - matrix = compute_build_matrix(meta, forge_config.get('matrix')) + matrix = compute_build_matrix( + meta, + forge_config.get('matrix'), + forge_config.get('channels', {}).get('sources', tuple()) + ) cases_not_skipped = [] for case in matrix: