From f856c65cb828c24bf22ecb75e579c36625162fcd Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Sun, 18 Dec 2016 18:47:44 -0500 Subject: [PATCH 1/6] Clean up various AppVeyor hacks and kludge. Some of the changes included in here install a very particular version of `conda`. Since then newer version have been released that we use successfully. So it makes sense to remove the pinning. However, the update is still needed as AppVeyor's included `conda` is so old. Also we found ourselves in a situation where Python 2.7 jobs where running out of memory with particular configurations of the root environment when running `conda-forge-build-setup`. Some hacks were added to adjust the `root` environment to fix this. Though these appear to be no longer necessary. --- conda_smithy/templates/appveyor.yml.tmpl | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/conda_smithy/templates/appveyor.yml.tmpl b/conda_smithy/templates/appveyor.yml.tmpl index 12c69ebda..3ab455a61 100644 --- a/conda_smithy/templates/appveyor.yml.tmpl +++ b/conda_smithy/templates/appveyor.yml.tmpl @@ -41,17 +41,10 @@ install: # Cywing's git breaks conda-build. (See https://github.com/conda-forge/conda-smithy-feedstock/pull/2.) - cmd: rmdir C:\cygwin /s /q - # Add our channels. + # Add a hack to update conda as the included copy is too old. - cmd: set "OLDPATH=%PATH%" - cmd: set "PATH=%CONDA_INSTALL_LOCN%\\Scripts;%CONDA_INSTALL_LOCN%\\Library\\bin;%PATH%" - - cmd: conda config --set show_channel_urls true - {%- for channel in channels.get('sources', [])|reverse %} - - cmd: conda config --add channels {{ channel }}{% endfor %} - - # Add a hack to switch to `conda` version `4.1.12` before activating. - # This is required to handle a long path activation issue. - # Please see PR ( https://github.com/conda/conda/pull/3349 ). - - cmd: conda install --yes --quiet conda=4.1.12 + - cmd: conda update --yes --quiet conda - cmd: set "PATH=%OLDPATH%" - cmd: set "OLDPATH=" @@ -59,6 +52,12 @@ install: - cmd: call %CONDA_INSTALL_LOCN%\Scripts\activate.bat - cmd: set PYTHONUNBUFFERED=1 + # Add our channels. + - cmd: conda config --set show_channel_urls true + {%- for channel in channels.get('sources', [])|reverse %} + - cmd: conda config --add channels {{ channel }}{% endfor %} + + # Configure the VM. - cmd: conda install -n root --quiet --yes obvious-ci - cmd: conda install -n root --quiet --yes conda-forge-build-setup {% if build_setup -%} From abe3bbe8866924752cdd0548e9b67cc6f10d3bae Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Sun, 18 Dec 2016 03:17:05 -0500 Subject: [PATCH 2/6] Add `defaults` to the source channels explicitly. --- conda_smithy/configure_feedstock.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conda_smithy/configure_feedstock.py b/conda_smithy/configure_feedstock.py index ff0c1e0c2..da19b3437 100755 --- a/conda_smithy/configure_feedstock.py +++ b/conda_smithy/configure_feedstock.py @@ -529,7 +529,8 @@ def main(forge_file_directory): 'travis': {}, 'circle': {}, 'appveyor': {}, - 'channels': {'sources': ['conda-forge'], 'targets': [['conda-forge', 'main']]}, + 'channels': {'sources': ['conda-forge', 'defaults'], + 'targets': [['conda-forge', 'main']]}, 'github': {'user_or_org': 'conda-forge', 'repo_name': ''}, 'recipe_dir': recipe_dir} forge_dir = os.path.abspath(forge_file_directory) From 795712ec3f861a0beb6b6e86b15b38d4eb2ec4cc Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Sun, 18 Dec 2016 03:37:21 -0500 Subject: [PATCH 3/6] Explicitly remove the `defaults` channel. It is possible that a user may not want the `defaults` channel. This may be desired as they wish to build a clean stack from scratch. Though our main concern is that including `defaults` implicitly or hard-coding its inclusion can result in differences between re-rendering and build time that will cause a variety of subtle bugs. Given `defaults` is included if the source channels are not specified, there is no harm in us dropping it in this way. Here we drop the hard-coding of `defaults` from the docker build `.condarc`. Also we explicitly remove `defaults` from the `.condarc` on Travis CI and AppVeyor before proceeding to add channels. As a result, we can be assured that only the channels that we specify in `conda-forge.yml` or the default set included in `conda-smithy` will be the only channels used as sources and in the order we have specified. --- conda_smithy/templates/appveyor.yml.tmpl | 1 + conda_smithy/templates/run_docker_build.tmpl | 1 - conda_smithy/templates/travis.yml.tmpl | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conda_smithy/templates/appveyor.yml.tmpl b/conda_smithy/templates/appveyor.yml.tmpl index 3ab455a61..45406f51b 100644 --- a/conda_smithy/templates/appveyor.yml.tmpl +++ b/conda_smithy/templates/appveyor.yml.tmpl @@ -54,6 +54,7 @@ install: # Add our channels. - cmd: conda config --set show_channel_urls true + - cmd: conda config --remove channels defaults {%- for channel in channels.get('sources', [])|reverse %} - cmd: conda config --add channels {{ channel }}{% endfor %} diff --git a/conda_smithy/templates/run_docker_build.tmpl b/conda_smithy/templates/run_docker_build.tmpl index 451b45282..6b7a009b1 100644 --- a/conda_smithy/templates/run_docker_build.tmpl +++ b/conda_smithy/templates/run_docker_build.tmpl @@ -16,7 +16,6 @@ channels: {%- for channel in channels.get('sources', []) %} - {{ channel }} {%- endfor %} - - defaults # As we need conda-build conda-build: root-dir: /feedstock_root/build_artefacts diff --git a/conda_smithy/templates/travis.yml.tmpl b/conda_smithy/templates/travis.yml.tmpl index 36e8a6a3d..b1f30dab8 100644 --- a/conda_smithy/templates/travis.yml.tmpl +++ b/conda_smithy/templates/travis.yml.tmpl @@ -44,7 +44,7 @@ install: bash $MINICONDA_FILE -b source /Users/travis/miniconda3/bin/activate root - + conda config --remove channels defaults {%- for channel in channels.get('sources', [])|reverse %} conda config --add channels {{ channel }} {%- endfor %} From 2aa06adec9493dfa0d27387d6f5f69db2ec1b339 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Sun, 18 Dec 2016 03:55:29 -0500 Subject: [PATCH 4/6] Optionally provide channels when getting the index. The index will try pick up channels from `.condarc` normally. However, that may not be the ideal behavior. This allows us to override the channels to use when filling the index. As a result, we should get consistent behavior on different user's machines regardless of their own channel preferences. --- conda_smithy/configure_feedstock.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/conda_smithy/configure_feedstock.py b/conda_smithy/configure_feedstock.py index da19b3437..c9b26e817 100755 --- a/conda_smithy/configure_feedstock.py +++ b/conda_smithy/configure_feedstock.py @@ -500,8 +500,9 @@ def meta_of_feedstock(forge_dir, config=None): return meta -def compute_build_matrix(meta, existing_matrix=None): - index = conda.api.get_index() +def compute_build_matrix(meta, existing_matrix=None, channel_sources=tuple()): + channel_sources = tuple(channel_sources) + index = conda.api.get_index(channel_urls=channel_sources) mtx = special_case_version_matrix(meta, index) mtx = list(filter_cases(mtx, ['python >=2.7,<3|>=3.4', 'numpy >=1.10'])) if existing_matrix: From a293c33bd75a1894affbf111c1710dc8b2aa43a8 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Sun, 18 Dec 2016 03:55:57 -0500 Subject: [PATCH 5/6] Override the definition of `defaults`. When getting the index, `conda` has a notion of `defaults` (at least in older versions) that is dependent on the platform it is run on. Unfortunately, this may result in using things like the `msys2` channel when re-rendering on Linux even though there are no Linux packages in this case. To fix this we add a hack that detects the `defaults` channel if listed and redefines it based on the platform we are masquerading as. This hopefully should allow Windows users to re-render natively. Also this should ensure that we do use `msys2` on Windows so that feedstocks requiring it will be re-rendered properly. --- conda_smithy/configure_feedstock.py | 36 +++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/conda_smithy/configure_feedstock.py b/conda_smithy/configure_feedstock.py index c9b26e817..dfc872a3f 100755 --- a/conda_smithy/configure_feedstock.py +++ b/conda_smithy/configure_feedstock.py @@ -11,6 +11,25 @@ import conda.api import conda.config + +try: + from conda.base.constants import ( + DEFAULT_CHANNELS_UNIX, + DEFAULT_CHANNELS_WIN, + ) +except ImportError: + # old conda + DEFAULT_CHANNELS_UNIX = ( + 'https://repo.continuum.io/pkgs/free', + 'https://repo.continuum.io/pkgs/pro', + ) + + DEFAULT_CHANNELS_WIN = ( + 'https://repo.continuum.io/pkgs/free', + 'https://repo.continuum.io/pkgs/pro', + 'https://repo.continuum.io/pkgs/msys2', + ) + import conda_build.metadata try: import conda_build.api @@ -502,6 +521,23 @@ def meta_of_feedstock(forge_dir, config=None): def compute_build_matrix(meta, existing_matrix=None, channel_sources=tuple()): channel_sources = tuple(channel_sources) + + # Override what `defaults` means depending on the platform used. + try: + i = channel_sources.index("defaults") + + defaults = DEFAULT_CHANNELS_UNIX + if meta_config(meta).subdir.startswith("win"): + defaults = DEFAULT_CHANNELS_WIN + + channel_sources = ( + channel_sources[:i] + + defaults + + channel_sources[i+1:] + ) + except ValueError: + pass + index = conda.api.get_index(channel_urls=channel_sources) mtx = special_case_version_matrix(meta, index) mtx = list(filter_cases(mtx, ['python >=2.7,<3|>=3.4', 'numpy >=1.10'])) From e434f611f06c24545d734e6b9f87c40d3700298e Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Sun, 18 Dec 2016 03:56:53 -0500 Subject: [PATCH 6/6] Use `channels/sources` for our index. Instead of using whatever `conda` finds to generate our index (e.g. `.condarc`), this uses the `channels/sources` from the feedstock's `conda-forge.yml` to determine what should constitute the index. This matches more closely with what the CIs would do anyways as they get all dependencies from this list. --- conda_smithy/configure_feedstock.py | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/conda_smithy/configure_feedstock.py b/conda_smithy/configure_feedstock.py index dfc872a3f..44a9f1285 100755 --- a/conda_smithy/configure_feedstock.py +++ b/conda_smithy/configure_feedstock.py @@ -63,7 +63,11 @@ def render_run_docker_build(jinja_env, forge_config, forge_dir): meta = forge_config['package'] with fudge_subdir('linux-64', build_config=meta_config(meta)): meta.parse_again() - matrix = compute_build_matrix(meta, forge_config.get('matrix')) + matrix = compute_build_matrix( + meta, + forge_config.get('matrix'), + forge_config.get('channels', {}).get('sources', tuple()) + ) cases_not_skipped = [] for case in matrix: pkgs, vars = split_case(case) @@ -164,7 +168,11 @@ def render_circle(jinja_env, forge_config, forge_dir): meta = forge_config['package'] with fudge_subdir('linux-64', build_config=meta_config(meta)): meta.parse_again() - matrix = compute_build_matrix(meta, forge_config.get('matrix')) + matrix = compute_build_matrix( + meta, + forge_config.get('matrix'), + forge_config.get('channels', {}).get('sources', tuple()) + ) cases_not_skipped = [] for case in matrix: @@ -217,7 +225,11 @@ def render_travis(jinja_env, forge_config, forge_dir): meta = forge_config['package'] with fudge_subdir('osx-64', build_config=meta_config(meta)): meta.parse_again() - matrix = compute_build_matrix(meta, forge_config.get('matrix')) + matrix = compute_build_matrix( + meta, + forge_config.get('matrix'), + forge_config.get('channels', {}).get('sources', tuple()) + ) cases_not_skipped = [] for case in matrix: @@ -349,7 +361,11 @@ def render_appveyor(jinja_env, forge_config, forge_dir): for platform, arch in [['win-32', 'x86'], ['win-64', 'x64']]: with fudge_subdir(platform, build_config=meta_config(meta)): meta.parse_again() - matrix = compute_build_matrix(meta, forge_config.get('matrix')) + matrix = compute_build_matrix( + meta, + forge_config.get('matrix'), + forge_config.get('channels', {}).get('sources', tuple()) + ) cases_not_skipped = [] for case in matrix: