Merge pull request #2933 from yuvipanda/cartopy-again

Get python packages from conda-forge, use mamba
berkeley-dsep-infra · Oct 27, 2021 · 638b22f · 638b22f
2 parents b3dec6a + 6bafd50
commit 638b22f
Show file tree

Hide file tree

Showing 16 changed files with 161 additions and 153 deletions.
diff --git a/deployments/biology/image/Dockerfile b/deployments/biology/image/Dockerfile
@@ -127,9 +127,9 @@ RUN apt-get update -qq --yes > /dev/null && \
 
 WORKDIR /home/jovyan
 
-COPY install-miniforge.bash /tmp/install-miniforge.bash
-RUN chmod 777 /tmp/install-miniforge.bash
-RUN /tmp/install-miniforge.bash
+COPY install-mambaforge.bash /tmp/install-mambaforge.bash
+RUN chmod 777 /tmp/install-mambaforge.bash
+RUN /tmp/install-mambaforge.bash
 
 # Needed by RStudio
 RUN apt-get update -qq --yes && \

diff --git a/...ents/biology/image/install-miniforge.bash → ...nts/biology/image/install-mambaforge.bash b/...ents/biology/image/install-miniforge.bash → ...nts/biology/image/install-mambaforge.bash
@@ -1,12 +1,12 @@
 #!/bin/bash
-# This downloads and installs a pinned version of miniconda
+# This downloads and installs a pinned version of mambaforge
 set -ex
 
 cd $(dirname $0)
-MINIFORGE_VERSION=4.10.3-4
+MAMBAFORGE_VERSION=4.10.3-7
 
-URL="https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/Miniforge3-${MINIFORGE_VERSION}-Linux-x86_64.sh"
-INSTALLER_PATH=/tmp/miniforge-installer.sh
+URL="https://github.com/conda-forge/miniforge/releases/download/${MAMBAFORGE_VERSION}/Mambaforge-${MAMBAFORGE_VERSION}-Linux-x86_64.sh"
+INSTALLER_PATH=/tmp/mambaforge-installer.sh
 
 # make sure we don't do anything funky with user's $HOME
 # since this is run as root
@@ -34,7 +34,7 @@ conda clean --all -f -y
 # Remove the big installer so we don't increase docker image size too much
 rm ${INSTALLER_PATH}
 
-# Remove the pip cache created as part of installing miniconda
+# Remove the pip cache created as part of installing mambaforge
 rm -rf /root/.cache
 
 chown -R $NB_USER:$NB_USER ${CONDA_DIR}

diff --git a/deployments/data8/image/Dockerfile b/deployments/data8/image/Dockerfile
@@ -70,9 +70,9 @@ ENV PATH ${CONDA_DIR}/bin:$PATH
 
 USER root
 
-# Install miniforge as root - the script chowns to $NB_USER by the end
-COPY install-miniforge.bash /tmp/install-miniforge.bash
-RUN /tmp/install-miniforge.bash
+# Install mambaforgeas root - the script chowns to $NB_USER by the end
+COPY install-mambaforge.bash /tmp/install-mambaforge.bash
+RUN /tmp/install-mambaforge.bash
 
 USER ${NB_USER}
 

diff --git a/...yments/data8/image/install-miniforge.bash → ...ments/data8/image/install-mambaforge.bash b/...yments/data8/image/install-miniforge.bash → ...ments/data8/image/install-mambaforge.bash
@@ -1,12 +1,12 @@
 #!/bin/bash
-# This downloads and installs a pinned version of miniconda
+# This downloads and installs a pinned version of mambaforge
 set -ex
 
 cd $(dirname $0)
-MINIFORGE_VERSION=4.10.3-4
+MAMBAFORGE_VERSION=4.10.3-7
 
-URL="https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/Miniforge3-${MINIFORGE_VERSION}-Linux-x86_64.sh"
-INSTALLER_PATH=/tmp/miniforge-installer.sh
+URL="https://github.com/conda-forge/miniforge/releases/download/${MAMBAFORGE_VERSION}/Mambaforge-${MAMBAFORGE_VERSION}-Linux-x86_64.sh"
+INSTALLER_PATH=/tmp/mambaforge-installer.sh
 
 # make sure we don't do anything funky with user's $HOME
 # since this is run as root
@@ -34,7 +34,7 @@ conda clean --all -f -y
 # Remove the big installer so we don't increase docker image size too much
 rm ${INSTALLER_PATH}
 
-# Remove the pip cache created as part of installing miniconda
+# Remove the pip cache created as part of installing mambaforge
 rm -rf /root/.cache
 
 chown -R $NB_USER:$NB_USER ${CONDA_DIR}

diff --git a/deployments/datahub/images/default/Dockerfile b/deployments/datahub/images/default/Dockerfile
@@ -49,7 +49,7 @@ RUN apt-get update > /dev/null && \
     apt-get -qq install --yes \
             # for LS88-5 and modules basemap
             libspatialindex-dev \
-            # for cartopy
+            # for R sf packages
             libgeos-dev \
             libproj-dev \
             proj-data \
@@ -213,10 +213,10 @@ ENV PATH ${CONDA_DIR}/bin:$PATH:/usr/lib/rstudio-server/bin
 
 WORKDIR /home/${NB_USER}
 
-# Install miniforge as root
+# Install mambaforge as root
 USER root
-COPY install-miniforge.bash /tmp/install-miniforge.bash
-RUN /tmp/install-miniforge.bash
+COPY install-mambaforge.bash /tmp/install-mambaforge.bash
+RUN /tmp/install-mambaforge.bash
 
 # Install conda environment as our user
 USER ${NB_USER}

diff --git a/deployments/datahub/images/default/environment.yml b/deployments/datahub/images/default/environment.yml
@@ -1,7 +1,32 @@
 dependencies:
-- nodejs==15.*
-- pip==20.2.*
-- python==3.9.*
+- nodejs=15.*
+- pip=20.2.*
+- python=3.9.*
 
 # pymc3 needs this
-- mkl-service==2.4.*
+- mkl-service=2.4.*
+
+# Base scientific packages that other conda packages we install depend on
+# We don't want to have conda packages depend on pip packages if possible
+- numpy=1.21.*
+- matplotlib=3.4.*
+- scipy=1.7.*
+- ipympl=0.8.*
+- pandas=1.3.*
+- statsmodels=0.12.*
+- scikit-learn=0.24.*
+- seaborn=0.11.*
+- bokeh=2.3.*
+- decorator=5.0.*
+- networkx=2.6.*
+- spacy=3.1.*
+- nltk=3.6.*
+
+# EPS88, data100
+# https://github.com/berkeley-dsep-infra/datahub/issues/1796
+# https://github.com/berkeley-dsep-infra/datahub/issues/2824
+- shapely=1.8.*
+- cartopy=0.20.*
+
+# ls 88-3; neuro
+- pillow=8.3.*
diff --git a/...hub/images/default/install-miniforge.bash → ...ub/images/default/install-mambaforge.bash b/...hub/images/default/install-miniforge.bash → ...ub/images/default/install-mambaforge.bash
@@ -1,12 +1,12 @@
 #!/bin/bash
-# This downloads and installs a pinned version of miniconda
+# This downloads and installs a pinned version of mambaforge
 set -ex
 
 cd $(dirname $0)
-MINIFORGE_VERSION=4.10.3-4
+MAMBAFORGE_VERSION=4.10.3-7
 
-URL="https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/Miniforge3-${MINIFORGE_VERSION}-Linux-x86_64.sh"
-INSTALLER_PATH=/tmp/miniforge-installer.sh
+URL="https://github.com/conda-forge/miniforge/releases/download/${MAMBAFORGE_VERSION}/Mambaforge-${MAMBAFORGE_VERSION}-Linux-x86_64.sh"
+INSTALLER_PATH=/tmp/mambaforge-installer.sh
 
 # make sure we don't do anything funky with user's $HOME
 # since this is run as root
@@ -34,7 +34,7 @@ conda clean --all -f -y
 # Remove the big installer so we don't increase docker image size too much
 rm ${INSTALLER_PATH}
 
-# Remove the pip cache created as part of installing miniconda
+# Remove the pip cache created as part of installing mambaforge
 rm -rf /root/.cache
 
 chown -R $NB_USER:$NB_USER ${CONDA_DIR}

diff --git a/deployments/datahub/images/default/requirements.txt b/deployments/datahub/images/default/requirements.txt
@@ -1,10 +1,5 @@
 # data8; foundation
 datascience==0.17.0
-matplotlib==3.4.3
-ipympl==0.7.0
-pandas==1.3.2
-scipy==1.7.1
-statsmodels==0.12.2
 okpy==1.18.1
 folium==0.12.1
 #
@@ -16,28 +11,11 @@ jupyter-shiny-proxy==1.1
 # cogsci88;
 ggplot==0.11.5
 #
-# geog88; spring 2018
-mplleaflet==0.0.5
-# updating geopandas from 0.9.0 for issue #2838 (data100, fall 2021)
-geopandas==0.10.0
-geopy==2.2.0
-pysal==2.5.0
-rtree==0.9.7
-netCDF4==1.5.7
-#
 # cogsci131; spring 2018
 nose==1.3.7
 #
 # modules
-scikit-learn==0.24.2
-seaborn==0.11.2
-bokeh==2.3.3
-decorator==5.0.9
-networkx==2.6.2
-# pygame==1.9.6
-nltk==3.6.2
 beautifulsoup4==4.9.3
-spacy==3.1.1
 qgrid==1.3.1
 nb2pdf==0.6.2
 #
@@ -47,11 +25,18 @@ tqdm==4.62.1
 mne==0.23.0
 nibabel==3.2.1
 h5py==3.3.0
-pillow==8.3.1
 numexpr==2.7.3
 openpyxl==3.0.7
 nilearn==0.8.0
 
+# data100, geog88
+# https://github.com/berkeley-dsep-infra/datahub/issues/2838
+geopandas==0.10.*
+geopy==2.2.*
+pysal==2.5.*
+rtree==0.9.*
+netcdf4==1.5.*
+mplleaflet==0.0.5
 # phys 151;
 emcee==3.1.0
 daft==0.1.2
@@ -61,8 +46,8 @@ corner==2.2.1
 pymdptoolbox==4.0-b3
 #
 # data-x; DL
-tensorflow==2.6.0
-scikit-image==0.18.2
+tensorflow==2.6.*
+scikit-image==0.18.*
 tables==3.6.1
 opencv-python==4.5.3.56
 
@@ -88,8 +73,6 @@ gspread==4.0.1
 
 # eps 109; fall 2019
 ffmpeg-python==0.2.0
-# see https://github.com/berkeley-dsep-infra/datahub/issues/1796
-Cartopy==0.19.0.post1 --no-binary Cartopy
 
 # data 88e; spring 2021
 plotly==5.2.1
@@ -114,8 +97,6 @@ hdbscan==0.8.27
 
 # espm 125/bio 105; fall 2019
 # see https://github.com/berkeley-dsep-infra/datahub/issues/1796
-shapely==1.7.1 --no-binary shapely
-pyvcf==0.6.8
 bitarray==2.3.0
 nlmpy==1.0.1
 
@@ -211,3 +192,5 @@ ipython-sql==0.4.0
 metpy==1.1.0
 pooch==1.5.2
 
+# PS88 https://github.com/berkeley-dsep-infra/datahub/issues/2925
+linearmodels==4.24
diff --git a/deployments/ischool/image/Dockerfile b/deployments/ischool/image/Dockerfile
@@ -48,8 +48,8 @@ RUN curl --silent --location --fail ${RSTUDIO_URL} > /tmp/rstudio.deb && \
     dpkg -i /tmp/rstudio.deb && \
     rm /tmp/rstudio.deb
 
-COPY install-miniforge.bash /tmp/install-miniforge.bash
-RUN /tmp/install-miniforge.bash
+COPY install-mambaforge.bash /tmp/install-mambaforge.bash
+RUN /tmp/install-mambaforge.bash
 
 USER ${NB_USER}
 

diff --git a/...ents/ischool/image/install-miniforge.bash → ...nts/ischool/image/install-mambaforge.bash b/...ents/ischool/image/install-miniforge.bash → ...nts/ischool/image/install-mambaforge.bash
@@ -1,12 +1,12 @@
 #!/bin/bash
-# This downloads and installs a pinned version of miniconda
+# This downloads and installs a pinned version of mambaforge
 set -ex
 
 cd $(dirname $0)
-MINIFORGE_VERSION=4.10.2-0
+MAMBAFORGE_VERSION=4.10.3-7
 
-URL="https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/Miniforge3-${MINIFORGE_VERSION}-Linux-x86_64.sh"
-INSTALLER_PATH=/tmp/miniforge-installer.sh
+URL="https://github.com/conda-forge/miniforge/releases/download/${MAMBAFORGE_VERSION}/Mambaforge-${MAMBAFORGE_VERSION}-Linux-x86_64.sh"
+INSTALLER_PATH=/tmp/mambaforge-installer.sh
 
 # make sure we don't do anything funky with user's $HOME
 # since this is run as root
@@ -34,7 +34,7 @@ conda clean --all -f -y
 # Remove the big installer so we don't increase docker image size too much
 rm ${INSTALLER_PATH}
 
-# Remove the pip cache created as part of installing miniconda
+# Remove the pip cache created as part of installing mambaforge
 rm -rf /root/.cache
 
 chown -R $NB_USER:$NB_USER ${CONDA_DIR}

diff --git a/deployments/julia/image/Dockerfile b/deployments/julia/image/Dockerfile
@@ -38,8 +38,8 @@ RUN mkdir -p ${JULIA_DIR} && chown ${NB_USER}:${NB_USER} ${JULIA_DIR}
 
 WORKDIR /home/jovyan
 
-COPY install-miniforge.bash /tmp/install-miniforge.bash
-RUN /tmp/install-miniforge.bash
+COPY install-mambaforge.bash /tmp/install-mambaforge.bash
+RUN /tmp/install-mambaforge.bash
 
 USER ${NB_USER}
 

diff --git a/deployments/julia/image/install-mambaforge.bash b/deployments/julia/image/install-mambaforge.bash
@@ -0,0 +1,42 @@
+#!/bin/bash
+# This downloads and installs a pinned version of mambaforge
+set -ex
+
+cd $(dirname $0)
+MAMBAFORGE_VERSION=4.10.3-7
+
+URL="https://github.com/conda-forge/miniforge/releases/download/${MAMBAFORGE_VERSION}/Mambaforge-${MAMBAFORGE_VERSION}-Linux-x86_64.sh"
+INSTALLER_PATH=/tmp/mambaforge-installer.sh
+
+# make sure we don't do anything funky with user's $HOME
+# since this is run as root
+unset HOME
+
+wget --quiet $URL -O ${INSTALLER_PATH}
+chmod +x ${INSTALLER_PATH}
+
+bash ${INSTALLER_PATH} -b -p ${CONDA_DIR}
+export PATH="${CONDA_DIR}/bin:$PATH"
+
+# Do not attempt to auto update conda or dependencies
+conda config --system --set auto_update_conda false
+conda config --system --set show_channel_urls true
+
+# empty conda history file,
+# which seems to result in some effective pinning of packages in the initial env,
+# which we don't intend.
+# this file must not be *removed*, however
+echo '' > ${CONDA_DIR}/conda-meta/history
+
+# Clean things out!
+conda clean --all -f -y
+
+# Remove the big installer so we don't increase docker image size too much
+rm ${INSTALLER_PATH}
+
+# Remove the pip cache created as part of installing mambaforge
+rm -rf /root/.cache
+
+chown -R $NB_USER:$NB_USER ${CONDA_DIR}
+
+conda list -n root
diff --git a/deployments/julia/image/install-miniforge.bash b/deployments/julia/image/install-miniforge.bash