Trac #33671: Add devcontainer.json for development with VS Code in a …

…Docker container

https://code.visualstudio.com/docs/remote/containers

We add two new sections in the developer's guide:
https://47f8b984ef964a5aa34147393fbdc32e0dde88ad--sagemath-
tobias.netlify.app/developer/portability_testing.html#using-our-pre-
built-docker-images-published-on-ghcr-io

We also set up devcontainer configurations for the !CoCalc and
computop/sage Docker images, as well as downstream distribution
packaging of Sage.

- 3-manifolds/sagedocker#2

Tested `devcontainer.json` of

- `portability-ubuntu-jammy-standard`: builds well; runs well
- `develop-docker-computop`: builds well; runs well
- `downstream-docker-cocalc`: builds well; runs well (except machines
with the issue #32434)
- `downstream-docker-computop`: builds well; runs well
- `downstream-archlinux-latest`: builds well; runs well
- `downstream-conda-forge-latest`: builds well; runs well

Follow-ups:

- #34363 Installation guide: Document installation on Windows via
devcontainer
- #34286 Cookiecutter for Sage user projects with devcontainer
- #34403 !GitHub Codespaces - may also be able to test it by starting a
codespace at https://github.com/codespaces/new?hide_repo_select=true&ref
=public%2Fbuild%2Fdevcontainer&repo=302607352&machine=standardLinux32gb&
devcontainer_path=.devcontainer%2Fdevcontainer.json
- `downstream-*` for more distributions (generate with `tox` from the
info in `tox.ini, use `_sagemath` package)
- `develop-conda-forge-src-environment-dev`
(https://doc.sagemath.org/html/en/installation/conda.html#using-conda-
to-provide-all-dependencies-for-the-sage-library-experimental, similar
to our gitpod configuration)
- Restore `portability-debian-buster-i386-standard` and fix it
- Restore `develop-docker-cocalc` (was: builds well (after increasing
disk image space on the Docker daemon); fails to run)
- Restore - `portability-centos-7-devtoolset-gcc_11-standard` (was:
builds well; fails to run (exactly the same on WSL))
- Restore `downstream-docker-sagemath` (after #34242)
- add github workflows that verify that sage is at least compiling in
these containers. That should be relatively straightforward using
https://github.com/devcontainers/ci
- improve `portability-Dockerfile` as explained in comment:299

URL: https://trac.sagemath.org/33671
Reported by: mkoeppe
Ticket author(s): Tobias Diez, Matthias Koeppe, Kwankyu Lee
Reviewer(s): Kwankyu Lee, Matthias Koeppe

.devcontainer/develop-docker-computop/devcontainer.json

@@ -0,0 +1,18 @@
+// See https://aka.ms/devcontainer.json for format details.
+{
+    "name": "computop/sage Docker",
+    "image": "computop/sage",
+    "containerEnv": {
+        "MAKE": "make -j4"
+    },
+    // Install build tools, get rid of sourcing /sage/activate in non-login shells.
+    // libgmp.a is broken and leads to a build failure of ecm.
+    "onCreateCommand": ".devcontainer/onCreate.sh --sudo && sudo rm -f /sage/local/lib/libgmp.a && sed -i.bak '/sage.*activate/d' ~/.bashrc",
+    // Do not run configure within a sage-env (see #29485).
+    // The pari package is broken in the computop/sage 9.5 image, need to reinstall.
+    // Also libnauty is broken.
+    "updateContentCommand": "make configure && (export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin && unset CFLAGS LDFLAGS CXXFLAGS CPATH LIBRARY_PATH && ./configure --prefix=/sage/local --with-sage-venv) && make pari-clean nauty-clean build V=0",
+    "extensions": [
+        "ms-python.python"
+    ]
+}

.devcontainer/downstream-archlinux-latest/devcontainer.json

@@ -0,0 +1,12 @@
+// See https://aka.ms/devcontainer.json for format details.
+{
+    "name": "archlinux:latest downstream Sage",
+    "image": "archlinux:latest",
+    // Create an empty bashrc to avoid the error "No such file or directory" when opening a terminal.
+    "onCreateCommand": "EXTRA_SYSTEM_PACKAGES='sagemath sagemath-doc' EXTRA_SAGE_PACKAGES='notebook pip' .devcontainer/onCreate.sh && touch ~/.bashrc",
+    // There's no SAGE_LOCAL, so remove the symlink 'prefix'.
+    "updateContentCommand": "rm -f prefix && ln -sf /usr venv",
+    "extensions": [
+        "ms-python.python"
+    ]
+}

.devcontainer/downstream-conda-forge-latest/devcontainer.json

@@ -0,0 +1,19 @@
+// See https://aka.ms/devcontainer.json for format details.
+{
+    "name": "condaforge/mambaforge:latest downstream Sage",
+    "image": "condaforge/mambaforge:latest",
+    // Install Sage from the conda-forge package.
+    "onCreateCommand": "mamba install --yes sage",
+    // * If the workspace directory looks like a copy of the Sage source tree (SAGE_ROOT):
+    //   - it bootstraps and configures the Sage distribution,
+    //   - thus, the script ``./sage`` and the symlinks ``prefix``, ``venv`` are set as expected,
+    //   - the source tree is prepared for rebuilding Sage based from the source tree on
+    //     top of the existing installation from the Docker image.
+    //   - however, it does not start the build.
+    // * Otherwise, it does nothing. This is so that users can copy this devcontainer.json file as is
+    //   into their projects.
+    "updateContentCommand": "if [ -d pkgs/sagemath-standard ]; then make configure && ln -sf $CONDA_PREFIX venv; else echo 'Edit .devcontainer/devcontainer.json (updateContentCommand) to run project-specific startup commands'; fi",
+    "extensions": [
+        "ms-python.python"
+    ]
+}

.devcontainer/downstream-docker-cocalc/devcontainer.json

@@ -0,0 +1,19 @@
+// See https://aka.ms/devcontainer.json for format details.
+{
+    "name": "CoCalc Docker",
+    "image": "sagemathinc/cocalc",
+    "containerEnv": {
+        "MAKE": "make -j4"
+    },
+    // libgmp.a is broken and leads to a build failure of ecm.
+    "onCreateCommand": ".devcontainer/onCreate.sh && rm -f /usr/local/sage/local/lib/libgmp.a",
+    // * If the workspace directory looks like a copy of the Sage source tree (SAGE_ROOT):
+    //   - it bootstraps the Sage distribution,
+    //   - sets the symlink ``venv`` as expected,
+    // * Otherwise, it does nothing. This is so that users can copy this devcontainer.json file as is
+    //   into their projects.
+    "updateContentCommand": "if [ -d pkgs/sagemath-standard ]; then make configure && ./configure --enable-build-as-root --prefix=/usr/local/sage/local --with-sage-venv; else echo 'Edit .devcontainer/devcontainer.json (updateContentCommand) to run project-specific startup commands'; fi",
+    "extensions": [
+        "ms-python.python"
+    ]
+}

.devcontainer/downstream-docker-computop/devcontainer.json

@@ -0,0 +1,18 @@
+// See https://aka.ms/devcontainer.json for format details.
+{
+    "name": "computop/sage Docker",
+    "image": "computop/sage",
+    "containerEnv": {
+        "MAKE": "make -j4"
+    },
+    // Install build tools, get rid of sourcing /sage/activate in non-login shells.
+    // libgmp.a is broken and leads to a build failure of ecm.
+    "onCreateCommand": ".devcontainer/onCreate.sh --sudo && sudo rm -f /sage/local/lib/libgmp.a && sed -i.bak '/sage.*activate/d' ~/.bashrc",
+    // Do not run configure within a sage-env (see #29485).
+    // The pari package is broken in the computop/sage 9.5 image, need to reinstall.
+    // Also libnauty is broken.
+    "updateContentCommand": "make configure && (export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin && unset CFLAGS LDFLAGS CXXFLAGS CPATH LIBRARY_PATH && ./configure --prefix=/sage/local --with-sage-venv)",
+    "extensions": [
+        "ms-python.python"
+    ]
+}

.devcontainer/onCreate.sh

@@ -0,0 +1,9 @@
+#! /bin/sh
+# Run this script from SAGE_ROOT. Invoke with "--sudo" if sudo is needed.
+export PATH=$(pwd)/build/bin:$PATH
+SYSTEM=$(sage-guess-package-system)
+eval $(sage-print-system-package-command $SYSTEM "$@" update)
+eval $(sage-print-system-package-command $SYSTEM --yes "$@" --spkg install _bootstrap _prereq python3 git $EXTRA_SAGE_PACKAGES)
+if [ -n "$EXTRA_SYSTEM_PACKAGES" ]; then
+    eval $(sage-print-system-package-command $SYSTEM --yes "$@" install $EXTRA_SYSTEM_PACKAGES)
+fi

.devcontainer/portability-Dockerfile

@@ -0,0 +1,8 @@
+# This Dockerfile is used by all portability-.../devcontainer.json files,
+# which provide the actual values for the 4 arguments defined below, which
+# feed into the FROM statement that determines the base Docker image.
+ARG SYSTEM_FACTOR="ubuntu-jammy"
+ARG PACKAGE_FACTOR="standard"
+ARG DOCKER_TARGET="with-system-packages"
+ARG DOCKER_TAG="dev"
+FROM ghcr.io/sagemath/sage/sage-docker-${SYSTEM_FACTOR}-${PACKAGE_FACTOR}-${DOCKER_TARGET}:${DOCKER_TAG}

.devcontainer/portability-ubuntu-jammy-standard/devcontainer.json

@@ -0,0 +1,22 @@
+// See https://aka.ms/devcontainer.json for format details.
+{
+    "name": "Ubuntu jammy",
+    "build": {
+        "dockerfile": "portability-Dockerfile",
+        // See tox.ini for definitions
+        "args": {
+            "SYSTEM_FACTOR": "ubuntu-jammy",
+            "PACKAGE_FACTOR": "standard",
+            "DOCKER_TARGET": "with-targets",
+            "DOCKER_TAG": "dev"
+        }
+    },
+    "containerEnv": {
+        "MAKE": "make -j4"
+    },
+    "onCreateCommand": ".devcontainer/onCreate.sh",
+    "updateContentCommand": ".devcontainer/portability-updateContent.sh",
+    "extensions": [
+        "ms-python.python"
+    ]
+}

.devcontainer/portability-ubuntu-jammy-standard/portability-Dockerfile

@@ -0,0 +1 @@
+../portability-Dockerfile

.devcontainer/portability-updateContent.sh

@@ -0,0 +1,30 @@
+#! /bin/sh
+# The portability-.../devcontainer.json configurations run this script after
+# the container is started.
+#
+# The script assumes that it is run from SAGE_ROOT.
+#
+# If "config.log" or "logs" are symlinks (for example, created by 'tox -e local-...',
+# or after https://trac.sagemath.org/ticket/33262), they might point outside of
+# the dev container, so remove them. Likewise for upstream.
+for f in config.log logs upstream; do
+    if [ -L $f ]; then
+        rm -f $f
+    fi
+done
+# If possible (ensured after https://trac.sagemath.org/ticket/33262), keep the
+# logs in the container.
+if [ ! -d logs ]; then
+    ln -s /sage/logs logs
+fi
+# Bootstrap, configure, and build the Sage distribution, reusing the Sage
+# installation from the prebuilt image.
+set -e
+set -x
+make configure
+if [ -x /sage/config.status ]; then
+    eval ./configure $(/sage/config.status --config) --enable-build-as-root --prefix=/sage/local --with-sage-venv
+else
+    ./configure --enable-build-as-root --prefix=/sage/local --with-sage-venv
+fi
+make build V=0

.gitignore

@@ -217,6 +217,9 @@ src/ENV/
 src/env.bak/
 src/venv.bak/
 
+# devcontainer
+/.devcontainer/devcontainer.json
+
 # mypy
 **/.mypy_cache/
 

.vscode/settings.json

@@ -3,7 +3,7 @@
     "files.exclude": {
         "**/__pycache__": true,
         "src/**/*.cpp": true,
-        "src/**/*.so": true,
+        "src/**/*.so": true
     },
     "search.exclude": {
         "build/pkgs/sagemath_categories/src": true,
@@ -12,20 +12,21 @@
         "pkgs/sage-conf_pypi/sage_root/build": true,
         "pkgs/sagemath-categories/sage": true,
         "pkgs/sagemath-objects/sage": true,
-        "pkgs/sagemath-standard/sage": true,
+        "pkgs/sagemath-standard/sage": true
     },
     "python.testing.pytestEnabled": true,
     "python.testing.pytestArgs": [
         "--rootdir=src/sage",
         "-c=src/tox.ini",
-        "--doctest-modules",
+        "--doctest-modules"
     ],
     "python.testing.unittestEnabled": false,
     "python.linting.pycodestyleEnabled": true,
     "python.linting.enabled": true,
     "cSpell.words": [
         "furo",
         "Conda",
+        "sagemath",
         "Cython"
-    ],
+    ]
 }

build/bin/sage-print-system-package-command

@@ -142,7 +142,7 @@ case $system:$command in
         ;;
     *conda*:install)
         [ "$YES" = yes ] && options="$options --yes"
-        [ -n "$system_packages" ] && print_shell_command "conda install $system_packages"
+        [ -n "$system_packages" ] && print_shell_command "conda install $options $system_packages"
         ;;
     homebrew*:install)
         [ -n "$system_packages" ] && print_shell_command "brew install $system_packages"

build/make/Makefile.in

@@ -527,7 +527,7 @@ $(1)-$(4)-no-deps:
 	    echo "$$($(4)_DISABLED_MESSAGE)" 2>&1; \
 	    exit 1; \
 	else \
-	    sage-logger -p 'SAGE_CHECK=$$(SAGE_CHECK_$(1)) PATH=$$($(4))/bin:$$$$PATH $$(SAGE_SPKG) $$(SAGE_SPKG_OPTIONS) \
+	    sage-logger -p 'SAGE_CHECK=$$(SAGE_CHECK_$(1)) PATH=$$(SAGE_SRC)/bin:$$($(4))/bin:$$$$PATH $$(SAGE_SPKG) $$(SAGE_SPKG_OPTIONS) \
 		$(if $(filter $(1),$(TOOLCHAIN_DEPS)),--keep-existing) \
 		$(1)-$(2) $$($(4))' '$$(SAGE_LOGS)/$(1)-$(2).log'; \
 	fi

build/pkgs/notebook/distros/arch.txt

@@ -0,0 +1 @@
+jupyter-notebook

build/pkgs/pip/distros/arch.txt

@@ -0,0 +1 @@
+python-pip

build/pkgs/tox/distros/arch.txt

@@ -0,0 +1 @@
+python-tox

src/bin/sage-env

@@ -115,9 +115,8 @@ fi
 
 # Don't execute the commands more than once for the same version of
 # sage-env... for the same combination of SAGE_LOCAL and SAGE_VENV.
-# "5" indicates the version of the format of the value of SAGE_ENV_VERSION.
-# The current format was introduced in #32745.
-SAGE_ENV_VERSION="5:$SAGE_LOCAL:$SAGE_VENV"
+# "6" indicates the version of the format of the value of SAGE_ENV_VERSION.
+SAGE_ENV_VERSION="6:$SAGE_LOCAL:$SAGE_VENV:$SAGE_SRC"
 if [ "$SAGE_ENV_SOURCED" = "$SAGE_ENV_VERSION" ]; then
     # Already sourced, nothing to do.
     return 0