feat(dev_env): Dev env bootstrap support for two Python versions (#28213

) This change adds supports for bootstrapping two different Python versions. Python 3.8 will be used for Apple M1 since it's the next Python version for Sentry and it is the first one that supports Apple's M1 chip-set. Other miscellaneous fixes and changes included. Co-authored-by: Billy Vong <billyvg@users.noreply.github.com>
getsentry · Sep 1, 2021 · faa028a · faa028a
1 parent b717b22
commit faa028a
Show file tree

Hide file tree

Showing 11 changed files with 146 additions and 112 deletions.
diff --git a/.envrc b/.envrc
@@ -3,18 +3,13 @@
 # initialization (e.g. activating the venv) or giving recommendations on how to reach the desired state.
 # It also sets useful environment variables.
 # If you'd like to override or set any custom environment variables, this .envrc will read a .env file at the end.
-
 set -e
-HERE="$(
+SENTRY_ROOT="$(
     cd "$(dirname "${BASH_SOURCE[0]}")"
     pwd -P
 )"
 
-bold="$(tput bold)"
-red="$(tput setaf 1)"
-green="$(tput setaf 2)"
-yellow="$(tput setaf 3)"
-reset="$(tput sgr0)"
+source "${SENTRY_ROOT}/scripts/lib.sh"
 
 # XXX: we can't trap bash EXIT, because it'll override direnv's finalizing routines.
 #      consequently, using "exit" anywhere will skip this notice from showing.
@@ -58,10 +53,6 @@ report_to_sentry() {
     rm "$_SENTRY_LOG_FILE"
 }
 
-require() {
-    command -v "$1" >/dev/null 2>&1
-}
-
 debug() {
     if [ "${SENTRY_DIRENV_DEBUG}" ]; then
         echo -e "${@}"
@@ -86,6 +77,27 @@ die() {
     return 1
 }
 
+prompt_python_venv_creation() {
+    echo -e "${yellow}You are missing a Python virtualenv and we ${bold}need${reset}${yellow} to run a bootstrapping script (it can take a few minutes)"
+    info "About to create ${venv_name}..."
+    echo -e "\nContinue (y/N)?"
+    read -r resp
+    case "$resp" in
+    y | Y) echo "Okay, let's do this." ;;
+    *)
+        die "Aborted!"
+        ;;
+    esac
+}
+
+show_commands_info() {
+    echo -e "\n${red}Run the following commands to bring your environment up-to-date:"
+    for cmd in "${commands_to_run[@]}"; do
+        warn "    ${red}$cmd"
+    done
+    echo ""
+}
+
 ### Environment ###
 
 commands_to_run=()
@@ -111,9 +123,9 @@ export SENTRY_UI_HOT_RELOAD=1
 
 ### You can override the exported variables with a .env file
 # All exports should happen before here unless they're safeguarded (see devenv error reporting below)
-if [ -f "${HERE}/.env" ]; then
-    debug "Loading variables from ${HERE}/.env"
-    dotenv "${HERE}/.env"
+if [ -f "${SENTRY_ROOT}/.env" ]; then
+    info "Loading variables from ${SENTRY_ROOT}/.env"
+    dotenv "${SENTRY_ROOT}/.env"
 fi
 
 ## Notify of reporting to Sentry
@@ -156,15 +168,7 @@ make setup-git-config
 venv_name=".venv"
 
 if [ ! -f "${venv_name}/bin/activate" ]; then
-    warn "You are missing a Python virtualenv and we ${bold}need${reset} to run a bootstrapping script (it can take a few minutes)"
-    echo -e "\nContinue (y/N)?"
-    read -r resp
-    case "$resp" in
-    y | Y) echo "Okay, let's do this." ;;
-    *)
-        die "Aborted!"
-        ;;
-    esac
+    prompt_python_venv_creation
     # This is time consuming but it has to be done
     source ./scripts/bootstrap-py3-venv
 fi
@@ -181,7 +185,7 @@ source "${venv_name}/bin/activate"
 unset PS1
 
 debug "Ensuring proper virtualenv..."
-"${HERE}/scripts/ensure-venv.sh"
+"${SENTRY_ROOT}/scripts/ensure-venv.sh"
 
 if ! require sentry; then
     warn "Your virtualenv is activated, but sentry doesn't seem to be installed."
@@ -216,11 +220,7 @@ PATH_add node_modules/.bin
 
 # These are commands that can take a significant amount of time
 if [ ${#commands_to_run[@]} -ne 0 ]; then
-    echo -e "\n${red}Run the following commands to bring your environment up-to-date:"
-    for cmd in "${commands_to_run[@]}"; do
-        warn "    ${red}$cmd"
-    done
-    echo ""
+    show_commands_info
 fi
 
 if [ "${log_level}" != "info" ]; then

diff --git a/.github/actions/setup-python/action.yml b/.github/actions/setup-python/action.yml
@@ -16,7 +16,7 @@ inputs:
     description: 'pip cache version in order to bust cache'
     required: false
   python-version:
-    description: "python version to install"
+    description: 'python version to install'
     required: false
 
 outputs:
@@ -38,7 +38,7 @@ runs:
       shell: bash
       run: |
         if [ "${{ inputs.python-version }}" == "" ]; then
-          echo "::set-output name=python-version::$(cat .python-version)"
+          echo "::set-output name=python-version::$(SENTRY_NO_VENV_CHECK=1 ./scripts/do.sh get-pyenv-version)"
         else
           echo "::set-output name=python-version::${{ inputs.python-version }}"
         fi

diff --git a/.github/workflows/development-environment.yml b/.github/workflows/development-environment.yml
@@ -2,13 +2,13 @@ name: dev env
 on:
   pull_request:
     paths:
-    - 'Makefile'
-    - '.github/workflows/development-environment.yml'
-    - '.envrc'
-    - 'Brewfile'
-    - 'scripts/*'
-    - 'src/sentry/runner/commands/devserver.py'
-    - 'src/sentry/runner/commands/devservices.py'
+      - 'Makefile'
+      - '.github/workflows/development-environment.yml'
+      - '.envrc'
+      - 'Brewfile'
+      - 'scripts/*'
+      - 'src/sentry/runner/commands/devserver.py'
+      - 'src/sentry/runner/commands/devservices.py'
 
 jobs:
   dev-environment:
@@ -20,7 +20,7 @@ jobs:
         # macosx-11.0 is Big Sur, however, it takes long for jobs to get started
         # Using Ubuntu 18 until I figure out this error:
         # -> ImportError: libffi.so.6: cannot open shared object file: No such file or directory
-        os: [ macos-11.0, ubuntu-18.04 ]
+        os: [macos-11.0, ubuntu-18.04]
       fail-fast: false
     env:
       PIP_DISABLE_PIP_VERSION_CHECK: on
@@ -36,15 +36,15 @@ jobs:
         # Sometimes, brew needs to be updated before brew bundle would work
         # After installing Docker (via homebrew) we need to make sure that it is properly initialized on Mac
         run: |
-          brew update -q && brew bundle -q
+          brew update && brew bundle -q
           # This code is mentioned in our dev docs. Only remove if you adjust the docs as well
           SENTRY_NO_VENV_CHECK=1 ./scripts/do.sh init-docker
 
       # The next few steps are to set up the cache quickly
       - name: Set environment variables & others
         id: info
         run: |
-          echo "::set-output name=python-version::$(cat .python-version)"
+          echo "::set-output name=python-version::$(SENTRY_NO_VENV_CHECK=1 ./scripts/do.sh get-pyenv-version)"
           echo "::set-output name=pip-cache-dir::$(pip3 cache dir)"
           echo "::set-output name=pip-version::$(pip -V | awk -F ' ' '{print $2}')"
           echo "::set-output name=yarn-cache-dir::$(yarn cache dir)"

diff --git a/.github/workflows/python-deps.yml b/.github/workflows/python-deps.yml
@@ -2,8 +2,8 @@ name: python deps
 on:
   pull_request:
     paths:
-    - '.github/workflows/python-deps.yml'
-    - 'requirements*'
+      - '.github/workflows/python-deps.yml'
+      - 'requirements*'
 
 jobs:
   # This workflow makes sure that Python dependencies install correctly for
@@ -14,8 +14,8 @@ jobs:
     timeout-minutes: 20
     strategy:
       matrix:
-        os: [ macos-11.0, ubuntu-20.04 ]
-        python-version: [ 3.6.13, 3.8.10 ]
+        os: [macos-11.0, ubuntu-20.04]
+        python-version: [3.6.13, 3.8.11]
       fail-fast: false
     env:
       PIP_DISABLE_PIP_VERSION_CHECK: on
@@ -27,25 +27,16 @@ jobs:
       - uses: actions/checkout@v2
 
       - name: Install prerequisites
+        # Sometimes, brew needs to be updated before brew bundle would work
         run: |
-          HOMEBREW_NO_AUTO_UPDATE=1 brew bundle --no-upgrade
+          brew update && brew bundle -q
 
-      # Until GH composite actions can use `uses`, we need to setup python here
-      - uses: actions/setup-python@v2
+      - name: Setup python
+        id: setup-python
+        uses: ./.github/actions/setup-python
         with:
-          python-version: ${{ matrix.python-version }}
-
-      - name: Setup pip
-        uses: ./.github/actions/setup-pip
-        id: pip
-
-      - name: Cache
-        uses: actions/cache@v2
-        with:
-          path: |
-            ${{ steps.pip.outputs.pip-cache-dir }}
-          key: |
-            python-deps-${{ matrix.os }}-py${{ matrix.python-version }}-${{ hashFiles('requirements-*.txt') }}
+          # XXX: We need to pass this python-deps-${{ matrix.os }}-py${{ matrix.python-version }}-${{ hashFiles('requirements-*.txt') }}
+          cache-files-hash: ${{ hashFiles('requirements-*.txt') }}
 
       - name: Install dependencies
         run: |

diff --git a/.python-version b/.python-version
diff --git a/requirements-base.txt b/requirements-base.txt
@@ -20,6 +20,9 @@ google-cloud-core==1.5.0
 googleapis-common-protos==1.52.0
 google-cloud-pubsub==2.2.0
 google-cloud-storage==1.35.0
+# Only necessary to prevent installing the latest version
+# https://github.com/googleapis/python-crc32c/issues/83
+google-crc32c==1.1.2; python_version == '3.8'
 jsonschema==3.2.0
 lxml==4.6.3
 maxminddb==2.0.3

diff --git a/scripts/bootstrap-py3-venv b/scripts/bootstrap-py3-venv
@@ -11,6 +11,11 @@
 # trap "echo 'bootstrap FAILED.' && trap - ERR && return" ERR
 
 # So just going to sprinkle returns everywhere.
+SCRIPTS_DIR="$(
+    cd "$(dirname "${BASH_SOURCE[0]}")"
+    pwd -P
+)"
+source "${SCRIPTS_DIR}/lib.sh"
 
 command -v pyenv >/dev/null || {
     echo "You need to install pyenv. https://develop.sentry.dev/environment/#python"
@@ -24,7 +29,10 @@ command -v direnv >/dev/null || {
 
 gitroot="$(git rev-parse --show-toplevel)"
 cd "$gitroot"
-export venv_name="${PWD}/.venv"
+# This executes very quickly if the Python versions are already installed
+make setup-pyenv
+export PYENV_VERSION=$(SENTRY_NO_VENV_CHECK=1 ./scripts/do.sh get-pyenv-version)
+venv_name="${PWD}/.venv"
 
 if [[ -f "${venv_name}/bin/activate" ]]; then
     prompt_text="You have an existing virtualenv. This script will OVERWRITE it. Continue (y/N)?"
@@ -40,17 +48,15 @@ fi
 deactivate 2>/dev/null || true
 rm -rf "$venv_name"
 
-# XXX: You'll also need to temporarily pyenv local "$SENTRY_PYTHON_VERSION"
-#      and then revert changes to the .python-version after bootstrap finishes.
-if [[ -z "$SENTRY_PYTHON_VERSION" ]] && ! [[ "$(python3 -V 2>&1)" = "Python $(grep "3.6" .python-version)" ]]; then
+if [[ -z "$SENTRY_PYTHON_VERSION" ]] && ! query-valid-python-version; then
     echo "Your python3 version isn't as expected. Please run: make setup-pyenv"
     return 1
 fi
 
 python3 -m pip install -U pip           || { echo "bootstrap failed!"; return 1; }
 python3 -m venv "${venv_name}"          || { echo "bootstrap failed!"; return 1; }
 source "${venv_name}/bin/activate"      || { echo "bootstrap failed!"; return 1; }
-python3 -m pip install -U pip           || { echo "bootstrap failed!"; return 1; }
+python3 -m pip install -U pip wheel     || { echo "bootstrap failed!"; return 1; }
 make setup-git                          || { echo "bootstrap failed!"; return 1; }
 make install-py-dev                     || { echo "bootstrap failed!"; return 1; }
 deactivate                              || { echo "bootstrap failed!"; return 1; }

diff --git a/scripts/do.sh b/scripts/do.sh
@@ -2,8 +2,10 @@
 # This script is an interface to any of the methods of lib.sh
 # Call this script as "do.sh method_from_lib" to execute any function from that library
 set -eu
-
-HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")"; pwd -P)"
+HERE="$(
+    cd "$(dirname "${BASH_SOURCE[0]}")"
+    pwd -P
+)"
 # shellcheck disable=SC1090
 source "${HERE}/lib.sh"
 

diff --git a/scripts/ensure-venv.sh b/scripts/ensure-venv.sh
@@ -1,16 +1,17 @@
 #!/bin/bash
+HERE="$(
+    cd "$(dirname "${BASH_SOURCE[0]}")" || exit
+    pwd -P
+)"
+# shellcheck disable=SC1090
+source "${HERE}/lib.sh"
 
 # optionally opt out of virtualenv creation
 # WARNING: this will be removed (most likely renamed) soon!
 if [[ "$SENTRY_NO_VIRTUALENV_CREATION" == "1" ]]; then
     exit 0
 fi
 
-red="$(tput setaf 1)"
-yellow="$(tput setaf 3)"
-bold="$(tput bold)"
-reset="$(tput sgr0)"
-
 venv_name=".venv"
 
 die() {
@@ -21,33 +22,31 @@ EOF
 }
 
 if [[ -n "$VIRTUAL_ENV" ]]; then
-    minor=$(python -c "import sys; print(sys.version_info[1])")
-    # If .venv is less than Python 3.6 fail
-    [[ "$minor" -lt 6 ]] &&
-        die "Remove $VIRTUAL_ENV and try again since the Python version installed should be at least 3.6."
-    # If .venv is created with Python greater than 3.6 you might encounter problems and we want to ask you to downgrade
-    # unless you explicitely set an environment variable
-    if [[ "$minor" -gt 6 ]]; then
-        if [[ -n "$SENTRY_PYTHON_VERSION" ]]; then
-            cat <<EOF
+    # The developer is inside a virtualenv *and* has set a SENTRY_PYTHON_VERSION
+    # Let's assume that they know what they're doing
+    if [[ -n "$SENTRY_PYTHON_VERSION" ]]; then
+        cat <<EOF
 ${yellow}${bold}
-You have explicitly set a non-recommended Python version (${SENTRY_PYTHON_VERSION}). You're on your own.
+You have explicitly set a non-recommended Python version ($(python -V | awk '{print $2}')). You're on your own.
 ${reset}
 EOF
-        else
-            cat <<EOF
-${red}${bold}
-ERROR! You are running a virtualenv with a Python version different than 3.6
-We recommend you start with a fresh virtualenv or to set the variable SENTRY_PYTHON_VERSION
-to the Python version you want to use (e.g. 3.7).
-${reset}
+        exit 0
+    fi
+
+    # Let's make sure they know that they're not using a different version by mistake
+
+    if ! query-valid-python-version; then
+        cat <<EOF
+    ${yellow}${bold}
+    WARNING! You are running a virtualenv with a Python version ($(which python))
+    different than 3.6.13 or 3.8.11. We recommend you start with a fresh virtualenv OR"
+    use SENTRY_PYTHON_VERSION to by-pass this check.
+    ${reset}
 EOF
-            exit 1
-        fi
     fi
 else
     if [[ ! -f "${venv_name}/bin/activate" ]]; then
-        die "You don't seem to have a virtualenv. Please create one by running: python3.6 -m venv ${venv_name}"
+        die "You don't seem to have a virtualenv. Please create one by running: source ./scripts/bootstrap-py3-venv"
     fi
     die "You have a virtualenv, but it doesn't seem to be activated. Please run: source ${venv_name}/bin/activate"
 fi