From 7ce73d11549a7f6f0c693480bee2bd763e4ce0dc Mon Sep 17 00:00:00 2001 From: Armen Zambrano G Date: Wed, 1 Sep 2021 12:50:55 -0400 Subject: [PATCH] feat(dev_env): Dev env bootstrap support for two Python versions (#28213) This change adds supports for bootstrapping two different Python versions. Python 3.8 will be used for Apple M1 since it's the next Python version for Sentry and it is the first one that supports Apple's M1 chip-set. Other miscellaneous fixes and changes included. Co-authored-by: Billy Vong --- .envrc | 58 +++++++++---------- .github/actions/setup-python/action.yml | 4 +- .github/workflows/development-environment.yml | 20 +++---- .github/workflows/python-deps.yml | 31 ++++------ .python-version | 1 - requirements-base.txt | 3 + scripts/bootstrap-py3-venv | 16 +++-- scripts/do.sh | 6 +- scripts/ensure-venv.sh | 49 ++++++++-------- scripts/lib.sh | 34 ++++++++++- scripts/pyenv_setup.sh | 36 ++++++------ 11 files changed, 146 insertions(+), 112 deletions(-) delete mode 100644 .python-version diff --git a/.envrc b/.envrc index 75b29ed44ada29..2a6c34a1bf3b44 100644 --- a/.envrc +++ b/.envrc @@ -3,18 +3,13 @@ # initialization (e.g. activating the venv) or giving recommendations on how to reach the desired state. # It also sets useful environment variables. # If you'd like to override or set any custom environment variables, this .envrc will read a .env file at the end. - set -e -HERE="$( +SENTRY_ROOT="$( cd "$(dirname "${BASH_SOURCE[0]}")" pwd -P )" -bold="$(tput bold)" -red="$(tput setaf 1)" -green="$(tput setaf 2)" -yellow="$(tput setaf 3)" -reset="$(tput sgr0)" +source "${SENTRY_ROOT}/scripts/lib.sh" # XXX: we can't trap bash EXIT, because it'll override direnv's finalizing routines. # consequently, using "exit" anywhere will skip this notice from showing. @@ -58,10 +53,6 @@ report_to_sentry() { rm "$_SENTRY_LOG_FILE" } -require() { - command -v "$1" >/dev/null 2>&1 -} - debug() { if [ "${SENTRY_DIRENV_DEBUG}" ]; then echo -e "${@}" @@ -86,6 +77,27 @@ die() { return 1 } +prompt_python_venv_creation() { + echo -e "${yellow}You are missing a Python virtualenv and we ${bold}need${reset}${yellow} to run a bootstrapping script (it can take a few minutes)" + info "About to create ${venv_name}..." + echo -e "\nContinue (y/N)?" + read -r resp + case "$resp" in + y | Y) echo "Okay, let's do this." ;; + *) + die "Aborted!" + ;; + esac +} + +show_commands_info() { + echo -e "\n${red}Run the following commands to bring your environment up-to-date:" + for cmd in "${commands_to_run[@]}"; do + warn " ${red}$cmd" + done + echo "" +} + ### Environment ### commands_to_run=() @@ -111,9 +123,9 @@ export SENTRY_UI_HOT_RELOAD=1 ### You can override the exported variables with a .env file # All exports should happen before here unless they're safeguarded (see devenv error reporting below) -if [ -f "${HERE}/.env" ]; then - debug "Loading variables from ${HERE}/.env" - dotenv "${HERE}/.env" +if [ -f "${SENTRY_ROOT}/.env" ]; then + info "Loading variables from ${SENTRY_ROOT}/.env" + dotenv "${SENTRY_ROOT}/.env" fi ## Notify of reporting to Sentry @@ -156,15 +168,7 @@ make setup-git-config venv_name=".venv" if [ ! -f "${venv_name}/bin/activate" ]; then - warn "You are missing a Python virtualenv and we ${bold}need${reset} to run a bootstrapping script (it can take a few minutes)" - echo -e "\nContinue (y/N)?" - read -r resp - case "$resp" in - y | Y) echo "Okay, let's do this." ;; - *) - die "Aborted!" - ;; - esac + prompt_python_venv_creation # This is time consuming but it has to be done source ./scripts/bootstrap-py3-venv fi @@ -181,7 +185,7 @@ source "${venv_name}/bin/activate" unset PS1 debug "Ensuring proper virtualenv..." -"${HERE}/scripts/ensure-venv.sh" +"${SENTRY_ROOT}/scripts/ensure-venv.sh" if ! require sentry; then warn "Your virtualenv is activated, but sentry doesn't seem to be installed." @@ -216,11 +220,7 @@ PATH_add node_modules/.bin # These are commands that can take a significant amount of time if [ ${#commands_to_run[@]} -ne 0 ]; then - echo -e "\n${red}Run the following commands to bring your environment up-to-date:" - for cmd in "${commands_to_run[@]}"; do - warn " ${red}$cmd" - done - echo "" + show_commands_info fi if [ "${log_level}" != "info" ]; then diff --git a/.github/actions/setup-python/action.yml b/.github/actions/setup-python/action.yml index 24837ef5650d66..abd0e2f9789766 100644 --- a/.github/actions/setup-python/action.yml +++ b/.github/actions/setup-python/action.yml @@ -16,7 +16,7 @@ inputs: description: 'pip cache version in order to bust cache' required: false python-version: - description: "python version to install" + description: 'python version to install' required: false outputs: @@ -38,7 +38,7 @@ runs: shell: bash run: | if [ "${{ inputs.python-version }}" == "" ]; then - echo "::set-output name=python-version::$(cat .python-version)" + echo "::set-output name=python-version::$(SENTRY_NO_VENV_CHECK=1 ./scripts/do.sh get-pyenv-version)" else echo "::set-output name=python-version::${{ inputs.python-version }}" fi diff --git a/.github/workflows/development-environment.yml b/.github/workflows/development-environment.yml index 8e8c2dc58ce13c..00fb6bd36fdb31 100644 --- a/.github/workflows/development-environment.yml +++ b/.github/workflows/development-environment.yml @@ -2,13 +2,13 @@ name: dev env on: pull_request: paths: - - 'Makefile' - - '.github/workflows/development-environment.yml' - - '.envrc' - - 'Brewfile' - - 'scripts/*' - - 'src/sentry/runner/commands/devserver.py' - - 'src/sentry/runner/commands/devservices.py' + - 'Makefile' + - '.github/workflows/development-environment.yml' + - '.envrc' + - 'Brewfile' + - 'scripts/*' + - 'src/sentry/runner/commands/devserver.py' + - 'src/sentry/runner/commands/devservices.py' jobs: dev-environment: @@ -20,7 +20,7 @@ jobs: # macosx-11.0 is Big Sur, however, it takes long for jobs to get started # Using Ubuntu 18 until I figure out this error: # -> ImportError: libffi.so.6: cannot open shared object file: No such file or directory - os: [ macos-11.0, ubuntu-18.04 ] + os: [macos-11.0, ubuntu-18.04] fail-fast: false env: PIP_DISABLE_PIP_VERSION_CHECK: on @@ -36,7 +36,7 @@ jobs: # Sometimes, brew needs to be updated before brew bundle would work # After installing Docker (via homebrew) we need to make sure that it is properly initialized on Mac run: | - brew update -q && brew bundle -q + brew update && brew bundle -q # This code is mentioned in our dev docs. Only remove if you adjust the docs as well SENTRY_NO_VENV_CHECK=1 ./scripts/do.sh init-docker @@ -44,7 +44,7 @@ jobs: - name: Set environment variables & others id: info run: | - echo "::set-output name=python-version::$(cat .python-version)" + echo "::set-output name=python-version::$(SENTRY_NO_VENV_CHECK=1 ./scripts/do.sh get-pyenv-version)" echo "::set-output name=pip-cache-dir::$(pip3 cache dir)" echo "::set-output name=pip-version::$(pip -V | awk -F ' ' '{print $2}')" echo "::set-output name=yarn-cache-dir::$(yarn cache dir)" diff --git a/.github/workflows/python-deps.yml b/.github/workflows/python-deps.yml index 9ccbb822c948dd..654d9b8a3c7a28 100644 --- a/.github/workflows/python-deps.yml +++ b/.github/workflows/python-deps.yml @@ -2,8 +2,8 @@ name: python deps on: pull_request: paths: - - '.github/workflows/python-deps.yml' - - 'requirements*' + - '.github/workflows/python-deps.yml' + - 'requirements*' jobs: # This workflow makes sure that Python dependencies install correctly for @@ -14,8 +14,8 @@ jobs: timeout-minutes: 20 strategy: matrix: - os: [ macos-11.0, ubuntu-20.04 ] - python-version: [ 3.6.13, 3.8.10 ] + os: [macos-11.0, ubuntu-20.04] + python-version: [3.6.13, 3.8.11] fail-fast: false env: PIP_DISABLE_PIP_VERSION_CHECK: on @@ -27,25 +27,16 @@ jobs: - uses: actions/checkout@v2 - name: Install prerequisites + # Sometimes, brew needs to be updated before brew bundle would work run: | - HOMEBREW_NO_AUTO_UPDATE=1 brew bundle --no-upgrade + brew update && brew bundle -q - # Until GH composite actions can use `uses`, we need to setup python here - - uses: actions/setup-python@v2 + - name: Setup python + id: setup-python + uses: ./.github/actions/setup-python with: - python-version: ${{ matrix.python-version }} - - - name: Setup pip - uses: ./.github/actions/setup-pip - id: pip - - - name: Cache - uses: actions/cache@v2 - with: - path: | - ${{ steps.pip.outputs.pip-cache-dir }} - key: | - python-deps-${{ matrix.os }}-py${{ matrix.python-version }}-${{ hashFiles('requirements-*.txt') }} + # XXX: We need to pass this python-deps-${{ matrix.os }}-py${{ matrix.python-version }}-${{ hashFiles('requirements-*.txt') }} + cache-files-hash: ${{ hashFiles('requirements-*.txt') }} - name: Install dependencies run: | diff --git a/.python-version b/.python-version deleted file mode 100644 index 3609cf77078c22..00000000000000 --- a/.python-version +++ /dev/null @@ -1 +0,0 @@ -3.6.13 diff --git a/requirements-base.txt b/requirements-base.txt index 485ef639a2f235..8013198faa7322 100644 --- a/requirements-base.txt +++ b/requirements-base.txt @@ -20,6 +20,9 @@ google-cloud-core==1.5.0 googleapis-common-protos==1.52.0 google-cloud-pubsub==2.2.0 google-cloud-storage==1.35.0 +# Only necessary to prevent installing the latest version +# https://github.com/googleapis/python-crc32c/issues/83 +google-crc32c==1.1.2; python_version == '3.8' jsonschema==3.2.0 lxml==4.6.3 maxminddb==2.0.3 diff --git a/scripts/bootstrap-py3-venv b/scripts/bootstrap-py3-venv index f62ca0ea3f0ace..3b8f583e12fd5b 100644 --- a/scripts/bootstrap-py3-venv +++ b/scripts/bootstrap-py3-venv @@ -11,6 +11,11 @@ # trap "echo 'bootstrap FAILED.' && trap - ERR && return" ERR # So just going to sprinkle returns everywhere. +SCRIPTS_DIR="$( + cd "$(dirname "${BASH_SOURCE[0]}")" + pwd -P +)" +source "${SCRIPTS_DIR}/lib.sh" command -v pyenv >/dev/null || { echo "You need to install pyenv. https://develop.sentry.dev/environment/#python" @@ -24,7 +29,10 @@ command -v direnv >/dev/null || { gitroot="$(git rev-parse --show-toplevel)" cd "$gitroot" -export venv_name="${PWD}/.venv" +# This executes very quickly if the Python versions are already installed +make setup-pyenv +export PYENV_VERSION=$(SENTRY_NO_VENV_CHECK=1 ./scripts/do.sh get-pyenv-version) +venv_name="${PWD}/.venv" if [[ -f "${venv_name}/bin/activate" ]]; then prompt_text="You have an existing virtualenv. This script will OVERWRITE it. Continue (y/N)?" @@ -40,9 +48,7 @@ fi deactivate 2>/dev/null || true rm -rf "$venv_name" -# XXX: You'll also need to temporarily pyenv local "$SENTRY_PYTHON_VERSION" -# and then revert changes to the .python-version after bootstrap finishes. -if [[ -z "$SENTRY_PYTHON_VERSION" ]] && ! [[ "$(python3 -V 2>&1)" = "Python $(grep "3.6" .python-version)" ]]; then +if [[ -z "$SENTRY_PYTHON_VERSION" ]] && ! query-valid-python-version; then echo "Your python3 version isn't as expected. Please run: make setup-pyenv" return 1 fi @@ -50,7 +56,7 @@ fi python3 -m pip install -U pip || { echo "bootstrap failed!"; return 1; } python3 -m venv "${venv_name}" || { echo "bootstrap failed!"; return 1; } source "${venv_name}/bin/activate" || { echo "bootstrap failed!"; return 1; } -python3 -m pip install -U pip || { echo "bootstrap failed!"; return 1; } +python3 -m pip install -U pip wheel || { echo "bootstrap failed!"; return 1; } make setup-git || { echo "bootstrap failed!"; return 1; } make install-py-dev || { echo "bootstrap failed!"; return 1; } deactivate || { echo "bootstrap failed!"; return 1; } diff --git a/scripts/do.sh b/scripts/do.sh index 734aa045b1dcac..9a2f20c744b0fb 100755 --- a/scripts/do.sh +++ b/scripts/do.sh @@ -2,8 +2,10 @@ # This script is an interface to any of the methods of lib.sh # Call this script as "do.sh method_from_lib" to execute any function from that library set -eu - -HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")"; pwd -P)" +HERE="$( + cd "$(dirname "${BASH_SOURCE[0]}")" + pwd -P +)" # shellcheck disable=SC1090 source "${HERE}/lib.sh" diff --git a/scripts/ensure-venv.sh b/scripts/ensure-venv.sh index 92e33c392e5c67..8a0800eb9c19ba 100755 --- a/scripts/ensure-venv.sh +++ b/scripts/ensure-venv.sh @@ -1,4 +1,10 @@ #!/bin/bash +HERE="$( + cd "$(dirname "${BASH_SOURCE[0]}")" || exit + pwd -P +)" +# shellcheck disable=SC1090 +source "${HERE}/lib.sh" # optionally opt out of virtualenv creation # WARNING: this will be removed (most likely renamed) soon! @@ -6,11 +12,6 @@ if [[ "$SENTRY_NO_VIRTUALENV_CREATION" == "1" ]]; then exit 0 fi -red="$(tput setaf 1)" -yellow="$(tput setaf 3)" -bold="$(tput bold)" -reset="$(tput sgr0)" - venv_name=".venv" die() { @@ -21,33 +22,31 @@ EOF } if [[ -n "$VIRTUAL_ENV" ]]; then - minor=$(python -c "import sys; print(sys.version_info[1])") - # If .venv is less than Python 3.6 fail - [[ "$minor" -lt 6 ]] && - die "Remove $VIRTUAL_ENV and try again since the Python version installed should be at least 3.6." - # If .venv is created with Python greater than 3.6 you might encounter problems and we want to ask you to downgrade - # unless you explicitely set an environment variable - if [[ "$minor" -gt 6 ]]; then - if [[ -n "$SENTRY_PYTHON_VERSION" ]]; then - cat </dev/null; then return 0 fi return 1 } +query-apple-m1() { + query-mac && [[ $(uname -m) = 'arm64' ]] +} + +get-pyenv-version() { + local PYENV_VERSION + PYENV_VERSION=3.6.13 + if query-apple-m1; then + PYENV_VERSION=3.8.11 + fi + echo "${PYENV_VERSION}" +} + +query-valid-python-version() { + python_version=$(python3 -V 2>&1 | awk '{print $2}') + if [ "${python_version}" == 3.6.13 ] || [ "${python_version}" == 3.8.11 ]; then + return 0 + else + return 1 + fi +} + sudo-askpass() { if [ -z "${sudo-askpass-x}" ]; then sudo --askpass "$@" diff --git a/scripts/pyenv_setup.sh b/scripts/pyenv_setup.sh index db8975d39a7091..7743fd7e0ad983 100755 --- a/scripts/pyenv_setup.sh +++ b/scripts/pyenv_setup.sh @@ -3,15 +3,20 @@ # # Assumptions: # - This script assumes you're calling from the top directory of the repository +# +# shellcheck disable=SC2155 # Declare and assign separately to avoid masking return values. + set -eu HERE="$( cd "$(dirname "${BASH_SOURCE[0]}")" pwd -P )" -# shellcheck disable=SC1090 source "${HERE}/lib.sh" +# We can use PYENV_VERSION to define different Python versions, otherwise, determine load default values +[ -z ${PYENV_VERSION+x} ] && export PYENV_VERSION=$(get-pyenv-version) + get_shell_startup_script() { local _startup_script='' if [[ -n "$SHELL" ]]; then @@ -44,7 +49,8 @@ _append_to_startup_script() { case "$SHELL" in */bash) # shellcheck disable=SC2016 - echo "Visit https://github.com/pyenv/pyenv#installation on how to fully set up your Bash shell.";; + echo "Visit https://github.com/pyenv/pyenv#installation on how to fully set up your Bash shell." + ;; */zsh) # shellcheck disable=SC2016 echo -e '# It is assumed that pyenv is installed via Brew, so this is all we need to do.\n' \ @@ -52,7 +58,7 @@ _append_to_startup_script() { ;; */fish) # shellcheck disable=SC2016 - echo -e '\n# pyenv init\nstatus is-login; and pyenv init --path | source' >> "${1}" + echo -e '\n# pyenv init\nstatus is-login; and pyenv init --path | source' >>"${1}" ;; esac @@ -78,12 +84,8 @@ append_to_config() { install_pyenv() { if require pyenv; then - echo "Installing Python (if missing) via pyenv" - local pyenv_version - pyenv_version=$(pyenv -v | awk '{print $2}') - python_version=$(xargs -n1 <.python-version) # NOTE: We're dropping support for older pyenv versions - if [[ "$pyenv_version" < 2.0.0 ]]; then + if [[ "$(pyenv -v | awk '{print $2}')" < 2.0.0 ]]; then echo >&2 "!!! We've dropped support for pyenv v1." \ "Run the following (this is slow) and try again." # brew upgrade does not quite do the right thing @@ -92,14 +94,16 @@ install_pyenv() { exit 1 fi - # We need to patch the source code on Big Sur before building Python - # We can remove this once we upgrade to newer versions of Python - if query_big_sur; then + if query-apple-m1; then + pyenv install --skip-existing "${PYENV_VERSION}" + elif query-big-sur; then + # We need to patch the source code on Big Sur before building Python + # We can remove this once we upgrade to newer versions of Python # cat is used since pyenv would finish to soon when the Python version is already installed curl -sSL https://github.com/python/cpython/commit/8ea6353.patch | cat | - pyenv install --skip-existing --patch "$python_version" + pyenv install --skip-existing --patch "${PYENV_VERSION}" else - pyenv install --skip-existing "$python_version" + pyenv install --skip-existing "${PYENV_VERSION}" fi else echo >&2 "!!! pyenv not found, try running bootstrap script again or run \`brew bundle\` in the sentry repo" @@ -111,16 +115,14 @@ install_pyenv() { setup_pyenv() { configure-sentry-cli install_pyenv - _startup_script=$(get_shell_startup_script) - append_to_config "$_startup_script" + append_to_config "$(get_shell_startup_script)" # If the script is called with the "dot space right" approach (. ./scripts/pyenv_setup.sh), # the effects of this will be persistent outside of this script - echo "Activating pyenv and validating Python version" # Sets up PATH for pyenv eval "$(pyenv init --path)" python_version=$(python -V | sed s/Python\ //g) - [[ $python_version == $(cat .python-version) ]] || + [[ $python_version == "${PYENV_VERSION}" ]] || (echo "Wrong Python version: $python_version. Please report in #discuss-dev-tooling" && exit 1) }