From 3bd9b17b70612bc86d3c079e5c57cbbe8b171ac0 Mon Sep 17 00:00:00 2001 From: Max Howell Date: Tue, 2 Jan 2024 17:04:04 +0000 Subject: [PATCH] redo python venv machinery. Allow poetry Refs #211 --- .github/workflows/ci.cli.yml | 6 +- bin/cmd/build | 5 ++ lib/porcelain/build-script.ts | 15 ++++- libexec/bkpyvenv | 63 +++++++++++++++++++ {share/brewkit => libexec}/fix-shebangs.ts | 0 {share/brewkit => libexec}/install | 0 {share/brewkit => libexec}/make | 0 {share/brewkit => libexec}/patch | 0 {share/brewkit => libexec}/pkg-config | 0 {share/brewkit => libexec}/pkgx | 0 {share/brewkit => libexec}/python-venv.py | 0 {share/brewkit => libexec}/python-venv.sh | 0 {share/brewkit => libexec}/sed | 0 projects/git-clone.com/package.yml | 1 + projects/pyapp.com/1/package.yml | 27 -------- projects/pyapp.com/2/package.yml | 18 ------ projects/pyapp.com/2/requirements.txt | 1 - projects/pyapp.com/2/setup.py | 15 ----- .../pyapp.com/{1 => pip}/myapp/__init__.py | 0 projects/pyapp.com/{1 => pip}/myapp/main.py | 0 projects/pyapp.com/pip/package.yml | 26 ++++++++ .../pyapp.com/{1 => pip}/requirements.txt | 0 projects/pyapp.com/{1 => pip}/setup.py | 0 .../pyapp.com/{2 => poetry}/myapp/__init__.py | 0 .../pyapp.com/{2 => poetry}/myapp/main.py | 0 projects/pyapp.com/poetry/package.yml | 30 +++++++++ projects/pyapp.com/poetry/pyproject.toml | 16 +++++ ...on-venv-stubber.sh => python-venv-stub.py} | 26 +------- 28 files changed, 160 insertions(+), 89 deletions(-) create mode 100755 libexec/bkpyvenv rename {share/brewkit => libexec}/fix-shebangs.ts (100%) rename {share/brewkit => libexec}/install (100%) rename {share/brewkit => libexec}/make (100%) rename {share/brewkit => libexec}/patch (100%) rename {share/brewkit => libexec}/pkg-config (100%) rename {share/brewkit => libexec}/pkgx (100%) rename {share/brewkit => libexec}/python-venv.py (100%) mode change 100755 => 100644 rename {share/brewkit => libexec}/python-venv.sh (100%) mode change 100755 => 100644 rename {share/brewkit => libexec}/sed (100%) delete mode 100644 projects/pyapp.com/1/package.yml delete mode 100644 projects/pyapp.com/2/package.yml delete mode 100644 projects/pyapp.com/2/requirements.txt delete mode 100644 projects/pyapp.com/2/setup.py rename projects/pyapp.com/{1 => pip}/myapp/__init__.py (100%) rename projects/pyapp.com/{1 => pip}/myapp/main.py (100%) create mode 100644 projects/pyapp.com/pip/package.yml rename projects/pyapp.com/{1 => pip}/requirements.txt (100%) rename projects/pyapp.com/{1 => pip}/setup.py (100%) rename projects/pyapp.com/{2 => poetry}/myapp/__init__.py (100%) rename projects/pyapp.com/{2 => poetry}/myapp/main.py (100%) create mode 100644 projects/pyapp.com/poetry/package.yml create mode 100644 projects/pyapp.com/poetry/pyproject.toml rename share/brewkit/{python-venv-stubber.sh => python-venv-stub.py} (75%) mode change 100755 => 100644 diff --git a/.github/workflows/ci.cli.yml b/.github/workflows/ci.cli.yml index 4abbbbad..4afddf05 100644 --- a/.github/workflows/ci.cli.yml +++ b/.github/workflows/ci.cli.yml @@ -10,6 +10,7 @@ jobs: integration-tests: name: ${{matrix.pkg}} (${{matrix.platform.tag}}) strategy: + fail-fast: false matrix: platform: - os: ubuntu-latest @@ -20,8 +21,8 @@ jobs: pkg: - stark.com/foo - toolchain.com - - pyapp.com/1 - - pyapp.com/2 + - pyapp.com/pip + - pyapp.com/poetry - stark.com/foo@1.2.3 - git-clone.com - pc-cmake.com @@ -56,6 +57,7 @@ jobs: - run: test $(pkgx +stark.com/foo -- stark) = not_much_u - run: bin/bk audit + unit-tests: runs-on: ubuntu-latest env: diff --git a/bin/cmd/build b/bin/cmd/build index ef2d84ec..a270eb73 100755 --- a/bin/cmd/build +++ b/bin/cmd/build @@ -15,6 +15,7 @@ const { options, args } = await new Command() .name("build") .description("Build pkgx pantry pkgs with brewkit") .option("-C, --clean", "Clean everything first") + .option("-s, --stage", "Stage, do not run build") .arguments("[pkgspec]") .parse(); @@ -93,6 +94,10 @@ const script = new Path(`${config.path.build}.sh`) console.log('writing', script) script.write({text: script_content, force: true}).chmod(0o755) +if (options.stage) { + Deno.exit(0) +} + /// run script await gum('build') diff --git a/lib/porcelain/build-script.ts b/lib/porcelain/build-script.ts index 4aae8250..94c48379 100644 --- a/lib/porcelain/build-script.ts +++ b/lib/porcelain/build-script.ts @@ -17,7 +17,7 @@ export default async function(config: Config, PATH?: Path): Promise { const brewkitd = new Path(new URL(import.meta.url).pathname).parent().parent().parent() const brewkit_PATHs = [ - brewkitd.join("share/brewkit"), + brewkitd.join("libexec"), PATH ].compact(x => x?.string).join(':') @@ -26,6 +26,16 @@ export default async function(config: Config, PATH?: Path): Promise { FLAGS.push("export MACOSX_DEPLOYMENT_TARGET=11.0") } + const tmp = (() => { + switch (host().platform) { + case 'darwin': + case 'linux': + return `export TMPDIR="$HOME/tmp"; mkdir -p "$TMPDIR"` + case 'windows': + return `export TMP="$HOME/tmp"; export TEMP="$HOME/tmp"; mkdir -p "$TMP"` + } + }) + return undent` #!/${bash} @@ -39,8 +49,9 @@ export default async function(config: Config, PATH?: Path): Promise { set +a export PKGX="${pkgx}" - export SRCROOT=${config.path.build.string} export HOME=${config.path.home.string} + export SRCROOT=${config.path.build.string} + ${tmp()} if [ -n "$CI" ]; then export FORCE_UNSAFE_CONFIGURE=1 fi diff --git a/libexec/bkpyvenv b/libexec/bkpyvenv new file mode 100755 index 00000000..68d627b7 --- /dev/null +++ b/libexec/bkpyvenv @@ -0,0 +1,63 @@ +#!/usr/bin/env -S pkgx +git +install bash + +set -eo pipefail + +cd "$SRCROOT" + +d="$(cd "$(dirname "$0")"/.. && pwd)" + +CMD=$1 +shift + +if [ $1 = '--engine=poetry' ]; then + ENGINE=poetry + shift +fi + +PREFIX=$1 +shift + +set -x + +case $CMD in +stage) + VERSION=$1 + + if [ ! -d "$SRCROOT/.git" ]; then + GIT_DIR="$SRCROOT/.git" + git init + git config user.name 'pkgx[bot]' + git config user.email 'bot@pkgx.dev' + git commit -mnil --allow-empty + git tag -a "$VERSION" -m "Version $VERSION" --force + unset GIT_DIR + fi + + if [ "$ENGINE" = poetry ]; then + poetry config virtualenvs.create true + poetry config virtualenvs.in-project true + else + python -m venv "$PREFIX"/venv + fi + ;; +seal) + PYTHON=$(python --version | sed -n 's/Python \([0-9]\+\.[0-9]\+\).*/\1/p') + + if [ "$ENGINE" = poetry ]; then + # FIXME is there a more efficient way to do this? + # FIXME the glob is unfortunate + poetry build -f sdist + tar xzf \ + dist/*.tar.gz \ + --directory "$SRCROOT"/.venv/lib/python$PYTHON/site-packages \ + --strip-components=1 + mkdir -p "$PREFIX" + mv "$SRCROOT"/.venv "$PREFIX/venv" + fi + + for cmd in $@; do + install -D "$d/share/brewkit/python-venv-stub.py" $PREFIX/bin/$cmd + sed -i "1s|.*|#!/usr/bin/env -S pkgx python@$PYTHON|" $PREFIX/bin/$cmd + done + ;; +esac diff --git a/share/brewkit/fix-shebangs.ts b/libexec/fix-shebangs.ts similarity index 100% rename from share/brewkit/fix-shebangs.ts rename to libexec/fix-shebangs.ts diff --git a/share/brewkit/install b/libexec/install similarity index 100% rename from share/brewkit/install rename to libexec/install diff --git a/share/brewkit/make b/libexec/make similarity index 100% rename from share/brewkit/make rename to libexec/make diff --git a/share/brewkit/patch b/libexec/patch similarity index 100% rename from share/brewkit/patch rename to libexec/patch diff --git a/share/brewkit/pkg-config b/libexec/pkg-config similarity index 100% rename from share/brewkit/pkg-config rename to libexec/pkg-config diff --git a/share/brewkit/pkgx b/libexec/pkgx similarity index 100% rename from share/brewkit/pkgx rename to libexec/pkgx diff --git a/share/brewkit/python-venv.py b/libexec/python-venv.py old mode 100755 new mode 100644 similarity index 100% rename from share/brewkit/python-venv.py rename to libexec/python-venv.py diff --git a/share/brewkit/python-venv.sh b/libexec/python-venv.sh old mode 100755 new mode 100644 similarity index 100% rename from share/brewkit/python-venv.sh rename to libexec/python-venv.sh diff --git a/share/brewkit/sed b/libexec/sed similarity index 100% rename from share/brewkit/sed rename to libexec/sed diff --git a/projects/git-clone.com/package.yml b/projects/git-clone.com/package.yml index d5e3ed8e..16cebcb0 100644 --- a/projects/git-clone.com/package.yml +++ b/projects/git-clone.com/package.yml @@ -11,6 +11,7 @@ build: script: - touch foo - git add foo + # test run can be an array - run: - mkdir -p {{prefix}} - mv foo {{prefix}} diff --git a/projects/pyapp.com/1/package.yml b/projects/pyapp.com/1/package.yml deleted file mode 100644 index 510ac935..00000000 --- a/projects/pyapp.com/1/package.yml +++ /dev/null @@ -1,27 +0,0 @@ -distributable: - url: git+https://github.com/pkgxdev/brewkit - ref: v{{version}} - -versions: - - 1.0.0 - -dependencies: - python.org: ^3.11 - -build: | - mv props/* . - rmdir props - python-venv.py {{prefix}}/bin/myapp - -provides: - - bin/myapp - -test: - - myapp - - - run: | - test $(basename $FIXTURE .rb).rb = $(basename $FIXTURE) - test $(cat $FIXTURE) = foo - fixture: - content: foo - extname: .rb diff --git a/projects/pyapp.com/2/package.yml b/projects/pyapp.com/2/package.yml deleted file mode 100644 index 18135f72..00000000 --- a/projects/pyapp.com/2/package.yml +++ /dev/null @@ -1,18 +0,0 @@ -distributable: null - -versions: - - 1.0.0 - -dependencies: - python.org: ^3.11 - -build: | - mv props/* . - rmdir props - python-venv.sh {{prefix}}/bin/myapp - -provides: - - bin/myapp - -test: - myapp diff --git a/projects/pyapp.com/2/requirements.txt b/projects/pyapp.com/2/requirements.txt deleted file mode 100644 index 663bd1f6..00000000 --- a/projects/pyapp.com/2/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -requests \ No newline at end of file diff --git a/projects/pyapp.com/2/setup.py b/projects/pyapp.com/2/setup.py deleted file mode 100644 index 4d94c22e..00000000 --- a/projects/pyapp.com/2/setup.py +++ /dev/null @@ -1,15 +0,0 @@ -from setuptools import setup, find_packages - -setup( - name='myapp', - version='1.0.0', - packages=find_packages(), - install_requires=[ - 'requests', - ], - entry_points={ - 'console_scripts': [ - 'myapp=myapp.main:main', - ], - }, -) diff --git a/projects/pyapp.com/1/myapp/__init__.py b/projects/pyapp.com/pip/myapp/__init__.py similarity index 100% rename from projects/pyapp.com/1/myapp/__init__.py rename to projects/pyapp.com/pip/myapp/__init__.py diff --git a/projects/pyapp.com/1/myapp/main.py b/projects/pyapp.com/pip/myapp/main.py similarity index 100% rename from projects/pyapp.com/1/myapp/main.py rename to projects/pyapp.com/pip/myapp/main.py diff --git a/projects/pyapp.com/pip/package.yml b/projects/pyapp.com/pip/package.yml new file mode 100644 index 00000000..a50bf595 --- /dev/null +++ b/projects/pyapp.com/pip/package.yml @@ -0,0 +1,26 @@ +distributable: + # test can fetch a git repo + url: git+https://github.com/pkgxdev/brewkit + ref: v{{version}} + +versions: + - 1.0.0 + +dependencies: + pkgx.sh: ^1 + +build: + dependencies: + python.org: ^3.11 + script: + - mv props/* . + + - bkpyvenv stage {{prefix}} {{version}} + - ${{prefix}}/venv/bin/pip install {{srcroot}} + - bkpyvenv seal {{prefix}} myapp + +provides: + - bin/myapp + +test: + - myapp diff --git a/projects/pyapp.com/1/requirements.txt b/projects/pyapp.com/pip/requirements.txt similarity index 100% rename from projects/pyapp.com/1/requirements.txt rename to projects/pyapp.com/pip/requirements.txt diff --git a/projects/pyapp.com/1/setup.py b/projects/pyapp.com/pip/setup.py similarity index 100% rename from projects/pyapp.com/1/setup.py rename to projects/pyapp.com/pip/setup.py diff --git a/projects/pyapp.com/2/myapp/__init__.py b/projects/pyapp.com/poetry/myapp/__init__.py similarity index 100% rename from projects/pyapp.com/2/myapp/__init__.py rename to projects/pyapp.com/poetry/myapp/__init__.py diff --git a/projects/pyapp.com/2/myapp/main.py b/projects/pyapp.com/poetry/myapp/main.py similarity index 100% rename from projects/pyapp.com/2/myapp/main.py rename to projects/pyapp.com/poetry/myapp/main.py diff --git a/projects/pyapp.com/poetry/package.yml b/projects/pyapp.com/poetry/package.yml new file mode 100644 index 00000000..ff60ffea --- /dev/null +++ b/projects/pyapp.com/poetry/package.yml @@ -0,0 +1,30 @@ +versions: + - 1.0.0 + +dependencies: + pkgx.sh: ^1 + +build: + dependencies: + python.org: ~3.11 + python-poetry.org: ^1.7 + script: + - mv props/* . + + - bkpyvenv stage --engine=poetry {{prefix}} {{version}} + - poetry install + - bkpyvenv seal --engine=poetry {{prefix}} myapp + +provides: + - bin/myapp + +test: + - myapp + + # test fixtures can have extnames + - run: | + test $(basename $FIXTURE .rb).rb = $(basename $FIXTURE) + test $(cat $FIXTURE) = foo + fixture: + content: foo + extname: .rb diff --git a/projects/pyapp.com/poetry/pyproject.toml b/projects/pyapp.com/poetry/pyproject.toml new file mode 100644 index 00000000..79919902 --- /dev/null +++ b/projects/pyapp.com/poetry/pyproject.toml @@ -0,0 +1,16 @@ +[tool.poetry] +name = "myapp" +version = "1.0.0" +authors = ['foo'] +description = 'bar' + +[tool.poetry.scripts] +myapp = 'myapp.main:main' + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" + +[tool.poetry.dependencies] +python = ">=3.8.1,<3.12" # updated for llama-index dependency +requests = '*' diff --git a/share/brewkit/python-venv-stubber.sh b/share/brewkit/python-venv-stub.py old mode 100755 new mode 100644 similarity index 75% rename from share/brewkit/python-venv-stubber.sh rename to share/brewkit/python-venv-stub.py index 8a10a7bd..812fb6aa --- a/share/brewkit/python-venv-stubber.sh +++ b/share/brewkit/python-venv-stub.py @@ -1,34 +1,14 @@ -#!/usr/bin/env -S pkgx bash -# shellcheck shell=bash - -# python virtual-envs are not relocatable -# our only working choice is to rewrite these files and symlinks every time -# because we promise that pkgx is relocatable *at any time* -#FIXME requiring sed is a bit lame - -if test -z "$VIRTUAL_ENV"; then - echo "error: VIRTUAL_ENV not set" >&2 - exit 1 -fi - -mkdir -p "$VIRTUAL_ENV/../bin" - -CMD_NAME="$1" - -cat < "$VIRTUAL_ENV"/../bin/"$CMD_NAME" #!/usr/bin/env python import os import sys import glob -import shutil -from pathlib import Path # Determine directories and paths script_dir = os.path.dirname(os.path.realpath(__file__)) virtual_env = os.path.normpath(os.path.join(script_dir, '..', 'venv')) arg0 = os.path.basename(sys.argv[0]) -python_path = shutil.which('python') +python_path = sys.executable python_home = os.path.dirname(python_path) # Write pyvenv.cfg file @@ -64,9 +44,7 @@ # Execute the corresponding script in the virtual environment arg0 = os.path.join(virtual_env, 'bin', arg0) + args = sys.argv[1:] args.insert(0, arg0) os.execv(arg0, args) -EOF - -chmod +x "$VIRTUAL_ENV/../bin/$CMD_NAME"