add more scripts (#2)

.github/dependabot.yml

@@ -12,3 +12,8 @@ updates:
       interval: "monthly"
     rebase-strategy: "disabled"
 
+  - package-ecosystem: "docker"
+    directory: "/docker"
+    schedule:
+      interval: "monthly"
+

.github/workflows/build.yaml

@@ -36,12 +36,18 @@ jobs:
           venv/bin/ruff format --check obs_common tests
           venv/bin/ruff check obs_common tests
       - name: Run tests
+        env:
+          SENTRY_DSN: http://public@localhost:8090/1
+          STORAGE_EMULATOR_HOST: http://localhost:8001
         run: |
+          docker compose up -d fakesentry gcs-emulator
           venv/bin/pytest tests/
+          # stop services immediate and ignore errors
+          docker compose down -t0 || true
       - name: License Check
         if: ${{ matrix.python-version == '3.11' }}
         run: |
-          venv/bin/pip install -e .
+          venv/bin/pip install -e . --no-deps
           venv/bin/license-check
 
   build-and-release:

docker-compose.yml

@@ -0,0 +1,32 @@
+services:
+  # https://github.com/willkg/kent
+  fakesentry:
+    build:
+      context: docker/images/fakesentry
+    image: local/tecken_fakesentry
+    ports:
+      - "${EXPOSE_SENTRY_PORT:-8090}:8090"
+    command: run --host 0.0.0.0 --port 8090
+
+  # https://github.com/fsouza/fake-gcs-server
+  # Fake GCP GCS server for local development and testing
+  gcs-emulator:
+    image: fsouza/fake-gcs-server:1.49.2
+    command: -port 8001 -scheme http
+    ports:
+      - "${EXPOSE_GCS_EMULATOR_PORT:-8001}:8001"
+
+  # https://cloud.google.com/sdk/docs/downloads-docker
+  # official pubsub emulator
+  pubsub:
+    # also available as google/cloud-sdk:<version>-emulators
+    image: gcr.io/google.com/cloudsdktool/google-cloud-cli:463.0.0-emulators
+    command:
+      - gcloud
+      - beta
+      - emulators
+      - pubsub
+      - start
+      - --host-port=0.0.0.0:5010
+    ports:
+      - "${EXPOSE_GCS_EMULATOR_PORT:-5010}:5010"

docker/images/fakesentry/Dockerfile

@@ -0,0 +1,25 @@
+FROM python:3.11.9-slim-bullseye@sha256:320da7887b542fee80af7fac52146047a980d767abb9b8fe69d86eaa9113bcc4
+
+ARG groupid=5000
+ARG userid=5000
+
+WORKDIR /app/
+
+RUN groupadd -r kent && useradd --no-log-init -r -g kent kent
+
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends curl tini && \
+    apt-get autoremove -y && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1
+
+RUN pip install -U 'pip>=20' && \
+    pip install --no-cache-dir 'kent==2.0.0'
+
+USER kent
+
+ENTRYPOINT ["tini", "--", "/usr/local/bin/kent-server"]
+CMD ["run"]

obs_common/gcs_cli.py

@@ -0,0 +1,170 @@
+#!/usr/bin/env python
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+# Manipulate emulated GCS storage.
+
+# Usage: ./bin/gcs_cli.py CMD
+
+from pathlib import Path, PurePosixPath
+
+import click
+
+from google.auth.credentials import AnonymousCredentials
+from google.cloud import storage
+from google.cloud.exceptions import Conflict, NotFound
+
+
+def get_client():
+    return storage.Client(credentials=AnonymousCredentials())
+
+
+@click.group()
+def gcs_group():
+    """Local dev environment GCS manipulation script"""
+
+
+@gcs_group.command("create")
+@click.argument("bucket_name")
+def create_bucket(bucket_name):
+    """Creates a bucket
+
+    Specify BUCKET_NAME.
+
+    """
+    client = get_client()
+
+    try:
+        client.create_bucket(bucket_name)
+    except Conflict:
+        click.echo(f"GCS bucket {bucket_name!r} already exists.")
+    else:
+        click.echo(f"GCS bucket {bucket_name!r} created.")
+
+
+@gcs_group.command("delete")
+@click.argument("bucket_name")
+def delete_bucket(bucket_name):
+    """Deletes a bucket
+
+    Specify BUCKET_NAME.
+
+    """
+    client = get_client()
+
+    bucket = None
+
+    try:
+        bucket = client.get_bucket(bucket_name)
+    except NotFound:
+        click.echo(f"GCS bucket {bucket_name!r} does not exist.")
+        return
+
+    # delete blobs before deleting bucket, because bucket.delete(force=True) doesn't
+    # work if there are more than 256 blobs in the bucket.
+    for blob in bucket.list_blobs():
+        blob.delete()
+
+    bucket.delete()
+    click.echo(f"GCS bucket {bucket_name!r} deleted.")
+
+
+@gcs_group.command("list_buckets")
+@click.option("--details/--no-details", default=True, type=bool, help="With details")
+def list_buckets(details):
+    """List GCS buckets"""
+
+    client = get_client()
+
+    buckets = client.list_buckets()
+    for bucket in buckets:
+        if details:
+            # https://cloud.google.com/storage/docs/json_api/v1/buckets#resource-representations
+            click.echo(f"{bucket.name}\t{bucket.time_created}")
+        else:
+            click.echo(f"{bucket.name}")
+
+
+@gcs_group.command("list_objects")
+@click.option("--details/--no-details", default=True, type=bool, help="With details")
+@click.argument("bucket_name")
+def list_objects(bucket_name, details):
+    """List contents of a bucket"""
+
+    client = get_client()
+
+    try:
+        client.get_bucket(bucket_name)
+    except NotFound:
+        click.echo(f"GCS bucket {bucket_name!r} does not exist.")
+        return
+
+    blobs = list(client.list_blobs(bucket_name))
+    if blobs:
+        for blob in blobs:
+            # https://cloud.google.com/storage/docs/json_api/v1/objects#resource-representations
+            if details:
+                click.echo(f"{blob.name}\t{blob.size}\t{blob.updated}")
+            else:
+                click.echo(f"{blob.name}")
+    else:
+        click.echo("No objects in bucket.")
+
+
+@gcs_group.command("upload")
+@click.argument("source")
+@click.argument("destination")
+def upload(source, destination):
+    """Upload files to a bucket
+
+    SOURCE is a path to a file or directory of files. will recurse on directory trees
+
+    DESTINATION is a path to a file or directory in the bucket. If SOURCE is a
+    directory or DESTINATION ends with "/", then DESTINATION is treated as a directory.
+    """
+
+    client = get_client()
+
+    # remove protocol from destination if present
+    destination = destination.split("://", 1)[-1]
+    bucket_name, _, prefix = destination.partition("/")
+    prefix_path = PurePosixPath(prefix)
+
+    try:
+        bucket = client.get_bucket(bucket_name)
+    except NotFound as e:
+        raise click.ClickException(f"GCS bucket {bucket_name!r} does not exist.") from e
+
+    source_path = Path(source)
+    if not source_path.exists():
+        raise click.ClickException(f"local path {source!r} does not exist.")
+    source_is_dir = source_path.is_dir()
+    if source_is_dir:
+        sources = [p for p in source_path.rglob("*") if not p.is_dir()]
+    else:
+        sources = [source_path]
+    if not sources:
+        raise click.ClickException(f"No files in directory {source!r}.")
+    for path in sources:
+        if source_is_dir:
+            # source is a directory so treat destination as a directory
+            key = str(prefix_path / path.relative_to(source_path))
+        elif prefix == "" or prefix.endswith("/"):
+            # source is a file but destination is a directory, preserve file name
+            key = str(prefix_path / path.name)
+        else:
+            key = prefix
+        blob = bucket.blob(key)
+        blob.upload_from_filename(path)
+        click.echo(f"Uploaded gs://{bucket_name}/{key}")
+
+
+def main(argv=None):
+    argv = argv or []
+    gcs_group(argv)
+
+
+if __name__ == "__main__":
+    gcs_group()

obs_common/license_check.py

@@ -91,15 +91,15 @@ def has_license_header(path: pathlib.Path):
     return False
 
 
-def main(*args):
+def main(args=None):
     parser = argparse.ArgumentParser(description=DESCRIPTION)
     parser.add_argument(
         "-l", "--file-only", action="store_true", help="print files only"
     )
     parser.add_argument("--verbose", action="store_true", help="verbose output")
     parser.add_argument("target", help="file or directory tree to check", nargs="?")
 
-    parsed = parser.parse_args(*args)
+    parsed = parser.parse_args(args)
 
     if parsed.target:
         target = pathlib.Path(parsed.target)