Google analytics

.circleci/api-load-test.sh

@@ -0,0 +1,116 @@
+#!/bin/bash
+#
+# Copyright 2018-2023 contributors to the Marquez project
+# SPDX-License-Identifier: Apache-2.0
+#
+# A script used in CI to load test HTTP API server by:
+#   (1) Starting HTTP API server
+#   (2) Generating random dataset, job, and run metadata
+#   (3) Running load test using k6
+#   (4) Writing load test results to 'k6/results' for analysis
+#
+# Usage: $ ./api-load-test.sh
+
+set -e
+
+# Build version of Marquez
+readonly MARQUEZ_VERSION=0.51.0-SNAPSHOT
+# Fully qualified path to marquez.jar
+readonly MARQUEZ_JAR="api/build/libs/marquez-api-${MARQUEZ_VERSION}.jar"
+
+readonly MARQUEZ_HOST="localhost"
+readonly MARQUEZ_ADMIN_PORT=8081
+readonly MARQUEZ_URL="http://${MARQUEZ_HOST}:${MARQUEZ_ADMIN_PORT}"
+readonly MARQUEZ_DB="marquez-db"
+readonly POSTGRES_PORT=5432
+readonly API_PORT=5000
+readonly API_ADMIN_PORT=5001
+
+readonly METADATA_FILE="api/load-testing/metadata.json"
+readonly METADATA_STATS_QUERY=$(cat <<-END
+  SELECT run_uuid,COUNT(*)
+    FROM lineage_events
+   GROUP BY run_uuid;
+END
+)
+
+# marquez.yml
+cat > marquez.yml <<EOF
+server:
+  applicationConnectors:
+  - type: http
+    port: 8080
+    httpCompliance: RFC7230_LEGACY
+  adminConnectors:
+  - type: http
+    port: 8081
+
+db:
+  driverClass: org.postgresql.Driver
+  url: jdbc:postgresql://localhost:5432/marquez
+  user: marquez
+  password: marquez
+
+migrateOnStartup: true
+EOF
+
+log() {
+  echo -e "\033[1m>>\033[0m ${1}"
+}
+
+cpu_and_mem_info() {
+  log "CPU info:"
+  cat /proc/cpuinfo
+  log "MEM info:"
+  cat /proc/meminfo
+}
+
+metadata_stats() {
+  # Query db for metadata stats
+  log "load test metadata stats:"
+  docker exec "${MARQUEZ_DB}" \
+    psql -U marquez -c "${METADATA_STATS_QUERY}"
+}
+
+# Change working directory to project root
+project_root=$(git rev-parse --show-toplevel)
+cd "${project_root}"
+
+# (1) Start db
+log "start db:"
+docker-compose -f docker-compose.db.yml up --detach
+
+# (2) Build HTTP API server
+log "build http API server..."
+./gradlew --no-daemon :api:build -x test > /dev/null 2>&1
+
+# (3) Start HTTP API server
+log "start http API server..."
+mkdir marquez && \
+  java -jar "${MARQUEZ_JAR}" server marquez.yml > marquez/http.log 2>&1 &
+
+# (4) Wait for HTTP API server
+log "waiting for http API server (${MARQUEZ_URL})..."
+until curl --output /dev/null --silent --head --fail "${MARQUEZ_URL}/ping"; do
+    sleep 5
+done
+# When available, print status
+log "http API server is ready!"
+
+# (5) Use metadata command to generate random dataset, job, and run metadata
+log "generate load test metadata (${METADATA_FILE}):"
+java -jar "${MARQUEZ_JAR}" metadata --jobs 100 --bytes-per-event 16384 --output "${METADATA_FILE}"
+
+# Display CPU/MEM
+cpu_and_mem_info
+
+# (6) Run load test
+log "start load test:"
+mkdir -p k6/results && \
+  k6 run --vus 25 --duration 30s api/load-testing/http.js \
+    --out json=k6/results/full.json --summary-export=k6/results/summary.json
+
+# Display metadata stats
+metadata_stats
+
+echo "DONE!"

.circleci/config.yml

@@ -0,0 +1,264 @@
+version: 2.1
+
+# Define reusable config (see: https://yaml.org/spec/1.2/spec.html#id2765878)
+checkout_project_root: &checkout_project_root
+  # Override checkout path to project root (see: https://circleci.com/docs/2.0/configuration-reference/#checkout)
+  checkout:
+    path: ~/marquez
+
+# Only trigger CI job on release (=X.Y.Z)
+only-on-release: &only-on-release
+  filters:
+    tags:
+      only: /^[0-9]+(\.[0-9]+){2}(-rc\.[0-9]+)?$/
+    branches:
+      ignore: /.*/
+
+orbs:
+  codecov: codecov/codecov@3.2.3
+
+jobs:
+  build-api:
+    working_directory: ~/marquez
+    machine:
+      image: ubuntu-2004:2023.10.1
+    environment:
+      TESTCONTAINERS_RYUK_DISABLED: true
+    steps:
+      - checkout
+      - restore_cache:
+          keys:
+            - v1-api-{{ .Branch }}-{{ .Revision }}
+            - v1-api-{{ .Branch }}
+      - run: ./.circleci/get-jdk17.sh
+      - run: echo "checks.disable=true" > ~/.testcontainers.properties
+      - run: ./gradlew --no-daemon --stacktrace api:javadoc
+      - run: ./gradlew --no-daemon --stacktrace api:build
+      - run: ./gradlew --no-daemon api:jacocoTestReport
+      - codecov/upload
+      - store_test_results:
+          path: api/build/test-results/test
+      - store_artifacts:
+          path: build/reports/tests/test
+          destination: test-report
+      - save_cache:
+          key: v1-api-{{ .Branch }}-{{ .Revision }}
+          paths:
+            - ~/.gradle
+
+  build-image-api:
+    working_directory: ~/marquez
+    machine:
+      image: ubuntu-2004:2023.10.1
+    steps:
+      - checkout
+      - run: docker build --no-cache --tag "marquezproject/marquez:${CIRCLE_SHA1}" .
+      - run: docker save -o marquez.tar "marquezproject/marquez:${CIRCLE_SHA1}"
+      - store_artifacts:
+          path: marquez.tar
+
+  build-image-web:
+    working_directory: ~/marquez/web
+    machine:
+      image: ubuntu-2004:2023.10.1
+    steps:
+      - *checkout_project_root
+      - run: docker build --no-cache --tag "marquezproject/marquez-web:${CIRCLE_SHA1}" .
+      - run: docker save -o marquez-web.tar "marquezproject/marquez-web:${CIRCLE_SHA1}"
+      - store_artifacts:
+          path: marquez-web.tar
+
+  unit-test-web:
+    working_directory: ~/marquez/web
+    docker:
+      - image: cimg/node:22.9.0
+    resource_class: large
+    environment:
+      TZ: 'America/New_York'
+    steps:
+      - *checkout_project_root
+      - restore_cache:
+          keys:
+            - v1-web-{{ .Branch }}-{{ .Revision }}
+            - v1-web-{{ .Branch }}
+      - run: npm install
+      - run: npm run test
+      - run: npm run eslint-fix
+      - run: npm run build
+      - save_cache:
+          paths:
+            - node_modules
+          key: v1-web-{{ .Branch }}-{{ .Revision }}
+
+  build-client-java:
+    working_directory: ~/marquez
+    machine:
+      image: ubuntu-2004:2023.10.1
+    steps:
+      - checkout
+      - restore_cache:
+          keys:
+            - v1-client-java-{{ .Branch }}-{{ .Revision }}
+            - v1-client-java-{{ .Branch }}
+      - run: ./.circleci/get-jdk17.sh
+      - run: ./gradlew --no-daemon --stacktrace clients:java:javadoc
+      - run: ./gradlew --no-daemon --stacktrace clients:java:build
+      - run: ./gradlew --no-daemon clients:java:jacocoTestReport
+      - codecov/upload
+      - store_test_results:
+          path: clients/java/build/test-results/test
+      - store_artifacts:
+          path: build/reports/tests/test
+          destination: test-report
+      - save_cache:
+          key: v1-client-java-{{ .Branch }}-{{ .Revision }}
+          paths:
+            - ~/.gradle
+
+  unit-test-client-python:
+    working_directory: ~/marquez/clients/python
+    docker:
+      - image: circleci/python:3.6
+    steps:
+      - *checkout_project_root
+      - run: pip install -e .[dev]
+      - run: flake8
+      - run: pytest --cov=marquez_python tests/
+      - codecov/upload
+
+  build-client-python:
+    working_directory: ~/marquez/clients/python
+    docker:
+      - image: circleci/python:3.6
+    steps:
+      - *checkout_project_root
+      - run: python setup.py sdist bdist_wheel
+      - persist_to_workspace:
+          root: .
+          paths:
+            - ./dist/*.whl
+            - ./dist/*.tar.gz
+
+  lint-spec-api:
+    working_directory: ~/marquez
+    docker:
+      - image: cimg/node:12.22.7
+    steps:
+      - checkout
+      - run: npm install --prefix=${HOME}/.local --global redoc-cli
+      - run: redoc-cli bundle spec/openapi.yml
+
+  load-test-api:
+    working_directory: ~/marquez
+    machine:
+      image: ubuntu-2004:2023.10.1
+    steps:
+      - checkout
+      - run: ./.circleci/get-docker-compose.sh
+      - run: ./.circleci/get-jdk17.sh
+      - run: ./.circleci/get-k6.sh
+      - run: ./.circleci/api-load-test.sh
+      - store_artifacts:
+          path: marquez
+      - store_artifacts:
+          path: k6
+    environment:
+      - POSTGRES_PORT=5432
+
+  migrate-db:
+    working_directory: ~/marquez
+    machine:
+      image: ubuntu-2004:2023.10.1
+    resource_class: large
+    steps:
+      - checkout
+      - run: ./.circleci/get-docker-compose.sh
+      - run: ./.circleci/db-migration.sh
+    environment:
+      - POSTGRES_PORT=5432
+      - API_PORT=5000
+      - API_ADMIN_PORT=5001
+
+  publish-snapshot:
+    working_directory: ~/marquez
+    machine:
+      image: ubuntu-2004:2023.10.1
+    steps:
+      - checkout
+      - run: ./.circleci/get-jdk17.sh
+      - run: ./.circleci/publish.sh # *SNAPSHOT.jar
+
+  release-java:
+    working_directory: ~/marquez
+    machine:
+      image: ubuntu-2004:2023.10.1
+    steps:
+      - checkout
+      - run: ./.circleci/get-jdk17.sh
+      - run: ./.circleci/publish.sh # *.jar
+
+  release-python:
+    working_directory: ~/marquez
+    docker:
+      - image: circleci/python:3.6
+    steps:
+      - checkout
+      - attach_workspace:
+          at: .
+      - run: pip install wheel twine
+      - run: python -m twine upload --non-interactive --verbose --repository pypi dist/*
+
+  release-docker:
+    working_directory: ~/marquez
+    machine:
+      image: ubuntu-2004:2023.10.1
+    steps:
+      - checkout
+      - run: ./docker/login.sh
+      - run: ./docker/build-and-push.sh $CIRCLE_TAG
+
+workflows:
+  marquez:
+    jobs:
+      - build-api
+      - build-image-api:
+          requires:
+            - build-api
+      - build-image-web:
+          requires:
+            - unit-test-web
+      - build-client-java
+      - unit-test-web
+      - unit-test-client-python
+      - lint-spec-api
+      - load-test-api:
+          requires:
+            - build-api
+      - migrate-db:
+          requires:
+            - build-api
+      - publish-snapshot:
+          filters:
+            branches:
+              only: main
+          context: release
+          requires:
+            - build-image-api
+            - load-test-api
+            - migrate-db
+
+  release:
+    jobs:
+      - build-client-python:
+          <<: *only-on-release
+      - release-java:
+          <<: *only-on-release
+          context: release
+      - release-python:
+          <<: *only-on-release
+          context: release
+          requires:
+            - build-client-python
+      - release-docker:
+          <<: *only-on-release
+          context: release