Merge branch 'main' into ol-facets/improve-dataset-facets-access

MarquezProject · Feb 22, 2023 · be4e649 · be4e649
2 parents 67babf4 + 89bb97a
commit be4e649
Show file tree

Hide file tree

Showing 50 changed files with 855 additions and 16,552 deletions.
diff --git a/.circleci/api-load-test.sh b/.circleci/api-load-test.sh
@@ -14,7 +14,7 @@
 set -e
 
 # Build version of Marquez
-readonly MARQUEZ_VERSION="0.31.0-SNAPSHOT"
+readonly MARQUEZ_VERSION="0.32.0-SNAPSHOT"
 # Fully qualified path to marquez.jar
 readonly MARQUEZ_JAR="api/build/libs/marquez-api-${MARQUEZ_VERSION}.jar"
 

diff --git a/.circleci/db-migration.sh b/.circleci/db-migration.sh
@@ -13,7 +13,7 @@
 # Version of PostgreSQL
 readonly POSTGRES_VERSION="12.1"
 # Version of Marquez
-readonly MARQUEZ_VERSION=0.30.0
+readonly MARQUEZ_VERSION=0.31.0
 # Build version of Marquez
 readonly MARQUEZ_BUILD_VERSION="$(git log --pretty=format:'%h' -n 1)" # SHA1
 

diff --git a/.env.example b/.env.example
@@ -1,4 +1,4 @@
 API_PORT=5000
 API_ADMIN_PORT=5001
 WEB_PORT=3000
-TAG=0.30.0
+TAG=0.31.0
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,6 +1,29 @@
 # Changelog
 
-## [Unreleased](https://github.com/MarquezProject/marquez/compare/0.30.0...HEAD)
+## [Unreleased](https://github.com/MarquezProject/marquez/compare/0.31.0...HEAD)
+
+## [0.31.0](https://github.com/MarquezProject/marquez/compare/0.30.0...0.31.0) - 2023-02-16
+
+### Added
+
+* UI: add facet view enhancements [`#2336`](https://github.com/MarquezProject/marquez/pull/2336) [@tito12](https://github.com/tito12)  
+    *Creates a dynamic component offering the ability to navigate and search the JSON, expand sections and click on links.*
+* UI: highlight selected path on graph and display status of jobs and datasets based on last 14 runs or latest quality facets [`#2384`](https://github.com/MarquezProject/marquez/pull/2384) [@tito12](https://github.com/tito12)  
+    *Adds highlighting of the visual graph based on upstream and downstream dependencies of selected nodes, makes displayed status reflect last 14 runs the case of jobs and latest quality facets in the case of datasets.*
+* UI: enable auto-accessibility feature on graph nodes [`#2388`](https://github.com/MarquezProject/marquez/pull/2400) [@merobi-hub](https://github.com/merobi-hub)  
+    *Adds attributes to the `FontAwesomeIcon`s to enable a built-in accessibility feature.*
+
+### Fixed
+
+* API: add index to `jobs_fqn` table using `namespace_name` and `job_fqn` columns [`#2357`](https://github.com/MarquezProject/marquez/pull/2357) [@collado-mike](https://github.com/collado-mike)  
+    *Optimizes read queries by adding an index to this table.*  
+* API: add missing indices to `column_lineage`, `dataset_facets`, `job_facets` tables [`#2419`](https://github.com/MarquezProject/marquez/pull/2419) [@pawel-big-lebowski](https://github.com/pawel-big-lebowski)  
+    *Creates missing indices on reference columns in a number of database tables.*
+* Spec: make data version and dataset types the same [`#2400`](https://github.com/MarquezProject/marquez/pull/2400) [@phixme](https://github.com/phixMe)  
+    *Makes the `fields` property the same for datasets and dataset versions, allowing type-generating systems to treat them the same way.*
+* UI: show location button only when link to code exists [`#2409`](https://github.com/MarquezProject/marquez/pull/2409) [@tito12](https://github.com/tito12)  
+    *Makes the button visible only if the link is not empty.*
+
 
 * Improve dataset facets access [`#2407`](https://github.com/MarquezProject/marquez/pull/2407) [@pawel-big-lebowski](https://github.com/pawel-big-lebowski)
   * Improves database query performance for accessing datasets and datasets' versions.*

diff --git a/api/build.gradle b/api/build.gradle
@@ -21,7 +21,7 @@ plugins {
 }
 
 ext {
-    jdbi3Version = '3.36.0'
+    jdbi3Version = '3.37.1'
     prometheusVersion = '0.16.0'
     testcontainersVersion = '1.17.6'
     sentryVersion = '6.13.0'

diff --git a/api/src/main/resources/marquez/db/migration/V58__job_fqn_name_index.sql b/api/src/main/resources/marquez/db/migration/V58__job_fqn_name_index.sql
@@ -0,0 +1 @@
+CREATE INDEX ON jobs_fqn (namespace_name, job_fqn);
diff --git a/api/src/main/resources/marquez/db/migration/V59.1__column_lineage_add_indexes.sql b/api/src/main/resources/marquez/db/migration/V59.1__column_lineage_add_indexes.sql
@@ -0,0 +1,11 @@
+create index column_lineage_output_dataset_version_uuid_index
+    on column_lineage (output_dataset_version_uuid);
+
+create index column_lineage_output_dataset_field_uuid_index
+    on column_lineage (output_dataset_field_uuid);
+
+create index column_lineage_input_dataset_version_uuid_index
+    on column_lineage (input_dataset_version_uuid);
+
+create index column_lineage_input_dataset_field_uuid_index
+    on column_lineage (input_dataset_field_uuid);
diff --git a/api/src/main/resources/marquez/db/migration/V59.2__facet_tables_indexes.sql b/api/src/main/resources/marquez/db/migration/V59.2__facet_tables_indexes.sql
@@ -0,0 +1,14 @@
+create index dataset_facets_dataset_uuid_index
+    on dataset_facets (dataset_uuid);
+
+create index dataset_facets_dataset_version_uuid_index
+    on dataset_facets (dataset_version_uuid);
+
+create index dataset_facets_run_uuid_index
+    on dataset_facets (run_uuid);
+
+create index job_facets_job_uuid_index
+    on job_facets (job_uuid);
+
+create index job_facets_run_uuid_index
+    on job_facets (run_uuid);
diff --git a/build.gradle b/build.gradle
@@ -21,7 +21,7 @@ buildscript {
     dependencies {
         classpath 'com.adarshr:gradle-test-logger-plugin:3.2.0'
         classpath 'gradle.plugin.com.github.johnrengelman:shadow:7.1.2'
-        classpath 'com.diffplug.spotless:spotless-plugin-gradle:6.14.0'
+        classpath 'com.diffplug.spotless:spotless-plugin-gradle:6.15.0'
     }
 }
 
@@ -58,11 +58,11 @@ subprojects {
         dropwizardVersion = '2.1.4'
         jacocoVersion = '0.8.8'
         junit5Version = '5.9.2'
-        lombokVersion = '1.18.24'
-        mockitoVersion = '5.1.0'
-        openlineageVersion = '0.19.2'
+        lombokVersion = '1.18.26'
+        mockitoVersion = '5.1.1'
+        openlineageVersion = '0.20.6'
         slf4jVersion = '1.7.36'
-        postgresqlVersion = '42.5.1'
+        postgresqlVersion = '42.5.3'
         isReleaseVersion = !version.endsWith('SNAPSHOT')
     }
 

diff --git a/chart/Chart.yaml b/chart/Chart.yaml
@@ -29,4 +29,4 @@ name: marquez
 sources:
   - https://github.com/MarquezProject/marquez
   - https://marquezproject.github.io/marquez/
-version: 0.30.0
+version: 0.31.0
diff --git a/chart/values.yaml b/chart/values.yaml
@@ -17,7 +17,7 @@ marquez:
   image:
     registry: docker.io
     repository: marquezproject/marquez
-    tag: 0.30.0
+    tag: 0.31.0
     pullPolicy: IfNotPresent
   ## Name of the existing secret containing credentials for the Marquez installation.
   ## When this is specified, it will take precedence over the values configured in the 'db' section.
@@ -75,7 +75,7 @@ web:
   image:
     registry: docker.io
     repository: marquezproject/marquez-web
-    tag: 0.30.0
+    tag: 0.31.0
     pullPolicy: IfNotPresent
   ## Marquez website will run on this port
   ##

diff --git a/clients/java/README.md b/clients/java/README.md
@@ -10,14 +10,14 @@ Maven:
 <dependency>
     <groupId>io.github.marquezproject</groupId>
     <artifactId>marquez-java</artifactId>
-    <version>0.30.0</version>
+    <version>0.31.0</version>
 </dependency>
 ```
 
 or Gradle:
 
 ```groovy
-implementation 'io.github.marquezproject:marquez-java:0.30.0
+implementation 'io.github.marquezproject:marquez-java:0.31.0
 ```
 
 ## Usage

diff --git a/clients/python/marquez_client/__init__.py b/clients/python/marquez_client/__init__.py
@@ -4,7 +4,7 @@
 # -*- coding: utf-8 -*-
 
 __author__ = """Marquez Project"""
-__version__ = "0.31.0"
+__version__ = "0.32.0"
 
 from marquez_client.client import MarquezClient              # noqa: F401
 from marquez_client.clients import Clients                   # noqa: F401
diff --git a/clients/python/setup.cfg b/clients/python/setup.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.31.0
+current_version = 0.32.0
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(?P<rc>.*)

diff --git a/clients/python/setup.py b/clients/python/setup.py
@@ -24,7 +24,7 @@
 
 setup(
     name="marquez-python",
-    version="0.31.0",
+    version="0.32.0",
     description="Marquez Python Client",
     long_description=readme,
     long_description_content_type="text/markdown",

diff --git a/docker/metadata.json b/docker/metadata.json
@@ -1358,6 +1358,22 @@
             "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/DatasourceDatasetFacet.json",
             "name": "food_delivery_db",
             "uri": "postgres://food_delivery:food_delivery@postgres:5432/food_delivery"
+          },
+          "dataQualityAssertions": {
+            "_producer": "https://github.com/MarquezProject/marquez/blob/main/docker/metadata.json",
+            "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/DataQualityAssertionsDatasetFacet.json",
+            "assertions": [
+              {
+                "assertion": "not_null",
+                "success": false,
+                "column": "driver_id"
+              },
+              {
+                "assertion": "is_string",
+                "success": true,
+                "column": "customer_address"
+              }
+            ]
           }
         }
       }
@@ -1824,4 +1840,4 @@
     },
     "producer": "https://github.com/MarquezProject/marquez/blob/main/docker/metadata.json"
   }
-]
+]
diff --git a/docker/up.sh b/docker/up.sh
@@ -8,9 +8,9 @@
 set -e
 
 # Version of Marquez
-readonly VERSION=0.30.0
+readonly VERSION=0.31.0
 # Build version of Marquez
-readonly BUILD_VERSION=0.30.0
+readonly BUILD_VERSION=0.31.0
 
 title() {
   echo -e "\033[1m${1}\033[0m"

diff --git a/docs/openapi.html b/docs/openapi.html
diff --git a/gradle.properties b/gradle.properties
@@ -7,4 +7,4 @@ org.gradle.jvmargs=--add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAME
   --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
   --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
 
-version=0.31.0-SNAPSHOT
+version=0.32.0-SNAPSHOT
diff --git a/renovate.json b/renovate.json
@@ -1,5 +1,5 @@
 {
-  "bumpVersion": "patch",
+  "bumpVersion": "minor",
   "extends": [
     "config:base",
     "docker:disable",

diff --git a/spec/openapi.yml b/spec/openapi.yml
@@ -3,7 +3,7 @@
 openapi: 3.0.2
 info:
   title: Marquez
-  version: 0.31.0-SNAPSHOT
+  version: 0.32.0-SNAPSHOT
   description: Marquez is an open source **metadata service** for the **collection**, **aggregation**,
     and **visualization** of a data ecosystem's metadata.
   license:
@@ -1288,10 +1288,15 @@ components:
                 type: string
               tags:
                 description: List of tags.
-                type: string
+                type: array
+                items:
+                  type: string
               description:
                 description: The description of the field.
                 type: string
+            required:
+              - name
+              - type
         tags:
           description: List of tags.
           type: string

diff --git a/web/.eslintrc.js b/web/.eslintrc.js
@@ -67,5 +67,4 @@ module.exports = {
       }
     }
   ]
-
 }
diff --git a/web/Dockerfile b/web/Dockerfile
@@ -1,6 +1,7 @@
 FROM node:12-alpine
 WORKDIR /usr/src/app
 RUN apk update && apk add --virtual bash coreutils
+RUN apk add --no-cache git
 COPY package*.json ./
 RUN npm install
 COPY . .

diff --git a/web/jest.config.js b/web/jest.config.js
@@ -10,6 +10,7 @@ module.exports = {
   setupTestFrameworkScriptFile: '<rootDir>/setupEnzyme.ts',
   globalSetup: '<rootDir>globalSetup.ts',
   setupFiles: ['<rootDir>setupJest.ts'],
+  testEnvironment: 'jsdom',
   globals: {
     'ts-jest': {
       isolatedModules: 'true'
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		CREATE INDEX ON jobs_fqn (namespace_name, job_fqn);
-Original file line number
+Diff line change
@@ Expand Up / @@ -67,5 +67,4 @@ module.exports = { @@
           }
         }
       ]
     }