diff --git a/.circleci/api-load-test.sh b/.circleci/api-load-test.sh index f641b4144c..69bf930d64 100755 --- a/.circleci/api-load-test.sh +++ b/.circleci/api-load-test.sh @@ -14,7 +14,7 @@ set -e # Build version of Marquez -readonly MARQUEZ_VERSION="0.31.0-SNAPSHOT" +readonly MARQUEZ_VERSION="0.32.0-SNAPSHOT" # Fully qualified path to marquez.jar readonly MARQUEZ_JAR="api/build/libs/marquez-api-${MARQUEZ_VERSION}.jar" diff --git a/.circleci/db-migration.sh b/.circleci/db-migration.sh index c4db760b4f..df7fcee5f5 100755 --- a/.circleci/db-migration.sh +++ b/.circleci/db-migration.sh @@ -13,7 +13,7 @@ # Version of PostgreSQL readonly POSTGRES_VERSION="12.1" # Version of Marquez -readonly MARQUEZ_VERSION=0.30.0 +readonly MARQUEZ_VERSION=0.31.0 # Build version of Marquez readonly MARQUEZ_BUILD_VERSION="$(git log --pretty=format:'%h' -n 1)" # SHA1 diff --git a/.env.example b/.env.example index 504393731e..ba8849098e 100644 --- a/.env.example +++ b/.env.example @@ -1,4 +1,4 @@ API_PORT=5000 API_ADMIN_PORT=5001 WEB_PORT=3000 -TAG=0.30.0 +TAG=0.31.0 diff --git a/CHANGELOG.md b/CHANGELOG.md index 9937f40f62..5050a05ee6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,29 @@ # Changelog -## [Unreleased](https://github.com/MarquezProject/marquez/compare/0.30.0...HEAD) +## [Unreleased](https://github.com/MarquezProject/marquez/compare/0.31.0...HEAD) + +## [0.31.0](https://github.com/MarquezProject/marquez/compare/0.30.0...0.31.0) - 2023-02-16 + +### Added + +* UI: add facet view enhancements [`#2336`](https://github.com/MarquezProject/marquez/pull/2336) [@tito12](https://github.com/tito12) + *Creates a dynamic component offering the ability to navigate and search the JSON, expand sections and click on links.* +* UI: highlight selected path on graph and display status of jobs and datasets based on last 14 runs or latest quality facets [`#2384`](https://github.com/MarquezProject/marquez/pull/2384) [@tito12](https://github.com/tito12) + *Adds highlighting of the visual graph based on upstream and downstream dependencies of selected nodes, makes displayed status reflect last 14 runs the case of jobs and latest quality facets in the case of datasets.* +* UI: enable auto-accessibility feature on graph nodes [`#2388`](https://github.com/MarquezProject/marquez/pull/2400) [@merobi-hub](https://github.com/merobi-hub) + *Adds attributes to the `FontAwesomeIcon`s to enable a built-in accessibility feature.* + +### Fixed + +* API: add index to `jobs_fqn` table using `namespace_name` and `job_fqn` columns [`#2357`](https://github.com/MarquezProject/marquez/pull/2357) [@collado-mike](https://github.com/collado-mike) + *Optimizes read queries by adding an index to this table.* +* API: add missing indices to `column_lineage`, `dataset_facets`, `job_facets` tables [`#2419`](https://github.com/MarquezProject/marquez/pull/2419) [@pawel-big-lebowski](https://github.com/pawel-big-lebowski) + *Creates missing indices on reference columns in a number of database tables.* +* Spec: make data version and dataset types the same [`#2400`](https://github.com/MarquezProject/marquez/pull/2400) [@phixme](https://github.com/phixMe) + *Makes the `fields` property the same for datasets and dataset versions, allowing type-generating systems to treat them the same way.* +* UI: show location button only when link to code exists [`#2409`](https://github.com/MarquezProject/marquez/pull/2409) [@tito12](https://github.com/tito12) + *Makes the button visible only if the link is not empty.* + * Improve dataset facets access [`#2407`](https://github.com/MarquezProject/marquez/pull/2407) [@pawel-big-lebowski](https://github.com/pawel-big-lebowski) * Improves database query performance for accessing datasets and datasets' versions.* diff --git a/api/build.gradle b/api/build.gradle index 2322285cf8..2ae7458418 100644 --- a/api/build.gradle +++ b/api/build.gradle @@ -21,7 +21,7 @@ plugins { } ext { - jdbi3Version = '3.36.0' + jdbi3Version = '3.37.1' prometheusVersion = '0.16.0' testcontainersVersion = '1.17.6' sentryVersion = '6.13.0' diff --git a/api/src/main/resources/marquez/db/migration/V58__job_fqn_name_index.sql b/api/src/main/resources/marquez/db/migration/V58__job_fqn_name_index.sql new file mode 100644 index 0000000000..ed03171a62 --- /dev/null +++ b/api/src/main/resources/marquez/db/migration/V58__job_fqn_name_index.sql @@ -0,0 +1 @@ +CREATE INDEX ON jobs_fqn (namespace_name, job_fqn); \ No newline at end of file diff --git a/api/src/main/resources/marquez/db/migration/V59.1__column_lineage_add_indexes.sql b/api/src/main/resources/marquez/db/migration/V59.1__column_lineage_add_indexes.sql new file mode 100644 index 0000000000..29f294b551 --- /dev/null +++ b/api/src/main/resources/marquez/db/migration/V59.1__column_lineage_add_indexes.sql @@ -0,0 +1,11 @@ +create index column_lineage_output_dataset_version_uuid_index + on column_lineage (output_dataset_version_uuid); + +create index column_lineage_output_dataset_field_uuid_index + on column_lineage (output_dataset_field_uuid); + +create index column_lineage_input_dataset_version_uuid_index + on column_lineage (input_dataset_version_uuid); + +create index column_lineage_input_dataset_field_uuid_index + on column_lineage (input_dataset_field_uuid); \ No newline at end of file diff --git a/api/src/main/resources/marquez/db/migration/V59.2__facet_tables_indexes.sql b/api/src/main/resources/marquez/db/migration/V59.2__facet_tables_indexes.sql new file mode 100644 index 0000000000..3fb183e743 --- /dev/null +++ b/api/src/main/resources/marquez/db/migration/V59.2__facet_tables_indexes.sql @@ -0,0 +1,14 @@ +create index dataset_facets_dataset_uuid_index + on dataset_facets (dataset_uuid); + +create index dataset_facets_dataset_version_uuid_index + on dataset_facets (dataset_version_uuid); + +create index dataset_facets_run_uuid_index + on dataset_facets (run_uuid); + +create index job_facets_job_uuid_index + on job_facets (job_uuid); + +create index job_facets_run_uuid_index + on job_facets (run_uuid); \ No newline at end of file diff --git a/build.gradle b/build.gradle index fb0347c907..9520c36e9e 100644 --- a/build.gradle +++ b/build.gradle @@ -21,7 +21,7 @@ buildscript { dependencies { classpath 'com.adarshr:gradle-test-logger-plugin:3.2.0' classpath 'gradle.plugin.com.github.johnrengelman:shadow:7.1.2' - classpath 'com.diffplug.spotless:spotless-plugin-gradle:6.14.0' + classpath 'com.diffplug.spotless:spotless-plugin-gradle:6.15.0' } } @@ -58,11 +58,11 @@ subprojects { dropwizardVersion = '2.1.4' jacocoVersion = '0.8.8' junit5Version = '5.9.2' - lombokVersion = '1.18.24' - mockitoVersion = '5.1.0' - openlineageVersion = '0.19.2' + lombokVersion = '1.18.26' + mockitoVersion = '5.1.1' + openlineageVersion = '0.20.6' slf4jVersion = '1.7.36' - postgresqlVersion = '42.5.1' + postgresqlVersion = '42.5.3' isReleaseVersion = !version.endsWith('SNAPSHOT') } diff --git a/chart/Chart.yaml b/chart/Chart.yaml index ebf98a6f3b..6b2468c09c 100644 --- a/chart/Chart.yaml +++ b/chart/Chart.yaml @@ -29,4 +29,4 @@ name: marquez sources: - https://github.com/MarquezProject/marquez - https://marquezproject.github.io/marquez/ -version: 0.30.0 +version: 0.31.0 diff --git a/chart/values.yaml b/chart/values.yaml index f4ac57c278..dabc4c7dca 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -17,7 +17,7 @@ marquez: image: registry: docker.io repository: marquezproject/marquez - tag: 0.30.0 + tag: 0.31.0 pullPolicy: IfNotPresent ## Name of the existing secret containing credentials for the Marquez installation. ## When this is specified, it will take precedence over the values configured in the 'db' section. @@ -75,7 +75,7 @@ web: image: registry: docker.io repository: marquezproject/marquez-web - tag: 0.30.0 + tag: 0.31.0 pullPolicy: IfNotPresent ## Marquez website will run on this port ## diff --git a/clients/java/README.md b/clients/java/README.md index f7cad8de68..5f7538d47e 100644 --- a/clients/java/README.md +++ b/clients/java/README.md @@ -10,14 +10,14 @@ Maven: io.github.marquezproject marquez-java - 0.30.0 + 0.31.0 ``` or Gradle: ```groovy -implementation 'io.github.marquezproject:marquez-java:0.30.0 +implementation 'io.github.marquezproject:marquez-java:0.31.0 ``` ## Usage diff --git a/clients/python/marquez_client/__init__.py b/clients/python/marquez_client/__init__.py index 181b44116e..e37efdabec 100644 --- a/clients/python/marquez_client/__init__.py +++ b/clients/python/marquez_client/__init__.py @@ -4,7 +4,7 @@ # -*- coding: utf-8 -*- __author__ = """Marquez Project""" -__version__ = "0.31.0" +__version__ = "0.32.0" from marquez_client.client import MarquezClient # noqa: F401 from marquez_client.clients import Clients # noqa: F401 diff --git a/clients/python/setup.cfg b/clients/python/setup.cfg index 6c1f7ce03c..d9baaed7f0 100644 --- a/clients/python/setup.cfg +++ b/clients/python/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.31.0 +current_version = 0.32.0 commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(?P.*) diff --git a/clients/python/setup.py b/clients/python/setup.py index b54ddeb95a..2219ecc477 100644 --- a/clients/python/setup.py +++ b/clients/python/setup.py @@ -24,7 +24,7 @@ setup( name="marquez-python", - version="0.31.0", + version="0.32.0", description="Marquez Python Client", long_description=readme, long_description_content_type="text/markdown", diff --git a/docker/metadata.json b/docker/metadata.json index 0b6c656cb6..65322f89cb 100644 --- a/docker/metadata.json +++ b/docker/metadata.json @@ -1358,6 +1358,22 @@ "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/DatasourceDatasetFacet.json", "name": "food_delivery_db", "uri": "postgres://food_delivery:food_delivery@postgres:5432/food_delivery" + }, + "dataQualityAssertions": { + "_producer": "https://github.com/MarquezProject/marquez/blob/main/docker/metadata.json", + "_schemaURL": "https://openlineage.io/spec/facets/1-0-0/DataQualityAssertionsDatasetFacet.json", + "assertions": [ + { + "assertion": "not_null", + "success": false, + "column": "driver_id" + }, + { + "assertion": "is_string", + "success": true, + "column": "customer_address" + } + ] } } } @@ -1824,4 +1840,4 @@ }, "producer": "https://github.com/MarquezProject/marquez/blob/main/docker/metadata.json" } -] +] \ No newline at end of file diff --git a/docker/up.sh b/docker/up.sh index a682b36a7e..afbbba5240 100755 --- a/docker/up.sh +++ b/docker/up.sh @@ -8,9 +8,9 @@ set -e # Version of Marquez -readonly VERSION=0.30.0 +readonly VERSION=0.31.0 # Build version of Marquez -readonly BUILD_VERSION=0.30.0 +readonly BUILD_VERSION=0.31.0 title() { echo -e "\033[1m${1}\033[0m" diff --git a/docs/openapi.html b/docs/openapi.html index 9407c7ad38..14d5d24a64 100644 --- a/docs/openapi.html +++ b/docs/openapi.html @@ -2174,7 +2174,7 @@ 55.627 l 55.6165,55.627 -231.245496,231.24803 c -127.185,127.1864 -231.5279,231.248 -231.873,231.248 -0.3451,0 -104.688, -104.0616 -231.873,-231.248 z - " fill="currentColor">

Marquez (0.30.0)

Download OpenAPI specification:Download

License: Apache 2.0

Marquez is an open source metadata service for the collection, aggregation, and visualization of a data ecosystem's metadata.

+ " fill="currentColor">

Marquez (0.31.0)

Download OpenAPI specification:Download

License: Apache 2.0

Marquez is an open source metadata service for the collection, aggregation, and visualization of a data ecosystem's metadata.

Namespaces

Create a namespace

Creates a new namespace object. A namespace enables the contextual grouping of related jobs and datasets. Namespaces must contain only letters (a-z, A-Z), numbers (0-9), underscores (_), dashes (-), colons (:), slashes (/), or dots (.). A namespace is case-insensitive with a maximum length of 1024 characters. Note jobs and datasets will be unique within a namespace, but not across namespaces.

path Parameters
namespace
required
string <= 1024 characters
Example: my-namespace

The name of the namespace.

Request Body schema: application/json
ownerName
required
string

The owner of the namespace.

@@ -2383,7 +2383,7 @@

Response samples

Content type
application/json
{
  • "totalCount": 1,
  • "results": [
    ]
}