Merge branch 'main' into feature/drop-job-context

.circleci/api-load-test.sh

@@ -14,7 +14,7 @@
 set -e
 
 # Build version of Marquez
-readonly MARQUEZ_VERSION=0.41.0-SNAPSHOT
+readonly MARQUEZ_VERSION=0.42.0-SNAPSHOT
 # Fully qualified path to marquez.jar
 readonly MARQUEZ_JAR="api/build/libs/marquez-api-${MARQUEZ_VERSION}.jar"
 

.circleci/db-migration.sh

@@ -13,7 +13,7 @@
 # Version of PostgreSQL
 readonly POSTGRES_VERSION="14"
 # Version of Marquez
-readonly MARQUEZ_VERSION=0.40.0
+readonly MARQUEZ_VERSION=0.41.0
 # Build version of Marquez
 readonly MARQUEZ_BUILD_VERSION="$(git log --pretty=format:'%h' -n 1)" # SHA1
 

.env.example

@@ -1,4 +1,4 @@
 API_PORT=5000
 API_ADMIN_PORT=5001
 WEB_PORT=3000
-TAG=0.40.0
+TAG=0.41.0

CHANGELOG.md

@@ -1,6 +1,37 @@
 # Changelog
 
-## [Unreleased](https://github.com/MarquezProject/marquez/compare/0.40.0...HEAD)
+## [Unreleased](https://github.com/MarquezProject/marquez/compare/0.41.0...HEAD)
+
+## [0.41.0](https://github.com/MarquezProject/marquez/compare/0.40.0...0.41.0) - 2023-09-20
+### Added
+* API: add support for the following parameters in the `SearchDao` [`#2556`](https://github.com/MarquezProject/marquez/pull/2556) [@tati](https://github.com/tati) [@wslulciuc](https://github.com/wslulciuc)  
+    *This PR updates the search endpoint to enforce `YYYY-MM-DD` for query params, use `YYYY-MM-DD` as `LocalDate`, and support the following query params:*
+    - *`namespace` - matches jobs or datasets within the given namespace.*
+    - *`before` - matches jobs or datasets before `YYYY-MM-DD`.*
+    - *`after` - matches jobs or datasets after `YYYY-MM-DD`.*  
+* Web: add paging on jobs and datasets [`#2614`](https://github.com/MarquezProject/marquez/pull/2614) [@phixme](https://github.com/phixMe)  
+    *Adds paging to jobs and datasets just like we already have on the lineage events page.*
+* Web: add tag descriptions to tooltips [`#2612`](https://github.com/MarquezProject/marquez/pull/2612) [@davidsharp7](https://github.com/davidsharp7)  
+    *Get the tag descriptions from the tags endpoint and when a column has a tag display the corresponding description on hover over. Context can be found [here](https://docs.github.com/en/issues/tracking-your-work-with-issues/linking-a-pull-request-to-an-issue).*
+* Web: add available column-level tags [`#2606`](https://github.com/MarquezProject/marquez/pull/2606) [@davidsharp7](https://github.com/davidsharp7)  
+    *Adds a new column called "tags" to the dataset column view along with the tags associated with the dataset column.*
+* Web: add HTML Tool Tip [`#2601`](https://github.com/MarquezProject/marquez/pull/2601) [@davidsharp7](https://github.com/davidsharp7)  
+    *Adds a Tool Tip to display basic node details.*
+
+### Fixed 
+* Web: fix dataset saga for paging [`#2615`](https://github.com/MarquezProject/marquez/pull/2615) [@phixme](https://github.com/phixMe)  
+    *Updates the saga, changes the default page size.*
+* API: perf/improve `jobdao` query [`#2609`](https://github.com/MarquezProject/marquez/pull/2609) [@algorithmy1](https://github.com/algorithmy1)  
+    *Optimizes the query to make use of Common Table Expressions to fetch the required data more efficiently and before the join, fixing a significant bottleneck.*
+
+### Changed
+* Docker: Postgres `14` [`#2607`](https://github.com/MarquezProject/marquez/pull/2607) [@wslulciuc](https://github.com/wslulciuc)  
+    *Bumps the recommended version of Postgres to 14.*
+    *When deploying locally, you might need to run `./docker/down.sh` to clean existing volumes.*
+
+### Removed
+* Client: tolerate null transformation attrs in field model [`#2600`](https://github.com/MarquezProject/marquez/pull/2600) [@davidjgoss](https://github.com/davidjgoss)  
+    *Removes the @NonNull annotation from the client class and the @NotNull from the model class.*
 
 ## [0.40.0](https://github.com/MarquezProject/marquez/compare/0.39.0...0.40.0) - 2023-08-15
 ### Added

chart/Chart.yaml

@@ -29,4 +29,4 @@ name: marquez
 sources:
   - https://github.com/MarquezProject/marquez
   - https://marquezproject.github.io/marquez/
-version: 0.40.0
+version: 0.41.0

chart/values.yaml

@@ -17,7 +17,7 @@ marquez:
   image:
     registry: docker.io
     repository: marquezproject/marquez
-    tag: 0.40.0
+    tag: 0.41.0
     pullPolicy: IfNotPresent
   ## Name of the existing secret containing credentials for the Marquez installation.
   ## When this is specified, it will take precedence over the values configured in the 'db' section.
@@ -75,7 +75,7 @@ web:
   image:
     registry: docker.io
     repository: marquezproject/marquez-web
-    tag: 0.40.0
+    tag: 0.41.0
     pullPolicy: IfNotPresent
   ## Marquez website will run on this port
   ##

clients/java/README.md

@@ -10,14 +10,14 @@ Maven:
 <dependency>
     <groupId>io.github.marquezproject</groupId>
     <artifactId>marquez-java</artifactId>
-    <version>0.40.0</version>
+    <version>0.41.0</version>
 </dependency>
 ```
 
 or Gradle:
 
 ```groovy
-implementation 'io.github.marquezproject:marquez-java:0.40.0
+implementation 'io.github.marquezproject:marquez-java:0.41.0
 ```
 
 ## Usage

clients/python/marquez_client/__init__.py

@@ -4,7 +4,7 @@
 # -*- coding: utf-8 -*-
 
 __author__ = """Marquez Project"""
-__version__ = "0.41.0"
+__version__ = "0.42.0"
 
 from marquez_client.client import MarquezClient              # noqa: F401
 from marquez_client.clients import Clients                   # noqa: F401

clients/python/setup.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.41.0
+current_version = 0.42.0
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(?P<rc>.*)

clients/python/setup.py

@@ -24,7 +24,7 @@
 
 setup(
     name="marquez-python",
-    version="0.41.0",
+    version="0.42.0",
     description="Marquez Python Client",
     long_description=readme,
     long_description_content_type="text/markdown",

docker/up.sh

@@ -8,9 +8,9 @@
 set -e
 
 # Version of Marquez
-readonly VERSION=0.40.0
+readonly VERSION=0.41.0
 # Build version of Marquez
-readonly BUILD_VERSION=0.40.0
+readonly BUILD_VERSION=0.41.0
 
 title() {
   echo -e "\033[1m${1}\033[0m"

gradle.properties

@@ -7,4 +7,4 @@ org.gradle.jvmargs=--add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAME
   --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
   --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
 
-version=0.41.0-SNAPSHOT
+version=0.42.0-SNAPSHOT

spec/openapi.yml

@@ -3,7 +3,7 @@
 openapi: 3.0.2
 info:
   title: Marquez
-  version: 0.41.0-SNAPSHOT
+  version: 0.42.0-SNAPSHOT
   description: Marquez is an open source **metadata service** for the **collection**, **aggregation**,
     and **visualization** of a data ecosystem's metadata.
   license:

web/src/components/core/text/MqText.tsx

@@ -30,6 +30,7 @@ interface OwnProps {
   small?: boolean
   bottomMargin?: boolean
   children: ReactElement | (string | ReactElement)[] | string | string[] | number | undefined | null
+  onClick?: () => void
 }
 
 type MqTextProps = OwnProps
@@ -53,6 +54,7 @@ const MqText: React.FC<MqTextProps> = ({
   highlight,
   color,
   small,
+  onClick,
 }) => {
   const theme = createTheme(useTheme())
 
@@ -143,6 +145,7 @@ const MqText: React.FC<MqTextProps> = ({
   if (heading) {
     return (
       <Typography
+        onClick={onClick}
         variant='h4'
         sx={Object.assign(classesObject.root, classesObject.heading, conditionalClasses)}
         style={style}
@@ -152,7 +155,12 @@ const MqText: React.FC<MqTextProps> = ({
     )
   } else if (link && linkTo) {
     return (
-      <LinkRouter to={linkTo} aria-disabled={disabled} style={{ textDecoration: 'none' }}>
+      <LinkRouter
+        to={linkTo}
+        aria-disabled={disabled}
+        style={{ textDecoration: 'none' }}
+        onClick={onClick}
+      >
         <Box
           component='span'
           sx={Object.assign(classesObject.root, classesObject.link, conditionalClasses)}
@@ -164,6 +172,7 @@ const MqText: React.FC<MqTextProps> = ({
   } else if (link && href) {
     return (
       <Link
+        onClick={onClick}
         href={href}
         target={'_blank'}
         rel='noopener noreferrer'
@@ -174,7 +183,11 @@ const MqText: React.FC<MqTextProps> = ({
     )
   } else {
     return (
-      <Box sx={Object.assign(classesObject.root, conditionalClasses)} style={style}>
+      <Box
+        onClick={onClick}
+        sx={Object.assign(classesObject.root, conditionalClasses)}
+        style={style}
+      >
         {children}
       </Box>
     )

web/src/components/datasets/DatasetDetailPage.tsx

@@ -17,6 +17,7 @@ import {
   fetchDatasetVersions,
   resetDataset,
   resetDatasetVersions,
+  setTabIndex,
 } from '../../store/actionCreators'
 import { useNavigate } from 'react-router-dom'
 import CloseIcon from '@mui/icons-material/Close'
@@ -29,14 +30,16 @@ import MqStatus from '../core/status/MqStatus'
 import MqText from '../core/text/MqText'
 
 import { useTheme } from '@emotion/react'
-import React, { ChangeEvent, FunctionComponent, SetStateAction, useEffect } from 'react'
+import Io from '../io/Io'
+import React, { ChangeEvent, FunctionComponent, useEffect } from 'react'
 
 interface StateProps {
   lineageDataset: LineageDataset
   versions: DatasetVersion[]
   versionsLoading: boolean
   datasets: IState['datasets']
   display: IState['display']
+  tabIndex: IState['lineage']['tabIndex']
 }
 
 interface DispatchProps {
@@ -45,6 +48,7 @@ interface DispatchProps {
   resetDataset: typeof resetDataset
   deleteDataset: typeof deleteDataset
   dialogToggle: typeof dialogToggle
+  setTabIndex: typeof setTabIndex
 }
 
 type IProps = StateProps & DispatchProps
@@ -68,6 +72,8 @@ const DatasetDetailPage: FunctionComponent<IProps> = (props) => {
     versions,
     versionsLoading,
     lineageDataset,
+    tabIndex,
+    setTabIndex,
   } = props
   const navigate = useNavigate()
   const i18next = require('i18next')
@@ -92,9 +98,8 @@ const DatasetDetailPage: FunctionComponent<IProps> = (props) => {
     []
   )
 
-  const [tab, setTab] = React.useState(0)
-  const handleChange = (event: ChangeEvent, newValue: SetStateAction<number>) => {
-    setTab(newValue)
+  const handleChange = (_: ChangeEvent, newValue: number) => {
+    setTabIndex(newValue)
   }
 
   if (versionsLoading) {
@@ -149,20 +154,26 @@ const DatasetDetailPage: FunctionComponent<IProps> = (props) => {
         )}
         <Box display={'flex'} justifyContent={'space-between'} mb={2}>
           <Box sx={{ borderBottom: 1, borderColor: 'divider' }}>
-            <Tabs value={tab} onChange={handleChange} textColor='primary' indicatorColor='primary'>
+            <Tabs
+              value={tabIndex}
+              onChange={handleChange}
+              textColor='primary'
+              indicatorColor='primary'
+            >
               <Tab
                 label={i18next.t('datasets.latest_tab')}
                 {...a11yProps(0)}
                 disableRipple={true}
               />
+              <Tab label={'I/O'} {...a11yProps(1)} disableRipple={true} />
               <Tab
                 label={i18next.t('datasets.history_tab')}
-                {...a11yProps(1)}
+                {...a11yProps(2)}
                 disableRipple={true}
               />
               <Tab
                 label={i18next.t('datasets.column_lineage_tab')}
-                {...a11yProps(1)}
+                {...a11yProps(3)}
                 disableRipple={true}
               />
             </Tabs>
@@ -214,15 +225,16 @@ const DatasetDetailPage: FunctionComponent<IProps> = (props) => {
           <MqText subdued>{description}</MqText>
         </Box>
       </Box>
-      {tab === 0 && (
+      {tabIndex === 0 && (
         <DatasetInfo
           datasetFields={firstVersion.fields}
           facets={firstVersion.facets}
           run={firstVersion.createdByRun}
         />
       )}
-      {tab === 1 && <DatasetVersions versions={props.versions} />}
-      {tab === 2 && <DatasetColumnLineage lineageDataset={props.lineageDataset} />}
+      {tabIndex === 1 && <Io />}
+      {tabIndex === 2 && <DatasetVersions versions={props.versions} />}
+      {tabIndex === 3 && <DatasetColumnLineage lineageDataset={props.lineageDataset} />}
     </Box>
   )
 }
@@ -232,6 +244,7 @@ const mapStateToProps = (state: IState) => ({
   display: state.display,
   versions: state.datasetVersions.result.versions,
   versionsLoading: state.datasetVersions.isLoading,
+  tabIndex: state.lineage.tabIndex,
 })
 
 const mapDispatchToProps = (dispatch: Redux.Dispatch) =>
@@ -242,6 +255,7 @@ const mapDispatchToProps = (dispatch: Redux.Dispatch) =>
       resetDataset: resetDataset,
       deleteDataset: deleteDataset,
       dialogToggle: dialogToggle,
+      setTabIndex: setTabIndex,
     },
     dispatch
   )