Merge branch 'develop' into misc/bump-fastapi-dependency

.github/workflows/release_prepare.yml

@@ -81,7 +81,7 @@ jobs:
       # Update the README, pyproject.toml, version and helm files
       - name: Update main files
         run: |
-          sed -i "s/${{ needs.fetch-versions.outputs.old_version }}/${{ needs.fetch-versions.outputs.new_version }}/g" README.md pyproject.toml helm/Chart.yaml helm/README.md
+          sed -i "s/${{ needs.fetch-versions.outputs.old_version }}/${{ needs.fetch-versions.outputs.new_version }}/g" README.md pyproject.toml src/zenml/VERSION helm/Chart.yaml helm/README.md
           git add README.md pyproject.toml src/zenml/VERSION helm/Chart.yaml helm/README.md
       # Update the Quickstart references
       - name: Replace the references in the quickstart example

README.md

@@ -333,7 +333,7 @@ the Apache License Version 2.0.
     <a href="https://github.com/zenml-io/zenml-projects">Projects Showcase</a>
     <br />
     <br />
-    🎉 Version 0.73.0 is out. Check out the release notes
+    🎉 Version 0.74.0 is out. Check out the release notes
     <a href="https://github.com/zenml-io/zenml/releases">here</a>.
     <br />
     🖥️ Download our VS Code Extension <a href="https://marketplace.visualstudio.com/items?itemName=ZenML.zenml-vscode">here</a>.

RELEASE_NOTES.md

@@ -1,5 +1,100 @@
 <!-- markdown-link-check-disable -->
 
+# 0.74.0
+
+The `0.74.0` release introduces several major features including [SageMaker 
+pipeline scheduling capabilities](https://docs.zenml.io/stack-components/orchestrators/sagemaker#scheduling-pipelines), 
+[Azure Container Registry (ACR) implicit authentication support](https://docs.zenml.io/stack-components/container-registries/azure#authentication-methods), 
+and [Vertex AI persistent resource handling for step operators](https://docs.zenml.io/stack-components/step-operators/vertex#using-persistent-resources-for-faster-development). 
+Additionally, this release includes comprehensive improvements 
+to timezone handling and significant enhancements to database performance.
+
+## Features
+
+- API Tokens support in the dashboard for time-boxed API authentication
+- [SageMaker pipeline scheduling capabilities](https://docs.zenml.io/stack-components/orchestrators/sagemaker#scheduling-pipelines)
+- [Azure Container Registry (ACR) and Storage Account implicit authentication](https://docs.zenml.io/stack-components/container-registries/azure#authentication-methods)
+- [Vertex AI persistent resource support](https://docs.zenml.io/stack-components/step-operators/vertex#using-persistent-resources-for-faster-development) for step operators
+- Support for [custom log formats](https://docs.zenml.io/how-to/control-logging/set-logging-format)
+- Run metadata and tag indices for improved performance
+- [Core concepts video added to documentation](https://docs.zenml.io/getting-started/core-concepts)
+
+## Improvements
+
+- Comprehensive timezone consistency improvements across the platform
+- Enhanced database query performance for pipelines, run templates, models, 
+and artifacts
+- Better handling of configured parameters during pipeline preparation
+- Support for passing run configurations as dictionaries when triggering 
+pipelines
+- Enhanced sorting capabilities for columns with empty values in the dashboard
+- Improved queries for pipelines, run templates, models, and artifacts
+- Better filtering functionality for run metadata
+- More efficient artifact filtering
+- Various Helm chart improvements and reorganization
+- Updated materializer support for newer PyTorch versions
+- Improved code repository management and downloading
+- Better handling of `SecretStr` values in store configurations
+
+## Fixes
+
+- Kubernetes service connector issues resolved
+- Fixed sorting for columns with potentially empty values
+- Corrected timestamp utilization for better timezone consistency
+- Resolved issues with vLLM pipeline config file usage
+- Fixed code download functionality for custom flavor components
+- Addressed various documentation and broken links
+- Corrected MySQL database connection warnings
+- Fixed issues with Vertex AI experiment tracker documentation
+
+## What's Changed
+
+* Fix some docs by @htahir1 in https://github.com/zenml-io/zenml/pull/3302
+* Replace deprecated `datetime.utcnow()` with `datetime.now(timezone.utc)`  by @aiakide in https://github.com/zenml-io/zenml/pull/3265
+* Adding the missing VertexAI experiment tracker docs by @bcdurak in https://github.com/zenml-io/zenml/pull/3308
+* Create Sagemaker pipeline schedules if specified by @htahir1 in https://github.com/zenml-io/zenml/pull/3271
+* Formatting by @schustmi in https://github.com/zenml-io/zenml/pull/3307
+* Remove trailing slashes from zenml login URLs by @stefannica in https://github.com/zenml-io/zenml/pull/3312
+* Fix Kubernetes service connector by @stefannica in https://github.com/zenml-io/zenml/pull/3313
+* Add notes on missing features for on-prem ZenML Pro deployments by @stefannica in https://github.com/zenml-io/zenml/pull/3301
+* Fix wrong warning log when directly connecting to MySQL DB by @schustmi in https://github.com/zenml-io/zenml/pull/3311
+* Fix typo by @schustmi in https://github.com/zenml-io/zenml/pull/3316
+* Minor fix for Sagemaker by @bcdurak in https://github.com/zenml-io/zenml/pull/3318
+* Rework timestamp utilization for timezone consistency by @stefannica in https://github.com/zenml-io/zenml/pull/3314
+* Add broken links checker by @htahir1 in https://github.com/zenml-io/zenml/pull/3305
+* Schedule timezone fixes by @schustmi in https://github.com/zenml-io/zenml/pull/3315
+* Misc code repository improvements by @schustmi in https://github.com/zenml-io/zenml/pull/3306
+* Add core concepts video by @htahir1 in https://github.com/zenml-io/zenml/pull/3324
+* Fix code download for custom flavor components by @schustmi in https://github.com/zenml-io/zenml/pull/3323
+* Allow passing run configuration as dict when triggering pipelines by @schustmi in https://github.com/zenml-io/zenml/pull/3326
+* Fix sorting by columns with potentially empty values by @schustmi in https://github.com/zenml-io/zenml/pull/3325
+* Allow custom log formats by @schustmi in https://github.com/zenml-io/zenml/pull/3288
+* Add vertex persistent resource to settings for step operator by @htahir1 in https://github.com/zenml-io/zenml/pull/3304
+* Fix use of config file in vLLM pipelines by @wjayesh in https://github.com/zenml-io/zenml/pull/3322
+* Fixing the CI with the new `huggingface-hub` version by @bcdurak in https://github.com/zenml-io/zenml/pull/3329
+* Handling string values as SecretStrs in store configurations by @bcdurak in https://github.com/zenml-io/zenml/pull/3319
+* More code repository improvements by @schustmi in https://github.com/zenml-io/zenml/pull/3327
+* Fix materializer for new pytorch version by @schustmi in https://github.com/zenml-io/zenml/pull/3331
+* Add some nicer docs by @htahir1 in https://github.com/zenml-io/zenml/pull/3328
+* Add run metadata and tag indices by @schustmi in https://github.com/zenml-io/zenml/pull/3310
+* Fix markdown link checker for external PRs by @schustmi in https://github.com/zenml-io/zenml/pull/3333
+* feat: implement implicit authentication for ACR and Storage Account by @lukas-reining in https://github.com/zenml-io/zenml/pull/3274
+* Add support for symlinks in GH download by @schustmi in https://github.com/zenml-io/zenml/pull/3332
+* ZenML Helm chart improvements by @stefannica in https://github.com/zenml-io/zenml/pull/3320
+* Move helm chart out of the source tree by @stefannica in https://github.com/zenml-io/zenml/pull/3338
+* Add option to skip stack validation by @schustmi in https://github.com/zenml-io/zenml/pull/3337
+* Improve queries for pipelines, run templates, models and artifacts by @schustmi in https://github.com/zenml-io/zenml/pull/3335
+* Improve configured parameter detection when preparing pipeline by @schustmi in https://github.com/zenml-io/zenml/pull/3339
+* Minor fix for the Artifact filter model by @bcdurak in https://github.com/zenml-io/zenml/pull/3334
+* Allow (un)installing integrations with system-wide uv installations by @schustmi in https://github.com/zenml-io/zenml/pull/3342
+* Fix filtering by run metadata by @schustmi in https://github.com/zenml-io/zenml/pull/3344
+
+## New Contributors
+
+* @lukas-reining made their first contribution in https://github.com/zenml-io/zenml/pull/3274
+
+**Full Changelog**: https://github.com/zenml-io/zenml/compare/0.73.0...0.74.0
+
 # 0.73.0
 
 The `0.73.0` release contains various changes and improvements, but most 

examples/quickstart/configs/training_aws.yaml

@@ -1,7 +1,7 @@
 # Environment configuration
 settings:
   docker:
-    parent_image: "715803424590.dkr.ecr.eu-central-1.amazonaws.com/zenml-public-pipelines:quickstart-0.73.0-py3.11-aws"
+    parent_image: "715803424590.dkr.ecr.eu-central-1.amazonaws.com/zenml-public-pipelines:quickstart-0.74.0-py3.11-aws"
     skip_build: True  # If you switch this to False remove the parent_image
     requirements: requirements.txt
     environment:

examples/quickstart/configs/training_azure.yaml

@@ -1,7 +1,7 @@
 # Environment configuration
 settings:
   docker:
-    parent_image: "zenmldocker/zenml-public-pipelines:quickstart-0.73.0-py3.11-azure"
+    parent_image: "zenmldocker/zenml-public-pipelines:quickstart-0.74.0-py3.11-azure"
     skip_build: True
     requirements: requirements.txt
     environment:

examples/quickstart/configs/training_gcp.yaml

@@ -1,7 +1,7 @@
 # Environment configuration
 settings:
   docker:
-    parent_image: "zenmldocker/zenml-public-pipelines:quickstart-0.73.0-py3.11-gcp"
+    parent_image: "zenmldocker/zenml-public-pipelines:quickstart-0.74.0-py3.11-gcp"
     skip_build: True
     requirements: requirements.txt
     environment:

examples/quickstart/quickstart.ipynb

@@ -499,7 +499,7 @@
         "# Common imports and setup\n",
         "if CLOUD_PROVIDER.lower() == \"gcp\":\n",
         "    parent_image = (\n",
-        "        \"zenmldocker/zenml-public-pipelines:quickstart-0.73.0-py3.11-gcp\"\n",
+        "        \"zenmldocker/zenml-public-pipelines:quickstart-0.74.0-py3.11-gcp\"\n",
         "    )\n",
         "    skip_build = True\n",
         "\n",
@@ -508,7 +508,7 @@
         "        SagemakerOrchestratorSettings,\n",
         "    )\n",
         "\n",
-        "    parent_image = \"339712793861.dkr.ecr.eu-central-1.amazonaws.com/zenml-public-pipelines:quickstart-0.73.0-py3.11-aws\"\n",
+        "    parent_image = \"339712793861.dkr.ecr.eu-central-1.amazonaws.com/zenml-public-pipelines:quickstart-0.74.0-py3.11-aws\"\n",
         "    skip_build = True  # if you switch this to False, you need to remove the parent image\n",
         "\n",
         "    settings[\"orchestrator.sagemaker\"] = SagemakerOrchestratorSettings(\n",
@@ -517,7 +517,7 @@
         "\n",
         "elif CLOUD_PROVIDER.lower() == \"azure\":\n",
         "    parent_image = (\n",
-        "        \"zenmldocker/zenml-public-pipelines:quickstart-0.73.0-py3.11-azure\"\n",
+        "        \"zenmldocker/zenml-public-pipelines:quickstart-0.74.0-py3.11-azure\"\n",
         "    )\n",
         "    skip_build = True\n",
         "\n",

examples/quickstart/requirements.txt

@@ -1,4 +1,4 @@
-zenml[server]==0.73.0
+zenml[server]==0.74.0
 notebook
 pyarrow
 datasets

examples/quickstart/requirements_aws.txt

@@ -1,4 +1,4 @@
-zenml[server]==0.73.0
+zenml[server]==0.74.0
 notebook
 pyarrow
 datasets

examples/quickstart/requirements_azure.txt

@@ -1,4 +1,4 @@
-zenml[server]==0.73.0
+zenml[server]==0.74.0
 notebook
 pyarrow
 datasets

examples/quickstart/requirements_gcp.txt

@@ -1,4 +1,4 @@
-zenml[server]==0.73.0
+zenml[server]==0.74.0
 notebook
 pyarrow
 datasets

helm/Chart.yaml

@@ -1,6 +1,6 @@
 apiVersion: v2
 name: zenml
-version: "0.73.0"
+version: "0.74.0"
 description: Open source MLOps framework for portable production ready ML pipelines
 keywords:
 - mlops

helm/README.md

@@ -20,8 +20,8 @@ ZenML is an open-source MLOps framework designed to help you create robust, main
 To install the ZenML chart directly from Amazon ECR, use the following command:
 
 ```bash
-# example command for version 0.73.0
-helm install my-zenml oci://public.ecr.aws/zenml/zenml --version 0.73.0
+# example command for version 0.74.0
+helm install my-zenml oci://public.ecr.aws/zenml/zenml --version 0.74.0
 ```
 
 Note: Ensure you have OCI support enabled in your Helm client and that you are authenticated with Amazon ECR.

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "zenml"
-version = "0.73.0"
+version = "0.74.0"
 packages = [{ include = "zenml", from = "src" }]
 description = "ZenML: Write production-ready ML code."
 authors = ["ZenML GmbH <info@zenml.io>"]

scripts/test-migrations.sh

@@ -23,7 +23,7 @@ else
 fi
 
 # List of versions to test
-VERSIONS=("0.40.3" "0.43.0" "0.44.3" "0.45.6" "0.47.0" "0.50.0" "0.51.0" "0.52.0" "0.53.1" "0.54.1" "0.55.5" "0.56.4" "0.57.1" "0.60.0" "0.61.0" "0.62.0" "0.63.0" "0.64.0" "0.65.0" "0.68.0" "0.70.0" "0.71.0" "0.72.0")
+VERSIONS=("0.40.3" "0.43.0" "0.44.3" "0.45.6" "0.47.0" "0.50.0" "0.51.0" "0.52.0" "0.53.1" "0.54.1" "0.55.5" "0.56.4" "0.57.1" "0.60.0" "0.61.0" "0.62.0" "0.63.0" "0.64.0" "0.65.0" "0.68.0" "0.70.0" "0.71.0" "0.72.0" "0.74.0")
 
 # Try to get the latest version using pip index
 version=$(pip index versions zenml 2>/dev/null | grep -v YANKED | head -n1 | awk '{print $2}' | tr -d '()')

src/zenml/VERSION

@@ -1 +1 @@
-0.73.0
+0.74.0

src/zenml/cli/utils.py

@@ -18,6 +18,7 @@
 import os
 import platform
 import re
+import shutil
 import subprocess
 import sys
 from typing import (
@@ -1048,22 +1049,18 @@ def install_packages(
         # just return without doing anything
         return
 
-    pip_command = ["uv", "pip"] if use_uv else ["pip"]
-    if upgrade:
-        command = (
-            [
-                sys.executable,
-                "-m",
-            ]
-            + pip_command
-            + [
-                "install",
-                "--upgrade",
-            ]
-            + packages
-        )
+    if use_uv and not is_installed_in_python_environment("uv"):
+        # If uv is installed globally, don't run as a python module
+        command = []
     else:
-        command = [sys.executable, "-m"] + pip_command + ["install"] + packages
+        command = [sys.executable, "-m"]
+
+    command += ["uv", "pip", "install"] if use_uv else ["pip", "install"]
+
+    if upgrade:
+        command += ["--upgrade"]
+
+    command += packages
 
     if not IS_DEBUG_ENV:
         quiet_flag = "-q" if use_uv else "-qqq"
@@ -1094,62 +1091,54 @@ def uninstall_package(package: str, use_uv: bool = False) -> None:
         package: The package to uninstall.
         use_uv: Whether to use uv for package uninstallation.
     """
-    pip_command = ["uv", "pip"] if use_uv else ["pip"]
-    quiet_flag = "-q" if use_uv else "-qqq"
-
-    if use_uv:
-        subprocess.check_call(
-            [
-                sys.executable,
-                "-m",
-            ]
-            + pip_command
-            + [
-                "uninstall",
-                quiet_flag,
-                package,
-            ]
-        )
+    if use_uv and not is_installed_in_python_environment("uv"):
+        # If uv is installed globally, don't run as a python module
+        command = []
     else:
-        subprocess.check_call(
-            [
-                sys.executable,
-                "-m",
-            ]
-            + pip_command
-            + [
-                "uninstall",
-                quiet_flag,
-                "-y",
-                package,
-            ]
-        )
+        command = [sys.executable, "-m"]
 
+    command += (
+        ["uv", "pip", "uninstall", "-q"]
+        if use_uv
+        else ["pip", "uninstall", "-y", "-qqq"]
+    )
+    command += [package]
 
-def is_uv_installed() -> bool:
-    """Check if uv is installed in the current environment.
+    subprocess.check_call(command)
+
+
+def is_installed_in_python_environment(package: str) -> bool:
+    """Check if a package is installed in the current python environment.
+
+    Args:
+        package: The package to check.
 
     Returns:
-        True if uv is installed, False otherwise.
+        True if the package is installed, False otherwise.
     """
     try:
-        pkg_resources.get_distribution("uv")
+        pkg_resources.get_distribution(package)
         return True
     except pkg_resources.DistributionNotFound:
         return False
 
 
+def is_uv_installed() -> bool:
+    """Check if uv is installed.
+
+    Returns:
+        True if uv is installed, False otherwise.
+    """
+    return shutil.which("uv") is not None
+
+
 def is_pip_installed() -> bool:
     """Check if pip is installed in the current environment.
 
     Returns:
         True if pip is installed, False otherwise.
     """
-    try:
-        pkg_resources.get_distribution("pip")
-        return True
-    except pkg_resources.DistributionNotFound:
-        return False
+    return is_installed_in_python_environment("pip")
 
 
 def pretty_print_secret(