From 22cd2314a42d1c86c50836791e2ebc9863de332c Mon Sep 17 00:00:00 2001 From: iameskild Date: Wed, 27 Sep 2023 15:50:17 -0700 Subject: [PATCH 01/14] Reduce project_name length to 16 --- src/_nebari/subcommands/init.py | 4 ++-- src/nebari/schema.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/_nebari/subcommands/init.py b/src/_nebari/subcommands/init.py index 4c3412358a..0b3f29f01b 100644 --- a/src/_nebari/subcommands/init.py +++ b/src/_nebari/subcommands/init.py @@ -693,7 +693,7 @@ def guided_init_wizard(ctx: typer.Context, guided_init: str): name_guidelines = """ The project name must adhere to the following requirements: - Letters from A to Z (upper and lower case), numbers, hyphens, and dashes - - Length from 3 to 32 characters + - Length from 3 to 16 characters - Begin and end with a letter """ @@ -773,7 +773,7 @@ def guided_init_wizard(ctx: typer.Context, guided_init: str): qmark=qmark, auto_enter=False, ).unsafe_ask(): - repo_url = "http://{git_provider}/{org_name}/{repo_name}" + repo_url = "https://{git_provider}/{org_name}/{repo_name}" git_provider = questionary.select( "Which git provider would you like to use?", diff --git a/src/nebari/schema.py b/src/nebari/schema.py index c18488a96e..89339bc647 100644 --- a/src/nebari/schema.py +++ b/src/nebari/schema.py @@ -7,7 +7,7 @@ from _nebari.version import __version__, rounded_ver_parse # Regex for suitable project names -project_name_regex = r"^[A-Za-z][A-Za-z0-9\-_]{1,30}[A-Za-z0-9]$" +project_name_regex = r"^[A-Za-z][A-Za-z0-9\-_]{1,14}[A-Za-z0-9]$" project_name_pydantic = pydantic.constr(regex=project_name_regex) # Regex for suitable namespaces From f19ecf804498b5b1415a41af4a863a92a9dc9eed Mon Sep 17 00:00:00 2001 From: iameskild Date: Wed, 27 Sep 2023 15:50:32 -0700 Subject: [PATCH 02/14] Update README, checklists --- .github/ISSUE_TEMPLATE/release-checklist.md | 8 ++++++-- .github/ISSUE_TEMPLATE/testing-checklist.md | 18 +++++++++--------- README.md | 15 +++++++-------- 3 files changed, 22 insertions(+), 19 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/release-checklist.md b/.github/ISSUE_TEMPLATE/release-checklist.md index a82409dd71..4e2c116e53 100644 --- a/.github/ISSUE_TEMPLATE/release-checklist.md +++ b/.github/ISSUE_TEMPLATE/release-checklist.md @@ -55,12 +55,16 @@ Release captain responsible - <@gh_username> ## Cut the official release +_We bundle a handful of other packages alongside Nebari that need to be released prior to the core Nebari release_ +- [ ] [Update and cut release for `nebari-dask` meta package on Conda-Forge.](https://github.com/conda-forge/nebari-dask-feedstock) +- [ ] [Cut PyPI release for `nebari-workflow-controller`](https://github.com/nebari-dev/nebari-workflow-controller) +- [ ] [Cut PyPI release for `argo-jupyter-scheduler`](https://github.com/nebari-dev/argo-jupyter-scheduler) + _These steps must be actioned in the order they appear in this checklist._ - [ ] [Tag, build and push docker images](https://github.com/nebari-dev/nebari-docker-images/releases/new) -- [ ] [Update and cut release for `nebari-dask` meta package on Conda-Forge.](https://github.com/conda-forge/nebari-dask-feedstock) - [ ] [Cut PyPI release via GHA release workflow.](https://github.com/nebari-dev/nebari/releases/new) - Avoid appending `v` to tag. - Copy release notes from `RELEASE.md`. -- [ ] Merge release branch into `main` - [ ] [Merge automated release PR for `nebari` on Conda-Forge.](https://github.com/conda-forge/nebari-feedstock) +- [ ] Merge release branch into `main` diff --git a/.github/ISSUE_TEMPLATE/testing-checklist.md b/.github/ISSUE_TEMPLATE/testing-checklist.md index a811239881..c3b925d91f 100644 --- a/.github/ISSUE_TEMPLATE/testing-checklist.md +++ b/.github/ISSUE_TEMPLATE/testing-checklist.md @@ -18,14 +18,14 @@ However, the following core services still need to be manually validated (until At minimum, the following services will need to be tested: -- [ ] [Log into keycloak as root user](https://nebari-docs.netlify.app/how-tos/configuring-keycloak#change-keycloak-root-password) - - [ ] [Add a user](https://nebari-docs.netlify.app/how-tos/configuring-keycloak#adding-a-nebari-user) -- [ ] [Log into conda-store and create](https://nebari-docs.netlify.app/tutorials/creating-new-environments) +- [ ] [Log into keycloak as root user](https://www.nebari.dev/docs/how-tos/configuring-keycloak/#change-keycloak-root-password) + - [ ] [Add a user](https://www.nebari.dev/docs/how-tos/configuring-keycloak/#adding-a-nebari-user) +- [ ] [Log into conda-store and create](https://www.nebari.dev/docs/tutorials/creating-new-environments) - [ ] a conda environment in a shared namespace and, - [ ] a conda environment in your personal namespace -- [ ] [Launch dask-gateway cluster, test auto-scaler and](https://nebari-docs.netlify.app/tutorials/using_dask) - - [ ] [Validate that the dask-labextention is working](https://nebari-docs.netlify.app/tutorials/using_dask#step-5---viewing-the-dashboard-inside-of-jupyterlab) -- [ ] [Create a basic CDS Dashboard](https://nebari-docs.netlify.app/tutorials/creating-cds-dashboard) -- [ ] [Open VS-Code extension](https://nebari-docs.netlify.app/tutorials/using-vscode) - - [ ] [Add the Python extension](https://nebari-docs.netlify.app/tutorials/using-vscode#adding-extensions) - - [ ] [Create a `.py` file and run it](https://nebari-docs.netlify.app/tutorials/using-vscode#running-python-code) +- [ ] [Launch dask-gateway cluster, test auto-scaler and](https://www.nebari.dev/docs/tutorials/using_dask) + - [ ] [Validate that the dask-labextention is working](https://www.nebari.dev/docs/tutorials/using_dask/#step-4---understand-dasks-diagnostic-tools) +- [ ] [Confirm that a notebook can be submitted via Jupyter-Scheduler](https://nebari.dev/docs/tutorials/jupyter-scheduler) +- [ ] [Open VS-Code extension](https://www.nebari.dev/docs/how-tos/using-vscode) + - [ ] [Add the Python extension](https://www.nebari.dev/docs/how-tos/using-vscode#adding-extensions) + - [ ] [Create a `.py` file and run it](https://www.nebari.dev/docs/how-tos/using-vscode#running-python-code) diff --git a/README.md b/README.md index e96b370ed2..6fd8c90c7f 100644 --- a/README.md +++ b/README.md @@ -34,10 +34,12 @@ - [License](#license) > **⚠️ Warning ⚠️** -> The project has recently been renamed from QHub to Nebari. If your deployment is still managed by `qhub`, performing an in place upgrade will **IRREVOCABLY BREAK** your deployment. -> This will cause you to lose any data stored on the platform, including but not limited to, NFS (file system) data, -> `conda-store` environments, Keycloak users and groups, etc. -> Make sure to [back up your data before attempting an upgrade](https://www.nebari.dev/docs/how-tos/manual-backup). +> The `2023.9.1` release includes the initial implementation of a [Pluggy-based](https://pluggy.readthedocs.io/en/stable/) extension mechanism, for more details refer [here](). +> This version also fully deprecates CDS Dashboards as it is no longer compatible with the newer versions of JupyterHub. +> For more details on all of changes included in this release, please refer to our [release notes](./RELEASE.md). +> After you've installed version `2023.9.1`, you can update your `nebari-config.yaml` by running `nebari upgrade -c nebari-config.yaml`, please +> follow the upgrades instructions output by this command. +> And please make sure to [back up your data before attempting an upgrade](https://www.nebari.dev/docs/how-tos/manual-backup). Automated data science platform. From [JupyterHub](https://jupyter.org/hub "Multi-user version of the Notebook") to Cloud environments with [Dask Gateway](https://docs.dask.org/ "Parallel computing in Python"). @@ -168,10 +170,7 @@ To guarantee a welcoming and friendly community, we require all community member ## Ongoing Support -The `v0.4.0` release introduced many changes that will irrevocably break your deployment if you attempt an in-place upgrade; for details, see our -[RELEASE](RELEASE.md#release-v040---march-17-2022) notes. To focus on the future direction of the project, we have decided as a team that we will provide **limited** support for older versions. Any new user is encouraged to use `v0.4.0` or greater. - -If you're using an older version of Nebari and would like professional support, please get in touch with the Nebari development team. +If you're using Nebari and would like professional support, please get in touch with the Nebari development team. ## License From 07576743d47c1171953ba904382062b8aedf8c37 Mon Sep 17 00:00:00 2001 From: iameskild Date: Thu, 28 Sep 2023 01:06:10 -0700 Subject: [PATCH 03/14] Update RELEASE notes --- .github/ISSUE_TEMPLATE/release-checklist.md | 4 +- RELEASE.md | 86 +++++++++++++++++++++ 2 files changed, 88 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/release-checklist.md b/.github/ISSUE_TEMPLATE/release-checklist.md index 4e2c116e53..ea27537a27 100644 --- a/.github/ISSUE_TEMPLATE/release-checklist.md +++ b/.github/ISSUE_TEMPLATE/release-checklist.md @@ -55,14 +55,14 @@ Release captain responsible - <@gh_username> ## Cut the official release -_We bundle a handful of other packages alongside Nebari that need to be released prior to the core Nebari release_ -- [ ] [Update and cut release for `nebari-dask` meta package on Conda-Forge.](https://github.com/conda-forge/nebari-dask-feedstock) +_If there were changes to the following packages, handle their releases before cutting a new release for Nebari_ - [ ] [Cut PyPI release for `nebari-workflow-controller`](https://github.com/nebari-dev/nebari-workflow-controller) - [ ] [Cut PyPI release for `argo-jupyter-scheduler`](https://github.com/nebari-dev/argo-jupyter-scheduler) _These steps must be actioned in the order they appear in this checklist._ - [ ] [Tag, build and push docker images](https://github.com/nebari-dev/nebari-docker-images/releases/new) +- [ ] [Update and cut release for `nebari-dask` meta package on Conda-Forge.](https://github.com/conda-forge/nebari-dask-feedstock) - [ ] [Cut PyPI release via GHA release workflow.](https://github.com/nebari-dev/nebari/releases/new) - Avoid appending `v` to tag. - Copy release notes from `RELEASE.md`. diff --git a/RELEASE.md b/RELEASE.md index 9eb17a1cd9..23e84a6540 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -11,6 +11,92 @@ This file is copied to nebari-dev/nebari-docs using a GitHub Action. --> ## Upcoming Release + +## Release 2023.9.1 - September 28, 2023 + +This release includes a major refactor which introduces a Pluggy-based extension mechanism which allow developers to build new stages. This is the initial implementation +of the extension mechanism and we expect the interface to be refined overtime. If you're interested in developing your own stage plugin, please refer to [our documentation](https://www.nebari.dev/docs/how-tos/nebari-extension-system#developing-an-extension). When you're ready to upgrade, please download this version from either PyPI or Conda-Forge and run the `nebari upgrade -c nebari-config.yaml` +command and follow the instructions + +> WARNING: This release deprecates support for CDS Dashboards and upgrading to this version will result in CDS Dashboards being uninstalled. A replacement dashboarding solution is currently in the works +> and will be integrated soon. + +> WARNING: Given the scope of changes in this release, we highly recommend backing up your system before upgrading. Please refer to our [Manual Backup](https://www.nebari.dev/docs/how-tos/manual-backup) documentation for more details. + +### Feature changes and enhancements + +* Extension Mechanism Implementation in [PR 1833](https://github.com/nebari-dev/nebari/pull/1833) + * This also includes much stricter schema validation. +* JupyterHub upgraded to 3.1 in [PR 1856](https://github.com/nebari-dev/nebari/pull/1856)' + +### Breaking Changes + +* While we have tried our best to avoid breaking changes when introducing the extension mechanism, the scope of the changes is too large for us to confidently say there won't be breaking changes. + + +### What's Changed +* Spinup spot instance for CI with cirun by @aktech in https://github.com/nebari-dev/nebari/pull/1882 +* Fix argo-viewer service account reference by @iameskild in https://github.com/nebari-dev/nebari/pull/1881 +* Framework for Nebari deployment via pytest for extensive testing by @aktech in https://github.com/nebari-dev/nebari/pull/1867 +* [pre-commit.ci] pre-commit autoupdate by @pre-commit-ci in https://github.com/nebari-dev/nebari/pull/1878 +* Test GCP/AWS Deployment with Pytest by @aktech in https://github.com/nebari-dev/nebari/pull/1871 +* Bump DigitalOcean provider to latest by @aktech in https://github.com/nebari-dev/nebari/pull/1891 +* Ensure path is Path object by @iameskild in https://github.com/nebari-dev/nebari/pull/1888 +* enabling viewing hidden files in jupyterlab file explorer by @kalpanachinnappan in https://github.com/nebari-dev/nebari/pull/1893 +* Extension Mechanism Implementation by @costrouc in https://github.com/nebari-dev/nebari/pull/1833 +* Fix import path in deployment tests & misc by @aktech in https://github.com/nebari-dev/nebari/pull/1908 +* pytest:ensure failure on warnings by @costrouc in https://github.com/nebari-dev/nebari/pull/1907 +* workaround for mixed string/posixpath error by @sblair-metrostar in https://github.com/nebari-dev/nebari/pull/1915 +* ENH: Remove aws cli, use boto3 by @fangchenli in https://github.com/nebari-dev/nebari/pull/1920 +* paginator for boto3 ec2 instance types by @sblair-metrostar in https://github.com/nebari-dev/nebari/pull/1923 +* Update README.md -- fix typo. by @teoliphant in https://github.com/nebari-dev/nebari/pull/1925 +* Add more unit tests, add cleanup step for Digital Ocean integration test by @iameskild in https://github.com/nebari-dev/nebari/pull/1910 +* Add cleanup step for AWS integration test, ensure diable_prompt is passed through by @iameskild in https://github.com/nebari-dev/nebari/pull/1921 +* K8s 1.25 + More Improvements by @Adam-D-Lewis in https://github.com/nebari-dev/nebari/pull/1856 +* adding lifecycle ignore to eks node group by @sblair-metrostar in https://github.com/nebari-dev/nebari/pull/1905 +* nebari init unit tests by @sblair-metrostar in https://github.com/nebari-dev/nebari/pull/1931 +* Bug fix - JH singleuser environment getting overwritten by @kenafoster in https://github.com/nebari-dev/nebari/pull/1933 +* Allow users to specify the Azure RG to deploy into by @iameskild in https://github.com/nebari-dev/nebari/pull/1927 +* nebari validate unit tests by @sblair-metrostar in https://github.com/nebari-dev/nebari/pull/1938 +* adding openid connect provider to enable irsa feature by @sblair-metrostar in https://github.com/nebari-dev/nebari/pull/1903 +* nebari upgrade CLI tests by @sblair-metrostar in https://github.com/nebari-dev/nebari/pull/1963 +* CI: Add test coverage by @fangchenli in https://github.com/nebari-dev/nebari/pull/1959 +* nebari cli environment variable handling, support, keycloak, dev tests by @sblair-metrostar in https://github.com/nebari-dev/nebari/pull/1968 +* CI: remove empty notebook to fix pre-commit json check by @fangchenli in https://github.com/nebari-dev/nebari/pull/1976 +* TYP: fix typing error in plugins by @fangchenli in https://github.com/nebari-dev/nebari/pull/1973 +* TYP: fix return class type in hookimpl by @fangchenli in https://github.com/nebari-dev/nebari/pull/1975 +* Allow users to specify Azure tags by @iameskild in https://github.com/nebari-dev/nebari/pull/1967 +* [pre-commit.ci] pre-commit autoupdate by @pre-commit-ci in https://github.com/nebari-dev/nebari/pull/1979 +* Do not try and add argo envs when disabled by @iameskild in https://github.com/nebari-dev/nebari/pull/1926 +* Handle region with care, updates to test suite by @iameskild in https://github.com/nebari-dev/nebari/pull/1930 +* remove custom auth from config schema by @sblair-metrostar in https://github.com/nebari-dev/nebari/pull/1994 +* CLI: handle removed dns options in deploy command by @fangchenli in https://github.com/nebari-dev/nebari/pull/1992 +* Add API docs by @kcpevey in https://github.com/nebari-dev/nebari/pull/1634 +* Upgrade images for jupyterhub-ssh, kbatch by @iameskild in https://github.com/nebari-dev/nebari/pull/1997 +* Add permissions to generate_cli_docs workflow by @iameskild in https://github.com/nebari-dev/nebari/pull/2005 +* standardize regex and messaging for names by @kenafoster in https://github.com/nebari-dev/nebari/pull/2003 +* ENH: specify required fields when retrieving available gcp projects by @fangchenli in https://github.com/nebari-dev/nebari/pull/2008 +* Modify JupyterHub networkPolicy to match existing policy by @iameskild in https://github.com/nebari-dev/nebari/pull/1991 +* Update package dependencies by @iameskild in https://github.com/nebari-dev/nebari/pull/1986 +* CI: Add AWS integration test workflow, clean up by @iameskild in https://github.com/nebari-dev/nebari/pull/1977 +* BUG: fix unboundlocalerror in integration test by @fangchenli in https://github.com/nebari-dev/nebari/pull/1999 +* Auth0/Github auth-provider config validation fix by @sblair-metrostar in https://github.com/nebari-dev/nebari/pull/2009 +* terraform upgrade to 1.5.7 by @sblair-metrostar in https://github.com/nebari-dev/nebari/pull/1998 +* cli init repo auto provision fix by @sblair-metrostar in https://github.com/nebari-dev/nebari/pull/2012 +* Add gcp_cleanup, minor changes by @iameskild in https://github.com/nebari-dev/nebari/pull/2010 +* Fix #2024 by @dcmcand in https://github.com/nebari-dev/nebari/pull/2025 +* Upgrade conda-store to 2023.9.2 by @iameskild in https://github.com/nebari-dev/nebari/pull/2028 + +### New Contributors +* @kalpanachinnappan made their first contribution in https://github.com/nebari-dev/nebari/pull/1893 +* @fangchenli made their first contribution in https://github.com/nebari-dev/nebari/pull/1920 +* @teoliphant made their first contribution in https://github.com/nebari-dev/nebari/pull/1925 +* @kenafoster made their first contribution in https://github.com/nebari-dev/nebari/pull/1933 +* @dcmcand made their first contribution in https://github.com/nebari-dev/nebari/pull/2025 + +**Full Changelog**: https://github.com/nebari-dev/nebari/compare/2023.7.2...2023.9.1 + + ## Release 2023.7.2 - August 3, 2023 This is a hot-fix release that resolves an issue whereby users in the `analyst` group are unable to launch their JupyterLab server because the name of the viewer-specific `ARGO_TOKEN` was mislabeled; see [PR 1881](https://github.com/nebari-dev/nebari/pull/1881) for more details. From 42b685c4f9c57a5f09a6771712453ee43f122454 Mon Sep 17 00:00:00 2001 From: iameskild Date: Thu, 28 Sep 2023 01:08:44 -0700 Subject: [PATCH 04/14] Update --- src/_nebari/subcommands/init.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/_nebari/subcommands/init.py b/src/_nebari/subcommands/init.py index 0b3f29f01b..5e2fcb48c1 100644 --- a/src/_nebari/subcommands/init.py +++ b/src/_nebari/subcommands/init.py @@ -484,7 +484,7 @@ def init( "-p", callback=typer_validate_regex( schema.project_name_regex, - "Project name must (1) consist of only letters, numbers, hyphens, and underscores, (2) begin and end with a letter, and (3) contain between 3 and 32 characters.", + "Project name must (1) consist of only letters, numbers, hyphens, and underscores, (2) begin and end with a letter, and (3) contain between 3 and 16 characters.", ), ), domain_name: typing.Optional[str] = typer.Option( From cf67595d8154a41675a379b72ef0038d1dc54b47 Mon Sep 17 00:00:00 2001 From: iameskild Date: Thu, 28 Sep 2023 01:16:22 -0700 Subject: [PATCH 05/14] Update DEFAULT_NEBARI_WORKFLOW_CONTROLLER_IMAGE_TAG --- src/_nebari/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/_nebari/constants.py b/src/_nebari/constants.py index edc313e671..92e4ca9981 100644 --- a/src/_nebari/constants.py +++ b/src/_nebari/constants.py @@ -13,7 +13,7 @@ DEFAULT_NEBARI_DASK_VERSION = CURRENT_RELEASE DEFAULT_NEBARI_IMAGE_TAG = CURRENT_RELEASE -DEFAULT_NEBARI_WORKFLOW_CONTROLLER_IMAGE_TAG = CURRENT_RELEASE +DEFAULT_NEBARI_WORKFLOW_CONTROLLER_IMAGE_TAG = "2023.9.1" DEFAULT_CONDA_STORE_IMAGE_TAG = "2023.9.2" From 431bc22f4216adf2fd7d21f702b21d2f31d6c76f Mon Sep 17 00:00:00 2001 From: iameskild Date: Thu, 5 Oct 2023 21:41:38 -0700 Subject: [PATCH 06/14] Update tests --- src/_nebari/constants.py | 2 +- tests/tests_unit/test_cli_validate.py | 18 +++++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/_nebari/constants.py b/src/_nebari/constants.py index 8762752e1f..16ff2ccc12 100644 --- a/src/_nebari/constants.py +++ b/src/_nebari/constants.py @@ -13,7 +13,7 @@ DEFAULT_NEBARI_DASK_VERSION = CURRENT_RELEASE DEFAULT_NEBARI_IMAGE_TAG = CURRENT_RELEASE -DEFAULT_NEBARI_WORKFLOW_CONTROLLER_IMAGE_TAG = "2023.9.1" +DEFAULT_NEBARI_WORKFLOW_CONTROLLER_IMAGE_TAG = "2023.7.2" DEFAULT_CONDA_STORE_IMAGE_TAG = "2023.9.2" diff --git a/tests/tests_unit/test_cli_validate.py b/tests/tests_unit/test_cli_validate.py index 0894da9273..00c46c2cd6 100644 --- a/tests/tests_unit/test_cli_validate.py +++ b/tests/tests_unit/test_cli_validate.py @@ -1,4 +1,5 @@ import re +import shutil import tempfile from pathlib import Path from typing import Any, Dict, List @@ -74,14 +75,17 @@ def test_cli_validate_local_happy_path(config_yaml: str): test_file = TEST_DATA_DIR / config_yaml assert test_file.exists() is True - # update the test file with the current version - _update_yaml_file(test_file, "nebari_version", __version__) + with tempfile.TemporaryDirectory() as tmpdirname: + temp_test_file = shutil.copy(test_file, tmpdirname) - app = create_cli() - result = runner.invoke(app, ["validate", "--config", test_file]) - assert not result.exception - assert 0 == result.exit_code - assert "Successfully validated configuration" in result.stdout + # update the copied test file with the current version if necessary + _update_yaml_file(temp_test_file, "nebari_version", __version__) + + app = create_cli() + result = runner.invoke(app, ["validate", "--config", temp_test_file]) + assert not result.exception + assert 0 == result.exit_code + assert "Successfully validated configuration" in result.stdout def test_cli_validate_from_env(): From 80c447209d1b39fe09882e81022606e81bbd5176 Mon Sep 17 00:00:00 2001 From: iameskild Date: Mon, 9 Oct 2023 17:03:34 -0700 Subject: [PATCH 07/14] Update based on review --- RELEASE.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/RELEASE.md b/RELEASE.md index 23e84a6540..a70e25e24e 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -18,7 +18,7 @@ This release includes a major refactor which introduces a Pluggy-based extension of the extension mechanism and we expect the interface to be refined overtime. If you're interested in developing your own stage plugin, please refer to [our documentation](https://www.nebari.dev/docs/how-tos/nebari-extension-system#developing-an-extension). When you're ready to upgrade, please download this version from either PyPI or Conda-Forge and run the `nebari upgrade -c nebari-config.yaml` command and follow the instructions -> WARNING: This release deprecates support for CDS Dashboards and upgrading to this version will result in CDS Dashboards being uninstalled. A replacement dashboarding solution is currently in the works +> WARNING: CDS Dashboards was removed in this release and upgrading to this version will result in CDS Dashboards being uninstalled. A replacement dashboarding solution is currently in the works > and will be integrated soon. > WARNING: Given the scope of changes in this release, we highly recommend backing up your system before upgrading. Please refer to our [Manual Backup](https://www.nebari.dev/docs/how-tos/manual-backup) documentation for more details. @@ -32,7 +32,7 @@ command and follow the instructions ### Breaking Changes * While we have tried our best to avoid breaking changes when introducing the extension mechanism, the scope of the changes is too large for us to confidently say there won't be breaking changes. - +> WARNING: CDS Dashboards was removed in this release and upgrading to this version will result in CDS Dashboards being uninstalled. A replacement dashboarding solution is currently in the work and will be integrated soon. ### What's Changed * Spinup spot instance for CI with cirun by @aktech in https://github.com/nebari-dev/nebari/pull/1882 From 9c78aee2804178affb379eb2272eeb79d8c2299c Mon Sep 17 00:00:00 2001 From: iameskild Date: Wed, 18 Oct 2023 21:30:18 -0700 Subject: [PATCH 08/14] Update RELEASE notes --- README.md | 4 ++-- RELEASE.md | 20 ++++++++++++++++++-- src/_nebari/constants.py | 2 +- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index fd94520c2b..5e78927064 100644 --- a/README.md +++ b/README.md @@ -34,10 +34,10 @@ - [License](#license) > **⚠️ Warning ⚠️** -> The `2023.9.1` release includes the initial implementation of a [Pluggy-based](https://pluggy.readthedocs.io/en/stable/) extension mechanism, for more details refer [here](). +> The `2023.10.1` release includes the initial implementation of a [Pluggy-based](https://pluggy.readthedocs.io/en/stable/) extension mechanism, for more details refer [here](). > This version also fully deprecates CDS Dashboards as it is no longer compatible with the newer versions of JupyterHub. > For more details on all of changes included in this release, please refer to our [release notes](./RELEASE.md). -> After you've installed version `2023.9.1`, you can update your `nebari-config.yaml` by running `nebari upgrade -c nebari-config.yaml`, please +> After you've installed version `2023.10.1`, you can update your `nebari-config.yaml` by running `nebari upgrade -c nebari-config.yaml`, please > follow the upgrades instructions output by this command. > And please make sure to [back up your data before attempting an upgrade](https://www.nebari.dev/docs/how-tos/manual-backup). diff --git a/RELEASE.md b/RELEASE.md index 25eb98931a..f01bf2eba6 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -12,7 +12,7 @@ This file is copied to nebari-dev/nebari-docs using a GitHub Action. --> ## Upcoming Release -## Release 2023.9.1 - September 28, 2023 +## Release 2023.10.1 - October 19, 2023 This release includes a major refactor which introduces a Pluggy-based extension mechanism which allow developers to build new stages. This is the initial implementation of the extension mechanism and we expect the interface to be refined overtime. If you're interested in developing your own stage plugin, please refer to [our documentation](https://www.nebari.dev/docs/how-tos/nebari-extension-system#developing-an-extension). When you're ready to upgrade, please download this version from either PyPI or Conda-Forge and run the `nebari upgrade -c nebari-config.yaml` @@ -86,6 +86,22 @@ command and follow the instructions * Add gcp_cleanup, minor changes by @iameskild in https://github.com/nebari-dev/nebari/pull/2010 * Fix #2024 by @dcmcand in https://github.com/nebari-dev/nebari/pull/2025 * Upgrade conda-store to 2023.9.2 by @iameskild in https://github.com/nebari-dev/nebari/pull/2028 +* Add upgrade steps, instructions for 2023.9.1 by @iameskild in https://github.com/nebari-dev/nebari/pull/2029 +* CI: add gcp integration test by @fangchenli in https://github.com/nebari-dev/nebari/pull/2049 +* CLN: remove flake8 from dependencies by @fangchenli in https://github.com/nebari-dev/nebari/pull/2044 +* [pre-commit.ci] pre-commit autoupdate by @pre-commit-ci in https://github.com/nebari-dev/nebari/pull/2047 +* fix typo in guided init for Digital Ocean by @dcmcand in https://github.com/nebari-dev/nebari/pull/2059 +* CI: add do integration by @fangchenli in https://github.com/nebari-dev/nebari/pull/2060 +* TYP: make all subfolders under kubernetes_services/template non-module by @fangchenli in https://github.com/nebari-dev/nebari/pull/2043 +* TYP: fix most typing errors in provider by @fangchenli in https://github.com/nebari-dev/nebari/pull/2038 +* Fix link to documentation on Nebari Deployment home page by @aktech in https://github.com/nebari-dev/nebari/pull/2063 +* TST: enable timeout config in playwright notebook test by @fangchenli in https://github.com/nebari-dev/nebari/pull/1996 +* DEPS: sync supported python version by @fangchenli in https://github.com/nebari-dev/nebari/pull/2065 +* Test support for Python 3.12 by @aktech in https://github.com/nebari-dev/nebari/pull/2046 +* BUG: fix validation error related to `provider` #2054 by @fangchenli in https://github.com/nebari-dev/nebari/pull/2056 +* CI: improve unit test workflow in CI, revert #2046 by @fangchenli in https://github.com/nebari-dev/nebari/pull/2071 +* TST: enable exact_match config in playwright notebook test by @fangchenli in https://github.com/nebari-dev/nebari/pull/2027 +* CI: move conda build test to separate job by @fangchenli in https://github.com/nebari-dev/nebari/pull/2073 ### New Contributors * @kalpanachinnappan made their first contribution in https://github.com/nebari-dev/nebari/pull/1893 @@ -94,7 +110,7 @@ command and follow the instructions * @kenafoster made their first contribution in https://github.com/nebari-dev/nebari/pull/1933 * @dcmcand made their first contribution in https://github.com/nebari-dev/nebari/pull/2025 -**Full Changelog**: https://github.com/nebari-dev/nebari/compare/2023.7.2...2023.9.1 +**Full Changelog**: https://github.com/nebari-dev/nebari/compare/2023.7.2...2023.10.1 ## Release 2023.7.2 - August 3, 2023 diff --git a/src/_nebari/constants.py b/src/_nebari/constants.py index 16ff2ccc12..926376d1c8 100644 --- a/src/_nebari/constants.py +++ b/src/_nebari/constants.py @@ -1,4 +1,4 @@ -CURRENT_RELEASE = "2023.7.2" +CURRENT_RELEASE = "2023.10.1" # NOTE: Terraform cannot be upgraded further due to Hashicorp licensing changes # implemented in August 2023. From cace085e10964b686f27177bb48a4b20ab9949a1 Mon Sep 17 00:00:00 2001 From: iameskild Date: Wed, 18 Oct 2023 21:32:45 -0700 Subject: [PATCH 09/14] Update based on review --- src/nebari/schema.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/nebari/schema.py b/src/nebari/schema.py index 89339bc647..ab90f8ebc7 100644 --- a/src/nebari/schema.py +++ b/src/nebari/schema.py @@ -86,9 +86,6 @@ def escaped_project_name(self): if self.provider == ProviderEnum.aws and project_name.startswith("aws"): project_name = "a" + project_name - if len(project_name) > 16: - project_name = project_name[:16] - return project_name From ab236fd09acaa9d394333e42fb09cc0ea62f2a9d Mon Sep 17 00:00:00 2001 From: iameskild Date: Thu, 19 Oct 2023 18:52:21 -0700 Subject: [PATCH 10/14] Add deprecation warning --- RELEASE.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/RELEASE.md b/RELEASE.md index f01bf2eba6..6b1be93127 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -12,7 +12,7 @@ This file is copied to nebari-dev/nebari-docs using a GitHub Action. --> ## Upcoming Release -## Release 2023.10.1 - October 19, 2023 +## Release 2023.10.1 - October 20, 2023 This release includes a major refactor which introduces a Pluggy-based extension mechanism which allow developers to build new stages. This is the initial implementation of the extension mechanism and we expect the interface to be refined overtime. If you're interested in developing your own stage plugin, please refer to [our documentation](https://www.nebari.dev/docs/how-tos/nebari-extension-system#developing-an-extension). When you're ready to upgrade, please download this version from either PyPI or Conda-Forge and run the `nebari upgrade -c nebari-config.yaml` @@ -32,8 +32,11 @@ command and follow the instructions ### Breaking Changes * While we have tried our best to avoid breaking changes when introducing the extension mechanism, the scope of the changes is too large for us to confidently say there won't be breaking changes. + > WARNING: CDS Dashboards was removed in this release and upgrading to this version will result in CDS Dashboards being uninstalled. A replacement dashboarding solution is currently in the work and will be integrated soon. +> WARNING: We will be removing and ending support for ClearML, Prefect and kbatch in the next release. The kbatch has been functionally replaced by Argo-Jupyter-Scheduler. We have seen little interest in ClearML and Prefect in recent years, and removing makes sense at this point. However if you wish to continue using them with Nebari we encourage you to [write your own Nebari extension](https://www.nebari.dev/docs/how-tos/nebari-extension-system#developing-an-extension). + ### What's Changed * Spinup spot instance for CI with cirun by @aktech in https://github.com/nebari-dev/nebari/pull/1882 * Fix argo-viewer service account reference by @iameskild in https://github.com/nebari-dev/nebari/pull/1881 From c1ec173f021dec36adb476d6dfb00a0df17e087e Mon Sep 17 00:00:00 2001 From: iameskild Date: Fri, 20 Oct 2023 10:50:47 -0700 Subject: [PATCH 11/14] Add link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5e78927064..54ad76dabf 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ - [License](#license) > **⚠️ Warning ⚠️** -> The `2023.10.1` release includes the initial implementation of a [Pluggy-based](https://pluggy.readthedocs.io/en/stable/) extension mechanism, for more details refer [here](). +> The `2023.10.1` release includes the initial implementation of a [Pluggy-based](https://pluggy.readthedocs.io/en/stable/) extension mechanism, for more details refer [here](https://www.nebari.dev/docs/community/plugins). > This version also fully deprecates CDS Dashboards as it is no longer compatible with the newer versions of JupyterHub. > For more details on all of changes included in this release, please refer to our [release notes](./RELEASE.md). > After you've installed version `2023.10.1`, you can update your `nebari-config.yaml` by running `nebari upgrade -c nebari-config.yaml`, please From 67c98f26165bf2f7bb90b7a7e15145c9a6eaa6c5 Mon Sep 17 00:00:00 2001 From: iameskild Date: Fri, 20 Oct 2023 11:06:52 -0700 Subject: [PATCH 12/14] Fix link to keycloak docs --- src/_nebari/deploy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/_nebari/deploy.py b/src/_nebari/deploy.py index 8ee8af0e4e..46cc20179b 100644 --- a/src/_nebari/deploy.py +++ b/src/_nebari/deploy.py @@ -71,7 +71,7 @@ def deploy_configuration( print(f"Kubecloak master realm username={username} password={password}") print( - "Additional administration docs can be found at https://docs.nebari.dev/en/stable/source/admin_guide/" + "Additional administration docs can be found at https://www.nebari.dev/docs/how-tos/configuring-keycloak" ) return stage_outputs From 09acbffb53457085759534362abd0e3ad730a2a2 Mon Sep 17 00:00:00 2001 From: iameskild Date: Fri, 20 Oct 2023 23:32:55 -0700 Subject: [PATCH 13/14] Remove image from DaskWorkerProfile --- src/_nebari/stages/kubernetes_services/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/_nebari/stages/kubernetes_services/__init__.py b/src/_nebari/stages/kubernetes_services/__init__.py index 376d52155c..2b8bb6fd80 100644 --- a/src/_nebari/stages/kubernetes_services/__init__.py +++ b/src/_nebari/stages/kubernetes_services/__init__.py @@ -112,7 +112,6 @@ class DaskWorkerProfile(schema.Base): worker_memory_limit: str worker_memory: str worker_threads: int = 1 - image: str = f"quay.io/nebari/nebari-dask-worker:{set_docker_image_tag()}" class Config: extra = "allow" From ac969986410c0c2bc671e1ece3826d6cba31499f Mon Sep 17 00:00:00 2001 From: iameskild Date: Sat, 21 Oct 2023 00:26:25 -0700 Subject: [PATCH 14/14] Update RELEASE.md --- RELEASE.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/RELEASE.md b/RELEASE.md index 6b1be93127..b5f7d5467a 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -105,6 +105,10 @@ command and follow the instructions * CI: improve unit test workflow in CI, revert #2046 by @fangchenli in https://github.com/nebari-dev/nebari/pull/2071 * TST: enable exact_match config in playwright notebook test by @fangchenli in https://github.com/nebari-dev/nebari/pull/2027 * CI: move conda build test to separate job by @fangchenli in https://github.com/nebari-dev/nebari/pull/2073 +* Revert conda-store to v0.4.14, #2028 by @iameskild in https://github.com/nebari-dev/nebari/pull/2074 +* ENH/CI: add mypy config, and CI workflow by @fangchenli in https://github.com/nebari-dev/nebari/pull/2066 +* Update upgrade for 2023.10.1 by @kenfoster in https://github.com/nebari-dev/nebari/pull/2080 +* Update RELEASE notes, minor fixes by @iameskild in https://github.com/nebari-dev/nebari/pull/2039 ### New Contributors * @kalpanachinnappan made their first contribution in https://github.com/nebari-dev/nebari/pull/1893