diff --git a/.github/ISSUE_TEMPLATE/docs_issue.md b/.github/ISSUE_TEMPLATE/docs_issue.md index 42b12ea2e2..22c13edeea 100644 --- a/.github/ISSUE_TEMPLATE/docs_issue.md +++ b/.github/ISSUE_TEMPLATE/docs_issue.md @@ -2,7 +2,7 @@ name: Docs bug/update request about: Suggest an improvement for Flyte docs title: "[Docs]" -labels: docs, untriaged +labels: documentation, untriaged assignees: '' --- diff --git a/.github/ISSUE_TEMPLATE/housekeeping_template.md b/.github/ISSUE_TEMPLATE/housekeeping_template.md new file mode 100644 index 0000000000..e176190c4a --- /dev/null +++ b/.github/ISSUE_TEMPLATE/housekeeping_template.md @@ -0,0 +1,17 @@ +--- +name: Housekeeping +about: Maintenance and cleanup tasks, that need to be performed. This may include tech-debt, library upgrades, CI/CD improvements etc +title: "[Housekeeping] " +labels: housekeeping +assignees: '' + +--- + +**Describe the Issue** +A clear and concise description of what the housekeeping item is. + +**What if we do not do this?** +Short description of potential consequences + +***Related component*** +Either Specific / all diff --git a/.github/config.yml b/.github/config.yml new file mode 100644 index 0000000000..f99bcd78f1 --- /dev/null +++ b/.github/config.yml @@ -0,0 +1,15 @@ +# Comment to be posted on PRs from first-time contributors in your repository +newPRWelcomeComment: > + Thank you for opening this pull request! 🙌 + + These tips will help get your PR across the finish line: + - Most of the repos have a PR template; if not, fill it out to the best of your knowledge. + - Sign off your commits (Reference: [DCO Guide](https://github.com/src-d/guide/blob/master/developer-community/fix-DCO.md)). + +# Comment to be posted to on pull requests merged by a first time user +firstPRMergeComment: > + Congrats on merging your first pull request! 🎉 + +# Comment to be posted on first-time issues +newIssueWelcomeComment: > + Thank you for opening your first issue here! 🛠 diff --git a/.github/workflows/publish-menifest.yml b/.github/workflows/publish-menifest.yml new file mode 100644 index 0000000000..834e3f51d6 --- /dev/null +++ b/.github/workflows/publish-menifest.yml @@ -0,0 +1,40 @@ +name: Create a flyte release +on: + workflow_dispatch: + inputs: + version: + description: 'version name. example v0.1.1' + required: true + +jobs: + goreleaser: + name: Goreleaser + runs-on: ubuntu-latest + outputs: + version: ${{ steps.bump-version.outputs.tag }} + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: "0" + - name: Bump version and push tag + id: bump-version + uses: anothrNick/github-tag-action@1.17.2 + env: + GITHUB_TOKEN: ${{ secrets.FLYTE_BOT_PAT }} + WITH_V: true + CUSTOM_TAG: ${{ github.event.inputs.version }} + RELEASE_BRANCHES: master + - uses: actions/checkout@v2 + with: + fetch-depth: "0" + ref: ${{ github.event.inputs.version }} + - name: build Release Menifest + run: | + make prepare_artifacts + - name: Run GoReleaser + uses: goreleaser/goreleaser-action@v2 + with: + version: latest + args: release --rm-dist + env: + GITHUB_TOKEN: ${{ secrets.FLYTE_BOT_PAT }} \ No newline at end of file diff --git a/.github/workflows/release-kustomize.yml b/.github/workflows/release-kustomize.yml new file mode 100644 index 0000000000..ca0f014afa --- /dev/null +++ b/.github/workflows/release-kustomize.yml @@ -0,0 +1,42 @@ +name: Publish flyte defination file + +on: + workflow_dispatch: + +jobs: + update-flyte-releases: + name: Update Flyte components + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: "0" + - name: Build kustomize + run: | + make release_automation + make kustomize + - name: Create Pull Request + id: cpr + uses: peter-evans/create-pull-request@v3 + with: + token: ${{ secrets.FLYTE_BOT_PAT }} + commit-message: Update Flyte Components + committer: Flyte-Bot + author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> + signoff: true + branch: flyte-bot-update-releases + delete-branch: true + title: 'Update Flyte components' + body: | + Updated flyte deployment + - Updated GCP Flyte deployment + - Updated EKS Flyte deployment + - Updated Sandbox Flyte deployment + - Updated TEST Flyte deployment + - Auto-generated by [flyte-bot] + labels: | + kustomize + team-reviewers: | + owners + maintainers + draft: false \ No newline at end of file diff --git a/.github/workflows/sandbox.yml b/.github/workflows/sandbox.yml index 43e06851f3..4fb2db51ff 100644 --- a/.github/workflows/sandbox.yml +++ b/.github/workflows/sandbox.yml @@ -25,7 +25,7 @@ jobs: build_extra_args: "--target=default --compress=true" context: ./ dockerfile: docker/sandbox/Dockerfile - push_image_and_stages: ${{ github.event.name == 'release' }} + push_image_and_stages: ${{ github.event_name == 'release' }} push-sandbox-dind-image: name: Push sandbox DinD image to GHCR runs-on: ubuntu-latest @@ -44,4 +44,4 @@ jobs: build_extra_args: "--target=dind --compress=true" context: ./ dockerfile: docker/sandbox/Dockerfile - push_image_and_stages: ${{ github.event.name == 'release' }} + push_image_and_stages: ${{ github.event_name == 'release' }} diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b11d184b82..6c690be8be 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -2,23 +2,25 @@ name: tests on: push: branches: - - master + - master pull_request: jobs: end-to-end: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 - - name: Kustomize and diff - run: DELTA_CHECK=true make kustomize - - uses: engineerd/setup-kind@v0.5.0 - - name: End2End - env: - DOCKER_USERNAME: ${{ github.actor }} - DOCKER_PASSWORD: "${{ secrets.GITHUB_TOKEN }}" - run: | - kubectl cluster-info - kubectl get pods -n kube-system - echo "current-context:" $(kubectl config current-context) - echo "environment-kubeconfig:" ${KUBECONFIG} - make end2end_execute + - uses: actions/checkout@v1 + - name: Kustomize and diff + run: DELTA_CHECK=true make kustomize + - uses: engineerd/setup-kind@v0.5.0 + with: + version: "v0.10.0" + - name: End2End + env: + DOCKER_USERNAME: ${{ github.actor }} + DOCKER_PASSWORD: "${{ secrets.GITHUB_TOKEN }}" + run: | + kubectl cluster-info + kubectl get pods -n kube-system + echo "current-context:" $(kubectl config current-context) + echo "environment-kubeconfig:" ${KUBECONFIG} + make end2end_execute diff --git a/.gitignore b/.gitignore index 25dff9d99e..bd8ac5ca2c 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,5 @@ docs/_sources/ docs/flytekit/flytekit.interfaces.html docs/searchindex.js docs/ +release/ +__pycache__/ diff --git a/.goreleaser.yml b/.goreleaser.yml new file mode 100644 index 0000000000..a69fd22b81 --- /dev/null +++ b/.goreleaser.yml @@ -0,0 +1,14 @@ +project_name: flyte +builds: + - skip: true +release: + github: + owner: flyteorg + name: flyte + draft: false + prerelease: auto + discussion_category_name: General + name_template: "{{.ProjectName}}-v{{.Version}}" + disable: false + extra_files: + - glob: ./release/* \ No newline at end of file diff --git a/CHANGELOG/CHANGELOG-v0.11.0.md b/CHANGELOG/CHANGELOG-v0.11.0.md new file mode 100644 index 0000000000..ab19f1e21a --- /dev/null +++ b/CHANGELOG/CHANGELOG-v0.11.0.md @@ -0,0 +1,21 @@ +# Flyte v0.11.0 + +## Flyte Platform +* New to flyte? https://start.flyte.org takes you through first run experience. (Thanks to @jeevb) +* [Grafana templates](https://docs.flyte.org/en/latest/howto/monitoring/index.html) for monitoring Flyte System and User Workflows. +* [Extend Flyte](https://docs.flyte.org/en/latest/plugins/index.html) docs. +* [FlyteIdl Docs](https://docs.flyte.org/projects/flyteidl/en/latest/) are published! You can learn about the core language that makes it all work. +* [Additional knob](https://github.com/flyteorg/flytepropeller/pull/219/files#diff-91657d6448dfbf87f4cecf126ad02bd668ea233edcf74e860ef4f54bdd4cb552R78) for fine tuning flyte propeller performance that speeds up executions drastically. +* OidC support for Google Idp (And other OidC compliant Idps) +* Various stabilization bugs. + +## Flytekit +Since v0.16.0a2, the last flytekit milestone release, all effort has been towards stabilizing the new API. Please see the individual [releases](https://github.com/flyteorg/flytekit/releases) for detailed information. The highlights are + +* Serialization/registration processes have been firmed up and utilities to ease that process introduced (not having to build a container to serialize for instance). +* Plugins structure revamped (eventually we'll move to a separate new repo entirely) +* User-facing imports have been organized into three top-level subpackages (`flytekit`, `flytekit.testing`, and `flytekit.extend`) +* Retries added to read-only Admin calls in client +* Lots of cleanup and additions to the [cookbook](https://flytecookbook.readthedocs.io/en/latest/) and documentation generally. +* Bug fixes. + diff --git a/CHANGELOG/CHANGELOG-v0.12.0.md b/CHANGELOG/CHANGELOG-v0.12.0.md new file mode 100644 index 0000000000..742cb4a12f --- /dev/null +++ b/CHANGELOG/CHANGELOG-v0.12.0.md @@ -0,0 +1,35 @@ +# General +* Added CoPilot configuration to sandbox manifest +* Updated and streamlined documentation navigation and themes. Better content organization coming soon! + +# Performance +* Improved execution performance including: + - Reduced cache lookups + - Improved GetWorkflowExecution performance + - Capped max number of nodes in each propeller round + - Misc. propeller performance tweaks +* TaskTemplate offloading + +# Housekeeping +* Migrated Datacatalog protobuf definitions to flyteidl [thanks @tnsetting] +* Upgraded stow version used in flytestdlib +* Moved off lyft kubernetes forks and onto official kubernetes library dependencies +* Revamped pod tasks to use official kubernetes python client library for defining PodSpecs + +# Events +* Richer event metadata for task executions +* Better merging of custom info across task events + +# Bug fixes +* Resolved non-backwards protobuf role changes that prevented launching single task executions [thanks @kanterov] +* Better handling of large workflows and errors in flytepropeller + +# Flytekit (Python) +* Access to secrets +* Bug fixes around the 0.16 release. + * Use original FlyteFile/FlyteDirectory + * Fix serialization of pod specs in pod plugin [thanks @jeevb] + * Accept auth role arg in single task execution + * Fixed task resolver in map task + * Requests and limits added to ContainerTask [thanks @migueltol22] + diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000000..f774b84d05 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,3 @@ +# Contributing to Flyte + +For information related to contributing to Flyte, please check out the [Contributing to Flyte](https://docs.flyte.org/en/latest/community/contribute.html) section of the documentation at the Flyte site. diff --git a/Makefile b/Makefile index 9081cf87d3..6b38fd26ff 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,21 @@ +export REPOSITORY=flyte + define PIP_COMPILE pip-compile $(1) --upgrade --verbose endef +.PHONY: update_boilerplate +update_boilerplate: + @boilerplate/update.sh + .PHONY: kustomize -kustomize: +kustomize: KUSTOMIZE_VERSION=3.9.2 bash script/generate_kustomize.sh +.PHONY: release_automation +release_automation: + bash script/release.sh + .PHONY: deploy_sandbox deploy_sandbox: bash script/deploy.sh @@ -20,11 +30,6 @@ end2end: end2end_execute: @end2end/execute.sh -# updates referenced docs from other repositories (e.g. flyteidl, flytekit) -.PHONY: generate-dependent-repo-docs -generate-dependent-repo-docs: - @FLYTEKIT_VERSION=0.15.4 FLYTEIDL_VERSION=0.18.11 ./script/update_ref_docs.sh - .PHONY: install-piptools install-piptools: pip install -U pip-tools @@ -33,3 +38,16 @@ install-piptools: doc-requirements.txt: doc-requirements.in install-piptools $(call PIP_COMPILE,doc-requirements.in) +.PHONY: requirements.txt +requirements.txt: requirements.in install-piptools + $(call PIP_COMPILE,requirements.in) + +.PHONY: stats +stats: + @generate-dashboard -o deployment/stats/prometheus/flytepropeller-dashboard.json stats/flytepropeller_dashboard.py + @generate-dashboard -o deployment/stats/prometheus/flyteadmin-dashboard.json stats/flyteadmin_dashboard.py + @generate-dashboard -o deployment/stats/prometheus/flyteuser-dashboard.json stats/flyteuser_dashboard.py + +.PHONY: prepare_artifacts +prepare_artifacts: + bash script/prepare_artifacts.sh \ No newline at end of file diff --git a/README.md b/README.md index a8d6171797..90dcaac6fe 100644 --- a/README.md +++ b/README.md @@ -1,188 +1,271 @@ -![Flyte Logo](rsts/images/flyte_lockup_gradient_on_light.png "Flyte Logo") - -[![Current Release](https://img.shields.io/github/release/lyft/flyte.svg)](https://github.com/lyft/flyte/releases/latest) -[![Build Status](https://travis-ci.org/lyft/flyte.svg?branch=master)](https://travis-ci.org/lyft/flyte) -[![License](https://img.shields.io/badge/LICENSE-Apache2.0-ff69b4.svg)](http://www.apache.org/licenses/LICENSE-2.0.html) -![Commit activity](https://img.shields.io/github/commit-activity/w/lyft/flyte.svg?style=plastic) -![Commit since last release](https://img.shields.io/github/commits-since/lyft/flyte/latest.svg?style=plastic) -![GitHub milestones Completed](https://img.shields.io/github/milestones/closed/lyft/flyte?style=plastic) -![GitHub next milestone percentage](https://img.shields.io/github/milestones/progress-percent/lyft/flyte/11?style=plastic) -[![Docs](https://readthedocs.org/projects/flyte/badge/?version=latest&style=plastic)](https://flyte.rtfd.io) -![Twitter Follow](https://img.shields.io/twitter/follow/flyteorg?label=Follow&style=social) -[![Slack Status](https://img.shields.io/badge/slack-join_chat-white.svg?logo=slack&style=social)](https://forms.gle/UVuek9WfBoweiqcJA) - -Flyte is a container-native, type-safe workflow and pipelines platform optimized for large scale processing and machine learning written in Golang. Workflows can be written in any language, with out of the box support for Python, Java and Scala. - -# Homepage -- [flyte.org](https://flyte.org) -- [Docs](https://flyte.readthedocs.io/en/latest/) - -# Introduction -Flyte is a fabric that connects disparate computation backends using a type safe data dependency graph. It records all changes to a pipeline, making it possible to rewind time. It also stores + +

+ Flyte and LF AI & Data Logo +

+ +

+ Flyte +

+ +

+Flyte is a production-grade, container-native, type-safe workflow and pipelines platform optimized for large scale processing and machine learning written in Golang +

+ +

+ + Current Release + + + Sandbox Build + + + End-to-End Tests + + + License + + Commit Activity + Commits since Last Release + GitHub Milestones Completed + GitHub Next Milestone Percentage + + Docs + + Twitter Follow + + Slack Status + +

+ +

+ Home Page + · + Quick Start + · + Documentation + · + Features + · + Community & Resources + · + Changelogs + · + Components +

+ + + +## 💥 Introduction + +Flyte is a structured programming and distributed processing platform that enables highly concurrent, scalable and maintainable workflows for `Machine Learning` and `Data Processing`. It is a fabric that connects disparate computation backends using a type safe data dependency graph. It records all changes to a pipeline, making it possible to rewind time. It also stores a history of all executions and provides an intuitive UI, CLI and REST/gRPC API to interact with the computation. -Flyte is more than a workflow engine, it provides workflows as a core concepts, but it also provides a single unit of execution - tasks, as a top level concept. Multiple tasks arranged in a data -producer-consumer order creates a workflow. Flyte workflows are pure specification and can be created using any language. Every task can also by any language. We do provide first class support for -python, making it perfect for modern Machine Learning and Data processing pipelines. +Flyte is more than a workflow engine -- it provides `workflow` as a core concept and a single unit of execution called `task` as a top level concept. Multiple tasks arranged in a data +producer-consumer order create a workflow. + +`Workflows` and `Tasks` can be written in any language, with out of the box support for [Python](https://github.com/flyteorg/flytekit), [Java and Scala](https://github.com/spotify/flytekit-java). + + +

+ 🚀 Quick Start +

+ + +With [docker installed](https://docs.docker.com/get-docker/), run the following command: + +```bash + docker run --rm --privileged -p 30081:30081 -p 30084:30084 ghcr.io/flyteorg/flyte-sandbox +``` + +This creates a local Flyte sandbox. Once the sandbox is ready, you should see the following message: `Flyte is ready! Flyte UI is available at http://localhost:30081/console`. + +Go ahead and visit http://localhost:30081/console to view the Flyte dashboard. + +Here's a quick visual tour of the console. + +![Flyte console Example](https://github.com/flyteorg/flyte/raw/static-resources/img/first-run-console-2.gif) + +To dig deeper into Flyte, refer to the [Documentation](https://docs.flyte.org/en/latest/index.html). + +## ⭐️ Current Deployments + +- [Freenome](https://www.freenome.com/) +- [Lyft Rideshare, Mapping](https://www.lyft.com/) +- [Lyft L5 autonomous](https://self-driving.lyft.com/level5/) +- [Spotify](https://www.spotify.com/) +- [USU Group](https://www.usu.com/) +- [Striveworks](https://striveworks.us/) + + +

+ 🔥 Features +

+ + +- Used at _Scale_ in production by **500+** users at Lyft with more than **1 million** executions and **40+ million** container executions per month +- Enables **collaboration across your organization**, as in: + - Execute distributed data pipelines/workflows + - Reuse tasks across projects, users, and workflows + - Backtrace to a specified workflow + - Compare results of training workflows over time and across pipelines + - Share workflows and tasks across your teams +- **[Quick registration](https://docs.flyte.org/projects/cookbook/en/latest/tutorial.html)** -- start locally and scale to the cloud instantly +- **Centralized Inventory** constituting Tasks, Workflows and Executions +- **gRPC / REST** interface to define and execute tasks and workflows +- **Type safe** construction of pipelines -- each task has an interface which is characterized by its input and output; thus, illegal construction of pipelines fails during declaration rather than at runtime +- Supports multiple **[data types](https://docs.flyte.org/projects/cookbook/en/latest/core.html)** for machine learning and data processing pipelines, such as Blobs (images, arbitrary files), Directories, Schema (columnar structured data), collections, maps etc. +- Memoization and Lineage tracking +- Workflow features: + - Start with one task, convert to a pipeline, attach **[multiple schedules](https://docs.flyte.org/projects/cookbook/en/latest/auto_core_remote_flyte/lp_schedules.html)**, trigger using a programmatic API, or on-demand + - Parallel step execution + - Extensible backend to add [customized plugin](https://docs.flyte.org/projects/cookbook/en/latest/auto_core_advanced/custom_task_plugin.html) experience (with simplified user experience) + - **[Branching](https://docs.flyte.org/projects/cookbook/en/latest/auto_core_intermediate/run_conditions.html)** + - Inline **[subworkflows](https://docs.flyte.org/projects/cookbook/en/latest/auto_core_intermediate/subworkflows.html)** (a workflow can be embeded within one node of the top level workflow) + - Distributed **remote child workflows** (a remote workflow can be triggered and statically verified at compile time) + - **[Array Tasks](https://docs.flyte.org/projects/cookbook/en/latest/auto_core_intermediate/map_task.html)** (map a function over a large dataset -- ensures controlled execution of thousands of containers) + - **[Dynamic workflow](https://docs.flyte.org/projects/cookbook/en/latest/auto_core_intermediate/dynamics.html)** creation and execution with runtime type safety + - Container side [plugins](https://docs.flyte.org/projects/cookbook/en/latest/plugins.html) with first class support in Python + - _PreAlpha_: Arbitrary flytekit-less containers supported ([RawContainer](https://docs.flyte.org/projects/cookbook/en/latest/auto_core_intermediate/raw_container.html)) +- Guaranteed **[reproducibility](https://docs.flyte.org/projects/cookbook/en/latest/auto_core_basic/task_cache.html)** of pipelines via: + - Versioned data, code and models + - Automatically tracked executions + - Declarative pipelines +- **Multi cloud support** (AWS, GCP and others) +- Extensible core, modularized, and deep observability +- Automated notifications to Slack, Email, and Pagerduty +- [Multi K8s cluster support](https://docs.flyte.org/projects/cookbook/en/latest/auto_plugins_pod/index.html) +- Out of the box support to run **[Spark jobs on K8s](https://docs.flyte.org/projects/cookbook/en/latest/auto_plugins_k8s_spark/index.html)**, **[Hive queries](https://docs.flyte.org/projects/cookbook/en/latest/auto_plugins_hive/index.html)**, etc. +- Snappy Console +- Python CLI and Golang CLI (flytectl) +- Written in **Golang** and optimized for large running jobs' performance + +### In Progress + +- Grafana templates (user/system observability) +- Helm chart for Flyte +- Performance optimization +- Flink-K8s + +## 🔌 Available Plugins + +- Containers +- [K8s Pods](https://docs.flyte.org/projects/cookbook/en/latest/auto_plugins_pod/index.html) +- AWS Batch Arrays +- K8s Pod Arrays +- K8s Spark (native [Pyspark](https://docs.flyte.org/projects/cookbook/en/latest/auto_plugins_k8s_spark/index.html) and Java/Scala) +- AWS Athena +- [Qubole Hive](https://docs.flyte.org/projects/cookbook/en/latest/auto_plugins_hive/index.html) +- Presto Queries +- Distributed Pytorch (K8s Native) -- [Pytorch Operator](https://docs.flyte.org/projects/cookbook/en/latest/auto_plugins_kfpytorch/index.html) +- Sagemaker([builtin algorithms](https://docs.flyte.org/projects/cookbook/en/latest/auto_plugins_sagemaker_training/sagemaker_builtin_algo_training.html) & [custom models](https://docs.flyte.org/projects/cookbook/en/latest/auto_plugins_sagemaker_training/sagemaker_custom_training.html)) +- Distributed Tensorflow (K8s Native) - TFOperator +- Papermill notebook execution ([Python](https://github.com/lyft/flytekit/tree/master/plugins/papermill) and Spark) +- Type safe and data checking for Pandas dataframe using Pandera + +### In Queue + +- Reactive pipelines +- A lot more integrations! + + +

+ 📦 Component Repos +

+ + +| Repo | Language | Purpose | Status | +| --------------------------------------------------------- | ------------- | ---------------------------------------------- | ---------------- | +| [flyte](https://github.com/lyft/flyte) | Kustomize,RST | deployment, documentation, issues | Production-grade | +| [flyteidl](https://github.com/lyft/flyteidl) | Protobuf | interface definitions | Production-grade | +| [flytepropeller](https://github.com/lyft/flytepropeller) | Go | execution engine | Production-grade | +| [flyteadmin](https://github.com/lyft/flyteadmin) | Go | control plane | Production-grade | +| [flytekit](https://github.com/lyft/flytekit) | Python | python SDK and tools | Production-grade | +| [flyteconsole](https://github.com/lyft/flyteconsole) | Typescript | admin console | Production-grade | +| [datacatalog](https://github.com/lyft/datacatalog) | Go | manage input & output artifacts | Production-grade | +| [flyteplugins](https://github.com/lyft/flyteplugins) | Go | flyte plugins | Production-grade | +| [flytestdlib](https://github.com/lyft/flytestdlib) | Go | standard library | Production-grade | +| [flytesnacks](https://github.com/lyft/flytesnacks) | Python | examples, tips, and tricks | Incubating | +| [flytekit-java](https://github.com/spotify/flytekit-java) | Java/Scala | Java & scala SDK for authoring Flyte workflows | Incubating | +| [flytectl](https://github.com/lyft/flytectl) | Go | A standalone Flyte CLI | Incomplete | + +## 🔩 Production K8s Operators + +| Repo | Language | Purpose | +| --------------------------------------------------------------------- | -------- | ---------------------- | +| [Spark](https://github.com/GoogleCloudPlatform/spark-on-k8s-operator) | Go | Apache Spark batch | +| [Flink](https://github.com/lyft/flinkk8soperator) | Go | Apache Flink streaming | -# Resources -Resources that would help you get a better understanding of Flyte. + +

+ 🤝 Community & Resources +

+ + +Here are the resources that would help you get a better understanding of Flyte. + +### Communication Channels -# Communication channels - [Slack Org](https://forms.gle/UVuek9WfBoweiqcJA) - [Email list](https://groups.google.com/a/flyte.org/g/users) -# Biweekly Community Sync -- Starting April 21 2020, the Flyte community meets every other Tuesday at 9:00 AM PST (US West coast time). -- You can join the [zoom link]( https://us04web.zoom.us/j/71298741279?pwd=TDR1RUppQmxGaDRFdzBOa2lHN1dsZz09). -- Meeting notes are captured in [Doc](https://docs.google.com/document/d/1Jb6eOPOzvTaHjtPEVy7OR2O5qK1MhEs3vv56DX2dacM/edit#heading=h.c5ha25xc546e) -- Demo Signup [Sheet](https://docs.google.com/spreadsheets/d/1Bz-s3SpMm1hGl6Z5y3HcxrNBcg9ZhSgqVgmCxK_eEhs/edit#gid=0) -- [Video Recordings](https://www.youtube.com/channel/UCNduEoLOToNo3nFVly-vUTQ) +### Biweekly Community Sync + +- 📣 **Flyte OSS Community Sync** happens every alternate Tuesday, 9am-10am PDT ([Checkout the events calendar & subscribe](https://calendar.google.com/calendar/embed?src=admin%40flyte.org&ctz=America%2FLos_Angeles)). Here's the [zoom link](https://us04web.zoom.us/j/71298741279?pwd=TDR1RUppQmxGaDRFdzBOa2lHN1dsZz09). +- Meeting notes and backlog of topics are captured in [doc](https://docs.google.com/document/d/1Jb6eOPOzvTaHjtPEVy7OR2O5qK1MhEs3vv56DX2dacM/edit#heading=h.c5ha25xc546e). +- If you'd like to revisit any community sync meeting that has happened, you can access the [video recordings](https://www.youtube.com/channel/UCNduEoLOToNo3nFVly-vUTQ). + +### Conference Talks -## Conference Talks - Kubecon 2019 - Flyte: Cloud Native Machine Learning and Data Processing Platform [video](https://www.youtube.com/watch?v=KdUJGSP1h9U) | [deck](https://kccncna19.sched.com/event/UaYY/flyte-cloud-native-machine-learning-data-processing-platform-ketan-umare-haytham-abuelfutuh-lyft) - Kubecon 2019 - Running LargeScale Stateful workloads on Kubernetes at Lyft [video](https://www.youtube.com/watch?v=ECeVQoble0g) -- re:invent 2019 - Implementing ML workflows with Kubernetes and Amazon Sagemaker [video](https://youtu.be/G-wzIQQJKaE) +- re:invent 2019 - Implementing ML workflows with Kubernetes and Amazon Sagemaker [video](https://youtu.be/G-wzIQQJKaE) - Cloud-native machine learning at Lyft with AWS Batch and Amazon EKS [video](https://youtu.be/n_rRb8u1GSM) - OSS + ELC NA 2020 [splash](https://ossna2020.sched.com/event/313cec91aa38a430a25f9571039874b8) - Datacouncil [splash](https://docs.google.com/document/d/1ZsCDOZ5ZJBPWzCNc45FhNtYQOxYHz0PAu9lrtDVnUpw/edit) - FB AI@Scale [Making MLOps & DataOps a reality](https://www.facebook.com/atscaleevents/videos/ai-scale-flyte-making-mlops-and-dataops-a-reality/1047312585732459/) - [GAIC 2020](http://www.globalbigdataconference.com/seattle/global-artificial-intelligence-virtual-conference-122/speaker-details/ketan-umare-113746.html) -## Blog Posts - 1. [Introducing Flyte: A Cloud Native Machine Learning and Data Processing Platform](https://eng.lyft.com/introducing-flyte-cloud-native-machine-learning-and-data-processing-platform-fb2bb3046a59) - 2. [Building a Gateway to Flyte](https://eng.lyft.com/building-a-gateway-to-flyte-474b451b32c8) +### Blog Posts + +1. [Introducing Flyte: A Cloud Native Machine Learning and Data Processing Platform](https://eng.lyft.com/introducing-flyte-cloud-native-machine-learning-and-data-processing-platform-fb2bb3046a59) +2. [Building a Gateway to Flyte](https://eng.lyft.com/building-a-gateway-to-flyte-474b451b32c8) + +### Podcasts -## Podcasts -- TWIML&AI - [Scalable and Maintainable ML Workflows at Lyft - Flyte](https://twimlai.com/twiml-talk-343-scalable-and-maintainable-workflows-at-lyft-with-flyte-w-haytham-abuelfutuh-and-ketan-umare/) +- TWIML&AI - [Scalable and Maintainable ML Workflows at Lyft - Flyte](https://twimlai.com/twiml-talk-343-scalable-and-maintainable-workflows-at-lyft-with-flyte-w-haytham-abuelfutuh-and-ketan-umare/) - Software Engineering Daily - [Flyte: Lyft Data Processing Platform](https://softwareengineeringdaily.com/2020/03/12/flyte-lyft-data-processing-platform-with-allyson-gale-and-ketan-umare/) -- MLOps Coffee session - [Flyte: an open-source tool for scalable, extensible , and portable workflows](https://anchor.fm/mlops/episodes/MLOps-Coffee-Sessions-12-Flyte-an-open-source-tool-for-scalable--extensible---and-portable-workflows-eksa5k) - - -# Features - - Used at Scale in production by 500+ users at Lyft with more than *900k* workflow executed a month and more than *30+* million container executions per month - - Fast registration - from local to remote in one second. - - Centralized Inventory of Tasks, Workflows and Executions - - Single Task Execution support - Start executing a task and then convert it to a workflow - - gRPC / REST interface to define and executes tasks and workflows - - Type safe construction of pipelines, each task has an interface which is characterized by its input and outputs. Thus illegal construction of pipelines fails during declaration rather than at - runtime - - Types that help in creating machine learning and data processing pipelines like - Blobs (images, arbitrary files), Directories, Schema (columnar structured data), collections, maps etc - - Memoization and Lineage tracking - - Workflows features - * Multiple Schedules for every workflow - * Parallel step execution - * Extensible Backend to add customized plugin experiences (with simplified User experiences) - * Arbitrary container execution - * Branching - * Inline Subworkflows (a workflow can be embeded within one node of the top level workflow) - * Distributed Remote Child workflows (a remote workflow can be triggered and statically verified at compile time) - * Array Tasks (map some function over a large dataset, controlled execution of 1000's of containers) - * Dynamic Workflow creation and execution - with runtime type safety - * Container side plugins with first class support in python - * PreAlpha: Arbitrary flytekit less containers supported (RawContainer) - - Maintain an inventory of tasks and workflows - - Record history of all executions and executions (as long as they follow convention) are completely repeatable - - Multi Cloud support (AWS, GCP and others) - - Extensible core - - Modularized - - Automated notifications to Slack, Email, Pagerduty - - Deep observability - - Multi K8s cluster support - - Comes with many system supported out of the box on K8s like Spark etc. - - Snappy Console - - Python CLI - - Written in Golang and optimized for performance of large running jobs - -## In Progress - - Golang CLI - flytectl - -## Coming Soon - - Reactive pipelines - - Grafana templates (user/system observability) - - More integrations - -# Available Plugins - - Containers - - K8s Pods - - AWS Batch Arrays - - K8s Pod arrays - - K8s Spark (native pyspark and java/scala) - - Qubole Hive - - Presto Queries - - Distributed Pytorch (K8s Native) - Pytorch Operator - - Sagemaker (builtin algorithms & custom models) - - Distributed Tensorflow (K8s Native) - TFOperator - - Papermill Notebook execution (python and spark) - -## Coming soon - - Flink-K8s - -# Current Usage -- [Freenome](https://www.freenome.com/) -- [Lyft Rideshare, Mapping](https://www.lyft.com/) -- [Lyft L5 autonomous](https://self-driving.lyft.com/level5/) -- [Spotify](https://www.spotify.com/) -- [USU Group](https://www.usu.com/) +- MLOps Coffee session - [Flyte: an open-source tool for scalable, extensible, and portable workflows](https://anchor.fm/mlops/episodes/MLOps-Coffee-Sessions-12-Flyte-an-open-source-tool-for-scalable--extensible---and-portable-workflows-eksa5k) -# Changelogs -[Changelogs](CHANGELOG/README.md) - - -# Component Repos -Repo | Language | Purpose | Status ---- | --- | --- | --- -[flyte](https://github.com/lyft/flyte) | Kustomize,RST | deployment, documentation, issues | Production-grade -[flyteidl](https://github.com/lyft/flyteidl) | Protobuf | interface definitions | Production-grade -[flytepropeller](https://github.com/lyft/flytepropeller) | Go | execution engine | Production-grade -[flyteadmin](https://github.com/lyft/flyteadmin) | Go | control plane | Production-grade -[flytekit](https://github.com/lyft/flytekit) | Python | python SDK and tools | Production-grade -[flyteconsole](https://github.com/lyft/flyteconsole) | Typescript | admin console | Production-grade -[datacatalog](https://github.com/lyft/datacatalog) | Go | manage input & output artifacts | Production-grade -[flyteplugins](https://github.com/lyft/flyteplugins) | Go | flyte plugins | Production-grade -[flytestdlib](https://github.com/lyft/flytestdlib) | Go | standard library | Production-grade -[flytesnacks](https://github.com/lyft/flytesnacks) | Python | examples, tips, and tricks | Incubating -[flytekit-java](https://github.com/spotify/flytekit-java) | Java/Scala | Java & scala SDK for authoring Flyte workflows | Incubating -[flytectl](https://github.com/lyft/flytectl) | Go | A standalone Flyte CLI | Incomplete - - -# Production K8s Operators - -Repo | Language | Purpose ---- | --- | --- -[Spark](https://github.com/GoogleCloudPlatform/spark-on-k8s-operator) | Go | Apache Spark batch -[Flink](https://github.com/lyft/flinkk8soperator) | Go | Apache Flink streaming - -# Top Contributors -Thank you to the community for making Flyte possible. - - [@wild-endeavor](https://github.com/wild-endeavor) - - [@katrogan](https://github.com/katrogan) - - [@EngHabu](https://github.com/EngHabu) - - [@akhurana001](https://github.com/akhurana001) - - [@anandswaminathan](https://github.com/anandswaminathan) - - [@kanterov](https://github.com/kanterov) - - [@honnix](https://github.com/honnix) - - [@jeevb](https://github.com/jeevb) - - [@jonathanburns](https://github.com/jonathanburns) - - [@migueltol22](https://github.com/migueltol22) - - [@varshaparthay](https://github.com/varshaparthay) - - [@pingsutw](https://github.com/pingsutw) - - [@narape](https://github.com/narape) - - [@lu4nm3](https://github.com/lu4nm3) - - [@bnsblue](https://github.com/bnsblue) - - [@RubenBarragan](https://github.com/RubenBarragan) - - [@schottra](https://github.com/schottra) - - [@evalsocket](https://github.com/evalsocket) - - [@matthewphsmith](https://github.com/matthewphsmith) - - [@slai](https://github.com/slai) - - [@derwiki](https://github.com/derwiki) - - [@tnsetting](https://github.com/tnsetting) - - [@jbrambleDC](https://github.com/jbrambleDC) - - [@igorvalko](https://github.com/igorvalko) - - [@chanadian](https://github.com/chanadian) - - [@surindersinghp](https://github.com/surindersinghp) - - [@vsbus](https://github.com/vsbus) - - [@catalinii](https://github.com/catalinii) - - [@kumare3](https://github.com/kumare3) +## 💖 Top Contributors +A big thank you to the community for making Flyte possible! +- [@wild-endeavor](https://github.com/wild-endeavor) +- [@katrogan](https://github.com/katrogan) +- [@EngHabu](https://github.com/EngHabu) +- [@akhurana001](https://github.com/akhurana001) +- [@anandswaminathan](https://github.com/anandswaminathan) +- [@kanterov](https://github.com/kanterov) +- [@honnix](https://github.com/honnix) +- [@jeevb](https://github.com/jeevb) +- [@jonathanburns](https://github.com/jonathanburns) +- [@migueltol22](https://github.com/migueltol22) +- [@varshaparthay](https://github.com/varshaparthay) +- [@pingsutw](https://github.com/pingsutw) +- [@narape](https://github.com/narape) +- [@lu4nm3](https://github.com/lu4nm3) +- [@bnsblue](https://github.com/bnsblue) +- [@RubenBarragan](https://github.com/RubenBarragan) +- [@schottra](https://github.com/schottra) +- [@evalsocket](https://github.com/evalsocket) +- [@matthewphsmith](https://github.com/matthewphsmith) +- [@slai](https://github.com/slai) +- [@derwiki](https://github.com/derwiki) +- [@tnsetting](https://github.com/tnsetting) +- [@jbrambleDC](https://github.com/jbrambleDC) +- [@igorvalko](https://github.com/igorvalko) +- [@chanadian](https://github.com/chanadian) +- [@surindersinghp](https://github.com/surindersinghp) +- [@vsbus](https://github.com/vsbus) +- [@catalinii](https://github.com/catalinii) +- [@kumare3](https://github.com/kumare3) diff --git a/boilerplate/flyte/Readme.rst b/boilerplate/flyte/Readme.rst new file mode 100644 index 0000000000..ea18781185 --- /dev/null +++ b/boilerplate/flyte/Readme.rst @@ -0,0 +1,8 @@ +Config File -- Welcome Bot +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Provides a ``config.yml`` file. + +**To Enable:** + +Add ``flyte/config.yml`` to your ``boilerplate/update.cfg`` file. \ No newline at end of file diff --git a/boilerplate/flyte/config.yml b/boilerplate/flyte/config.yml new file mode 100644 index 0000000000..f99bcd78f1 --- /dev/null +++ b/boilerplate/flyte/config.yml @@ -0,0 +1,15 @@ +# Comment to be posted on PRs from first-time contributors in your repository +newPRWelcomeComment: > + Thank you for opening this pull request! 🙌 + + These tips will help get your PR across the finish line: + - Most of the repos have a PR template; if not, fill it out to the best of your knowledge. + - Sign off your commits (Reference: [DCO Guide](https://github.com/src-d/guide/blob/master/developer-community/fix-DCO.md)). + +# Comment to be posted to on pull requests merged by a first time user +firstPRMergeComment: > + Congrats on merging your first pull request! 🎉 + +# Comment to be posted on first-time issues +newIssueWelcomeComment: > + Thank you for opening your first issue here! 🛠 diff --git a/boilerplate/flyte/update.sh b/boilerplate/flyte/update.sh new file mode 100755 index 0000000000..506c3221b7 --- /dev/null +++ b/boilerplate/flyte/update.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. +# ONLY EDIT THIS FILE FROM WITHIN THE 'LYFT/BOILERPLATE' REPOSITORY: +# +# TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst + +set -e + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" + +# Clone the config.yml file +echo " - copying ${DIR}/config.yml to the root directory." +cp ${DIR}/config.yml ${DIR}/../../.github/config.yml diff --git a/boilerplate/update.cfg b/boilerplate/update.cfg new file mode 100644 index 0000000000..90e8b6daac --- /dev/null +++ b/boilerplate/update.cfg @@ -0,0 +1 @@ +flyte/ diff --git a/boilerplate/update.sh b/boilerplate/update.sh new file mode 100755 index 0000000000..374e4b930e --- /dev/null +++ b/boilerplate/update.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash + +# WARNING: THIS FILE IS MANAGED IN THE 'BOILERPLATE' REPO AND COPIED TO OTHER REPOSITORIES. +# ONLY EDIT THIS FILE FROM WITHIN THE 'LYFT/BOILERPLATE' REPOSITORY: +# +# TO OPT OUT OF UPDATES, SEE https://github.com/lyft/boilerplate/blob/master/Readme.rst + +set -e + +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" + +OUT="$(mktemp -d)" +trap "rm -fr $OUT" EXIT + +git clone git@github.com:flyteorg/boilerplate.git "${OUT}" + +echo "Updating the update.sh script." +cp "${OUT}/boilerplate/update.sh" "${DIR}/update.sh" +echo "" + + +CONFIG_FILE="${DIR}/update.cfg" +README="https://github.com/flyteorg/boilerplate/blob/master/Readme.rst" + +if [ ! -f "$CONFIG_FILE" ]; then + echo "$CONFIG_FILE not found." + echo "This file is required in order to select which features to include." + echo "See $README for more details." + exit 1 +fi + +if [ -z "$REPOSITORY" ]; then + echo '$REPOSITORY is required to run this script' + echo "See $README for more details." + exit 1 +fi + +while read directory junk; do + # Skip comment lines (which can have leading whitespace) + if [[ "$directory" == '#'* ]]; then + continue + fi + # Skip blank or whitespace-only lines + if [[ "$directory" == "" ]]; then + continue + fi + # Lines like + # valid/path other_junk + # are not acceptable, unless `other_junk` is a comment + if [[ "$junk" != "" ]] && [[ "$junk" != '#'* ]]; then + echo "Invalid config! Only one directory is allowed per line. Found '$junk'" + exit 1 + fi + + dir_path="${OUT}/boilerplate/${directory}" + # Make sure the directory exists + if ! [[ -d "$dir_path" ]]; then + echo "Invalid boilerplate directory: '$directory'" + exit 1 + fi + + echo "***********************************************************************************" + echo "$directory is configured in update.cfg." + echo "-----------------------------------------------------------------------------------" + echo "syncing files from source." + rm -rf "${DIR}/${directory}" + mkdir -p $(dirname "${DIR}/${directory}") + cp -r "$dir_path" "${DIR}/${directory}" + if [ -f "${DIR}/${directory}/update.sh" ]; then + echo "executing ${DIR}/${directory}/update.sh" + "${DIR}/${directory}/update.sh" + fi + echo "***********************************************************************************" + echo "" +done < "$CONFIG_FILE" \ No newline at end of file diff --git a/deployment/eks/flyte_generated.yaml b/deployment/eks/flyte_generated.yaml index f0a55da47b..4f65a1985e 100644 --- a/deployment/eks/flyte_generated.yaml +++ b/deployment/eks/flyte_generated.yaml @@ -3,6 +3,16 @@ kind: Namespace metadata: name: flyte --- +apiVersion: v1 +kind: Namespace +metadata: + name: kubeflow +--- +apiVersion: v1 +kind: Namespace +metadata: + name: sparkoperator +--- apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -7600,6 +7610,12 @@ metadata: --- apiVersion: v1 kind: ServiceAccount +metadata: + name: flyte-pod-webhook + namespace: flyte +--- +apiVersion: v1 +kind: ServiceAccount metadata: annotations: eks.amazonaws.com/role-arn: arn:aws:iam::111222333456:role/flyte-operator @@ -7621,13 +7637,31 @@ metadata: app: pytorch-operator kustomize.component: pytorch-operator name: pytorch-operator - namespace: flyte + namespace: kubeflow --- apiVersion: v1 kind: ServiceAccount metadata: name: sparkoperator + namespace: sparkoperator +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: flyte-pod-webhook namespace: flyte +rules: +- apiGroups: + - '*' + resources: + - mutatingwebhookconfigurations + - secrets + - pods + verbs: + - get + - create + - update + - patch --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole @@ -7703,6 +7737,7 @@ rules: - flyte.lyft.com resources: - flyteworkflows + - flyteworkflows/finalizers verbs: - get - list @@ -7882,6 +7917,20 @@ rules: verbs: - '*' --- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: flyte-pod-webhook + namespace: flyte +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: flyte-pod-webhook +subjects: +- kind: ServiceAccount + name: flyte-pod-webhook + namespace: flyte +--- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRoleBinding metadata: @@ -7924,7 +7973,7 @@ roleRef: subjects: - kind: ServiceAccount name: pytorch-operator - namespace: flyte + namespace: kubeflow --- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRoleBinding @@ -7937,7 +7986,7 @@ roleRef: subjects: - kind: ServiceAccount name: sparkoperator - namespace: flyte + namespace: sparkoperator --- apiVersion: v1 data: @@ -8032,14 +8081,12 @@ data: storage: type: stow stow: - kind: google + kind: s3 config: - json: "" - # TODO: replace with the GCP project ID - project_id: - scopes: https://www.googleapis.com/auth/devstorage.read_write - # TODO replace with the container (bucket) in GCS used by Flyte as intermediate store - container: "flyte" + auth_type: iam + region: us-east-2 + # TODO replace with the container (bucket) in s3 used by Flyte as intermediate store + container: "flyte-demo" # NOTE this cache configuration is purely for propeller. But since we are having a common storage # config, we are configuring this value. In production create a separate storage config for # propeller and increase the cache size @@ -8050,7 +8097,7 @@ data: maxDownloadMBs: 10 kind: ConfigMap metadata: - name: datacatalog-config-2bb94tc5b5 + name: datacatalog-config-fktmhdgfk8 namespace: flyte --- apiVersion: v1 @@ -8096,19 +8143,14 @@ data: logger: show-source: true level: 2 - remote_data.yaml: | - remoteData: - # TODO change this to match the region of the s3 bucket - region: "us-west-2" - scheme: aws - signedUrls: - durationMinutes: 3 server.yaml: | server: httpPort: 8088 grpcPort: 8089 security: + # Controls whether to serve requests over SSL/TLS. secure: false + # Controls whether to enforce authentication. Follow the guide in https://docs.flyte.org/ on how to setup authentication. useAuth: false allowCors: true allowedOrigins: @@ -8116,28 +8158,45 @@ data: - "*" allowedHeaders: - "Content-Type" + - "flyte-authorization" + auth: + authorizedUris: + # This should point at your public http Uri. + - https://localhost:30081 + # This will be used by internal services in the same namespace as flyteadmin + - http://flyteadmin:80 + # This will be used by internal services in the same cluster but different namespaces + - http://flyteadmin.flyte.svc.cluster.local:80 + + # Controls app authentication config + appAuth: + thirdPartyConfig: + flyteClient: + clientId: flytectl + redirectUri: https://localhost:53593/callback + scopes: + - offline + - all + # Controls user authentication + userAuth: + openId: + baseUrl: https://accounts.google.com + scopes: + - profile + - openid + clientId: 657465813211-6eog7ek7li5k7i7fvgv2921075063hpe.apps.googleusercontent.com flyteadmin: roleNameKey: "iam.amazonaws.com/role" - profilerPort: 10254 - metricsScope: "flyte:" - metadataStoragePrefix: - - "metadata" - - "admin" - eventVersion: 1 - testing: - host: http://flyteadmin storage.yaml: | storage: type: stow stow: - kind: google + kind: s3 config: - json: "" - # TODO: replace with the GCP project ID - project_id: - scopes: https://www.googleapis.com/auth/devstorage.read_write - # TODO replace with the container (bucket) in GCS used by Flyte as intermediate store - container: "flyte" + auth_type: iam + region: us-east-2 + # TODO replace with the container (bucket) in s3 used by Flyte as intermediate store + container: "flyte-demo" # NOTE this cache configuration is purely for propeller. But since we are having a common storage # config, we are configuring this value. In production create a separate storage config for # propeller and increase the cache size @@ -8159,7 +8218,7 @@ data: gpu: 1 kind: ConfigMap metadata: - name: flyte-admin-config-t2f9g68k65 + name: flyte-admin-config-fh8cdtgk68 namespace: flyte --- apiVersion: v1 @@ -8181,6 +8240,7 @@ data: admin: endpoint: flyteadmin:81 insecure: true + useAuth: false catalog.yaml: | catalog-cache: endpoint: datacatalog:89 @@ -8192,7 +8252,7 @@ data: k8s: co-pilot: name: "flyte-copilot-" - image: "ghcr.io/lyft/flyteplugins/flytecopilot:dc4bdbd61cac88a39a5ff43e40f026bdbc2c78a2" + image: "ghcr.io/flyteorg/flytecopilot:v0.5.28" start-timeout: "30s" core.yaml: | propeller: @@ -8302,14 +8362,12 @@ data: storage: type: stow stow: - kind: google + kind: s3 config: - json: "" - # TODO: replace with the GCP project ID - project_id: - scopes: https://www.googleapis.com/auth/devstorage.read_write - # TODO replace with the container (bucket) in GCS used by Flyte as intermediate store - container: "flyte" + auth_type: iam + region: us-east-2 + # TODO replace with the container (bucket) in s3 used by Flyte as intermediate store + container: "flyte-demo" # NOTE this cache configuration is purely for propeller. But since we are having a common storage # config, we are configuring this value. In production create a separate storage config for # propeller and increase the cache size @@ -8331,7 +8389,7 @@ data: cloudwatch-log-group: kind: ConfigMap metadata: - name: flyte-propeller-config-ct29cf4cch + name: flyte-propeller-config-bk6dmfc8tm namespace: flyte --- apiVersion: v1 @@ -8344,6 +8402,22 @@ metadata: type: Opaque --- apiVersion: v1 +kind: Secret +metadata: + name: flyte-admin-auth + namespace: flyte +type: Opaque +--- +apiVersion: v1 +kind: Secret +metadata: + name: flyte-propeller-auth + namespace: flyte +stringData: + client_secret: foobar +type: Opaque +--- +apiVersion: v1 kind: Service metadata: annotations: @@ -8365,6 +8439,38 @@ spec: --- apiVersion: v1 kind: Service +metadata: + annotations: + cloud.google.com/load-balancer-type: Internal + name: datacatalog-metrics + namespace: flyte +spec: + ports: + - name: http-metrics + port: 10254 + protocol: TCP + selector: + app: datacatalog + type: LoadBalancer +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + projectcontour.io/upstream-protocol.h2c: grpc + name: flyte-pod-webhook + namespace: flyte +spec: + ports: + - name: https + port: 443 + protocol: TCP + targetPort: 9443 + selector: + app: flyte-pod-webhook +--- +apiVersion: v1 +kind: Service metadata: annotations: external-dns.alpha.kubernetes.io/hostname: flyteadmin.subdomain.mydomain.com @@ -8414,23 +8520,17 @@ apiVersion: v1 kind: Service metadata: annotations: - prometheus.io/path: /metrics - prometheus.io/port: "8443" - prometheus.io/scrape: "true" - labels: - app: pytorch-operator - kustomize.component: pytorch-operator - name: pytorch-operator + cloud.google.com/load-balancer-type: Internal + name: flytepropeller namespace: flyte spec: ports: - - name: monitoring-port - port: 8443 - targetPort: 8443 + - name: http-metrics + port: 10254 + protocol: TCP selector: - kustomize.component: pytorch-operator - name: pytorch-operator - type: ClusterIP + app: flytepropeller + type: LoadBalancer --- apiVersion: v1 kind: Service @@ -8451,9 +8551,31 @@ spec: --- apiVersion: v1 kind: Service +metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: "8443" + prometheus.io/scrape: "true" + labels: + app: pytorch-operator + kustomize.component: pytorch-operator + name: pytorch-operator + namespace: kubeflow +spec: + ports: + - name: monitoring-port + port: 8443 + targetPort: 8443 + selector: + kustomize.component: pytorch-operator + name: pytorch-operator + type: ClusterIP +--- +apiVersion: v1 +kind: Service metadata: name: spark-webhook - namespace: flyte + namespace: sparkoperator spec: ports: - name: webhook @@ -8483,16 +8605,15 @@ spec: labels: app: datacatalog app.kubernetes.io/name: datacatalog - app.kubernetes.io/version: 0.2.2 + app.kubernetes.io/version: 0.3.0 spec: containers: - command: - datacatalog - - --logtostderr - --config - /etc/datacatalog/config/*.yaml - serve - image: ghcr.io/lyft/datacatalog:v0.2.130 + image: ghcr.io/flyteorg/datacatalog:v0.3.2 imagePullPolicy: IfNotPresent name: datacatalog ports: @@ -8511,12 +8632,11 @@ spec: initContainers: - command: - datacatalog - - --logtostderr - --config - /etc/datacatalog/config/*.yaml - migrate - run - image: ghcr.io/lyft/datacatalog:v0.2.130 + image: ghcr.io/flyteorg/datacatalog:v0.3.2 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -8529,7 +8649,7 @@ spec: - emptyDir: {} name: shared-data - configMap: - name: datacatalog-config-2bb94tc5b5 + name: datacatalog-config-fktmhdgfk8 name: config-volume - name: db-pass secret: @@ -8537,6 +8657,63 @@ spec: --- apiVersion: apps/v1 kind: Deployment +metadata: + labels: + app: flyte-pod-webhook + name: flyte-pod-webhook + namespace: flyte +spec: + selector: + matchLabels: + app: flyte-pod-webhook + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: "10254" + prometheus.io/scrape: "true" + labels: + app: flyte-pod-webhook + app.kubernetes.io/name: flyte-pod-webhook + app.kubernetes.io/version: 0.5.13 + spec: + containers: + - args: + - webhook + - --config + - /etc/flyte/config/*.yaml + command: + - flytepropeller + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: ghcr.io/flyteorg/flytepropeller:v0.10.6 + imagePullPolicy: IfNotPresent + name: webhook + volumeMounts: + - mountPath: /etc/flyte/config + name: config-volume + readOnly: true + - mountPath: /etc/webhook/certs + name: webhook-certs + readOnly: true + serviceAccountName: flyte-pod-webhook + volumes: + - configMap: + name: flyte-propeller-config-bk6dmfc8tm + name: config-volume + - name: webhook-certs + secret: + secretName: flyte-pod-webhook +--- +apiVersion: apps/v1 +kind: Deployment metadata: labels: app: flyteadmin @@ -8556,16 +8733,15 @@ spec: labels: app: flyteadmin app.kubernetes.io/name: flyteadmin - app.kubernetes.io/version: 0.3.29 + app.kubernetes.io/version: 0.4.13 spec: containers: - command: - flyteadmin - - --logtostderr - --config - /etc/flyte/config/*.yaml - serve - image: ghcr.io/lyft/flyteadmin:v0.3.29 + image: ghcr.io/flyteorg/flyteadmin:v0.5.0 imagePullPolicy: IfNotPresent name: flyteadmin ports: @@ -8583,6 +8759,8 @@ spec: name: config-volume - mountPath: /etc/db name: db-pass + - mountPath: /etc/secrets/ + name: auth - command: - sh - -c @@ -8606,12 +8784,11 @@ spec: initContainers: - command: - flyteadmin - - --logtostderr - --config - /etc/flyte/config/*.yaml - migrate - run - image: ghcr.io/lyft/flyteadmin:v0.3.29 + image: ghcr.io/flyteorg/flyteadmin:v0.5.0 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -8621,7 +8798,6 @@ spec: name: db-pass - command: - flyteadmin - - --logtostderr - --config - /etc/flyte/config/*.yaml - migrate @@ -8629,7 +8805,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: ghcr.io/lyft/flyteadmin:v0.3.29 + image: ghcr.io/flyteorg/flyteadmin:v0.5.0 imagePullPolicy: IfNotPresent name: seed-projects volumeMounts: @@ -8639,12 +8815,11 @@ spec: name: db-pass - command: - flyteadmin - - --logtostderr - --config - /etc/flyte/config/*.yaml - clusterresource - sync - image: ghcr.io/lyft/flyteadmin:v0.3.29 + image: ghcr.io/flyteorg/flyteadmin:v0.5.0 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -8654,12 +8829,28 @@ spec: name: config-volume - mountPath: /etc/db name: db-pass + - args: + - flyteadmin --config=etc/flyte/config/*.yaml secrets init --localPath /etc/secrets/auth && flyteadmin --config=/etc/flyte/config/*.yaml secrets create --fromPath /etc/secrets/auth + command: + - /bin/sh + - -c + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: ghcr.io/flyteorg/flyteadmin:v0.5.0 + imagePullPolicy: IfNotPresent + name: generate-secrets + volumeMounts: + - mountPath: /etc/flyte/config + name: config-volume serviceAccountName: flyteadmin volumes: - emptyDir: {} name: shared-data - configMap: - name: flyte-admin-config-t2f9g68k65 + name: flyte-admin-config-fh8cdtgk68 name: config-volume - configMap: name: clusterresource-template-tkdkkt4cb5 @@ -8667,6 +8858,9 @@ spec: - name: db-pass secret: secretName: db-pass-bthd2588cc + - name: auth + secret: + secretName: flyte-admin-auth --- apiVersion: apps/v1 kind: Deployment @@ -8691,7 +8885,7 @@ spec: - envFrom: - configMapRef: name: flyte-console-config - image: ghcr.io/lyft/flyteconsole:v0.19.1 + image: ghcr.io/flyteorg/flyteconsole:v0.19.7 name: flyteconsole ports: - containerPort: 8080 @@ -8722,7 +8916,7 @@ spec: labels: app: flytepropeller app.kubernetes.io/name: flytepropeller - app.kubernetes.io/version: 0.5.13 + app.kubernetes.io/version: 0.7.1 spec: containers: - args: @@ -8735,7 +8929,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name - image: ghcr.io/flyteorg/flytepropeller:v0.5.13 + image: ghcr.io/flyteorg/flytepropeller:v0.10.6 imagePullPolicy: IfNotPresent name: flytepropeller ports: @@ -8748,11 +8942,16 @@ spec: volumeMounts: - mountPath: /etc/flyte/config name: config-volume + - mountPath: /etc/secrets/ + name: auth serviceAccountName: flytepropeller volumes: - configMap: - name: flyte-propeller-config-ct29cf4cch + name: flyte-propeller-config-bk6dmfc8tm name: config-volume + - name: auth + secret: + secretName: flyte-propeller-auth --- apiVersion: apps/v1 kind: Deployment @@ -8760,7 +8959,7 @@ metadata: labels: kustomize.component: pytorch-operator name: pytorch-operator - namespace: flyte + namespace: kubeflow spec: replicas: 1 selector: @@ -8799,7 +8998,7 @@ metadata: app.kubernetes.io/name: sparkoperator app.kubernetes.io/version: v2.4.5-v1beta2 name: sparkoperator - namespace: flyte + namespace: sparkoperator spec: replicas: 1 selector: @@ -8918,12 +9117,11 @@ spec: containers: - command: - flyteadmin - - --logtostderr - --config - /etc/flyte/config/*.yaml - clusterresource - sync - image: ghcr.io/lyft/flyteadmin:v0.3.29 + image: ghcr.io/flyteorg/flyteadmin:v0.5.0 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -8940,7 +9138,7 @@ spec: name: clusterresource-template-tkdkkt4cb5 name: resource-templates - configMap: - name: flyte-admin-config-t2f9g68k65 + name: flyte-admin-config-fh8cdtgk68 name: config-volume - name: db-pass secret: @@ -8949,12 +9147,52 @@ spec: --- apiVersion: batch/v1 kind: Job +metadata: + name: flyte-pod-webhook-secret + namespace: flyte +spec: + backoffLimit: 3 + template: + spec: + containers: + - args: + - webhook + - init-certs + - --config + - /etc/flyte/config/*.yaml + command: + - flytepropeller + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: ghcr.io/flyteorg/flytepropeller:v0.10.6 + imagePullPolicy: IfNotPresent + name: webhook + volumeMounts: + - mountPath: /etc/flyte/config + name: config-volume + restartPolicy: Never + serviceAccountName: flyte-pod-webhook + volumes: + - configMap: + name: flyte-propeller-config-bk6dmfc8tm + name: config-volume + ttlSecondsAfterFinished: 0 +--- +apiVersion: batch/v1 +kind: Job metadata: labels: app.kubernetes.io/name: sparkoperator app.kubernetes.io/version: v2.4.5-v1beta2 name: sparkoperator-init - namespace: flyte + namespace: sparkoperator spec: backoffLimit: 3 template: @@ -8967,7 +9205,7 @@ spec: - command: - /usr/bin/gencerts.sh - --namespace - - flyte + - sparkoperator - -p image: gcr.io/spark-operator/spark-operator:v1beta2-1.1.2-2.4.5 imagePullPolicy: IfNotPresent @@ -8975,14 +9213,19 @@ spec: restartPolicy: Never serviceAccountName: sparkoperator --- -apiVersion: extensions/v1beta1 +apiVersion: networking.k8s.io/v1 kind: Ingress metadata: annotations: + alb.ingress.kubernetes.io/actions.ssl-redirect: '{"Type": "redirect", "RedirectConfig": { "Protocol": "HTTPS", "Port": "443", "StatusCode": "HTTP_301"}}' + alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-east-2:111111111111:certificate/e92fefd8-6197-4249-a524-431d611c9af6 + alb.ingress.kubernetes.io/group.name: flytesystem + alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS":443}]' alb.ingress.kubernetes.io/scheme: internet-facing alb.ingress.kubernetes.io/tags: service_instance=production kubernetes.io/ingress.class: alb - nginx.ingress.kubernetes.io/ssl-redirect: "false" + labels: + app: flyteadmin name: flytesystem namespace: flyte spec: @@ -8990,30 +9233,171 @@ spec: - http: paths: - backend: - serviceName: flyteconsole - servicePort: 80 + service: + name: ssl-redirect + port: + name: use-annotation + path: /* + pathType: ImplementationSpecific + - backend: + service: + name: flyteconsole + port: + number: 80 path: /console + pathType: ImplementationSpecific - backend: - serviceName: flyteconsole - servicePort: 80 - path: /__webpack_hmr + service: + name: flyteconsole + port: + number: 80 + path: /console/* + pathType: ImplementationSpecific - backend: - serviceName: flyteadmin - servicePort: 80 - path: /api + service: + name: flyteadmin + port: + number: 80 + path: /api/* + pathType: ImplementationSpecific - backend: - serviceName: flyteadmin - servicePort: 80 + service: + name: flyteadmin + port: + number: 80 path: /healthcheck + pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 80 + path: /v1/* + pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 80 + path: /openapi/* + pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 80 + path: /.well-known/* + pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 80 + path: /.well-known + pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 80 + path: /login + pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 80 + path: /login/* + pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 80 + path: /logout + pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 80 + path: /logout/* + pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 80 + path: /callback + pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 80 + path: /callback/* + pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 80 + path: /me + pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 80 + path: /config + pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 80 + path: /config/* + pathType: ImplementationSpecific +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + alb.ingress.kubernetes.io/actions.ssl-redirect: '{"Type": "redirect", "RedirectConfig": { "Protocol": "HTTPS", "Port": "443", "StatusCode": "HTTP_301"}}' + alb.ingress.kubernetes.io/backend-protocol-version: GRPC + alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-east-2:111111111111:certificate/e92fefd8-6197-4249-a524-431d611c9af6 + alb.ingress.kubernetes.io/group.name: flytesystem + alb.ingress.kubernetes.io/listen-ports: '[{"HTTPS":443}]' + alb.ingress.kubernetes.io/scheme: internet-facing + alb.ingress.kubernetes.io/tags: service_instance=production + kubernetes.io/ingress.class: alb + nginx.ingress.kubernetes.io/ssl-redirect: "false" + labels: + app: flyteadmin + name: flytesystem-grpc + namespace: flyte +spec: + rules: + - http: + paths: - backend: - serviceName: flyteadmin - servicePort: 80 - path: /v1 + service: + name: flyteadmin + port: + number: 81 + path: /flyteidl.service.AdminService/* + pathType: ImplementationSpecific - backend: - serviceName: flyteadmin - servicePort: 81 - path: /flyteidl.service.AdminService + service: + name: flyteadmin + port: + number: 81 + path: /flyteidl.service.AuthMetadataService/* + pathType: ImplementationSpecific - backend: - serviceName: flyteadmin - servicePort: 87 - path: /openapi + service: + name: flyteadmin + port: + number: 81 + path: /flyteidl.service.IdentityService/* + pathType: ImplementationSpecific diff --git a/deployment/gcp/flyte_generated.yaml b/deployment/gcp/flyte_generated.yaml index 989d75e842..a45962aaa4 100644 --- a/deployment/gcp/flyte_generated.yaml +++ b/deployment/gcp/flyte_generated.yaml @@ -3,6 +3,16 @@ kind: Namespace metadata: name: flyte --- +apiVersion: v1 +kind: Namespace +metadata: + name: kubeflow +--- +apiVersion: v1 +kind: Namespace +metadata: + name: sparkoperator +--- apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: @@ -7600,6 +7610,12 @@ metadata: --- apiVersion: v1 kind: ServiceAccount +metadata: + name: flyte-pod-webhook + namespace: flyte +--- +apiVersion: v1 +kind: ServiceAccount metadata: name: flyteadmin namespace: flyte @@ -7617,13 +7633,31 @@ metadata: app: pytorch-operator kustomize.component: pytorch-operator name: pytorch-operator - namespace: flyte + namespace: kubeflow --- apiVersion: v1 kind: ServiceAccount metadata: name: sparkoperator + namespace: sparkoperator +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: flyte-pod-webhook namespace: flyte +rules: +- apiGroups: + - '*' + resources: + - mutatingwebhookconfigurations + - secrets + - pods + verbs: + - get + - create + - update + - patch --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole @@ -7699,6 +7733,7 @@ rules: - flyte.lyft.com resources: - flyteworkflows + - flyteworkflows/finalizers verbs: - get - list @@ -7878,6 +7913,20 @@ rules: verbs: - '*' --- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: flyte-pod-webhook + namespace: flyte +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: flyte-pod-webhook +subjects: +- kind: ServiceAccount + name: flyte-pod-webhook + namespace: flyte +--- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRoleBinding metadata: @@ -7920,7 +7969,7 @@ roleRef: subjects: - kind: ServiceAccount name: pytorch-operator - namespace: flyte + namespace: kubeflow --- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRoleBinding @@ -7933,7 +7982,7 @@ roleRef: subjects: - kind: ServiceAccount name: sparkoperator - namespace: flyte + namespace: sparkoperator --- apiVersion: v1 data: @@ -8091,15 +8140,14 @@ data: logger: show-source: true level: 2 - remote_data.yaml: | - remoteData: - scheme: "gcp" server.yaml: | server: httpPort: 8088 grpcPort: 8089 security: + # Controls whether to serve requests over SSL/TLS. secure: false + # Controls whether to enforce authentication. Follow the guide in https://docs.flyte.org/ on how to setup authentication. useAuth: false allowCors: true allowedOrigins: @@ -8107,16 +8155,35 @@ data: - "*" allowedHeaders: - "Content-Type" + - "flyte-authorization" + auth: + authorizedUris: + # This should point at your public http Uri. + - https://localhost:30081 + # This will be used by internal services in the same namespace as flyteadmin + - http://flyteadmin:80 + # This will be used by internal services in the same cluster but different namespaces + - http://flyteadmin.flyte.svc.cluster.local:80 + + # Controls app authentication config + appAuth: + thirdPartyConfig: + flyteClient: + clientId: flytectl + redirectUri: https://localhost:53593/callback + scopes: + - offline + - all + # Controls user authentication + userAuth: + openId: + baseUrl: https://accounts.google.com + scopes: + - profile + - openid + clientId: 657465813211-6eog7ek7li5k7i7fvgv2921075063hpe.apps.googleusercontent.com flyteadmin: roleNameKey: "iam.amazonaws.com/role" - profilerPort: 10254 - metricsScope: "flyte:" - metadataStoragePrefix: - - "metadata" - - "admin" - eventVersion: 1 - testing: - host: http://flyteadmin storage.yaml: | storage: type: stow @@ -8150,7 +8217,7 @@ data: gpu: 1 kind: ConfigMap metadata: - name: flyte-admin-config-7k2dmcb4tg + name: flyte-admin-config-2gh2k52tb8 namespace: flyte --- apiVersion: v1 @@ -8172,6 +8239,7 @@ data: admin: endpoint: flyteadmin:81 insecure: true + useAuth: false catalog.yaml: | catalog-cache: endpoint: datacatalog:89 @@ -8183,7 +8251,7 @@ data: k8s: co-pilot: name: "flyte-copilot-" - image: "ghcr.io/lyft/flyteplugins/flytecopilot:dc4bdbd61cac88a39a5ff43e40f026bdbc2c78a2" + image: "ghcr.io/flyteorg/flytecopilot:v0.5.28" start-timeout: "30s" core.yaml: | propeller: @@ -8318,7 +8386,7 @@ data: stackdriver-logresourcename: k8s_container kind: ConfigMap metadata: - name: flyte-propeller-config-7288ccf2f8 + name: flyte-propeller-config-b5bccdkh58 namespace: flyte --- apiVersion: v1 @@ -8331,6 +8399,22 @@ metadata: type: Opaque --- apiVersion: v1 +kind: Secret +metadata: + name: flyte-admin-auth + namespace: flyte +type: Opaque +--- +apiVersion: v1 +kind: Secret +metadata: + name: flyte-propeller-auth + namespace: flyte +stringData: + client_secret: foobar +type: Opaque +--- +apiVersion: v1 kind: Service metadata: name: cloudsqlproxy @@ -8381,6 +8465,22 @@ spec: --- apiVersion: v1 kind: Service +metadata: + annotations: + projectcontour.io/upstream-protocol.h2c: grpc + name: flyte-pod-webhook + namespace: flyte +spec: + ports: + - name: https + port: 443 + protocol: TCP + targetPort: 9443 + selector: + app: flyte-pod-webhook +--- +apiVersion: v1 +kind: Service metadata: annotations: cloud.google.com/load-balancer-type: Internal @@ -8449,6 +8549,23 @@ spec: --- apiVersion: v1 kind: Service +metadata: + labels: + app: redis-resource-manager + name: redis-resource-manager + namespace: flyte +spec: + ports: + - name: redis + port: 6379 + protocol: TCP + targetPort: redis + selector: + app: redis-resource-manager + type: ClusterIP +--- +apiVersion: v1 +kind: Service metadata: annotations: prometheus.io/path: /metrics @@ -8458,7 +8575,7 @@ metadata: app: pytorch-operator kustomize.component: pytorch-operator name: pytorch-operator - namespace: flyte + namespace: kubeflow spec: ports: - name: monitoring-port @@ -8471,26 +8588,9 @@ spec: --- apiVersion: v1 kind: Service -metadata: - labels: - app: redis-resource-manager - name: redis-resource-manager - namespace: flyte -spec: - ports: - - name: redis - port: 6379 - protocol: TCP - targetPort: redis - selector: - app: redis-resource-manager - type: ClusterIP ---- -apiVersion: v1 -kind: Service metadata: name: spark-webhook - namespace: flyte + namespace: sparkoperator spec: ports: - name: webhook @@ -8547,16 +8647,15 @@ spec: labels: app: datacatalog app.kubernetes.io/name: datacatalog - app.kubernetes.io/version: 0.2.2 + app.kubernetes.io/version: 0.3.0 spec: containers: - command: - datacatalog - - --logtostderr - --config - /etc/datacatalog/config/*.yaml - serve - image: ghcr.io/lyft/datacatalog:v0.2.130 + image: ghcr.io/flyteorg/datacatalog:v0.3.2 imagePullPolicy: IfNotPresent name: datacatalog ports: @@ -8575,12 +8674,11 @@ spec: initContainers: - command: - datacatalog - - --logtostderr - --config - /etc/datacatalog/config/*.yaml - migrate - run - image: ghcr.io/lyft/datacatalog:v0.2.130 + image: ghcr.io/flyteorg/datacatalog:v0.3.2 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -8601,6 +8699,63 @@ spec: --- apiVersion: apps/v1 kind: Deployment +metadata: + labels: + app: flyte-pod-webhook + name: flyte-pod-webhook + namespace: flyte +spec: + selector: + matchLabels: + app: flyte-pod-webhook + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: "10254" + prometheus.io/scrape: "true" + labels: + app: flyte-pod-webhook + app.kubernetes.io/name: flyte-pod-webhook + app.kubernetes.io/version: 0.5.13 + spec: + containers: + - args: + - webhook + - --config + - /etc/flyte/config/*.yaml + command: + - flytepropeller + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: ghcr.io/flyteorg/flytepropeller:v0.10.6 + imagePullPolicy: IfNotPresent + name: webhook + volumeMounts: + - mountPath: /etc/flyte/config + name: config-volume + readOnly: true + - mountPath: /etc/webhook/certs + name: webhook-certs + readOnly: true + serviceAccountName: flyte-pod-webhook + volumes: + - configMap: + name: flyte-propeller-config-b5bccdkh58 + name: config-volume + - name: webhook-certs + secret: + secretName: flyte-pod-webhook +--- +apiVersion: apps/v1 +kind: Deployment metadata: labels: app: flyteadmin @@ -8620,16 +8775,15 @@ spec: labels: app: flyteadmin app.kubernetes.io/name: flyteadmin - app.kubernetes.io/version: 0.3.29 + app.kubernetes.io/version: 0.4.13 spec: containers: - command: - flyteadmin - - --logtostderr - --config - /etc/flyte/config/*.yaml - serve - image: ghcr.io/lyft/flyteadmin:v0.3.29 + image: ghcr.io/flyteorg/flyteadmin:v0.5.0 imagePullPolicy: IfNotPresent name: flyteadmin ports: @@ -8647,6 +8801,8 @@ spec: name: config-volume - mountPath: /etc/db name: db-pass + - mountPath: /etc/secrets/ + name: auth - command: - sh - -c @@ -8670,12 +8826,11 @@ spec: initContainers: - command: - flyteadmin - - --logtostderr - --config - /etc/flyte/config/*.yaml - migrate - run - image: ghcr.io/lyft/flyteadmin:v0.3.29 + image: ghcr.io/flyteorg/flyteadmin:v0.5.0 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -8685,7 +8840,6 @@ spec: name: db-pass - command: - flyteadmin - - --logtostderr - --config - /etc/flyte/config/*.yaml - migrate @@ -8693,7 +8847,7 @@ spec: - flytesnacks - flytetester - flyteexamples - image: ghcr.io/lyft/flyteadmin:v0.3.29 + image: ghcr.io/flyteorg/flyteadmin:v0.5.0 imagePullPolicy: IfNotPresent name: seed-projects volumeMounts: @@ -8703,12 +8857,11 @@ spec: name: db-pass - command: - flyteadmin - - --logtostderr - --config - /etc/flyte/config/*.yaml - clusterresource - sync - image: ghcr.io/lyft/flyteadmin:v0.3.29 + image: ghcr.io/flyteorg/flyteadmin:v0.5.0 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -8718,12 +8871,28 @@ spec: name: config-volume - mountPath: /etc/db name: db-pass + - args: + - flyteadmin --config=etc/flyte/config/*.yaml secrets init --localPath /etc/secrets/auth && flyteadmin --config=/etc/flyte/config/*.yaml secrets create --fromPath /etc/secrets/auth + command: + - /bin/sh + - -c + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: ghcr.io/flyteorg/flyteadmin:v0.5.0 + imagePullPolicy: IfNotPresent + name: generate-secrets + volumeMounts: + - mountPath: /etc/flyte/config + name: config-volume serviceAccountName: flyteadmin volumes: - emptyDir: {} name: shared-data - configMap: - name: flyte-admin-config-7k2dmcb4tg + name: flyte-admin-config-2gh2k52tb8 name: config-volume - configMap: name: clusterresource-template-tkdkkt4cb5 @@ -8731,6 +8900,9 @@ spec: - name: db-pass secret: secretName: db-pass-bthd2588cc + - name: auth + secret: + secretName: flyte-admin-auth --- apiVersion: apps/v1 kind: Deployment @@ -8755,7 +8927,7 @@ spec: - envFrom: - configMapRef: name: flyte-console-config - image: ghcr.io/lyft/flyteconsole:v0.19.1 + image: ghcr.io/flyteorg/flyteconsole:v0.19.7 name: flyteconsole ports: - containerPort: 8080 @@ -8786,7 +8958,7 @@ spec: labels: app: flytepropeller app.kubernetes.io/name: flytepropeller - app.kubernetes.io/version: 0.5.13 + app.kubernetes.io/version: 0.7.1 spec: containers: - args: @@ -8799,7 +8971,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name - image: ghcr.io/flyteorg/flytepropeller:v0.5.13 + image: ghcr.io/flyteorg/flytepropeller:v0.10.6 imagePullPolicy: IfNotPresent name: flytepropeller ports: @@ -8812,11 +8984,16 @@ spec: volumeMounts: - mountPath: /etc/flyte/config name: config-volume + - mountPath: /etc/secrets/ + name: auth serviceAccountName: flytepropeller volumes: - configMap: - name: flyte-propeller-config-7288ccf2f8 + name: flyte-propeller-config-b5bccdkh58 name: config-volume + - name: auth + secret: + secretName: flyte-propeller-auth --- apiVersion: apps/v1 kind: Deployment @@ -8824,7 +9001,7 @@ metadata: labels: kustomize.component: pytorch-operator name: pytorch-operator - namespace: flyte + namespace: kubeflow spec: replicas: 1 selector: @@ -8863,7 +9040,7 @@ metadata: app.kubernetes.io/name: sparkoperator app.kubernetes.io/version: v2.4.5-v1beta2 name: sparkoperator - namespace: flyte + namespace: sparkoperator spec: replicas: 1 selector: @@ -8982,12 +9159,11 @@ spec: containers: - command: - flyteadmin - - --logtostderr - --config - /etc/flyte/config/*.yaml - clusterresource - sync - image: ghcr.io/lyft/flyteadmin:v0.3.29 + image: ghcr.io/flyteorg/flyteadmin:v0.5.0 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -9004,7 +9180,7 @@ spec: name: clusterresource-template-tkdkkt4cb5 name: resource-templates - configMap: - name: flyte-admin-config-7k2dmcb4tg + name: flyte-admin-config-2gh2k52tb8 name: config-volume - name: db-pass secret: @@ -9013,12 +9189,52 @@ spec: --- apiVersion: batch/v1 kind: Job +metadata: + name: flyte-pod-webhook-secret + namespace: flyte +spec: + backoffLimit: 3 + template: + spec: + containers: + - args: + - webhook + - init-certs + - --config + - /etc/flyte/config/*.yaml + command: + - flytepropeller + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: ghcr.io/flyteorg/flytepropeller:v0.10.6 + imagePullPolicy: IfNotPresent + name: webhook + volumeMounts: + - mountPath: /etc/flyte/config + name: config-volume + restartPolicy: Never + serviceAccountName: flyte-pod-webhook + volumes: + - configMap: + name: flyte-propeller-config-b5bccdkh58 + name: config-volume + ttlSecondsAfterFinished: 0 +--- +apiVersion: batch/v1 +kind: Job metadata: labels: app.kubernetes.io/name: sparkoperator app.kubernetes.io/version: v2.4.5-v1beta2 name: sparkoperator-init - namespace: flyte + namespace: sparkoperator spec: backoffLimit: 3 template: @@ -9031,7 +9247,7 @@ spec: - command: - /usr/bin/gencerts.sh - --namespace - - flyte + - sparkoperator - -p image: gcr.io/spark-operator/spark-operator:v1beta2-1.1.2-2.4.5 imagePullPolicy: IfNotPresent @@ -9039,7 +9255,7 @@ spec: restartPolicy: Never serviceAccountName: sparkoperator --- -apiVersion: extensions/v1beta1 +apiVersion: networking.k8s.io/v1 kind: Ingress metadata: annotations: @@ -9051,30 +9267,163 @@ spec: - http: paths: - backend: - serviceName: flyteconsole - servicePort: 80 + service: + name: flyteconsole + port: + number: 80 + path: /__webpack_hmr + pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 81 + path: /flyteidl.service.AdminService + pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 81 + path: /flyteidl.service.AuthMetadataService + pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 81 + path: /flyteidl.service.IdentityService + pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 87 + path: /openapi + pathType: ImplementationSpecific + - backend: + service: + name: flyteconsole + port: + number: 80 path: /console + pathType: ImplementationSpecific - backend: - serviceName: flyteconsole - servicePort: 80 - path: /__webpack_hmr + service: + name: flyteconsole + port: + number: 80 + path: /console/* + pathType: ImplementationSpecific - backend: - serviceName: flyteadmin - servicePort: 80 + service: + name: flyteadmin + port: + number: 80 path: /api + pathType: ImplementationSpecific - backend: - serviceName: flyteadmin - servicePort: 80 + service: + name: flyteadmin + port: + number: 80 + path: /api/* + pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 80 path: /healthcheck + pathType: ImplementationSpecific - backend: - serviceName: flyteadmin - servicePort: 80 - path: /v1 + service: + name: flyteadmin + port: + number: 80 + path: /v1/* + pathType: ImplementationSpecific - backend: - serviceName: flyteadmin - servicePort: 81 - path: /flyteidl.service.AdminService + service: + name: flyteadmin + port: + number: 80 + path: /openapi/* + pathType: ImplementationSpecific - backend: - serviceName: flyteadmin - servicePort: 87 - path: /openapi + service: + name: flyteadmin + port: + number: 80 + path: /.well-known + pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 80 + path: /.well-known/* + pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 80 + path: /login + pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 80 + path: /login/* + pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 80 + path: /logout + pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 80 + path: /logout/* + pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 80 + path: /callback + pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 80 + path: /callback/* + pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 80 + path: /me + pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 80 + path: /config + pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 80 + path: /config/* + pathType: ImplementationSpecific diff --git a/deployment/sandbox/flyte_generated.yaml b/deployment/sandbox/flyte_generated.yaml index d119bfd043..71b2f7422b 100644 --- a/deployment/sandbox/flyte_generated.yaml +++ b/deployment/sandbox/flyte_generated.yaml @@ -3,11 +3,21 @@ kind: Namespace metadata: name: flyte --- +apiVersion: v1 +kind: Namespace +metadata: + name: kubernetes-dashboard +--- +apiVersion: v1 +kind: Namespace +metadata: + name: projectcontour +--- apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.2.9 + controller-gen.kubebuilder.io/version: v0.4.1 creationTimestamp: null name: extensionservices.projectcontour.io spec: @@ -40,10 +50,28 @@ spec: description: ExtensionServiceSpec defines the desired state of an ExtensionService resource. properties: loadBalancerPolicy: - description: The policy for load balancing GRPC service requests. Note that the `Cookie` load balancing strategy cannot be used here. + description: The policy for load balancing GRPC service requests. Note that the `Cookie` and `RequestHash` load balancing strategies cannot be used here. properties: + requestHashPolicies: + description: RequestHashPolicies contains a list of hash policies to apply when the `RequestHash` load balancing strategy is chosen. If an element of the supplied list of hash policies is invalid, it will be ignored. If the list of hash policies is empty after validation, the load balancing strategy will fall back the the default `RoundRobin`. + items: + description: RequestHashPolicy contains configuration for an individual hash policy on a request attribute. + properties: + headerHashOptions: + description: HeaderHashOptions should be set when request header hash based load balancing is desired. It must be the only hash option field set, otherwise this request hash policy object will be ignored. + properties: + headerName: + description: HeaderName is the name of the HTTP request header that will be used to calculate the hash key. If the header specified is not present on a request, no hash will be produced. + minLength: 1 + type: string + type: object + terminal: + description: Terminal is a flag that allows for short-circuiting computing of a hash for a given request. If set to true, and the request attribute specified in the attribute hash options is present, no further hash policies will be used to calculate a hash for the request. + type: boolean + type: object + type: array strategy: - description: Strategy specifies the policy used to balance requests across the pool of backend pods. Valid policy names are `Random`, `RoundRobin`, `WeightedLeastRequest`, `Random` and `Cookie`. If an unknown strategy name is specified or no policy is supplied, the default `RoundRobin` policy is used. + description: Strategy specifies the policy used to balance requests across the pool of backend pods. Valid policy names are `Random`, `RoundRobin`, `WeightedLeastRequest`, `Cookie`, and `RequestHash`. If an unknown strategy name is specified or no policy is supplied, the default `RoundRobin` policy is used. type: string type: object protocol: @@ -244,7 +272,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.2.9 + controller-gen.kubebuilder.io/version: v0.4.1 creationTimestamp: null name: httpproxies.projectcontour.io spec: @@ -429,8 +457,26 @@ spec: loadBalancerPolicy: description: The load balancing policy for this route. properties: + requestHashPolicies: + description: RequestHashPolicies contains a list of hash policies to apply when the `RequestHash` load balancing strategy is chosen. If an element of the supplied list of hash policies is invalid, it will be ignored. If the list of hash policies is empty after validation, the load balancing strategy will fall back the the default `RoundRobin`. + items: + description: RequestHashPolicy contains configuration for an individual hash policy on a request attribute. + properties: + headerHashOptions: + description: HeaderHashOptions should be set when request header hash based load balancing is desired. It must be the only hash option field set, otherwise this request hash policy object will be ignored. + properties: + headerName: + description: HeaderName is the name of the HTTP request header that will be used to calculate the hash key. If the header specified is not present on a request, no hash will be produced. + minLength: 1 + type: string + type: object + terminal: + description: Terminal is a flag that allows for short-circuiting computing of a hash for a given request. If set to true, and the request attribute specified in the attribute hash options is present, no further hash policies will be used to calculate a hash for the request. + type: boolean + type: object + type: array strategy: - description: Strategy specifies the policy used to balance requests across the pool of backend pods. Valid policy names are `Random`, `RoundRobin`, `WeightedLeastRequest`, `Random` and `Cookie`. If an unknown strategy name is specified or no policy is supplied, the default `RoundRobin` policy is used. + description: Strategy specifies the policy used to balance requests across the pool of backend pods. Valid policy names are `Random`, `RoundRobin`, `WeightedLeastRequest`, `Cookie`, and `RequestHash`. If an unknown strategy name is specified or no policy is supplied, the default `RoundRobin` policy is used. type: string type: object pathRewritePolicy: @@ -457,6 +503,103 @@ spec: permitInsecure: description: Allow this path to respond to insecure requests over HTTP which are normally not permitted when a `virtualhost.tls` block is present. type: boolean + rateLimitPolicy: + description: The policy for rate limiting on the route. + properties: + global: + description: Global defines global rate limiting parameters, i.e. parameters defining descriptors that are sent to an external rate limit service (RLS) for a rate limit decision on each request. + properties: + descriptors: + description: Descriptors defines the list of descriptors that will be generated and sent to the rate limit service. Each descriptor contains 1+ key-value pair entries. + items: + description: RateLimitDescriptor defines a list of key-value pair generators. + properties: + entries: + description: Entries is the list of key-value pair generators. + items: + description: RateLimitDescriptorEntry is a key-value pair generator. Exactly one field on this struct must be non-nil. + properties: + genericKey: + description: GenericKey defines a descriptor entry with a static key and value. + properties: + key: + description: Key defines the key of the descriptor entry. If not set, the key is set to "generic_key". + type: string + value: + description: Value defines the value of the descriptor entry. + minLength: 1 + type: string + type: object + remoteAddress: + description: RemoteAddress defines a descriptor entry with a key of "remote_address" and a value equal to the client's IP address (from x-forwarded-for). + type: object + requestHeader: + description: RequestHeader defines a descriptor entry that's populated only if a given header is present on the request. The descriptor key is static, and the descriptor value is equal to the value of the header. + properties: + descriptorKey: + description: DescriptorKey defines the key to use on the descriptor entry. + minLength: 1 + type: string + headerName: + description: HeaderName defines the name of the header to look for on the request. + minLength: 1 + type: string + type: object + type: object + minItems: 1 + type: array + type: object + minItems: 1 + type: array + type: object + local: + description: Local defines local rate limiting parameters, i.e. parameters for rate limiting that occurs within each Envoy pod as requests are handled. + properties: + burst: + description: Burst defines the number of requests above the requests per unit that should be allowed within a short period of time. + format: int32 + type: integer + requests: + description: Requests defines how many requests per unit of time should be allowed before rate limiting occurs. + format: int32 + minimum: 1 + type: integer + responseHeadersToAdd: + description: ResponseHeadersToAdd is an optional list of response headers to set when a request is rate-limited. + items: + description: HeaderValue represents a header name/value pair + properties: + name: + description: Name represents a key of a header + minLength: 1 + type: string + value: + description: Value represents the value of a header specified by a key + minLength: 1 + type: string + required: + - name + - value + type: object + type: array + responseStatusCode: + description: ResponseStatusCode is the HTTP status code to use for responses to rate-limited requests. Codes must be in the 400-599 range (inclusive). If not specified, the Envoy default of 429 (Too Many Requests) is used. + format: int32 + maximum: 599 + minimum: 400 + type: integer + unit: + description: Unit defines the period of time within which requests over the limit will be rate limited. Valid values are "second", "minute" and "hour". + enum: + - second + - minute + - hour + type: string + required: + - requests + - unit + type: object + type: object requestHeadersPolicy: description: The policy for managing request headers during proxying. properties: @@ -716,10 +859,28 @@ spec: - name type: object loadBalancerPolicy: - description: The load balancing policy for the backend services. + description: The load balancing policy for the backend services. Note that the `Cookie` and `RequestHash` load balancing strategies cannot be used here. properties: + requestHashPolicies: + description: RequestHashPolicies contains a list of hash policies to apply when the `RequestHash` load balancing strategy is chosen. If an element of the supplied list of hash policies is invalid, it will be ignored. If the list of hash policies is empty after validation, the load balancing strategy will fall back the the default `RoundRobin`. + items: + description: RequestHashPolicy contains configuration for an individual hash policy on a request attribute. + properties: + headerHashOptions: + description: HeaderHashOptions should be set when request header hash based load balancing is desired. It must be the only hash option field set, otherwise this request hash policy object will be ignored. + properties: + headerName: + description: HeaderName is the name of the HTTP request header that will be used to calculate the hash key. If the header specified is not present on a request, no hash will be produced. + minLength: 1 + type: string + type: object + terminal: + description: Terminal is a flag that allows for short-circuiting computing of a hash for a given request. If set to true, and the request attribute specified in the attribute hash options is present, no further hash policies will be used to calculate a hash for the request. + type: boolean + type: object + type: array strategy: - description: Strategy specifies the policy used to balance requests across the pool of backend pods. Valid policy names are `Random`, `RoundRobin`, `WeightedLeastRequest`, `Random` and `Cookie`. If an unknown strategy name is specified or no policy is supplied, the default `RoundRobin` policy is used. + description: Strategy specifies the policy used to balance requests across the pool of backend pods. Valid policy names are `Random`, `RoundRobin`, `WeightedLeastRequest`, `Cookie`, and `RequestHash`. If an unknown strategy name is specified or no policy is supplied, the default `RoundRobin` policy is used. type: string type: object services: @@ -910,6 +1071,103 @@ spec: fqdn: description: The fully qualified domain name of the root of the ingress tree all leaves of the DAG rooted at this object relate to the fqdn. type: string + rateLimitPolicy: + description: The policy for rate limiting on the virtual host. + properties: + global: + description: Global defines global rate limiting parameters, i.e. parameters defining descriptors that are sent to an external rate limit service (RLS) for a rate limit decision on each request. + properties: + descriptors: + description: Descriptors defines the list of descriptors that will be generated and sent to the rate limit service. Each descriptor contains 1+ key-value pair entries. + items: + description: RateLimitDescriptor defines a list of key-value pair generators. + properties: + entries: + description: Entries is the list of key-value pair generators. + items: + description: RateLimitDescriptorEntry is a key-value pair generator. Exactly one field on this struct must be non-nil. + properties: + genericKey: + description: GenericKey defines a descriptor entry with a static key and value. + properties: + key: + description: Key defines the key of the descriptor entry. If not set, the key is set to "generic_key". + type: string + value: + description: Value defines the value of the descriptor entry. + minLength: 1 + type: string + type: object + remoteAddress: + description: RemoteAddress defines a descriptor entry with a key of "remote_address" and a value equal to the client's IP address (from x-forwarded-for). + type: object + requestHeader: + description: RequestHeader defines a descriptor entry that's populated only if a given header is present on the request. The descriptor key is static, and the descriptor value is equal to the value of the header. + properties: + descriptorKey: + description: DescriptorKey defines the key to use on the descriptor entry. + minLength: 1 + type: string + headerName: + description: HeaderName defines the name of the header to look for on the request. + minLength: 1 + type: string + type: object + type: object + minItems: 1 + type: array + type: object + minItems: 1 + type: array + type: object + local: + description: Local defines local rate limiting parameters, i.e. parameters for rate limiting that occurs within each Envoy pod as requests are handled. + properties: + burst: + description: Burst defines the number of requests above the requests per unit that should be allowed within a short period of time. + format: int32 + type: integer + requests: + description: Requests defines how many requests per unit of time should be allowed before rate limiting occurs. + format: int32 + minimum: 1 + type: integer + responseHeadersToAdd: + description: ResponseHeadersToAdd is an optional list of response headers to set when a request is rate-limited. + items: + description: HeaderValue represents a header name/value pair + properties: + name: + description: Name represents a key of a header + minLength: 1 + type: string + value: + description: Value represents the value of a header specified by a key + minLength: 1 + type: string + required: + - name + - value + type: object + type: array + responseStatusCode: + description: ResponseStatusCode is the HTTP status code to use for responses to rate-limited requests. Codes must be in the 400-599 range (inclusive). If not specified, the Envoy default of 429 (Too Many Requests) is used. + format: int32 + maximum: 599 + minimum: 400 + type: integer + unit: + description: Unit defines the period of time within which requests over the limit will be rate limited. Valid values are "second", "minute" and "hour". + enum: + - second + - minute + - hour + type: string + required: + - requests + - unit + type: object + type: object tls: description: If present the fields describes TLS properties of the virtual host. The SNI names that will be matched on are described in fqdn, the tls.secretName secret must contain a certificate that itself contains a name that matches the FQDN. properties: @@ -927,7 +1185,7 @@ spec: description: EnableFallbackCertificate defines if the vhost should allow a default certificate to be applied which handles all requests which don't match the SNI defined in this vhost. type: boolean minimumProtocolVersion: - description: Minimum TLS version this vhost should negotiate + description: MinimumProtocolVersion is the minimum TLS version this vhost should negotiate. Valid options are `1.2` (default) and `1.3`. Any other value defaults to TLS 1.2. type: string passthrough: description: Passthrough defines whether the encrypted TLS handshake will be passed through to the backing cluster. Either Passthrough or SecretName must be specified, but not both. @@ -1076,6 +1334,29 @@ spec: ip: description: IP is set for load-balancer ingress points that are IP based (typically GCE or OpenStack load-balancers) type: string + ports: + description: Ports is a list of records of service ports If used, every port defined in the service should have an entry in it + items: + properties: + error: + description: 'Error is to record the problem with the service port The format of the error shall comply with the following rules: - built-in error values shall be specified in this file and those shall use CamelCase names - cloud provider specific error values must have names that comply with the format foo.example.com/CamelCase. --- The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)' + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + port: + description: Port is the port number of the service port of which status is recorded here + format: int32 + type: integer + protocol: + default: TCP + description: 'Protocol is the protocol of the service port of which status is recorded here The supported values are: "TCP", "UDP", "SCTP"' + type: string + required: + - port + - protocol + type: object + type: array + x-kubernetes-list-type: atomic type: object type: array type: object @@ -1099,7 +1380,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.2.9 + controller-gen.kubebuilder.io/version: v0.4.1 creationTimestamp: null name: tlscertificatedelegations.projectcontour.io spec: @@ -1302,44 +1583,109 @@ spec: apiVersion: v1 kind: ServiceAccount metadata: - name: contour + name: datacatalog namespace: flyte --- apiVersion: v1 kind: ServiceAccount metadata: - name: contour-certgen + name: flyte-pod-webhook namespace: flyte --- apiVersion: v1 kind: ServiceAccount metadata: - name: datacatalog + name: flyteadmin namespace: flyte --- apiVersion: v1 kind: ServiceAccount metadata: - name: envoy + name: flytepropeller namespace: flyte --- apiVersion: v1 kind: ServiceAccount metadata: - name: flyteadmin - namespace: flyte + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard + namespace: kubernetes-dashboard --- apiVersion: v1 kind: ServiceAccount metadata: - name: flytepropeller - namespace: flyte + name: contour + namespace: projectcontour +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: contour-certgen + namespace: projectcontour +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: envoy + namespace: projectcontour +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard + namespace: kubernetes-dashboard +rules: +- apiGroups: + - "" + resourceNames: + - kubernetes-dashboard-key-holder + - kubernetes-dashboard-certs + - kubernetes-dashboard-csrf + resources: + - secrets + verbs: + - get + - update + - delete +- apiGroups: + - "" + resourceNames: + - kubernetes-dashboard-settings + resources: + - configmaps + verbs: + - get + - update +- apiGroups: + - "" + resourceNames: + - heapster + - dashboard-metrics-scraper + resources: + - services + verbs: + - proxy +- apiGroups: + - "" + resourceNames: + - heapster + - 'http:heapster:' + - 'https:heapster:' + - dashboard-metrics-scraper + - http:dashboard-metrics-scraper + resources: + - services/proxy + verbs: + - get --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: contour-certgen - namespace: flyte + namespace: projectcontour rules: - apiGroups: - "" @@ -1351,6 +1697,24 @@ rules: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole +metadata: + name: flyte-pod-webhook + namespace: flyte +rules: +- apiGroups: + - '*' + resources: + - mutatingwebhookconfigurations + - secrets + - pods + verbs: + - get + - create + - update + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole metadata: name: flyteadmin namespace: flyte @@ -1421,10 +1785,7 @@ rules: - apiGroups: - networking.k8s.io resources: - - gatewayclasses - - gateways - - httproutes - - tcproutes + - ingressclasses verbs: - get - list @@ -1445,6 +1806,17 @@ rules: - create - get - update +- apiGroups: + - networking.x-k8s.io + resources: + - backendpolicies + - gateways + - httproutes + - tlsroutes + verbs: + - get + - list + - watch - apiGroups: - projectcontour.io resources: @@ -1528,6 +1900,7 @@ rules: - flyte.lyft.com resources: - flyteworkflows + - flyteworkflows/finalizers verbs: - get - list @@ -1540,10 +1913,43 @@ rules: - deletecollection --- apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard +rules: +- apiGroups: + - metrics.k8s.io + resources: + - pods + - nodes + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard + namespace: kubernetes-dashboard +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: kubernetes-dashboard +subjects: +- kind: ServiceAccount + name: kubernetes-dashboard + namespace: kubernetes-dashboard +--- +apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding metadata: name: contour - namespace: flyte + namespace: projectcontour roleRef: apiGroup: rbac.authorization.k8s.io kind: Role @@ -1551,6 +1957,20 @@ roleRef: subjects: - kind: ServiceAccount name: contour-certgen + namespace: projectcontour +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: flyte-pod-webhook + namespace: flyte +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: flyte-pod-webhook +subjects: +- kind: ServiceAccount + name: flyte-pod-webhook namespace: flyte --- apiVersion: rbac.authorization.k8s.io/v1 @@ -1564,7 +1984,33 @@ roleRef: subjects: - kind: ServiceAccount name: contour - namespace: flyte + namespace: projectcontour +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubernetes-dashboard +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kubernetes-dashboard +subjects: +- kind: ServiceAccount + name: kubernetes-dashboard + namespace: kubernetes-dashboard +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubernetes-dashboard-admin +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: +- kind: ServiceAccount + name: kubernetes-dashboard + namespace: kubernetes-dashboard --- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRoleBinding @@ -1630,92 +2076,6 @@ metadata: namespace: flyte --- apiVersion: v1 -data: - contour.yaml: | - # - # server: - # determine which XDS Server implementation to utilize in Contour. - # xds-server-type: contour - # - # should contour expect to be running inside a k8s cluster - # incluster: true - # - # path to kubeconfig (if not running inside a k8s cluster) - # kubeconfig: /path/to/.kube/config - # - # disable HTTPProxy permitInsecure field - disablePermitInsecure: false - tls: - # minimum TLS version that Contour will negotiate - # minimum-protocol-version: "1.1" - # Defines the Kubernetes name/namespace matching a secret to use - # as the fallback certificate when requests which don't match the - # SNI defined for a vhost. - fallback-certificate: - # name: fallback-secret-name - # namespace: projectcontour - envoy-client-certificate: - # name: envoy-client-cert-secret-name - # namespace: projectcontour - # The following config shows the defaults for the leader election. - # leaderelection: - # configmap-name: leader-elect - # configmap-namespace: projectcontour - ### Logging options - # Default setting - accesslog-format: envoy - # To enable JSON logging in Envoy - # accesslog-format: json - # The default fields that will be logged are specified below. - # To customise this list, just add or remove entries. - # The canonical list is available at - # https://godoc.org/github.com/projectcontour/contour/internal/envoy#JSONFields - # json-fields: - # - "@timestamp" - # - "authority" - # - "bytes_received" - # - "bytes_sent" - # - "downstream_local_address" - # - "downstream_remote_address" - # - "duration" - # - "method" - # - "path" - # - "protocol" - # - "request_id" - # - "requested_server_name" - # - "response_code" - # - "response_flags" - # - "uber_trace_id" - # - "upstream_cluster" - # - "upstream_host" - # - "upstream_local_address" - # - "upstream_service_time" - # - "user_agent" - # - "x_forwarded_for" - # - # default-http-versions: - # - "HTTP/2" - # - "HTTP/1.1" - # - # The following shows the default proxy timeout settings. - # timeouts: - # request-timeout: infinity - # connection-idle-timeout: 60s - # stream-idle-timeout: 5m - # max-connection-duration: infinity - # connection-shutdown-grace-period: 5s - # - # Envoy cluster settings. - # cluster: - # configure the cluster dns lookup family - # valid options are: auto (default), v4, v6 - # dns-lookup-family: auto -kind: ConfigMap -metadata: - name: contour - namespace: flyte ---- -apiVersion: v1 data: db.yaml: | database: @@ -1793,18 +2153,14 @@ data: logger: show-source: true level: 2 - remote_data.yaml: | - remoteData: - region: "us-east-1" - scheme: "local" - signedUrls: - durationMinutes: 3 server.yaml: | server: httpPort: 8088 grpcPort: 8089 security: + # Controls whether to serve requests over SSL/TLS. secure: false + # Controls whether to enforce authentication. Follow the guide in https://docs.flyte.org/ on how to setup authentication. useAuth: false allowCors: true allowedOrigins: @@ -1812,16 +2168,35 @@ data: - "*" allowedHeaders: - "Content-Type" + - "flyte-authorization" + auth: + authorizedUris: + # This should point at your public http Uri. + - https://localhost:30081 + # This will be used by internal services in the same namespace as flyteadmin + - http://flyteadmin:80 + # This will be used by internal services in the same cluster but different namespaces + - http://flyteadmin.flyte.svc.cluster.local:80 + + # Controls app authentication config + appAuth: + thirdPartyConfig: + flyteClient: + clientId: flytectl + redirectUri: https://localhost:53593/callback + scopes: + - offline + - all + # Controls user authentication + userAuth: + openId: + baseUrl: https://accounts.google.com + scopes: + - profile + - openid + clientId: 657465813211-6eog7ek7li5k7i7fvgv2921075063hpe.apps.googleusercontent.com flyteadmin: roleNameKey: "iam.amazonaws.com/role" - profilerPort: 10254 - metricsScope: "flyte:" - metadataStoragePrefix: - - "metadata" - - "admin" - eventVersion: 1 - testing: - host: http://flyteadmin storage.yaml: |+ storage: type: minio @@ -1847,7 +2222,7 @@ data: gpu: 1 kind: ConfigMap metadata: - name: flyte-admin-config-5b5g7785h8 + name: flyte-admin-config-cbf8chmd2t namespace: flyte --- apiVersion: v1 @@ -1869,6 +2244,7 @@ data: admin: endpoint: flyteadmin:81 insecure: true + useAuth: false catalog.yaml: | catalog-cache: endpoint: datacatalog:89 @@ -1879,7 +2255,7 @@ data: k8s: co-pilot: name: "flyte-copilot-" - image: "ghcr.io/lyft/flyteplugins/flytecopilot:dc4bdbd61cac88a39a5ff43e40f026bdbc2c78a2" + image: "ghcr.io/flyteorg/flytecopilot:v0.5.28" start-timeout: "30s" core.yaml: | propeller: @@ -1915,6 +2291,9 @@ data: type: bucket rate: 10 capacity: 100 + webhook: + certDir: /etc/webhook/certs + serviceName: flyte-pod-webhook enabled_plugins.yaml: | tasks: task-plugins: @@ -1959,13 +2338,143 @@ data: plugins: logs: kubernetes-enabled: true - kubernetes-url: "http://localhost:30082" + kubernetes-template-uri: "http://localhost:30082/#/log/{{ .namespace }}/{{ .podName }}/pod?namespace={{ .namespace }}" kind: ConfigMap metadata: - name: flyte-propeller-config-94gbkmm6cm + name: flyte-propeller-config-4b26tfd275 namespace: flyte --- apiVersion: v1 +kind: ConfigMap +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard-settings + namespace: kubernetes-dashboard +--- +apiVersion: v1 +data: + contour.yaml: | + # + # server: + # determine which XDS Server implementation to utilize in Contour. + # xds-server-type: contour + # + # Specify the gateway-api Gateway Contour should watch. + # gateway: + # name: contour + # namespace: projectcontour + # + # should contour expect to be running inside a k8s cluster + # incluster: true + # + # path to kubeconfig (if not running inside a k8s cluster) + # kubeconfig: /path/to/.kube/config + # + # Disable RFC-compliant behavior to strip "Content-Length" header if + # "Tranfer-Encoding: chunked" is also set. + # disableAllowChunkedLength: false + # Disable HTTPProxy permitInsecure field + disablePermitInsecure: false + tls: + # minimum TLS version that Contour will negotiate + # minimum-protocol-version: "1.2" + # TLS ciphers to be supported by Envoy TLS listeners when negotiating + # TLS 1.2. + # cipher-suites: + # - '[ECDHE-ECDSA-AES128-GCM-SHA256|ECDHE-ECDSA-CHACHA20-POLY1305]' + # - '[ECDHE-RSA-AES128-GCM-SHA256|ECDHE-RSA-CHACHA20-POLY1305]' + # - 'ECDHE-ECDSA-AES256-GCM-SHA384' + # - 'ECDHE-RSA-AES256-GCM-SHA384' + # Defines the Kubernetes name/namespace matching a secret to use + # as the fallback certificate when requests which don't match the + # SNI defined for a vhost. + fallback-certificate: + # name: fallback-secret-name + # namespace: projectcontour + envoy-client-certificate: + # name: envoy-client-cert-secret-name + # namespace: projectcontour + # The following config shows the defaults for the leader election. + # leaderelection: + # configmap-name: leader-elect + # configmap-namespace: projectcontour + ### Logging options + # Default setting + accesslog-format: envoy + # To enable JSON logging in Envoy + # accesslog-format: json + # The default fields that will be logged are specified below. + # To customise this list, just add or remove entries. + # The canonical list is available at + # https://godoc.org/github.com/projectcontour/contour/internal/envoy#JSONFields + # json-fields: + # - "@timestamp" + # - "authority" + # - "bytes_received" + # - "bytes_sent" + # - "downstream_local_address" + # - "downstream_remote_address" + # - "duration" + # - "method" + # - "path" + # - "protocol" + # - "request_id" + # - "requested_server_name" + # - "response_code" + # - "response_flags" + # - "uber_trace_id" + # - "upstream_cluster" + # - "upstream_host" + # - "upstream_local_address" + # - "upstream_service_time" + # - "user_agent" + # - "x_forwarded_for" + # + # default-http-versions: + # - "HTTP/2" + # - "HTTP/1.1" + # + # The following shows the default proxy timeout settings. + # timeouts: + # request-timeout: infinity + # connection-idle-timeout: 60s + # stream-idle-timeout: 5m + # max-connection-duration: infinity + # delayed-close-timeout: 1s + # connection-shutdown-grace-period: 5s + # + # Envoy cluster settings. + # cluster: + # configure the cluster dns lookup family + # valid options are: auto (default), v4, v6 + # dns-lookup-family: auto + # + # Envoy network settings. + # network: + # Configure the number of additional ingress proxy hops from the + # right side of the x-forwarded-for HTTP header to trust. + # num-trusted-hops: 0 + # + # Configure an optional global rate limit service. + # rateLimitService: + # Identifies the extension service defining the rate limit service, + # formatted as /. + # extensionService: projectcontour/ratelimit + # Defines the rate limit domain to pass to the rate limit service. + # Acts as a container for a set of rate limit definitions within + # the RLS. + # domain: contour + # Defines whether to allow requests to proceed when the rate limit + # service fails to respond with a valid rate limit decision within + # the timeout defined on the extension service. + # failOpen: false +kind: ConfigMap +metadata: + name: contour + namespace: projectcontour +--- +apiVersion: v1 data: pass.txt: YXdlc29tZXNhdWNl kind: Secret @@ -1975,19 +2484,51 @@ metadata: type: Opaque --- apiVersion: v1 -kind: Service +kind: Secret metadata: - name: contour + name: flyte-admin-auth namespace: flyte -spec: - ports: - - name: xds - port: 8001 - protocol: TCP - targetPort: 8001 - selector: - app: contour - type: ClusterIP +type: Opaque +--- +apiVersion: v1 +kind: Secret +metadata: + name: flyte-propeller-auth + namespace: flyte +stringData: + client_secret: foobar +type: Opaque +--- +apiVersion: v1 +data: + password: bXl1c2Vy + user_secret: bXlzZWNyZXQ= + username: bXl1c2Vy +kind: Secret +metadata: + name: user-info + namespace: flyte +type: Opaque +--- +apiVersion: v1 +data: + csrf: "" +kind: Secret +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard-csrf + namespace: kubernetes-dashboard +type: Opaque +--- +apiVersion: v1 +kind: Secret +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard-key-holder + namespace: kubernetes-dashboard +type: Opaque --- apiVersion: v1 kind: Service @@ -2013,22 +2554,17 @@ apiVersion: v1 kind: Service metadata: annotations: - service.beta.kubernetes.io/aws-load-balancer-backend-protocol: tcp - name: envoy + projectcontour.io/upstream-protocol.h2c: grpc + name: flyte-pod-webhook namespace: flyte spec: - externalTrafficPolicy: Local ports: - - name: http - nodePort: 30081 - port: 80 - protocol: TCP - name: https port: 443 protocol: TCP + targetPort: 9443 selector: - app: envoy - type: LoadBalancer + app: flyte-pod-webhook --- apiVersion: v1 kind: Service @@ -2122,110 +2658,73 @@ spec: app: postgres type: NodePort --- -apiVersion: apps/v1 -kind: Deployment +apiVersion: v1 +kind: Service metadata: labels: - app: contour + k8s-app: dashboard-metrics-scraper + name: dashboard-metrics-scraper + namespace: kubernetes-dashboard +spec: + ports: + - port: 8000 + targetPort: 8000 + selector: + k8s-app: dashboard-metrics-scraper +--- +apiVersion: v1 +kind: Service +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard + namespace: kubernetes-dashboard +spec: + ports: + - name: http + nodePort: 30082 + port: 80 + protocol: TCP + targetPort: 9090 + selector: + k8s-app: kubernetes-dashboard + type: NodePort +--- +apiVersion: v1 +kind: Service +metadata: name: contour - namespace: flyte + namespace: projectcontour spec: - replicas: 2 + ports: + - name: xds + port: 8001 + protocol: TCP + targetPort: 8001 selector: - matchLabels: - app: contour - strategy: - rollingUpdate: - maxSurge: 50% - type: RollingUpdate - template: - metadata: - annotations: - prometheus.io/port: "8000" - prometheus.io/scrape: "true" - labels: - app: contour - spec: - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - podAffinityTerm: - labelSelector: - matchLabels: - app: contour - topologyKey: kubernetes.io/hostname - weight: 100 - containers: - - args: - - serve - - --incluster - - --xds-address=0.0.0.0 - - --xds-port=8001 - - --envoy-service-http-port=80 - - --envoy-service-https-port=443 - - --contour-cafile=/certs/ca.crt - - --contour-cert-file=/certs/tls.crt - - --contour-key-file=/certs/tls.key - - --config-path=/config/contour.yaml - command: - - contour - env: - - name: CONTOUR_NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - - name: POD_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.name - image: docker.io/projectcontour/contour:v1.11.0 - imagePullPolicy: IfNotPresent - livenessProbe: - httpGet: - path: /healthz - port: 8000 - name: contour - ports: - - containerPort: 8001 - name: xds - protocol: TCP - - containerPort: 8000 - name: metrics - protocol: TCP - - containerPort: 6060 - name: debug - protocol: TCP - readinessProbe: - initialDelaySeconds: 15 - periodSeconds: 10 - tcpSocket: - port: 8001 - volumeMounts: - - mountPath: /certs - name: contourcert - readOnly: true - - mountPath: /config - name: contour-config - readOnly: true - dnsPolicy: ClusterFirst - securityContext: - runAsGroup: 65534 - runAsNonRoot: true - runAsUser: 65534 - serviceAccountName: contour - volumes: - - name: contourcert - secret: - secretName: contourcert - - configMap: - defaultMode: 420 - items: - - key: contour.yaml - path: contour.yaml - name: contour - name: contour-config + app: contour + type: ClusterIP +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + service.beta.kubernetes.io/aws-load-balancer-backend-protocol: tcp + name: envoy + namespace: projectcontour +spec: + externalTrafficPolicy: Local + ports: + - name: http + nodePort: 30081 + port: 80 + protocol: TCP + - name: https + port: 443 + protocol: TCP + selector: + app: envoy + type: NodePort --- apiVersion: apps/v1 kind: Deployment @@ -2248,16 +2747,15 @@ spec: labels: app: datacatalog app.kubernetes.io/name: datacatalog - app.kubernetes.io/version: 0.2.2 + app.kubernetes.io/version: 0.3.0 spec: containers: - command: - datacatalog - - --logtostderr - --config - /etc/datacatalog/config/*.yaml - serve - image: ghcr.io/lyft/datacatalog:v0.2.130 + image: ghcr.io/flyteorg/datacatalog:v0.3.2 imagePullPolicy: IfNotPresent name: datacatalog ports: @@ -2271,29 +2769,91 @@ spec: initContainers: - command: - datacatalog - - --logtostderr - --config - - /etc/datacatalog/config/*.yaml - - migrate - - run - image: ghcr.io/lyft/datacatalog:v0.2.130 + - /etc/datacatalog/config/*.yaml + - migrate + - run + image: ghcr.io/flyteorg/datacatalog:v0.3.2 + imagePullPolicy: IfNotPresent + name: run-migrations + volumeMounts: + - mountPath: /etc/datacatalog/config + name: config-volume + - mountPath: /etc/db + name: db-pass + serviceAccountName: datacatalog + volumes: + - emptyDir: {} + name: shared-data + - configMap: + name: datacatalog-config-64k8dg9gck + name: config-volume + - name: db-pass + secret: + secretName: db-pass-9dgchhk2bm +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: flyte-pod-webhook + name: flyte-pod-webhook + namespace: flyte +spec: + selector: + matchLabels: + app: flyte-pod-webhook + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: "10254" + prometheus.io/scrape: "true" + labels: + app: flyte-pod-webhook + app.kubernetes.io/name: flyte-pod-webhook + app.kubernetes.io/version: 0.5.13 + spec: + containers: + - args: + - webhook + - --config + - /etc/flyte/config/*.yaml + command: + - flytepropeller + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: ghcr.io/flyteorg/flytepropeller:v0.10.6 imagePullPolicy: IfNotPresent - name: run-migrations + name: webhook volumeMounts: - - mountPath: /etc/datacatalog/config + - mountPath: /etc/secrets/user-info + name: sample-secrets + readOnly: true + - mountPath: /etc/flyte/config name: config-volume - - mountPath: /etc/db - name: db-pass - serviceAccountName: datacatalog + readOnly: true + - mountPath: /etc/webhook/certs + name: webhook-certs + readOnly: true + serviceAccountName: flyte-pod-webhook volumes: - - emptyDir: {} - name: shared-data + - name: sample-secrets + secret: + secretName: user-info - configMap: - name: datacatalog-config-64k8dg9gck + name: flyte-propeller-config-4b26tfd275 name: config-volume - - name: db-pass + - name: webhook-certs secret: - secretName: db-pass-9dgchhk2bm + secretName: flyte-pod-webhook --- apiVersion: apps/v1 kind: Deployment @@ -2316,16 +2876,15 @@ spec: labels: app: flyteadmin app.kubernetes.io/name: flyteadmin - app.kubernetes.io/version: 0.3.29 + app.kubernetes.io/version: 0.4.13 spec: containers: - command: - flyteadmin - - --logtostderr - --config - /etc/flyte/config/*.yaml - serve - image: ghcr.io/lyft/flyteadmin:v0.3.29 + image: ghcr.io/flyteorg/flyteadmin:v0.5.0 imagePullPolicy: IfNotPresent name: flyteadmin ports: @@ -2343,6 +2902,8 @@ spec: name: config-volume - mountPath: /etc/db name: db-pass + - mountPath: /etc/secrets/ + name: auth - command: - sh - -c @@ -2372,12 +2933,11 @@ spec: name: check-db-ready - command: - flyteadmin - - --logtostderr - --config - /etc/flyte/config/*.yaml - migrate - run - image: ghcr.io/lyft/flyteadmin:v0.3.29 + image: ghcr.io/flyteorg/flyteadmin:v0.5.0 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -2387,14 +2947,13 @@ spec: name: db-pass - command: - flyteadmin - - --logtostderr - --config - /etc/flyte/config/*.yaml - migrate - seed-projects - flytesnacks - flyteexamples - image: ghcr.io/lyft/flyteadmin:v0.3.29 + image: ghcr.io/flyteorg/flyteadmin:v0.5.0 imagePullPolicy: IfNotPresent name: seed-projects volumeMounts: @@ -2404,12 +2963,11 @@ spec: name: db-pass - command: - flyteadmin - - --logtostderr - --config - /etc/flyte/config/*.yaml - clusterresource - sync - image: ghcr.io/lyft/flyteadmin:v0.3.29 + image: ghcr.io/flyteorg/flyteadmin:v0.5.0 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -2419,6 +2977,22 @@ spec: name: config-volume - mountPath: /etc/db name: db-pass + - args: + - flyteadmin --config=etc/flyte/config/*.yaml secrets init --localPath /etc/secrets/auth && flyteadmin --config=/etc/flyte/config/*.yaml secrets create --fromPath /etc/secrets/auth + command: + - /bin/sh + - -c + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: ghcr.io/flyteorg/flyteadmin:v0.5.0 + imagePullPolicy: IfNotPresent + name: generate-secrets + volumeMounts: + - mountPath: /etc/flyte/config + name: config-volume serviceAccountName: flyteadmin volumes: - configMap: @@ -2427,11 +3001,14 @@ spec: - emptyDir: {} name: shared-data - configMap: - name: flyte-admin-config-5b5g7785h8 + name: flyte-admin-config-cbf8chmd2t name: config-volume - name: db-pass secret: secretName: db-pass-9dgchhk2bm + - name: auth + secret: + secretName: flyte-admin-auth --- apiVersion: apps/v1 kind: Deployment @@ -2456,7 +3033,7 @@ spec: - envFrom: - configMapRef: name: flyte-console-config - image: ghcr.io/lyft/flyteconsole:v0.19.1 + image: ghcr.io/flyteorg/flyteconsole:v0.19.7 name: flyteconsole ports: - containerPort: 8080 @@ -2487,7 +3064,7 @@ spec: labels: app: flytepropeller app.kubernetes.io/name: flytepropeller - app.kubernetes.io/version: 0.5.13 + app.kubernetes.io/version: 0.7.1 spec: containers: - args: @@ -2500,7 +3077,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name - image: ghcr.io/flyteorg/flytepropeller:v0.5.13 + image: ghcr.io/flyteorg/flytepropeller:v0.10.6 imagePullPolicy: IfNotPresent name: flytepropeller ports: @@ -2508,11 +3085,16 @@ spec: volumeMounts: - mountPath: /etc/flyte/config name: config-volume + - mountPath: /etc/secrets/ + name: auth serviceAccountName: flytepropeller volumes: - configMap: - name: flyte-propeller-config-94gbkmm6cm + name: flyte-propeller-config-4b26tfd275 name: config-volume + - name: auth + secret: + secretName: flyte-propeller-auth --- apiVersion: apps/v1 kind: Deployment @@ -2537,7 +3119,7 @@ spec: value: minio - name: MINIO_SECRET_KEY value: miniostorage - image: minio/minio:RELEASE.2020-12-16T05-05-17Z + image: minio/minio:RELEASE.2021-04-06T23-11-00Z name: minio ports: - containerPort: 9000 @@ -2545,6 +3127,15 @@ spec: volumeMounts: - mountPath: /data name: minio-storage + initContainers: + - command: + - mkdir + - /data/my-s3-bucket + image: minio/minio:RELEASE.2021-04-06T23-11-00Z + name: create-bucket + volumeMounts: + - mountPath: /data + name: minio-storage volumes: - emptyDir: {} name: minio-storage @@ -2579,6 +3170,215 @@ spec: - emptyDir: {} name: postgres-storage --- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + k8s-app: dashboard-metrics-scraper + name: dashboard-metrics-scraper + namespace: kubernetes-dashboard +spec: + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + k8s-app: dashboard-metrics-scraper + template: + metadata: + annotations: + seccomp.security.alpha.kubernetes.io/pod: runtime/default + labels: + k8s-app: dashboard-metrics-scraper + spec: + containers: + - image: kubernetesui/metrics-scraper:v1.0.6 + livenessProbe: + httpGet: + path: / + port: 8000 + scheme: HTTP + initialDelaySeconds: 30 + timeoutSeconds: 30 + name: dashboard-metrics-scraper + ports: + - containerPort: 8000 + protocol: TCP + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsGroup: 2001 + runAsUser: 1001 + volumeMounts: + - mountPath: /tmp + name: tmp-volume + nodeSelector: + kubernetes.io/os: linux + serviceAccountName: kubernetes-dashboard + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + volumes: + - emptyDir: {} + name: tmp-volume +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + k8s-app: kubernetes-dashboard + name: kubernetes-dashboard + namespace: kubernetes-dashboard +spec: + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + k8s-app: kubernetes-dashboard + template: + metadata: + labels: + k8s-app: kubernetes-dashboard + spec: + containers: + - args: + - --namespace=kubernetes-dashboard + - --enable-insecure-login + - --enable-skip-login + - --disable-settings-authorizer + image: kubernetesui/dashboard:v2.2.0 + livenessProbe: + httpGet: + path: / + port: 9090 + initialDelaySeconds: 30 + timeoutSeconds: 30 + name: kubernetes-dashboard + ports: + - containerPort: 9090 + protocol: TCP + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsGroup: 2001 + runAsUser: 1001 + volumeMounts: + - mountPath: /tmp + name: tmp-volume + nodeSelector: + kubernetes.io/os: linux + serviceAccountName: kubernetes-dashboard + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + volumes: + - emptyDir: {} + name: tmp-volume +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: contour + name: contour + namespace: projectcontour +spec: + replicas: 2 + selector: + matchLabels: + app: contour + strategy: + rollingUpdate: + maxSurge: 50% + type: RollingUpdate + template: + metadata: + annotations: + prometheus.io/port: "8000" + prometheus.io/scrape: "true" + labels: + app: contour + spec: + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + app: contour + topologyKey: kubernetes.io/hostname + weight: 100 + containers: + - args: + - serve + - --incluster + - --xds-address=0.0.0.0 + - --xds-port=8001 + - --envoy-service-http-port=80 + - --envoy-service-https-port=443 + - --contour-cafile=/certs/ca.crt + - --contour-cert-file=/certs/tls.crt + - --contour-key-file=/certs/tls.key + - --config-path=/config/contour.yaml + command: + - contour + env: + - name: CONTOUR_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + image: docker.io/projectcontour/contour:v1.13.1 + imagePullPolicy: IfNotPresent + livenessProbe: + httpGet: + path: /healthz + port: 8000 + name: contour + ports: + - containerPort: 8001 + name: xds + protocol: TCP + - containerPort: 8000 + name: metrics + protocol: TCP + - containerPort: 6060 + name: debug + protocol: TCP + readinessProbe: + initialDelaySeconds: 15 + periodSeconds: 10 + tcpSocket: + port: 8001 + volumeMounts: + - mountPath: /certs + name: contourcert + readOnly: true + - mountPath: /config + name: contour-config + readOnly: true + dnsPolicy: ClusterFirst + securityContext: + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + serviceAccountName: contour + volumes: + - name: contourcert + secret: + secretName: contourcert + - configMap: + defaultMode: 420 + items: + - key: contour.yaml + path: contour.yaml + name: contour + name: contour-config +--- apiVersion: batch/v1beta1 kind: CronJob metadata: @@ -2592,12 +3392,11 @@ spec: containers: - command: - flyteadmin - - --logtostderr - --config - /etc/flyte/config/*.yaml - clusterresource - sync - image: ghcr.io/lyft/flyteadmin:v0.3.29 + image: ghcr.io/flyteorg/flyteadmin:v0.5.0 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -2614,7 +3413,7 @@ spec: name: clusterresource-template-dtg8ff28mt name: resource-templates - configMap: - name: flyte-admin-config-5b5g7785h8 + name: flyte-admin-config-cbf8chmd2t name: config-volume - name: db-pass secret: @@ -2627,7 +3426,7 @@ metadata: labels: app: envoy name: envoy - namespace: flyte + namespace: projectcontour spec: selector: matchLabels: @@ -2648,7 +3447,7 @@ spec: - shutdown-manager command: - /bin/contour - image: docker.io/projectcontour/contour:v1.11.0 + image: docker.io/projectcontour/contour:v1.13.1 imagePullPolicy: IfNotPresent lifecycle: preStop: @@ -2683,7 +3482,7 @@ spec: fieldRef: apiVersion: v1 fieldPath: metadata.name - image: docker.io/envoyproxy/envoy:v1.16.2 + image: docker.io/envoyproxy/envoy:v1.17.1 imagePullPolicy: IfNotPresent lifecycle: preStop: @@ -2710,8 +3509,10 @@ spec: volumeMounts: - mountPath: /config name: envoy-config + readOnly: true - mountPath: /certs name: envoycert + readOnly: true initContainers: - args: - bootstrap @@ -2730,7 +3531,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: docker.io/projectcontour/contour:v1.11.0 + image: docker.io/projectcontour/contour:v1.13.1 imagePullPolicy: IfNotPresent name: envoy-initconfig volumeMounts: @@ -2756,8 +3557,48 @@ spec: apiVersion: batch/v1 kind: Job metadata: - name: contour-certgen-v1.11.0 + name: flyte-pod-webhook-secret namespace: flyte +spec: + backoffLimit: 3 + template: + spec: + containers: + - args: + - webhook + - init-certs + - --config + - /etc/flyte/config/*.yaml + command: + - flytepropeller + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: ghcr.io/flyteorg/flytepropeller:v0.10.6 + imagePullPolicy: IfNotPresent + name: webhook + volumeMounts: + - mountPath: /etc/flyte/config + name: config-volume + restartPolicy: Never + serviceAccountName: flyte-pod-webhook + volumes: + - configMap: + name: flyte-propeller-config-4b26tfd275 + name: config-volume + ttlSecondsAfterFinished: 0 +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: contour-certgen-v1.13.1 + namespace: projectcontour spec: backoffLimit: 1 completions: 1 @@ -2781,7 +3622,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - image: docker.io/projectcontour/contour:v1.11.0 + image: docker.io/projectcontour/contour:v1.13.1 imagePullPolicy: Always name: contour restartPolicy: Never @@ -2792,7 +3633,7 @@ spec: serviceAccountName: contour-certgen ttlSecondsAfterFinished: 0 --- -apiVersion: extensions/v1beta1 +apiVersion: networking.k8s.io/v1 kind: Ingress metadata: annotations: @@ -2804,35 +3645,168 @@ spec: - http: paths: - backend: - serviceName: flyteconsole - servicePort: 80 + service: + name: flyteconsole + port: + number: 80 + path: /__webpack_hmr + pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 81 + path: /flyteidl.service.AdminService + pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 81 + path: /flyteidl.service.AuthMetadataService + pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 81 + path: /flyteidl.service.IdentityService + pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 87 + path: /openapi + pathType: ImplementationSpecific + - backend: + service: + name: flyteconsole + port: + number: 80 path: /console + pathType: ImplementationSpecific - backend: - serviceName: flyteconsole - servicePort: 80 - path: /__webpack_hmr + service: + name: flyteconsole + port: + number: 80 + path: /console/* + pathType: ImplementationSpecific - backend: - serviceName: flyteadmin - servicePort: 80 + service: + name: flyteadmin + port: + number: 80 path: /api + pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 80 + path: /api/* + pathType: ImplementationSpecific - backend: - serviceName: flyteadmin - servicePort: 80 + service: + name: flyteadmin + port: + number: 80 path: /healthcheck + pathType: ImplementationSpecific - backend: - serviceName: flyteadmin - servicePort: 80 - path: /v1 + service: + name: flyteadmin + port: + number: 80 + path: /v1/* + pathType: ImplementationSpecific - backend: - serviceName: flyteadmin - servicePort: 81 - path: /flyteidl.service.AdminService + service: + name: flyteadmin + port: + number: 80 + path: /openapi/* + pathType: ImplementationSpecific - backend: - serviceName: flyteadmin - servicePort: 87 - path: /openapi + service: + name: flyteadmin + port: + number: 80 + path: /.well-known + pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 80 + path: /.well-known/* + pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 80 + path: /login + pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 80 + path: /login/* + pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 80 + path: /logout + pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 80 + path: /logout/* + pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 80 + path: /callback + pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 80 + path: /callback/* + pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 80 + path: /me + pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 80 + path: /config + pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 80 + path: /config/* + pathType: ImplementationSpecific --- -apiVersion: extensions/v1beta1 +apiVersion: networking.k8s.io/v1 kind: Ingress metadata: annotations: @@ -2844,6 +3818,9 @@ spec: - http: paths: - backend: - serviceName: minio - servicePort: 9000 + service: + name: minio + port: + number: 9000 path: /minio + pathType: ImplementationSpecific diff --git a/deployment/stats/prometheus/flyteadmin-dashboard.json b/deployment/stats/prometheus/flyteadmin-dashboard.json new file mode 100644 index 0000000000..a120ec974c --- /dev/null +++ b/deployment/stats/prometheus/flyteadmin-dashboard.json @@ -0,0 +1,18045 @@ +{ + "__inputs": [ + { + "description": "Prometheus server that connects to Flyte", + "label": "Prometheus", + "name": "DS_PROM", + "pluginId": "prometheus", + "pluginName": "Prometheus", + "type": "datasource" + } + ], + "annotations": { + "list": [] + }, + "description": "Flyte Admin/Control Plane Dashboard. This is great for monitoring FlyteAdmin and the Service API.", + "editable": false, + "gnetId": null, + "hideControls": false, + "id": null, + "links": [], + "refresh": "10s", + "rows": [ + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 1, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:executions:get_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "get Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 2, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:executions:list_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 3, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:executions:create_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "create Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 4, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:executions:update_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "update Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 5, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:executions:list_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 6, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:executions:list_identifiers_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list_identifiers Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 7, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:executions:delete_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "delete Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 8, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:executions:exists_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "exists Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "DB executions ops stats" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 9, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:executions:get_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "get Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 10, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:executions:list_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 11, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:executions:create_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "create Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 12, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:executions:update_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "update Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 13, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:executions:list_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 14, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:executions:list_identifiers_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list_identifiers Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 15, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:executions:delete_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "delete Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 16, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:executions:exists_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "exists Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "DB executions ops stats" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 17, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:task_executions:get_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "get Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 18, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:task_executions:list_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 19, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:task_executions:create_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "create Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 20, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:task_executions:update_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "update Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 21, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:task_executions:list_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 22, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:task_executions:list_identifiers_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list_identifiers Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 23, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:task_executions:delete_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "delete Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 24, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:task_executions:exists_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "exists Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "DB task_executions ops stats" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 25, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:task_executions:get_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "get Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 26, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:task_executions:list_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 27, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:task_executions:create_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "create Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 28, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:task_executions:update_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "update Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 29, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:task_executions:list_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 30, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:task_executions:list_identifiers_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list_identifiers Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 31, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:task_executions:delete_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "delete Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 32, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:task_executions:exists_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "exists Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "DB task_executions ops stats" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 33, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:node_executions:get_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "get Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 34, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:node_executions:list_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 35, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:node_executions:create_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "create Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 36, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:node_executions:update_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "update Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 37, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:node_executions:list_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 38, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:node_executions:list_identifiers_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list_identifiers Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 39, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:node_executions:delete_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "delete Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 40, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:node_executions:exists_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "exists Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "DB node_executions ops stats" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 41, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:node_executions:get_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "get Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 42, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:node_executions:list_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 43, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:node_executions:create_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "create Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 44, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:node_executions:update_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "update Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 45, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:node_executions:list_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 46, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:node_executions:list_identifiers_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list_identifiers Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 47, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:node_executions:delete_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "delete Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 48, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:node_executions:exists_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "exists Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "DB node_executions ops stats" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 49, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:workflows:get_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "get Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 50, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:workflows:list_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 51, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:workflows:create_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "create Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 52, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:workflows:update_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "update Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 53, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:workflows:list_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 54, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:workflows:list_identifiers_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list_identifiers Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 55, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:workflows:delete_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "delete Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 56, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:workflows:exists_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "exists Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "DB workflows ops stats" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 57, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:workflows:get_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "get Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 58, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:workflows:list_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 59, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:workflows:create_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "create Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 60, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:workflows:update_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "update Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 61, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:workflows:list_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 62, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:workflows:list_identifiers_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list_identifiers Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 63, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:workflows:delete_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "delete Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 64, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:workflows:exists_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "exists Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "DB workflows ops stats" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 65, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:launch_plans:get_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "get Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 66, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:launch_plans:list_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 67, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:launch_plans:create_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "create Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 68, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:launch_plans:update_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "update Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 69, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:launch_plans:list_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 70, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:launch_plans:list_identifiers_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list_identifiers Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 71, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:launch_plans:delete_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "delete Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 72, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:launch_plans:exists_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "exists Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "DB launch_plans ops stats" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 73, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:launch_plans:get_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "get Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 74, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:launch_plans:list_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 75, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:launch_plans:create_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "create Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 76, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:launch_plans:update_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "update Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 77, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:launch_plans:list_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 78, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:launch_plans:list_identifiers_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list_identifiers Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 79, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:launch_plans:delete_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "delete Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 80, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:launch_plans:exists_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "exists Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "DB launch_plans ops stats" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 81, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:project:get_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "get Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 82, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:project:list_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 83, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:project:create_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "create Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 84, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:project:update_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "update Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 85, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:project:list_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 86, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:project:list_identifiers_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list_identifiers Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 87, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:project:delete_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "delete Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 88, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:database:postgres:repositories:project:exists_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "exists Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "DB project ops stats" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 89, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:project:get_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "get Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 90, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:project:list_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 91, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:project:create_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "create Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 92, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:project:update_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "update Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 93, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:project:list_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 94, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:project:list_identifiers_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list_identifiers Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 95, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:project:delete_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "delete Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 96, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:admin:database:postgres:repositories:project:exists_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "exists Count Ops", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "DB project ops stats" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 97, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(irate(flyte:admin:create_execution:codes:OK[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "ok", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:create_execution:codes:InvalidArgument[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "invalid-args", + "metric": "", + "refId": "B", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:create_execution:codes:AlreadyExists[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "already-exists", + "metric": "", + "refId": "C", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:create_execution:codes:FailedPrecondition[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "failed-precondition", + "metric": "", + "refId": "D", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "create_execution return codes", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 98, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(irate(flyte:admin:create_execution:errors[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "errors", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:create_execution:success[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "success", + "metric": "", + "refId": "B", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "create_execution success vs errors", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 99, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:create_execution:duration_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "create_execution Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "create_execution stats" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 100, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(irate(flyte:admin:create_launch_plan:codes:OK[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "ok", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:create_launch_plan:codes:InvalidArgument[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "invalid-args", + "metric": "", + "refId": "B", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:create_launch_plan:codes:AlreadyExists[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "already-exists", + "metric": "", + "refId": "C", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:create_launch_plan:codes:FailedPrecondition[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "failed-precondition", + "metric": "", + "refId": "D", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "create_launch_plan return codes", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 101, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(irate(flyte:admin:create_launch_plan:errors[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "errors", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:create_launch_plan:success[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "success", + "metric": "", + "refId": "B", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "create_launch_plan success vs errors", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 102, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:create_launch_plan:duration_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "create_launch_plan Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "create_launch_plan stats" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 103, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(irate(flyte:admin:create_task:codes:OK[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "ok", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:create_task:codes:InvalidArgument[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "invalid-args", + "metric": "", + "refId": "B", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:create_task:codes:AlreadyExists[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "already-exists", + "metric": "", + "refId": "C", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:create_task:codes:FailedPrecondition[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "failed-precondition", + "metric": "", + "refId": "D", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "create_task return codes", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 104, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(irate(flyte:admin:create_task:errors[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "errors", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:create_task:success[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "success", + "metric": "", + "refId": "B", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "create_task success vs errors", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 105, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:create_task:duration_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "create_task Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "create_task stats" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 106, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(irate(flyte:admin:create_workflow:codes:OK[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "ok", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:create_workflow:codes:InvalidArgument[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "invalid-args", + "metric": "", + "refId": "B", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:create_workflow:codes:AlreadyExists[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "already-exists", + "metric": "", + "refId": "C", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:create_workflow:codes:FailedPrecondition[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "failed-precondition", + "metric": "", + "refId": "D", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "create_workflow return codes", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 107, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(irate(flyte:admin:create_workflow:errors[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "errors", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:create_workflow:success[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "success", + "metric": "", + "refId": "B", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "create_workflow success vs errors", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 108, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:create_workflow:duration_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "create_workflow Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "create_workflow stats" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 109, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(irate(flyte:admin:create_node_execution_event:codes:OK[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "ok", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:create_node_execution_event:codes:InvalidArgument[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "invalid-args", + "metric": "", + "refId": "B", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:create_node_execution_event:codes:AlreadyExists[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "already-exists", + "metric": "", + "refId": "C", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:create_node_execution_event:codes:FailedPrecondition[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "failed-precondition", + "metric": "", + "refId": "D", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "create_node_execution_event return codes", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 110, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(irate(flyte:admin:create_node_execution_event:errors[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "errors", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:create_node_execution_event:success[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "success", + "metric": "", + "refId": "B", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "create_node_execution_event success vs errors", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 111, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:create_node_execution_event:duration_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "create_node_execution_event Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "create_node_execution_event stats" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 112, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(irate(flyte:admin:create_task_execution_event:codes:OK[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "ok", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:create_task_execution_event:codes:InvalidArgument[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "invalid-args", + "metric": "", + "refId": "B", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:create_task_execution_event:codes:AlreadyExists[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "already-exists", + "metric": "", + "refId": "C", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:create_task_execution_event:codes:FailedPrecondition[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "failed-precondition", + "metric": "", + "refId": "D", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "create_task_execution_event return codes", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 113, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(irate(flyte:admin:create_task_execution_event:errors[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "errors", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:create_task_execution_event:success[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "success", + "metric": "", + "refId": "B", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "create_task_execution_event success vs errors", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 114, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:create_task_execution_event:duration_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "create_task_execution_event Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "create_task_execution_event stats" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 115, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(irate(flyte:admin:get_execution:codes:OK[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "ok", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:get_execution:codes:InvalidArgument[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "invalid-args", + "metric": "", + "refId": "B", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:get_execution:codes:AlreadyExists[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "already-exists", + "metric": "", + "refId": "C", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:get_execution:codes:FailedPrecondition[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "failed-precondition", + "metric": "", + "refId": "D", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "get_execution return codes", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 116, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(irate(flyte:admin:get_execution:errors[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "errors", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:get_execution:success[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "success", + "metric": "", + "refId": "B", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "get_execution success vs errors", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 117, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:get_execution:duration_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "get_execution Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "get_execution stats" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 118, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(irate(flyte:admin:get_launch_plan:codes:OK[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "ok", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:get_launch_plan:codes:InvalidArgument[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "invalid-args", + "metric": "", + "refId": "B", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:get_launch_plan:codes:AlreadyExists[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "already-exists", + "metric": "", + "refId": "C", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:get_launch_plan:codes:FailedPrecondition[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "failed-precondition", + "metric": "", + "refId": "D", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "get_launch_plan return codes", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 119, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(irate(flyte:admin:get_launch_plan:errors[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "errors", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:get_launch_plan:success[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "success", + "metric": "", + "refId": "B", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "get_launch_plan success vs errors", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 120, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:get_launch_plan:duration_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "get_launch_plan Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "get_launch_plan stats" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 121, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(irate(flyte:admin:get_task:codes:OK[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "ok", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:get_task:codes:InvalidArgument[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "invalid-args", + "metric": "", + "refId": "B", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:get_task:codes:AlreadyExists[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "already-exists", + "metric": "", + "refId": "C", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:get_task:codes:FailedPrecondition[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "failed-precondition", + "metric": "", + "refId": "D", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "get_task return codes", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 122, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(irate(flyte:admin:get_task:errors[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "errors", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:get_task:success[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "success", + "metric": "", + "refId": "B", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "get_task success vs errors", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 123, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:get_task:duration_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "get_task Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "get_task stats" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 124, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(irate(flyte:admin:get_workflow:codes:OK[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "ok", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:get_workflow:codes:InvalidArgument[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "invalid-args", + "metric": "", + "refId": "B", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:get_workflow:codes:AlreadyExists[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "already-exists", + "metric": "", + "refId": "C", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:get_workflow:codes:FailedPrecondition[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "failed-precondition", + "metric": "", + "refId": "D", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "get_workflow return codes", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 125, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(irate(flyte:admin:get_workflow:errors[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "errors", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:get_workflow:success[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "success", + "metric": "", + "refId": "B", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "get_workflow success vs errors", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 126, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:get_workflow:duration_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "get_workflow Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "get_workflow stats" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 127, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(irate(flyte:admin:get_node_execution:codes:OK[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "ok", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:get_node_execution:codes:InvalidArgument[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "invalid-args", + "metric": "", + "refId": "B", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:get_node_execution:codes:AlreadyExists[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "already-exists", + "metric": "", + "refId": "C", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:get_node_execution:codes:FailedPrecondition[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "failed-precondition", + "metric": "", + "refId": "D", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "get_node_execution return codes", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 128, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(irate(flyte:admin:get_node_execution:errors[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "errors", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:get_node_execution:success[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "success", + "metric": "", + "refId": "B", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "get_node_execution success vs errors", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 129, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:get_node_execution:duration_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "get_node_execution Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "get_node_execution stats" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 130, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(irate(flyte:admin:get_task_execution:codes:OK[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "ok", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:get_task_execution:codes:InvalidArgument[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "invalid-args", + "metric": "", + "refId": "B", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:get_task_execution:codes:AlreadyExists[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "already-exists", + "metric": "", + "refId": "C", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:get_task_execution:codes:FailedPrecondition[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "failed-precondition", + "metric": "", + "refId": "D", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "get_task_execution return codes", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 131, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(irate(flyte:admin:get_task_execution:errors[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "errors", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:get_task_execution:success[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "success", + "metric": "", + "refId": "B", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "get_task_execution success vs errors", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 132, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:get_task_execution:duration_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "get_task_execution Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "get_task_execution stats" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 133, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(irate(flyte:admin:get_active_launch_plan:codes:OK[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "ok", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:get_active_launch_plan:codes:InvalidArgument[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "invalid-args", + "metric": "", + "refId": "B", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:get_active_launch_plan:codes:AlreadyExists[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "already-exists", + "metric": "", + "refId": "C", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:get_active_launch_plan:codes:FailedPrecondition[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "failed-precondition", + "metric": "", + "refId": "D", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "get_active_launch_plan return codes", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 134, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(irate(flyte:admin:get_active_launch_plan:errors[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "errors", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:get_active_launch_plan:success[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "success", + "metric": "", + "refId": "B", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "get_active_launch_plan success vs errors", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 135, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:get_active_launch_plan:duration_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "get_active_launch_plan Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "get_active_launch_plan stats" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 136, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(irate(flyte:admin:list_execution:codes:OK[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "ok", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:list_execution:codes:InvalidArgument[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "invalid-args", + "metric": "", + "refId": "B", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:list_execution:codes:AlreadyExists[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "already-exists", + "metric": "", + "refId": "C", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:list_execution:codes:FailedPrecondition[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "failed-precondition", + "metric": "", + "refId": "D", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list_execution return codes", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 137, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(irate(flyte:admin:list_execution:errors[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "errors", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:list_execution:success[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "success", + "metric": "", + "refId": "B", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list_execution success vs errors", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 138, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:list_execution:duration_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list_execution Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "list_execution stats" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 139, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(irate(flyte:admin:list_launch_plan:codes:OK[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "ok", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:list_launch_plan:codes:InvalidArgument[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "invalid-args", + "metric": "", + "refId": "B", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:list_launch_plan:codes:AlreadyExists[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "already-exists", + "metric": "", + "refId": "C", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:list_launch_plan:codes:FailedPrecondition[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "failed-precondition", + "metric": "", + "refId": "D", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list_launch_plan return codes", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 140, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(irate(flyte:admin:list_launch_plan:errors[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "errors", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:list_launch_plan:success[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "success", + "metric": "", + "refId": "B", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list_launch_plan success vs errors", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 141, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:list_launch_plan:duration_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list_launch_plan Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "list_launch_plan stats" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 142, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(irate(flyte:admin:list_task:codes:OK[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "ok", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:list_task:codes:InvalidArgument[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "invalid-args", + "metric": "", + "refId": "B", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:list_task:codes:AlreadyExists[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "already-exists", + "metric": "", + "refId": "C", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:list_task:codes:FailedPrecondition[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "failed-precondition", + "metric": "", + "refId": "D", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list_task return codes", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 143, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(irate(flyte:admin:list_task:errors[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "errors", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:list_task:success[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "success", + "metric": "", + "refId": "B", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list_task success vs errors", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 144, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:list_task:duration_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list_task Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "list_task stats" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 145, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(irate(flyte:admin:list_workflow:codes:OK[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "ok", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:list_workflow:codes:InvalidArgument[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "invalid-args", + "metric": "", + "refId": "B", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:list_workflow:codes:AlreadyExists[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "already-exists", + "metric": "", + "refId": "C", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:list_workflow:codes:FailedPrecondition[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "failed-precondition", + "metric": "", + "refId": "D", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list_workflow return codes", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 146, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(irate(flyte:admin:list_workflow:errors[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "errors", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:list_workflow:success[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "success", + "metric": "", + "refId": "B", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list_workflow success vs errors", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 147, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:list_workflow:duration_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list_workflow Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "list_workflow stats" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 148, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(irate(flyte:admin:list_node_execution:codes:OK[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "ok", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:list_node_execution:codes:InvalidArgument[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "invalid-args", + "metric": "", + "refId": "B", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:list_node_execution:codes:AlreadyExists[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "already-exists", + "metric": "", + "refId": "C", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:list_node_execution:codes:FailedPrecondition[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "failed-precondition", + "metric": "", + "refId": "D", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list_node_execution return codes", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 149, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(irate(flyte:admin:list_node_execution:errors[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "errors", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:list_node_execution:success[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "success", + "metric": "", + "refId": "B", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list_node_execution success vs errors", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 150, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:list_node_execution:duration_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list_node_execution Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "list_node_execution stats" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 151, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(irate(flyte:admin:list_task_execution:codes:OK[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "ok", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:list_task_execution:codes:InvalidArgument[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "invalid-args", + "metric": "", + "refId": "B", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:list_task_execution:codes:AlreadyExists[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "already-exists", + "metric": "", + "refId": "C", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:list_task_execution:codes:FailedPrecondition[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "failed-precondition", + "metric": "", + "refId": "D", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list_task_execution return codes", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 152, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(irate(flyte:admin:list_task_execution:errors[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "errors", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:list_task_execution:success[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "success", + "metric": "", + "refId": "B", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list_task_execution success vs errors", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 153, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:list_task_execution:duration_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list_task_execution Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "list_task_execution stats" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 154, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(irate(flyte:admin:list_active_launch_plan:codes:OK[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "ok", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:list_active_launch_plan:codes:InvalidArgument[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "invalid-args", + "metric": "", + "refId": "B", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:list_active_launch_plan:codes:AlreadyExists[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "already-exists", + "metric": "", + "refId": "C", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:list_active_launch_plan:codes:FailedPrecondition[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "failed-precondition", + "metric": "", + "refId": "D", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list_active_launch_plan return codes", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 155, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(irate(flyte:admin:list_active_launch_plan:errors[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "errors", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(irate(flyte:admin:list_active_launch_plan:success[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "success", + "metric": "", + "refId": "B", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list_active_launch_plan success vs errors", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 156, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:admin:list_active_launch_plan:duration_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "list_active_launch_plan Latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "list_active_launch_plan stats" + } + ], + "schemaVersion": 12, + "sharedCrosshair": false, + "style": "dark", + "tags": [ + "flyte", + "prometheus", + "flyteadmin", + "flyte-controlplane" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Flyte Admin Dashboard (via Prometheus)", + "uid": null, + "version": 0 +} diff --git a/deployment/stats/prometheus/flytepropeller-dashboard.json b/deployment/stats/prometheus/flytepropeller-dashboard.json new file mode 100644 index 0000000000..457765c18d --- /dev/null +++ b/deployment/stats/prometheus/flytepropeller-dashboard.json @@ -0,0 +1,4349 @@ +{ + "__inputs": [ + { + "description": "Prometheus server that connects to Flyte", + "label": "Prometheus", + "name": "DS_PROM", + "pluginId": "prometheus", + "pluginName": "Prometheus", + "type": "datasource" + } + ], + "annotations": { + "list": [] + }, + "description": "Flyte Propeller Dashboard. This is great for monitoring FlytePropeller / Flyte data plane deployments. This is mostly useful for the Flyte deployment maintainer", + "editable": false, + "gnetId": null, + "hideControls": false, + "id": null, + "links": [], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 1, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:propeller:all:free_workers_count) by (kubernetes_pod_name)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Free workers count", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 2, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:round:abort_error[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "System errors", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 3, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(deriv(flyte:propeller:all:round:system_error_unlabeled[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "System errors", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 4, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "{__name__=~\"flyte:propeller:all:node:plugin:.*_failure_unlabeled\"}", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "{__name__=~\"flyte:propeller:all:node:plugin:.*_success_unlabeled\"}", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Plugin Failures", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 5, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:round:raw_unlabeled_ms[5m])) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "round Latency by quantile", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 6, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:round:raw_ms[5m])) by (wf)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "round Latency per workflow", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 7, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:round:panic_unlabeled[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Round panic", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 8, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:propeller:all:collector:flyteworkflow) by (project)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Running Workflows per project", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "Core metrics" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 9, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "(sum(rate(flyte:propeller:all:metastore:cache_hit[5m])) * 100) / (sum(rate(flyte:propeller:all:metastore:cache_miss[5m])) + sum(rate(flyte:propeller:all:metastore:cache_hit[5m])))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "cache hit percentage", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 10, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:metastore:head_failure_unlabeled[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "head-failure", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:metastore:bad_container_unlabeled[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "bad-container", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:metastore:bad_key_unlabeled[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "bad-key", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:metastore:read_failure_unlabeled[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "read-failure", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:metastore:write_failure_unlabeled[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "write-failure", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Failures from metastore", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "Metastore failures and cache" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 11, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:propeller:all:metastore:copy:overall_unlabeled_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Metastore copy latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 12, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:propeller:all:metastore:write_ms) by (quantile, wf)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Metastore write latency by workflow", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 13, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:propeller:all:metastore:read_open_ms) by (quantile, wf)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Metastore read open latency by workflow", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 14, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:propeller:all:metastore:head_ms) by (quantile, wf)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Metastore head latency by workflow", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 15, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:propeller:all:metastore:proto_fetch_ms) by (quantile, wf)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "proto-fetch", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(flyte:propeller:all:metastore:remote_fetch_ms) by (quantile, wf)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "remote-fetch", + "metric": "", + "refId": "B", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Metastore fetch latency by workflow", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "Metastore latencies" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 16, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:propeller:all:node:node_exec_latency_us) by (quantile, wf) / 1000", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Node Exec latency quantile and workflow", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 17, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:propeller:all:node:node_input_latency_ms) by (quantile, wf)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Node Input latency quantile and workflow", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 18, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:propeller:all:node:event_recording:success_duration_ms) by (quantile, wf)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(flyte:propeller:all:node:event_recording:failure_duration_ms) by (quantile, wf)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "B", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Node Event event recording latency quantile and workflow", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 19, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:node:perma_system_error_duration_unlabeled_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "system error", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:node:perma_user_error_duration_unlabeled_ms[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "user error", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:node:perma_unknown_error_duration_unlabeled_ms[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "user error", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "node event recording count", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "Node Metrics" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 20, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:propeller:all:workflow:event_recording:success_duration_ms) by (quantile, wf)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "wf event recording latency success", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 21, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:workflow:event_recording:success_duration_ms_count[5m])) by (wf)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "success", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:workflow:event_recording:failure_duration_ms_count[5m])) by (wf)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "failure", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "wf event recording count", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 22, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:propeller:all:node:event_recording:success_duration_ms) by (quantile, wf)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "node event recording latency success", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 23, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:node:event_recording:success_duration_ms_count[5m])) by (wf)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "success", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:node:event_recording:failure_duration_ms_count[5m])) by (wf)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "failure", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "node event recording count", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 24, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:propeller:all:task:event_recording:success_duration_ms) by (quantile, wf)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "task event recording latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 25, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:task:event_recording:success_duration_ms_count[5m])) by (wf)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "success wf", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:task:event_recording:failure_duration_ms_count[5m])) by (wf)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "failure", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "task event recording count", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 26, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:propeller:all:node:build_dynamic_workflow_us) by (quantile, wf) / 1000", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Dynamic workflow build latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 27, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:node:build_dynamic_workflow_us_count[5m])) by (wf)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Dynamic workflow build count", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 28, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:admin_launcher:cache_hit[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "hit", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:admin_launcher:cache_miss[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "miss", + "metric": "", + "refId": "B", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Admin Launcher cache", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "Perf metrics" + }, + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 29, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:propeller:all:wf_update_latency_ms) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "wf update etcD latency", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 30, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:wf_update_latency_ms_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "etcD writes", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 31, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:wf_update_conflict[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "etcD write conflicts", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 32, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 3, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:wf_update_failed[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "etcD write fail", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "etcD write metrics" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 33, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:main_adds[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "WF Adds to main queue", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 34, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:main_depth[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Unprocessed Queue depth", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 35, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:main_retries[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Item retries", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 36, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "flyte:propeller:all:main_unfinished_work_s", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Seconds of unfinished work in progress", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 37, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:main_work_duration_us_sum[5m]) / rate(flyte:propeller:all:main_work_duration_us_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Workqueue work average duration", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 38, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:main_queue_latency_us_sum[5m]) / rate(flyte:propeller:all:main_queue_latency_us_count[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Duration for which an item stays in queue - avg", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "FlytePropeller Queue metrics" + } + ], + "schemaVersion": 12, + "sharedCrosshair": false, + "style": "dark", + "tags": [ + "flyte", + "prometheus", + "flytepropeller", + "flyte-dataplane" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Flyte Propeller Dashboard (via Prometheus)", + "uid": null, + "version": 0 +} diff --git a/deployment/stats/prometheus/flyteuser-dashboard.json b/deployment/stats/prometheus/flyteuser-dashboard.json new file mode 100644 index 0000000000..3ec252f2e4 --- /dev/null +++ b/deployment/stats/prometheus/flyteuser-dashboard.json @@ -0,0 +1,1691 @@ +{ + "__inputs": [ + { + "description": "Prometheus server that connects to Flyte", + "label": "Prometheus", + "name": "DS_PROM", + "pluginId": "prometheus", + "pluginName": "Prometheus", + "type": "datasource" + } + ], + "annotations": { + "list": [] + }, + "description": "Flyte User Dashboard. This is great to get a birds-eye and drill down view of executions in your Flyte cluster. Useful for the user.", + "editable": false, + "gnetId": null, + "hideControls": false, + "id": null, + "links": [], + "refresh": "10s", + "rows": [ + { + "collapse": false, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 1, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:workflow:accepted{project=~\"$project\", domain=~\"$domain\", wf=~\"$workflow\"}[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Accepted Workflow", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 2, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:workflow:success_duration_ms_count{project=~\"$project\", domain=~\"$domain\", wf=~\"$workflow\"}[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Successful Workflow", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 3, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:workflow:failure_duration_ms_count{project=~\"$project\", domain=~\"$domain\", wf=~\"$workflow\"}[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Failed Workflow", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 4, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:workflow:workflow_aborted{project=~\"$project\", domain=~\"$domain\", wf=~\"$workflow\"}[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Aborted Workflow", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 5, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:propeller:all:workflow:success_duration_ms{project=~\"$project\", domain=~\"$domain\", wf=~\"$workflow\"}) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Successful workflow execution time by Quantile", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 6, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:propeller:all:workflow:failed_duration_ms{project=~\"$project\", domain=~\"$domain\", wf=~\"$workflow\"}) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Failed workflow execution time by Quantile", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 7, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 2, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(flyte:propeller:all:node:queueing_latency_ms{project=~\"$project\", domain=~\"$domain\", wf=~\"$workflow\"}) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Node queuing latency by Quantile", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "Workflow Stats" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 8, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "kube_resourcequota{resource=\"limits.cpu\", namespace=\"$project-$domain\", type=\"hard\"}", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "max cpu", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "kube_resourcequota{resource=\"limits.cpu\", namespace=\"$project-$domain\", type=\"used\"}", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "used cpu", + "metric": "", + "refId": "B", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Limits vs usage", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 9, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "kube_resourcequota{resource=\"limits.memory\", namespace=\"$project-$domain\", type=\"hard\"}", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "max mem", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + }, + { + "datasource": "", + "expr": "kube_resourcequota{resource=\"limits.memory\", namespace=\"$project-$domain\", type=\"used\"}", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "used mem", + "metric": "", + "refId": "B", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Mem Limits vs usage", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "ops", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "Kubernetes Quota Usage stats" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 10, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(kube_pod_container_status_waiting * on(pod) group_left(label_execution_id, label_task_name, label_node_id, label_workflow_name) kube_pod_labels{label_execution_id !=\"\",namespace=~\"$project-$domain\",label_workflow_name=~\"$workflow\"}) by (namespace, label_execution_id, label_task_name, label_node_id, label_workflow_name) > 0", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Pending tasks", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 11, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "(max(container_memory_rss{image!=\"\"} * on(pod) group_left(label_execution_id, label_task_name, label_node_id, label_workflow_name) kube_pod_labels{label_execution_id !=\"\",namespace=~\"$project-$domain\",label_workflow_name=~\"$workflow\"} * on(pod) group_left(phase) kube_pod_status_phase{phase=\"Running\"}) by (namespace, pod, label_execution_id, label_task_name, label_node_id, label_workflow_name) / max(kube_pod_container_resource_limits_memory_bytes{container!=\"\"} * on(pod) group_left(label_execution_id, label_task_name, label_node_id, label_workflow_name) kube_pod_labels{label_execution_id !=\"\"} * on(pod) group_left(phase) kube_pod_status_phase{phase=\"Running\"}) by (namespace, pod, label_execution_id, label_task_name, label_node_id, label_workflow_name)) > 0", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Memory Usage Percentage", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 12, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "(sum(rate(container_cpu_usage_seconds_total{image!=\"\"}[2m]) * on(pod) group_left(label_execution_id, label_task_name, label_node_id, label_workflow_name) kube_pod_labels{label_execution_id !=\"\",namespace=~\"$project-$domain\",label_workflow_name=~\"$workflow\"} * on(pod) group_left(phase) kube_pod_status_phase{phase=\"Running\"}) by (namespace, pod, label_execution_id, label_task_name, label_node_id, label_workflow_name) / sum(kube_pod_container_resource_limits_cpu_cores{container!=\"\"} * on(pod) group_left(label_execution_id, label_task_name, label_node_id, label_workflow_name) kube_pod_labels{label_execution_id !=\"\"} * on(pod) group_left(phase) kube_pod_status_phase{phase=\"Running\"}) by (namespace, pod, label_execution_id, label_task_name, label_node_id, label_workflow_name)) > 0", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage Percentage", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "Task stats" + }, + { + "collapse": true, + "editable": true, + "height": "250px", + "panels": [ + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 13, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:node:user_error_duration_ms_count{project=~\"$project\",domain=~\"$domain\",wf=~\"$project:$domain:$workflow\"}[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "User errors", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "cacheTimeout": null, + "datasource": "${DS_PROM}", + "description": null, + "editable": true, + "error": false, + "fill": 1, + "grid": { + "threshold1": null, + "threshold1Color": "rgba(216, 200, 27, 0.27)", + "threshold2": null, + "threshold2Color": "rgba(234, 112, 112, 0.22)" + }, + "height": null, + "hideTimeOverride": false, + "id": 14, + "interval": null, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "sort": null, + "sortDesc": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "maxDataPoints": 100, + "minSpan": null, + "nullPointMode": "connected", + "options": { + "alertThreshold": true, + "dataLinks": [] + }, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "datasource": "", + "expr": "sum(rate(flyte:propeller:all:node:system_error_duration_ms_count{project=~\"$project\",domain=~\"$domain\",wf=~\"$project:$domain:$workflow\"}[5m]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "System errors", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "decimals": null, + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + } + ], + "repeat": null, + "showTitle": true, + "title": "Error (System vs user)" + } + ], + "schemaVersion": 12, + "sharedCrosshair": false, + "style": "dark", + "tags": [ + "flyte", + "prometheus", + "flyteuser", + "flyte-user" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "tags": [], + "text": null, + "value": null + }, + "datasource": "${DS_PROM}", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "project", + "options": [], + "query": "label_values(flyte:propeller:all:collector:flyteworkflow, project)", + "refresh": 1, + "regex": null, + "sort": true, + "tagValuesQuery": null, + "tagsQuery": null, + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "tags": [], + "text": null, + "value": null + }, + "datasource": "${DS_PROM}", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "domain", + "options": [], + "query": "label_values(flyte:propeller:all:collector:flyteworkflow, domain)", + "refresh": 1, + "regex": null, + "sort": true, + "tagValuesQuery": null, + "tagsQuery": null, + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "tags": [], + "text": null, + "value": null + }, + "datasource": "${DS_PROM}", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "workflow", + "options": [], + "query": "label_values(flyte:propeller:all:collector:flyteworkflow, wf)", + "refresh": 1, + "regex": null, + "sort": true, + "tagValuesQuery": null, + "tagsQuery": null, + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Flyte User Dashboard (via Prometheus)", + "uid": null, + "version": 0 +} diff --git a/deployment/test/flyte_generated.yaml b/deployment/test/flyte_generated.yaml index 60c64b5def..069cb37355 100644 --- a/deployment/test/flyte_generated.yaml +++ b/deployment/test/flyte_generated.yaml @@ -18,151 +18,16 @@ spec: scope: Namespaced version: v1alpha1 --- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - labels: - component: ingressroute - name: ingressroutes.contour.heptio.com -spec: - additionalPrinterColumns: - - JSONPath: .spec.virtualhost.fqdn - description: Fully qualified domain name - name: FQDN - type: string - - JSONPath: .spec.virtualhost.tls.secretName - description: Secret with TLS credentials - name: TLS Secret - type: string - - JSONPath: .spec.routes[0].match - description: First routes defined - name: First route - type: string - - JSONPath: .status.currentStatus - description: The current status of the IngressRoute - name: Status - type: string - - JSONPath: .status.description - description: Description of the current status - name: Status Description - type: string - group: contour.heptio.com - names: - kind: IngressRoute - plural: ingressroutes - scope: Namespaced - validation: - openAPIV3Schema: - properties: - spec: - properties: - healthCheck: - properties: - healthyThresholdCount: - type: integer - intervalSeconds: - type: integer - path: - pattern: ^\/.*$ - type: string - timeoutSeconds: - type: integer - unhealthyThresholdCount: - type: integer - required: - - path - type: object - routes: - items: - properties: - delegate: - properties: - name: - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - namespace: - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - required: - - name - type: object - match: - pattern: ^\/.*$ - type: string - services: - items: - properties: - healthCheck: - properties: - healthyThresholdCount: - type: integer - intervalSeconds: - type: integer - path: - pattern: ^\/.*$ - type: string - timeoutSeconds: - type: integer - unhealthyThresholdCount: - type: integer - required: - - path - type: object - name: - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ - type: string - port: - type: integer - strategy: - enum: - - RoundRobin - - WeightedLeastRequest - - Random - - RingHash - - Maglev - type: string - weight: - type: integer - required: - - name - - port - type: object - type: array - required: - - match - type: array - strategy: - enum: - - RoundRobin - - LeastRequest - - Random - - RingHash - - Maglev - type: string - virtualhost: - properties: - aliases: - items: - pattern: ^([a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,}$ - type: string - type: array - fqdn: - pattern: ^([a-zA-Z0-9]+(-[a-zA-Z0-9]+)*\.)+[a-z]{2,}$ - type: string - required: - - routes - version: v1beta1 ---- apiVersion: v1 kind: ServiceAccount metadata: - name: contour + name: datacatalog namespace: flyte --- apiVersion: v1 kind: ServiceAccount metadata: - name: datacatalog + name: flyte-pod-webhook namespace: flyte --- apiVersion: v1 @@ -179,6 +44,24 @@ metadata: --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole +metadata: + name: flyte-pod-webhook + namespace: flyte +rules: +- apiGroups: + - '*' + resources: + - mutatingwebhookconfigurations + - secrets + - pods + verbs: + - get + - create + - update + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole metadata: name: flyteadmin namespace: flyte @@ -251,6 +134,7 @@ rules: - flyte.lyft.com resources: - flyteworkflows + - flyteworkflows/finalizers verbs: - get - list @@ -262,95 +146,46 @@ rules: - post - deletecollection --- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: contour -rules: -- apiGroups: - - "" - resources: - - configmaps - - endpoints - - nodes - - pods - - secrets - verbs: - - list - - watch -- apiGroups: - - "" - resources: - - nodes - verbs: - - get -- apiGroups: - - "" - resources: - - services - verbs: - - get - - list - - watch -- apiGroups: - - extensions - resources: - - ingresses - verbs: - - get - - list - - watch -- apiGroups: - - contour.heptio.com - resources: - - ingressroutes - verbs: - - get - - list - - watch - - put - - post - - patch ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: - name: flyteadmin-binding + name: flyte-pod-webhook namespace: flyte roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: flyteadmin + name: flyte-pod-webhook subjects: - kind: ServiceAccount - name: flyteadmin + name: flyte-pod-webhook namespace: flyte --- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRoleBinding metadata: - name: flytepropeller + name: flyteadmin-binding namespace: flyte roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: flytepropeller + name: flyteadmin subjects: - kind: ServiceAccount - name: flytepropeller + name: flyteadmin namespace: flyte --- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRoleBinding metadata: - name: contour + name: flytepropeller + namespace: flyte roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole - name: contour + name: flytepropeller subjects: - kind: ServiceAccount - name: contour + name: flytepropeller namespace: flyte --- apiVersion: v1 @@ -466,18 +301,14 @@ data: logger: show-source: true level: 2 - remote_data.yaml: | - remoteData: - region: "us-east-1" - scheme: "local" - signedUrls: - durationMinutes: 3 server.yaml: | server: httpPort: 8088 grpcPort: 8089 security: + # Controls whether to serve requests over SSL/TLS. secure: false + # Controls whether to enforce authentication. Follow the guide in https://docs.flyte.org/ on how to setup authentication. useAuth: false allowCors: true allowedOrigins: @@ -485,16 +316,35 @@ data: - "*" allowedHeaders: - "Content-Type" + - "flyte-authorization" + auth: + authorizedUris: + # This should point at your public http Uri. + - https://localhost:30081 + # This will be used by internal services in the same namespace as flyteadmin + - http://flyteadmin:80 + # This will be used by internal services in the same cluster but different namespaces + - http://flyteadmin.flyte.svc.cluster.local:80 + + # Controls app authentication config + appAuth: + thirdPartyConfig: + flyteClient: + clientId: flytectl + redirectUri: https://localhost:53593/callback + scopes: + - offline + - all + # Controls user authentication + userAuth: + openId: + baseUrl: https://accounts.google.com + scopes: + - profile + - openid + clientId: 657465813211-6eog7ek7li5k7i7fvgv2921075063hpe.apps.googleusercontent.com flyteadmin: roleNameKey: "iam.amazonaws.com/role" - profilerPort: 10254 - metricsScope: "flyte:" - metadataStoragePrefix: - - "metadata" - - "admin" - eventVersion: 1 - testing: - host: http://flyteadmin storage.yaml: |+ storage: type: minio @@ -520,7 +370,7 @@ data: gpu: 1 kind: ConfigMap metadata: - name: flyte-admin-config-k7f4k7g76c + name: flyte-admin-config-t8ct89hdm5 namespace: flyte --- apiVersion: v1 @@ -533,6 +383,7 @@ data: admin: endpoint: flyteadmin:81 insecure: true + useAuth: false catalog.yaml: | catalog-cache: endpoint: datacatalog:89 @@ -543,7 +394,7 @@ data: k8s: co-pilot: name: "flyte-copilot-" - image: "ghcr.io/lyft/flyteplugins/flytecopilot:dc4bdbd61cac88a39a5ff43e40f026bdbc2c78a2" + image: "ghcr.io/flyteorg/flytecopilot:v0.5.28" start-timeout: "30s" core.yaml: | propeller: @@ -579,6 +430,9 @@ data: type: bucket rate: 10 capacity: 100 + webhook: + certDir: /etc/webhook/certs + serviceName: flyte-pod-webhook enabled_plugins.yaml: | tasks: task-plugins: @@ -619,15 +473,9 @@ data: region: us-east-1 container: "my-s3-bucket" - task_logs.yaml: | - # TODO this is used to enable log links in the UI. - plugins: - logs: - kubernetes-enabled: true - kubernetes-url: "http://localhost:30082" kind: ConfigMap metadata: - name: flyte-propeller-config-569h4b5459 + name: flyte-propeller-config-68bh2f6c6f namespace: flyte --- apiVersion: v1 @@ -640,20 +488,20 @@ metadata: type: Opaque --- apiVersion: v1 -kind: Service +kind: Secret metadata: - labels: - app: contour - name: contour + name: flyte-admin-auth namespace: flyte -spec: - ports: - - nodePort: 30081 - port: 80 - protocol: TCP - selector: - app: contour - type: NodePort +type: Opaque +--- +apiVersion: v1 +kind: Secret +metadata: + name: flyte-propeller-auth + namespace: flyte +stringData: + client_secret: foobar +type: Opaque --- apiVersion: v1 kind: Service @@ -677,6 +525,22 @@ spec: --- apiVersion: v1 kind: Service +metadata: + annotations: + projectcontour.io/upstream-protocol.h2c: grpc + name: flyte-pod-webhook + namespace: flyte +spec: + ports: + - name: https + port: 443 + protocol: TCP + targetPort: 9443 + selector: + app: flyte-pod-webhook +--- +apiVersion: v1 +kind: Service metadata: annotations: projectcontour.io/upstream-protocol.h2c: grpc @@ -724,88 +588,6 @@ spec: --- apiVersion: apps/v1 kind: Deployment -metadata: - labels: - app: contour - name: contour - namespace: flyte -spec: - replicas: 1 - selector: - matchLabels: - app: contour - template: - metadata: - annotations: - prometheus.io/format: prometheus - prometheus.io/path: /stats - prometheus.io/port: "8002" - prometheus.io/scrape: "true" - labels: - app: contour - spec: - containers: - - args: - - -c - - /config/contour.yaml - - --service-cluster - - cluster0 - - --service-node - - node0 - command: - - envoy - image: docker.io/envoyproxy/envoy-alpine:v1.6.0 - name: envoy-envoyingressv1 - ports: - - containerPort: 80 - name: http - - containerPort: 8002 - name: statsd - resources: - limits: - cpu: 100m - memory: 100Mi - requests: - cpu: 100m - memory: 100Mi - volumeMounts: - - mountPath: /config - name: contour-config - - args: - - serve - - --incluster - - --envoy-http-port=80 - - --debug-http-port=6069 - command: - - contour - image: gcr.io/heptio-images/contour:v0.6.1 - imagePullPolicy: Always - name: contour-unknown - ports: - - containerPort: 8000 - name: contour - dnsPolicy: ClusterFirst - initContainers: - - args: - - bootstrap - - /config/contour.yaml - - --statsd-enabled - command: - - contour - image: gcr.io/heptio-images/contour:v0.6.1 - imagePullPolicy: Always - name: envoy-initconfig - volumeMounts: - - mountPath: /config - name: contour-config - serviceAccountName: contour - terminationGracePeriodSeconds: 30 - volumes: - - emptyDir: {} - name: contour-config ---- -apiVersion: apps/v1 -kind: Deployment metadata: labels: app: datacatalog @@ -825,16 +607,15 @@ spec: labels: app: datacatalog app.kubernetes.io/name: datacatalog - app.kubernetes.io/version: 0.2.2 + app.kubernetes.io/version: 0.3.0 spec: containers: - command: - datacatalog - - --logtostderr - --config - /etc/datacatalog/config/*.yaml - serve - image: ghcr.io/lyft/datacatalog:v0.2.130 + image: ghcr.io/flyteorg/datacatalog:v0.3.2 imagePullPolicy: IfNotPresent name: datacatalog ports: @@ -848,12 +629,11 @@ spec: initContainers: - command: - datacatalog - - --logtostderr - --config - /etc/datacatalog/config/*.yaml - migrate - run - image: ghcr.io/lyft/datacatalog:v0.2.130 + image: ghcr.io/flyteorg/datacatalog:v0.3.2 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -874,6 +654,63 @@ spec: --- apiVersion: apps/v1 kind: Deployment +metadata: + labels: + app: flyte-pod-webhook + name: flyte-pod-webhook + namespace: flyte +spec: + selector: + matchLabels: + app: flyte-pod-webhook + template: + metadata: + annotations: + prometheus.io/path: /metrics + prometheus.io/port: "10254" + prometheus.io/scrape: "true" + labels: + app: flyte-pod-webhook + app.kubernetes.io/name: flyte-pod-webhook + app.kubernetes.io/version: 0.5.13 + spec: + containers: + - args: + - webhook + - --config + - /etc/flyte/config/*.yaml + command: + - flytepropeller + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: ghcr.io/flyteorg/flytepropeller:v0.10.6 + imagePullPolicy: IfNotPresent + name: webhook + volumeMounts: + - mountPath: /etc/flyte/config + name: config-volume + readOnly: true + - mountPath: /etc/webhook/certs + name: webhook-certs + readOnly: true + serviceAccountName: flyte-pod-webhook + volumes: + - configMap: + name: flyte-propeller-config-68bh2f6c6f + name: config-volume + - name: webhook-certs + secret: + secretName: flyte-pod-webhook +--- +apiVersion: apps/v1 +kind: Deployment metadata: labels: app: flyteadmin @@ -893,16 +730,15 @@ spec: labels: app: flyteadmin app.kubernetes.io/name: flyteadmin - app.kubernetes.io/version: 0.3.29 + app.kubernetes.io/version: 0.4.13 spec: containers: - command: - flyteadmin - - --logtostderr - --config - /etc/flyte/config/*.yaml - serve - image: ghcr.io/lyft/flyteadmin:v0.3.29 + image: ghcr.io/flyteorg/flyteadmin:v0.5.0 imagePullPolicy: IfNotPresent name: flyteadmin ports: @@ -920,6 +756,8 @@ spec: name: config-volume - mountPath: /etc/db name: db-pass + - mountPath: /etc/secrets/ + name: auth - command: - sh - -c @@ -949,12 +787,11 @@ spec: name: check-db-ready - command: - flyteadmin - - --logtostderr - --config - /etc/flyte/config/*.yaml - migrate - run - image: ghcr.io/lyft/flyteadmin:v0.3.29 + image: ghcr.io/flyteorg/flyteadmin:v0.5.0 imagePullPolicy: IfNotPresent name: run-migrations volumeMounts: @@ -964,14 +801,13 @@ spec: name: db-pass - command: - flyteadmin - - --logtostderr - --config - /etc/flyte/config/*.yaml - migrate - seed-projects - flytetester - flytesnacks - image: ghcr.io/lyft/flyteadmin:v0.3.29 + image: ghcr.io/flyteorg/flyteadmin:v0.5.0 imagePullPolicy: IfNotPresent name: seed-projects volumeMounts: @@ -981,12 +817,11 @@ spec: name: db-pass - command: - flyteadmin - - --logtostderr - --config - /etc/flyte/config/*.yaml - clusterresource - sync - image: ghcr.io/lyft/flyteadmin:v0.3.29 + image: ghcr.io/flyteorg/flyteadmin:v0.5.0 imagePullPolicy: IfNotPresent name: sync-cluster-resources volumeMounts: @@ -996,6 +831,22 @@ spec: name: config-volume - mountPath: /etc/db name: db-pass + - args: + - flyteadmin --config=etc/flyte/config/*.yaml secrets init --localPath /etc/secrets/auth && flyteadmin --config=/etc/flyte/config/*.yaml secrets create --fromPath /etc/secrets/auth + command: + - /bin/sh + - -c + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: ghcr.io/flyteorg/flyteadmin:v0.5.0 + imagePullPolicy: IfNotPresent + name: generate-secrets + volumeMounts: + - mountPath: /etc/flyte/config + name: config-volume serviceAccountName: flyteadmin volumes: - configMap: @@ -1004,11 +855,14 @@ spec: - emptyDir: {} name: shared-data - configMap: - name: flyte-admin-config-k7f4k7g76c + name: flyte-admin-config-t8ct89hdm5 name: config-volume - name: db-pass secret: secretName: db-pass-9dgchhk2bm + - name: auth + secret: + secretName: flyte-admin-auth --- apiVersion: apps/v1 kind: Deployment @@ -1030,7 +884,7 @@ spec: labels: app: flytepropeller app.kubernetes.io/name: flytepropeller - app.kubernetes.io/version: 0.5.13 + app.kubernetes.io/version: 0.7.1 spec: containers: - args: @@ -1043,7 +897,7 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name - image: ghcr.io/flyteorg/flytepropeller:v0.5.13 + image: ghcr.io/flyteorg/flytepropeller:v0.10.6 imagePullPolicy: IfNotPresent name: flytepropeller ports: @@ -1051,11 +905,16 @@ spec: volumeMounts: - mountPath: /etc/flyte/config name: config-volume + - mountPath: /etc/secrets/ + name: auth serviceAccountName: flytepropeller volumes: - configMap: - name: flyte-propeller-config-569h4b5459 + name: flyte-propeller-config-68bh2f6c6f name: config-volume + - name: auth + secret: + secretName: flyte-propeller-auth --- apiVersion: apps/v1 kind: Deployment @@ -1080,7 +939,7 @@ spec: value: minio - name: MINIO_SECRET_KEY value: miniostorage - image: minio/minio:RELEASE.2020-12-16T05-05-17Z + image: minio/minio:RELEASE.2021-04-06T23-11-00Z name: minio ports: - containerPort: 9000 @@ -1088,6 +947,15 @@ spec: volumeMounts: - mountPath: /data name: minio-storage + initContainers: + - command: + - mkdir + - /data/my-s3-bucket + image: minio/minio:RELEASE.2021-04-06T23-11-00Z + name: create-bucket + volumeMounts: + - mountPath: /data + name: minio-storage volumes: - emptyDir: {} name: minio-storage @@ -1122,47 +990,47 @@ spec: - emptyDir: {} name: postgres-storage --- -apiVersion: extensions/v1beta1 -kind: Ingress +apiVersion: batch/v1 +kind: Job metadata: - annotations: - nginx.ingress.kubernetes.io/ssl-redirect: "false" - name: flytesystem + name: flyte-pod-webhook-secret namespace: flyte spec: - rules: - - http: - paths: - - backend: - serviceName: flyteconsole - servicePort: 80 - path: /console - - backend: - serviceName: flyteconsole - servicePort: 80 - path: /__webpack_hmr - - backend: - serviceName: flyteadmin - servicePort: 80 - path: /api - - backend: - serviceName: flyteadmin - servicePort: 80 - path: /healthcheck - - backend: - serviceName: flyteadmin - servicePort: 80 - path: /v1 - - backend: - serviceName: flyteadmin - servicePort: 81 - path: /flyteidl.service.AdminService - - backend: - serviceName: flyteadmin - servicePort: 87 - path: /openapi + backoffLimit: 3 + template: + spec: + containers: + - args: + - webhook + - init-certs + - --config + - /etc/flyte/config/*.yaml + command: + - flytepropeller + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: ghcr.io/flyteorg/flytepropeller:v0.10.6 + imagePullPolicy: IfNotPresent + name: webhook + volumeMounts: + - mountPath: /etc/flyte/config + name: config-volume + restartPolicy: Never + serviceAccountName: flyte-pod-webhook + volumes: + - configMap: + name: flyte-propeller-config-68bh2f6c6f + name: config-volume + ttlSecondsAfterFinished: 0 --- -apiVersion: extensions/v1beta1 +apiVersion: networking.k8s.io/v1 kind: Ingress metadata: annotations: @@ -1174,6 +1042,9 @@ spec: - http: paths: - backend: - serviceName: minio - servicePort: 9000 + service: + name: minio + port: + number: 9000 path: /minio + pathType: ImplementationSpecific diff --git a/doc-requirements.in b/doc-requirements.in index d84752e50a..6d288ac2e7 100644 --- a/doc-requirements.in +++ b/doc-requirements.in @@ -1,6 +1,7 @@ +git+git://github.com/flyteorg/furo@main +readthedocs-sphinx-search sphinx sphinx-prompt -sphinx-material sphinx-code-include sphinx-autoapi sphinx-copybutton diff --git a/doc-requirements.txt b/doc-requirements.txt index b7e13e0c63..cd18b68296 100644 --- a/doc-requirements.txt +++ b/doc-requirements.txt @@ -6,22 +6,22 @@ # alabaster==0.7.12 # via sphinx -astroid==2.4.2 +astroid==2.5.6 # via sphinx-autoapi -babel==2.9.0 +babel==2.9.1 # via sphinx beautifulsoup4==4.9.3 # via + # furo # sphinx-code-include - # sphinx-material certifi==2020.12.5 # via requests chardet==4.0.0 # via requests -css-html-js-minify==2.5.5 - # via sphinx-material docutils==0.16 # via sphinx +git+git://github.com/flyteorg/furo@main + # via -r doc-requirements.in idna==2.10 # via requests imagesize==1.2.0 @@ -30,38 +30,36 @@ jinja2==2.11.3 # via # sphinx # sphinx-autoapi -lazy-object-proxy==1.4.3 +lazy-object-proxy==1.6.0 # via astroid -lxml==4.6.2 - # via sphinx-material markupsafe==1.1.1 # via jinja2 packaging==20.9 # via sphinx -pygments==2.7.4 +pygments==2.8.1 # via # sphinx # sphinx-prompt # sphinx-tabs pyparsing==2.4.7 # via packaging -python-slugify[unidecode]==4.0.1 - # via sphinx-material pytz==2021.1 # via babel pyyaml==5.4.1 # via sphinx-autoapi +readthedocs-sphinx-search==0.1.0 + # via -r doc-requirements.in requests==2.25.1 # via sphinx six==1.15.0 # via - # astroid # sphinx-code-include + # sphinxext-remoteliteralinclude snowballstemmer==2.1.0 # via sphinx -soupsieve==2.2 +soupsieve==2.2.1 # via beautifulsoup4 -sphinx-autoapi==1.7.0 +sphinx-autoapi==1.8.1 # via -r doc-requirements.in sphinx-code-include==1.1.1 # via -r doc-requirements.in @@ -69,20 +67,18 @@ sphinx-copybutton==0.3.1 # via -r doc-requirements.in sphinx-issues==1.2.0 # via -r doc-requirements.in -sphinx-material==0.0.32 - # via -r doc-requirements.in -sphinx-prompt==1.3.0 +sphinx-prompt==1.4.0 # via -r doc-requirements.in -sphinx-tabs==2.0.1 +sphinx-tabs==2.1.0 # via -r doc-requirements.in -sphinx==3.4.3 +sphinx==3.5.4 # via # -r doc-requirements.in + # furo # sphinx-autoapi # sphinx-code-include # sphinx-copybutton # sphinx-issues - # sphinx-material # sphinx-prompt # sphinx-tabs # sphinxext-remoteliteralinclude @@ -98,15 +94,11 @@ sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.4 # via sphinx -sphinxext-remoteliteralinclude==0.1.0 +sphinxext-remoteliteralinclude==0.2.0 # via -r doc-requirements.in -text-unidecode==1.3 - # via python-slugify unidecode==1.2.0 - # via - # python-slugify - # sphinx-autoapi -urllib3==1.26.3 + # via sphinx-autoapi +urllib3==1.26.4 # via requests wrapt==1.12.1 # via astroid diff --git a/docker/sandbox/Dockerfile b/docker/sandbox/Dockerfile index 7211cb74fd..3d64453538 100644 --- a/docker/sandbox/Dockerfile +++ b/docker/sandbox/Dockerfile @@ -14,6 +14,9 @@ RUN git clone -b ${BUILDKIT_CLI_FOR_KUBECTL_VERSION} --single-branch --depth 1 h FROM alpine:3.13.2 AS base_ +# Install dependencies +RUN apk add --no-cache curl + # Make directory to store artifacts RUN mkdir -p /flyteorg/bin /flyteorg/share @@ -25,13 +28,16 @@ RUN wget -q -O /flyteorg/bin/k3s https://github.com/k3s-io/k3s/releases/download # Install buildkit-cli-for-kubectl COPY --from=go_builder_ /install/linux/ /flyteorg/bin/ -# Download flyte manifest +# Install flytectl +RUN curl https://raw.githubusercontent.com/flyteorg/flytectl/master/install.sh | sh + +# Copy flyte manifest COPY deployment/sandbox/flyte_generated.yaml /flyteorg/share/flyte_generated.yaml # Copy scripts +RUN cp /bin/flytectl /flyteorg/bin/ COPY docker/sandbox/kubectl docker/sandbox/wait-for-flyte.sh /flyteorg/bin/ - FROM base_ AS default # Install dependencies @@ -39,6 +45,7 @@ RUN apk add --no-cache bash git make tini # Copy entrypoints COPY docker/sandbox/flyte-entrypoint-default.sh /flyteorg/bin/flyte-entrypoint.sh +COPY docker/sandbox/bashrc /root/.bashrc # Update PATH variable ENV PATH "/flyteorg/bin:${PATH}" @@ -50,7 +57,7 @@ VOLUME /var/lib/cni VOLUME /var/log # Expose Flyte ports -EXPOSE 30081 30084 +EXPOSE 30081 30082 30084 ENTRYPOINT ["tini", "flyte-entrypoint.sh"] @@ -76,6 +83,6 @@ VOLUME /var/lib/cni VOLUME /var/log # Expose Flyte ports -EXPOSE 30081 30084 +EXPOSE 30081 30082 30084 ENTRYPOINT ["tini", "flyte-entrypoint.sh"] diff --git a/docker/sandbox/bashrc b/docker/sandbox/bashrc new file mode 100644 index 0000000000..550a1d4e9e --- /dev/null +++ b/docker/sandbox/bashrc @@ -0,0 +1,7 @@ +# Friendlier prompt +export PS1="\e[0;35m\u@flyte-sandbox \w> \e[m" + +# Useful aliases +alias k="kubectl" +alias kf="kubectl -n flyte" +alias l="ls -lthra" diff --git a/docker/sandbox/flyte-entrypoint-default.sh b/docker/sandbox/flyte-entrypoint-default.sh index ce1cc8a0c0..5b5c3ceb1d 100755 --- a/docker/sandbox/flyte-entrypoint-default.sh +++ b/docker/sandbox/flyte-entrypoint-default.sh @@ -6,7 +6,8 @@ trap 'pkill -P $$' EXIT # Start k3s echo "Starting k3s cluster..." -k3s server --no-deploy=traefik --no-deploy=servicelb --no-deploy=local-storage --no-deploy=metrics-server &> /var/log/k3s.log & +KUBERNETES_API_PORT=${KUBERNETES_API_PORT:-6443} +k3s server --no-deploy=traefik --no-deploy=servicelb --no-deploy=local-storage --no-deploy=metrics-server --https-listen-port=${KUBERNETES_API_PORT} &> /var/log/k3s.log & K3S_PID=$! timeout 600 sh -c "until k3s kubectl explain deployment &> /dev/null; do sleep 1; done" || ( echo >&2 "Timed out while waiting for the Kubernetes cluster to start"; exit 1 ) echo "Done." diff --git a/docker/sandbox/flyte-entrypoint-dind.sh b/docker/sandbox/flyte-entrypoint-dind.sh index 39b7d7ce48..9db5613c34 100755 --- a/docker/sandbox/flyte-entrypoint-dind.sh +++ b/docker/sandbox/flyte-entrypoint-dind.sh @@ -23,7 +23,8 @@ echo "Done." # Start k3s echo "Starting k3s cluster..." -k3s server --docker --no-deploy=traefik --no-deploy=servicelb --no-deploy=local-storage --no-deploy=metrics-server &> /var/log/k3s.log & +KUBERNETES_API_PORT=${KUBERNETES_API_PORT:-6443} +k3s server --docker --no-deploy=traefik --no-deploy=servicelb --no-deploy=local-storage --no-deploy=metrics-server --https-listen-port=${KUBERNETES_API_PORT} &> /var/log/k3s.log & K3S_PID=$! timeout 600 sh -c "until k3s kubectl explain deployment &> /dev/null; do sleep 1; done" || ( echo >&2 "Timed out while waiting for the Kubernetes cluster to start"; exit 1 ) echo "Done." diff --git a/docker/sandbox/wait-for-flyte.sh b/docker/sandbox/wait-for-flyte.sh index a5266e090f..c99e722944 100755 --- a/docker/sandbox/wait-for-flyte.sh +++ b/docker/sandbox/wait-for-flyte.sh @@ -16,6 +16,6 @@ timeout 600 sh -c "until k3s kubectl get namespace flyte &> /dev/null; do sleep k3s kubectl wait --for=condition=available deployment/datacatalog deployment/flyteadmin deployment/flyteconsole deployment/flytepropeller -n flyte --timeout=10m || ( echo >&2 "Timed out while waiting for the Flyte deployment to start"; exit 1 ) # Wait for envoy proxy to become ready -timeout 600 sh -c 'until [[ $(k3s kubectl get daemonset envoy -n flyte -o jsonpath="{.status.numberReady}") -eq 1 ]]; do sleep 1; done' || ( echo >&2 "Timed out while waiting for the Flyte envoy proxy to start"; exit 1 ) +timeout 600 sh -c 'until [[ $(k3s kubectl get daemonset envoy -n projectcontour -o jsonpath="{.status.numberReady}") -eq 1 ]]; do sleep 1; done' || ( echo >&2 "Timed out while waiting for the Flyte envoy proxy to start"; exit 1 ) echo "Flyte is ready! Flyte UI is available at http://localhost:30081/console." diff --git a/end2end/execute.sh b/end2end/execute.sh index e03a3e25de..a59c312390 100755 --- a/end2end/execute.sh +++ b/end2end/execute.sh @@ -35,8 +35,6 @@ function wait_for_flyte_deploys() { echo "" kubectl -n flyte rollout status deployment minio echo "" - kubectl -n flyte rollout status deployment contour - echo "" echo "Flyte deployed in $SECONDS seconds." } diff --git a/end2end/tests/endtoend.yaml b/end2end/tests/endtoend.yaml index 82aa687985..7cd2cb6a50 100644 --- a/end2end/tests/endtoend.yaml +++ b/end2end/tests/endtoend.yaml @@ -11,7 +11,7 @@ spec: command: - bash - -c - image: ghcr.io/flyteorg/flytetools:85ab24ff1025327a02cbd570ab96bbdded132ce9 + image: ghcr.io/flyteorg/flytetools:2638d4444add18096a688bc458d963cd3997e52e imagePullPolicy: IfNotPresent name: flytetester resources: diff --git a/kustomize/README.md b/kustomize/README.md index 4189f9a146..94f0f6ec56 100644 --- a/kustomize/README.md +++ b/kustomize/README.md @@ -1,5 +1,5 @@ # Install Flyte using Kustomize -Flyte can be deployed to a kubernetes cluster using a generated deployment yaml file. This file is generated using [Kustomize](https://kubectl.docs.kubernetes.io/pages/reference/kustomize.html). +Flyte can be deployed to a kubernetes cluster using a generated deployment yaml file. This file is generated using [Kustomize](https://kubectl.docs.kubernetes.io/guides/introduction/kustomize/). Please refer to Kustomize documentation to see how it works. In brief, Kustomize allows composing a deployment yaml using multiple components. In Flyte all the core components are listed under [Base Components](./base). The Base components also consist of diff --git a/kustomize/overlays/gcp/dependencies/cloudsqlproxy/deployment.yaml b/kustomize/base/addons/cloudsqlproxy/deployment.yaml similarity index 100% rename from kustomize/overlays/gcp/dependencies/cloudsqlproxy/deployment.yaml rename to kustomize/base/addons/cloudsqlproxy/deployment.yaml diff --git a/kustomize/overlays/gcp/dependencies/cloudsqlproxy/kustomization.yaml b/kustomize/base/addons/cloudsqlproxy/kustomization.yaml similarity index 100% rename from kustomize/overlays/gcp/dependencies/cloudsqlproxy/kustomization.yaml rename to kustomize/base/addons/cloudsqlproxy/kustomization.yaml diff --git a/kustomize/overlays/gcp/dependencies/cloudsqlproxy/service.yaml b/kustomize/base/addons/cloudsqlproxy/service.yaml similarity index 100% rename from kustomize/overlays/gcp/dependencies/cloudsqlproxy/service.yaml rename to kustomize/base/addons/cloudsqlproxy/service.yaml diff --git a/kustomize/base/addons/contour_ingress_controller/kustomization.yaml b/kustomize/base/addons/contour_ingress_controller/kustomization.yaml new file mode 100644 index 0000000000..e75e137804 --- /dev/null +++ b/kustomize/base/addons/contour_ingress_controller/kustomization.yaml @@ -0,0 +1,4 @@ +resources: +- https://raw.githubusercontent.com/projectcontour/contour/release-1.13/examples/render/contour.yaml +patchesStrategicMerge: +- service.yaml diff --git a/kustomize/overlays/sandbox/dependencies/contour_ingress_controller/service.yaml b/kustomize/base/addons/contour_ingress_controller/service.yaml similarity index 60% rename from kustomize/overlays/sandbox/dependencies/contour_ingress_controller/service.yaml rename to kustomize/base/addons/contour_ingress_controller/service.yaml index 79e68127b1..a848798e11 100644 --- a/kustomize/overlays/sandbox/dependencies/contour_ingress_controller/service.yaml +++ b/kustomize/base/addons/contour_ingress_controller/service.yaml @@ -2,15 +2,13 @@ apiVersion: v1 kind: Service metadata: - name: contour - labels: - app: contour + name: envoy + namespace: projectcontour spec: # use NodePort to make sure the service is accessible type: NodePort - selector: - app: contour ports: - - protocol: TCP - port: 80 + - port: 80 + name: http + protocol: TCP nodePort: 30081 diff --git a/kustomize/overlays/sandbox/dependencies/database/database.yaml b/kustomize/base/addons/database/database.yaml similarity index 100% rename from kustomize/overlays/sandbox/dependencies/database/database.yaml rename to kustomize/base/addons/database/database.yaml diff --git a/kustomize/overlays/sandbox/dependencies/database/kustomization.yaml b/kustomize/base/addons/database/kustomization.yaml similarity index 100% rename from kustomize/overlays/sandbox/dependencies/database/kustomization.yaml rename to kustomize/base/addons/database/kustomization.yaml diff --git a/kustomize/base/addons/kubernetes_dashboard/clusterrolebinding.yaml b/kustomize/base/addons/kubernetes_dashboard/clusterrolebinding.yaml new file mode 100644 index 0000000000..76fd3fc503 --- /dev/null +++ b/kustomize/base/addons/kubernetes_dashboard/clusterrolebinding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: kubernetes-dashboard-admin +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: +- kind: ServiceAccount + name: kubernetes-dashboard + namespace: kubernetes-dashboard diff --git a/kustomize/base/addons/kubernetes_dashboard/deployment.yaml b/kustomize/base/addons/kubernetes_dashboard/deployment.yaml new file mode 100644 index 0000000000..49299fec08 --- /dev/null +++ b/kustomize/base/addons/kubernetes_dashboard/deployment.yaml @@ -0,0 +1,15 @@ +kind: Deployment +apiVersion: apps/v1 +metadata: + name: kubernetes-dashboard + namespace: kubernetes-dashboard +spec: + template: + spec: + containers: + - name: kubernetes-dashboard + args: + - --namespace=kubernetes-dashboard + - --enable-insecure-login + - --enable-skip-login + - --disable-settings-authorizer diff --git a/kustomize/base/addons/kubernetes_dashboard/kustomization.yaml b/kustomize/base/addons/kubernetes_dashboard/kustomization.yaml new file mode 100644 index 0000000000..22384dfe0f --- /dev/null +++ b/kustomize/base/addons/kubernetes_dashboard/kustomization.yaml @@ -0,0 +1,7 @@ +resources: + # TODO (jeev): Figure out how to rev this automatically +- https://raw.githubusercontent.com/kubernetes/dashboard/v2.2.0/aio/deploy/alternative.yaml +- clusterrolebinding.yaml +patchesStrategicMerge: +- deployment.yaml +- service.yaml diff --git a/kustomize/base/addons/kubernetes_dashboard/service.yaml b/kustomize/base/addons/kubernetes_dashboard/service.yaml new file mode 100644 index 0000000000..c08a9308c8 --- /dev/null +++ b/kustomize/base/addons/kubernetes_dashboard/service.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: kubernetes-dashboard + namespace: kubernetes-dashboard +spec: + type: NodePort + ports: + - port: 80 + name: http + protocol: TCP + nodePort: 30082 diff --git a/kustomize/overlays/eks/dependencies/redis/deployment.yaml b/kustomize/base/addons/redis/deployment.yaml similarity index 100% rename from kustomize/overlays/eks/dependencies/redis/deployment.yaml rename to kustomize/base/addons/redis/deployment.yaml diff --git a/kustomize/overlays/eks/dependencies/redis/kustomization.yaml b/kustomize/base/addons/redis/kustomization.yaml similarity index 100% rename from kustomize/overlays/eks/dependencies/redis/kustomization.yaml rename to kustomize/base/addons/redis/kustomization.yaml diff --git a/kustomize/overlays/eks/dependencies/redis/service.yaml b/kustomize/base/addons/redis/service.yaml similarity index 100% rename from kustomize/overlays/eks/dependencies/redis/service.yaml rename to kustomize/base/addons/redis/service.yaml diff --git a/kustomize/overlays/sandbox/dependencies/storage/kustomization.yaml b/kustomize/base/addons/storage/kustomization.yaml similarity index 100% rename from kustomize/overlays/sandbox/dependencies/storage/kustomization.yaml rename to kustomize/base/addons/storage/kustomization.yaml diff --git a/kustomize/base/addons/storage/storage.yaml b/kustomize/base/addons/storage/storage.yaml new file mode 100644 index 0000000000..132a338cb8 --- /dev/null +++ b/kustomize/base/addons/storage/storage.yaml @@ -0,0 +1,75 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: minio + namespace: flyte +spec: + selector: + matchLabels: + app: minio + template: + metadata: + labels: + app: minio + spec: + volumes: + - name: minio-storage + emptyDir: {} + initContainers: + - image: minio/minio + name: create-bucket + command: + - mkdir + - /data/my-s3-bucket + volumeMounts: + - name: minio-storage + mountPath: /data + containers: + - image: minio/minio + name: minio + env: + - name: MINIO_ACCESS_KEY + value: minio + - name: MINIO_SECRET_KEY + value: miniostorage + args: + - server + - /data + ports: + - containerPort: 9000 + name: minio + volumeMounts: + - name: minio-storage + mountPath: /data +--- +apiVersion: v1 +kind: Service +metadata: + name: minio + namespace: flyte +spec: + externalName: minio + ports: + - port: 9000 + selector: + app: minio +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: minio + namespace: flyte + annotations: + nginx.ingress.kubernetes.io/ssl-redirect: "false" +spec: + rules: + - http: + paths: + - path: /minio + pathType: ImplementationSpecific + backend: + service: + name: minio + port: + number: 9000 diff --git a/kustomize/base/admindeployment/auth_secret.yaml b/kustomize/base/admindeployment/auth_secret.yaml new file mode 100644 index 0000000000..9a1281d776 --- /dev/null +++ b/kustomize/base/admindeployment/auth_secret.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Secret +metadata: + name: flyte-admin-auth + namespace: flyte +type: Opaque diff --git a/kustomize/base/admindeployment/clustersync/cron.yaml b/kustomize/base/admindeployment/clustersync/cron.yaml index 7651a93e0b..e3fbfdfa69 100644 --- a/kustomize/base/admindeployment/clustersync/cron.yaml +++ b/kustomize/base/admindeployment/clustersync/cron.yaml @@ -12,9 +12,9 @@ spec: serviceAccountName: flyteadmin containers: - name: sync-cluster-resources - image: flyteadmin:v0.3.29 + image: flyteadmin:v0.4.13 imagePullPolicy: IfNotPresent - command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/*.yaml", "clusterresource", "sync"] + command: ["flyteadmin", "--config", "/etc/flyte/config/*.yaml", "clusterresource", "sync"] volumeMounts: - name: resource-templates mountPath: /etc/flyte/clusterresource/templates diff --git a/kustomize/base/admindeployment/deployment.yaml b/kustomize/base/admindeployment/deployment.yaml index b7c476ccff..1c21394c91 100644 --- a/kustomize/base/admindeployment/deployment.yaml +++ b/kustomize/base/admindeployment/deployment.yaml @@ -16,7 +16,7 @@ spec: labels: app: flyteadmin app.kubernetes.io/name: flyteadmin - app.kubernetes.io/version: 0.3.29 + app.kubernetes.io/version: 0.4.13 annotations: prometheus.io/scrape: "true" prometheus.io/port: "10254" @@ -24,81 +24,126 @@ spec: spec: serviceAccountName: flyteadmin volumes: - - name: shared-data - emptyDir: {} - - name: config-volume - configMap: - name: flyte-admin-config - - name: resource-templates - configMap: - name: clusterresource-template - - name: db-pass - secret: - secretName: db-pass - initContainers: - - name: run-migrations - image: flyteadmin:v0.3.29 - imagePullPolicy: IfNotPresent - command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/*.yaml", "migrate", "run"] - volumeMounts: - - name: config-volume - mountPath: /etc/flyte/config - - name: db-pass - mountPath: /etc/db - # Optional, These just seed the project - TODO move them to only - - name: seed-projects - image: flyteadmin:v0.3.29 - imagePullPolicy: IfNotPresent - command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/*.yaml", - "migrate", "seed-projects", "flytesnacks", "flytetester", "flyteexamples"] - volumeMounts: - - name: config-volume - mountPath: /etc/flyte/config - - name: db-pass - mountPath: /etc/db - - name: sync-cluster-resources - image: flyteadmin:v0.3.29 - imagePullPolicy: IfNotPresent - command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/*.yaml", "clusterresource", "sync"] - volumeMounts: - - name: resource-templates - mountPath: /etc/flyte/clusterresource/templates - - name: config-volume - mountPath: /etc/flyte/config - - name: db-pass - mountPath: /etc/db - containers: - - name: flyteadmin - image: flyteadmin:v0.3.29 - imagePullPolicy: IfNotPresent - command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/*.yaml", "serve"] - ports: - - containerPort: 8088 - - containerPort: 8089 - volumeMounts: - name: shared-data - mountPath: /srv/flyte + emptyDir: {} - name: config-volume - mountPath: /etc/flyte/config + configMap: + name: flyte-admin-config + - name: resource-templates + configMap: + name: clusterresource-template - name: db-pass - mountPath: /etc/db - - name: redoc - image: docker.io/redocly/redoc - imagePullPolicy: IfNotPresent - ports: - - containerPort: 8087 - resources: - limits: - memory: "200Mi" - cpu: "0.1" - command: - - sh - - -c - - ln -s /usr/share/nginx/html /usr/share/nginx/html/openapi && sh /usr/local/bin/docker-run.sh - env: - - name: PAGE_TITLE - value: "Flyte Admin OpenAPI" - - name: SPEC_URL - value: "/api/v1/openapi" - - name: PORT - value: "8087" + secret: + secretName: db-pass + - name: auth + secret: + secretName: flyte-admin-auth + initContainers: + - name: run-migrations + image: flyteadmin:v0.4.0 + imagePullPolicy: IfNotPresent + command: + [ + "flyteadmin", + "--config", + "/etc/flyte/config/*.yaml", + "migrate", + "run", + ] + volumeMounts: + - name: config-volume + mountPath: /etc/flyte/config + - name: db-pass + mountPath: /etc/db + # Optional, These just seed the project - TODO move them to only + - name: seed-projects + image: flyteadmin:v0.4.0 + imagePullPolicy: IfNotPresent + command: + [ + "flyteadmin", + "--config", + "/etc/flyte/config/*.yaml", + "migrate", + "seed-projects", + "flytesnacks", + "flytetester", + "flyteexamples", + ] + volumeMounts: + - name: config-volume + mountPath: /etc/flyte/config + - name: db-pass + mountPath: /etc/db + - name: sync-cluster-resources + image: flyteadmin:v0.4.0 + imagePullPolicy: IfNotPresent + command: + [ + "flyteadmin", + "--config", + "/etc/flyte/config/*.yaml", + "clusterresource", + "sync", + ] + volumeMounts: + - name: resource-templates + mountPath: /etc/flyte/clusterresource/templates + - name: config-volume + mountPath: /etc/flyte/config + - name: db-pass + mountPath: /etc/db + - name: generate-secrets + image: flyteadmin:v0.4.0 + imagePullPolicy: IfNotPresent + command: ["/bin/sh", "-c"] + args: + [ + "flyteadmin --config=etc/flyte/config/*.yaml secrets init --localPath /etc/secrets/auth && flyteadmin --config=/etc/flyte/config/*.yaml secrets create --fromPath /etc/secrets/auth", + ] + volumeMounts: + - name: config-volume + mountPath: /etc/flyte/config + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + containers: + - name: flyteadmin + image: flyteadmin:v0.4.0 + imagePullPolicy: IfNotPresent + command: + ["flyteadmin", "--config", "/etc/flyte/config/*.yaml", "serve"] + ports: + - containerPort: 8088 + - containerPort: 8089 + volumeMounts: + - name: shared-data + mountPath: /srv/flyte + - name: config-volume + mountPath: /etc/flyte/config + - name: db-pass + mountPath: /etc/db + - name: auth + mountPath: /etc/secrets/ + - name: redoc + image: docker.io/redocly/redoc + imagePullPolicy: IfNotPresent + ports: + - containerPort: 8087 + resources: + limits: + memory: "200Mi" + cpu: "0.1" + command: + - sh + - -c + - ln -s /usr/share/nginx/html /usr/share/nginx/html/openapi && sh /usr/local/bin/docker-run.sh + env: + - name: PAGE_TITLE + value: "Flyte Admin OpenAPI" + - name: SPEC_URL + value: "/api/v1/openapi" + - name: PORT + value: "8087" diff --git a/kustomize/base/admindeployment/kustomization.yaml b/kustomize/base/admindeployment/kustomization.yaml index a944d005ca..5c3fedd15b 100644 --- a/kustomize/base/admindeployment/kustomization.yaml +++ b/kustomize/base/admindeployment/kustomization.yaml @@ -1,3 +1,4 @@ resources: +- auth_secret.yaml - deployment.yaml - service.yaml diff --git a/kustomize/base/datacatalog/deployment.yaml b/kustomize/base/datacatalog/deployment.yaml index 11d7febc0c..ebed3d6708 100644 --- a/kustomize/base/datacatalog/deployment.yaml +++ b/kustomize/base/datacatalog/deployment.yaml @@ -15,7 +15,7 @@ spec: labels: app: datacatalog app.kubernetes.io/name: datacatalog - app.kubernetes.io/version: 0.2.2 + app.kubernetes.io/version: 0.3.0 annotations: prometheus.io/scrape: "true" prometheus.io/port: "10254" @@ -33,9 +33,9 @@ spec: secretName: db-pass initContainers: - name: run-migrations - image: datacatalog:v0.2.2 + image: datacatalog:v0.3.0 imagePullPolicy: IfNotPresent - command: ["datacatalog", "--logtostderr", "--config", "/etc/datacatalog/config/*.yaml", "migrate", "run"] + command: ["datacatalog", "--config", "/etc/datacatalog/config/*.yaml", "migrate", "run"] volumeMounts: - name: config-volume mountPath: /etc/datacatalog/config @@ -43,9 +43,9 @@ spec: mountPath: /etc/db containers: - name: datacatalog - image: datacatalog:v0.2.2 + image: datacatalog:v0.3.0 imagePullPolicy: IfNotPresent - command: ["datacatalog", "--logtostderr", "--config", "/etc/datacatalog/config/*.yaml", "serve"] + command: ["datacatalog", "--config", "/etc/datacatalog/config/*.yaml", "serve"] ports: - containerPort: 8088 - containerPort: 8089 diff --git a/kustomize/base/ingress/ingress.yaml b/kustomize/base/ingress/ingress.yaml index 431cbede0a..9b0f257f0d 100644 --- a/kustomize/base/ingress/ingress.yaml +++ b/kustomize/base/ingress/ingress.yaml @@ -1,4 +1,4 @@ -apiVersion: extensions/v1beta1 +apiVersion: networking.k8s.io/v1 kind: Ingress metadata: name: flytesystem @@ -7,38 +7,172 @@ metadata: nginx.ingress.kubernetes.io/ssl-redirect: "false" spec: rules: - - http: - paths: - # NOTE: If you change this, you must update the BASE_URL value in flyteconsole.yaml - - path: /console - backend: - serviceName: flyteconsole - servicePort: 80 - # This is useful only for sandbox mode and should be templatized/removed in non-sandbox environments - - path: /__webpack_hmr - backend: - serviceName: flyteconsole - servicePort: 80 - - path: /api - backend: - serviceName: flyteadmin - servicePort: 80 - - path: /healthcheck - backend: - serviceName: flyteadmin - servicePort: 80 - - path: /v1 - backend: - serviceName: flyteadmin - servicePort: 80 - # NOTE: Port 81 in flyteadmin is the GRPC server port for - # FlyteAdmin. - - path: /flyteidl.service.AdminService - backend: - serviceName: flyteadmin - servicePort: 81 - # Port 87 in FlyteAdmin maps to the redoc container. - - path: /openapi - backend: - serviceName: flyteadmin - servicePort: 87 + - http: + paths: + # This is useful only for sandbox mode and should be templatized/removed in non-sandbox environments + - path: /__webpack_hmr + pathType: ImplementationSpecific + backend: + service: + name: flyteconsole + port: + number: 80 + # NOTE: Port 81 in flyteadmin is the GRPC server port for + # FlyteAdmin. + - path: /flyteidl.service.AdminService + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 81 + - path: /flyteidl.service.AuthMetadataService + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 81 + - path: /flyteidl.service.IdentityService + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 81 + # Port 87 in FlyteAdmin maps to the redoc container. + - path: /openapi + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 87 + # NOTE: If you change this, you must update the BASE_URL value in flyteconsole.yaml + - path: /console + pathType: ImplementationSpecific + backend: + service: + name: flyteconsole + port: + number: 80 + - path: /console/* + pathType: ImplementationSpecific + backend: + service: + name: flyteconsole + port: + number: 80 + - path: /api + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 80 + - path: /api/* + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 80 + - path: /healthcheck + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 80 + - path: /v1/* + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 80 + # Port 87 in FlyteAdmin maps to the redoc container. + - path: /openapi/* + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 80 + - path: /.well-known + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 80 + - path: /.well-known/* + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 80 + - path: /login + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 80 + - path: /login/* + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 80 + - path: /logout + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 80 + - path: /logout/* + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 80 + - path: /callback + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 80 + - path: /callback/* + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 80 + - path: /me + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 80 + - path: /config + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 80 + - path: /config/* + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 80 diff --git a/kustomize/base/ingress/kustomization.yaml b/kustomize/base/ingress/kustomization.yaml index 36e8c40ae5..14d8f3a54f 100644 --- a/kustomize/base/ingress/kustomization.yaml +++ b/kustomize/base/ingress/kustomization.yaml @@ -1,2 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + resources: -- ingress.yaml + - ingress.yaml diff --git a/kustomize/base/operators/kfoperators/namespace/kustomization.yaml b/kustomize/base/operators/kfoperators/namespace/kustomization.yaml new file mode 100644 index 0000000000..bf20f4df68 --- /dev/null +++ b/kustomize/base/operators/kfoperators/namespace/kustomization.yaml @@ -0,0 +1,2 @@ +resources: +- namespace.yaml diff --git a/kustomize/base/operators/kfoperators/namespace/namespace.yaml b/kustomize/base/operators/kfoperators/namespace/namespace.yaml new file mode 100644 index 0000000000..7a940e4673 --- /dev/null +++ b/kustomize/base/operators/kfoperators/namespace/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: kubeflow diff --git a/kustomize/base/operators/kfoperators/pytorch/kustomization.yaml b/kustomize/base/operators/kfoperators/pytorch/kustomization.yaml index 2cab6d5984..a75b8ad07c 100644 --- a/kustomize/base/operators/kfoperators/pytorch/kustomization.yaml +++ b/kustomize/base/operators/kfoperators/pytorch/kustomization.yaml @@ -1,3 +1,4 @@ bases: + - ../namespace - github.com/kubeflow/manifests/pytorch-job/pytorch-job-crds/base?ref=v1.0.0 - - github.com/kubeflow/manifests/pytorch-job/pytorch-operator/base?ref=v1.0.0 \ No newline at end of file + - github.com/kubeflow/manifests/pytorch-job/pytorch-operator/base?ref=v1.0.0 diff --git a/kustomize/base/operators/kfoperators/tensorflow/kustomization.yaml b/kustomize/base/operators/kfoperators/tensorflow/kustomization.yaml index 33b2883011..9d7d7b7def 100644 --- a/kustomize/base/operators/kfoperators/tensorflow/kustomization.yaml +++ b/kustomize/base/operators/kfoperators/tensorflow/kustomization.yaml @@ -1,3 +1,4 @@ bases: + - ../namespace - github.com/kubeflow/manifests/tf-training/tf-job-crds/base?ref=v1.0.0 - github.com/kubeflow/manifests/tf-training/tf-job-operator/base?ref=v1.0.0 diff --git a/kustomize/base/propeller/auth_secret.yaml b/kustomize/base/propeller/auth_secret.yaml new file mode 100644 index 0000000000..b6bbbe4a8c --- /dev/null +++ b/kustomize/base/propeller/auth_secret.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Secret +metadata: + name: flyte-propeller-auth + namespace: flyte +type: Opaque +stringData: + client_secret: foobar diff --git a/kustomize/base/propeller/deployment.yaml b/kustomize/base/propeller/deployment.yaml index da0b1179ed..c216295b60 100644 --- a/kustomize/base/propeller/deployment.yaml +++ b/kustomize/base/propeller/deployment.yaml @@ -15,7 +15,7 @@ spec: labels: app: flytepropeller app.kubernetes.io/name: flytepropeller - app.kubernetes.io/version: 0.5.13 + app.kubernetes.io/version: 0.7.1 annotations: prometheus.io/scrape: "true" prometheus.io/port: "10254" @@ -23,25 +23,30 @@ spec: spec: serviceAccountName: flytepropeller volumes: - - name: config-volume - configMap: - name: flyte-propeller-config - containers: - - name: flytepropeller - image: flytepropeller:v0.5.13 - command: - - flytepropeller - args: - - --config - - /etc/flyte/config/*.yaml - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - imagePullPolicy: IfNotPresent - ports: - - containerPort: 10254 - volumeMounts: - name: config-volume - mountPath: /etc/flyte/config + configMap: + name: flyte-propeller-config + - name: auth + secret: + secretName: flyte-propeller-auth + containers: + - name: flytepropeller + image: flytepropeller:v0.7.1 + command: + - flytepropeller + args: + - --config + - /etc/flyte/config/*.yaml + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + imagePullPolicy: IfNotPresent + ports: + - containerPort: 10254 + volumeMounts: + - name: config-volume + mountPath: /etc/flyte/config + - name: auth + mountPath: /etc/secrets/ diff --git a/kustomize/base/propeller/kustomization.yaml b/kustomize/base/propeller/kustomization.yaml index 3f090bdf48..d21d0b929c 100644 --- a/kustomize/base/propeller/kustomization.yaml +++ b/kustomize/base/propeller/kustomization.yaml @@ -1,3 +1,5 @@ resources: -- deployment.yaml -- rbac.yaml + - auth_secret.yaml + - deployment.yaml + - rbac.yaml + - webhook.yaml diff --git a/kustomize/base/propeller/rbac.yaml b/kustomize/base/propeller/rbac.yaml index 524740005e..951a48cc02 100644 --- a/kustomize/base/propeller/rbac.yaml +++ b/kustomize/base/propeller/rbac.yaml @@ -54,6 +54,7 @@ rules: - flyte.lyft.com resources: - flyteworkflows + - flyteworkflows/finalizers verbs: - get - list diff --git a/kustomize/base/propeller/webhook.yaml b/kustomize/base/propeller/webhook.yaml new file mode 100644 index 0000000000..23efff56d2 --- /dev/null +++ b/kustomize/base/propeller/webhook.yaml @@ -0,0 +1,161 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: flyte-pod-webhook-secret + namespace: flyte +spec: + backoffLimit: 3 + ttlSecondsAfterFinished: 0 + template: + spec: + serviceAccountName: flyte-pod-webhook + restartPolicy: Never + containers: + - name: webhook + image: flytepropeller:v0.5.13 + imagePullPolicy: IfNotPresent + command: + - flytepropeller + args: + - webhook + - init-certs + - --config + - /etc/flyte/config/*.yaml + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: config-volume + mountPath: /etc/flyte/config + volumes: + - name: config-volume + configMap: + name: flyte-propeller-config +--- +# Create the actual deployment +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flyte-pod-webhook + namespace: flyte + labels: + app: flyte-pod-webhook +spec: + selector: + matchLabels: + app: flyte-pod-webhook + template: + metadata: + labels: + app: flyte-pod-webhook + app.kubernetes.io/name: flyte-pod-webhook + app.kubernetes.io/version: 0.5.13 + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "10254" + prometheus.io/path: "/metrics" + spec: + serviceAccountName: flyte-pod-webhook + containers: + - name: webhook + image: flytepropeller:v0.5.13 + imagePullPolicy: IfNotPresent + command: + - flytepropeller + args: + - webhook + - --config + - /etc/flyte/config/*.yaml + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: config-volume + mountPath: /etc/flyte/config + readOnly: true + - name: webhook-certs + mountPath: /etc/webhook/certs + readOnly: true + volumes: + - name: config-volume + configMap: + name: flyte-propeller-config + - name: webhook-certs + secret: + secretName: flyte-pod-webhook +--- +# Service +apiVersion: v1 +kind: Service +metadata: + name: flyte-pod-webhook + namespace: flyte + annotations: + # This tells contour to use a H2 connection for the port associated + # with the name 'grpc' under spec/ports. + # For more information, refer to + # https://github.com/heptio/contour/blob/master/docs/annotations.md#contour-specific-service-annotations + # # Following this issue - the annotation was updated https://github.com/projectcontour/contour/issues/2092 + projectcontour.io/upstream-protocol.h2c: "grpc" +spec: + selector: + app: flyte-pod-webhook + ports: + - name: https + protocol: TCP + port: 443 + targetPort: 9443 +--- +# Create a ClusterRole for the webhook +# https://kubernetes.io/docs/admin/authorization/rbac/ +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: flyte-pod-webhook + namespace: flyte +rules: + - apiGroups: + - "*" + resources: + - mutatingwebhookconfigurations + - secrets + - pods + verbs: + - get + - create + - update + - patch +--- +# Create a Service Account for webhook +apiVersion: v1 +kind: ServiceAccount +metadata: + name: flyte-pod-webhook + namespace: flyte +--- +# Create a binding from Role -> ServiceAccount +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: flyte-pod-webhook + namespace: flyte +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: flyte-pod-webhook +subjects: + - kind: ServiceAccount + name: flyte-pod-webhook + namespace: flyte diff --git a/kustomize/base/single_cluster/headless/config/admin/remote_data.yaml b/kustomize/base/single_cluster/headless/config/admin/remote_data.yaml deleted file mode 100644 index 41719a0e66..0000000000 --- a/kustomize/base/single_cluster/headless/config/admin/remote_data.yaml +++ /dev/null @@ -1,5 +0,0 @@ -remoteData: - region: "us-east-1" - scheme: "local" - signedUrls: - durationMinutes: 3 diff --git a/kustomize/base/single_cluster/headless/config/admin/server.yaml b/kustomize/base/single_cluster/headless/config/admin/server.yaml index 253bf8c776..e13f3ad156 100644 --- a/kustomize/base/single_cluster/headless/config/admin/server.yaml +++ b/kustomize/base/single_cluster/headless/config/admin/server.yaml @@ -2,7 +2,9 @@ server: httpPort: 8088 grpcPort: 8089 security: + # Controls whether to serve requests over SSL/TLS. secure: false + # Controls whether to enforce authentication. Follow the guide in https://docs.flyte.org/ on how to setup authentication. useAuth: false allowCors: true allowedOrigins: @@ -10,13 +12,32 @@ server: - "*" allowedHeaders: - "Content-Type" + - "flyte-authorization" +auth: + authorizedUris: + # This should point at your public http Uri. + - https://localhost:30081 + # This will be used by internal services in the same namespace as flyteadmin + - http://flyteadmin:80 + # This will be used by internal services in the same cluster but different namespaces + - http://flyteadmin.flyte.svc.cluster.local:80 + + # Controls app authentication config + appAuth: + thirdPartyConfig: + flyteClient: + clientId: flytectl + redirectUri: https://localhost:53593/callback + scopes: + - offline + - all + # Controls user authentication + userAuth: + openId: + baseUrl: https://accounts.google.com + scopes: + - profile + - openid + clientId: 657465813211-6eog7ek7li5k7i7fvgv2921075063hpe.apps.googleusercontent.com flyteadmin: roleNameKey: "iam.amazonaws.com/role" - profilerPort: 10254 - metricsScope: "flyte:" - metadataStoragePrefix: - - "metadata" - - "admin" - eventVersion: 1 - testing: - host: http://flyteadmin diff --git a/kustomize/base/single_cluster/headless/config/propeller/admin.yaml b/kustomize/base/single_cluster/headless/config/propeller/admin.yaml index 55cf01b83b..63ee5c63fc 100644 --- a/kustomize/base/single_cluster/headless/config/propeller/admin.yaml +++ b/kustomize/base/single_cluster/headless/config/propeller/admin.yaml @@ -5,3 +5,4 @@ event: admin: endpoint: flyteadmin:81 insecure: true + useAuth: false diff --git a/kustomize/base/single_cluster/headless/config/propeller/core.yaml b/kustomize/base/single_cluster/headless/config/propeller/core.yaml index e1fce27b72..055e7a8840 100644 --- a/kustomize/base/single_cluster/headless/config/propeller/core.yaml +++ b/kustomize/base/single_cluster/headless/config/propeller/core.yaml @@ -31,3 +31,6 @@ propeller: type: bucket rate: 10 capacity: 100 +webhook: + certDir: /etc/webhook/certs + serviceName: flyte-pod-webhook diff --git a/kustomize/base/single_cluster/headless/config/propeller/plugins/copilot.yaml b/kustomize/base/single_cluster/headless/config/propeller/plugins/copilot.yaml index 09abe54a38..de03e858c6 100644 --- a/kustomize/base/single_cluster/headless/config/propeller/plugins/copilot.yaml +++ b/kustomize/base/single_cluster/headless/config/propeller/plugins/copilot.yaml @@ -2,5 +2,5 @@ plugins: k8s: co-pilot: name: "flyte-copilot-" - image: "ghcr.io/lyft/flyteplugins/flytecopilot:dc4bdbd61cac88a39a5ff43e40f026bdbc2c78a2" + image: "ghcr.io/flyteorg/flytecopilot:v0.5.28" start-timeout: "30s" diff --git a/kustomize/base/single_cluster/headless/config/propeller/plugins/task_logs.yaml b/kustomize/base/single_cluster/headless/config/propeller/plugins/task_logs.yaml deleted file mode 100644 index de402d2ed3..0000000000 --- a/kustomize/base/single_cluster/headless/config/propeller/plugins/task_logs.yaml +++ /dev/null @@ -1,5 +0,0 @@ -# TODO this is used to enable log links in the UI. -plugins: - logs: - kubernetes-enabled: true - kubernetes-url: "http://localhost:30082" diff --git a/kustomize/base/single_cluster/headless/kustomization.yaml b/kustomize/base/single_cluster/headless/kustomization.yaml index c9d371de72..d4cf4a7520 100644 --- a/kustomize/base/single_cluster/headless/kustomization.yaml +++ b/kustomize/base/single_cluster/headless/kustomization.yaml @@ -3,62 +3,60 @@ kind: Kustomization # All the resources that make up the deployment resources: -# global resources -- ../../ingress -- ../../admindeployment -- ../../datacatalog -- ../../wf_crd -- ../../propeller -- ../../adminserviceaccount + # global resources + - ../../namespace + - ../../admindeployment + - ../../datacatalog + - ../../wf_crd + - ../../propeller + - ../../adminserviceaccount configMapGenerator: -# the main admin configmap -- name: flyte-admin-config - files: - - ./config/admin/server.yaml - - ./config/admin/domain.yaml - - ./config/admin/db.yaml - - ./config/admin/cluster_resources.yaml - - ./config/admin/remote_data.yaml - - ./config/admin/task_resource_defaults.yaml - - ./config/common/storage.yaml - - ./config/common/logger.yaml + # the main admin configmap + - name: flyte-admin-config + files: + - ./config/admin/server.yaml + - ./config/admin/domain.yaml + - ./config/admin/db.yaml + - ./config/admin/cluster_resources.yaml + - ./config/admin/task_resource_defaults.yaml + - ./config/common/storage.yaml + - ./config/common/logger.yaml -# cluster resource templates -- name: clusterresource-template - files: -# Files are read in alphabetical order. To ensure that we create the namespace first, prefix the file name with "aa". - - ./config/clusterresource-templates/aa_namespace.yaml - - ./config/clusterresource-templates/ab_project-resource-quota.yaml - - ./config/clusterresource-templates/ac_project-copilot-dataconfig.yaml + # cluster resource templates + - name: clusterresource-template + files: + # Files are read in alphabetical order. To ensure that we create the namespace first, prefix the file name with "aa". + - ./config/clusterresource-templates/aa_namespace.yaml + - ./config/clusterresource-templates/ab_project-resource-quota.yaml + - ./config/clusterresource-templates/ac_project-copilot-dataconfig.yaml -# Flyte Propeller Configuration -- name: flyte-propeller-config - files: - - ./config/propeller/core.yaml - - ./config/propeller/admin.yaml - - ./config/propeller/catalog.yaml - - ./config/propeller/resource_manager.yaml - - ./config/propeller/enabled_plugins.yaml - - ./config/propeller/plugins/copilot.yaml - - ./config/propeller/plugins/k8s.yaml - - ./config/propeller/plugins/task_logs.yaml - - ./config/common/storage.yaml - - ./config/common/logger.yaml + # Flyte Propeller Configuration + - name: flyte-propeller-config + files: + - ./config/propeller/core.yaml + - ./config/propeller/admin.yaml + - ./config/propeller/catalog.yaml + - ./config/propeller/resource_manager.yaml + - ./config/propeller/enabled_plugins.yaml + - ./config/propeller/plugins/copilot.yaml + - ./config/propeller/plugins/k8s.yaml + - ./config/common/storage.yaml + - ./config/common/logger.yaml -# TODO Flyte Console Configuration -#- name: flyte-console-config -# files: -# - ./config/console.yaml + # TODO Flyte Console Configuration + #- name: flyte-console-config + # files: + # - ./config/console.yaml -- name: datacatalog-config - files: - - ./config/datacatalog/server.yaml - - ./config/datacatalog/db.yaml - - ./config/common/storage.yaml - - ./config/common/logger.yaml + - name: datacatalog-config + files: + - ./config/datacatalog/server.yaml + - ./config/datacatalog/db.yaml + - ./config/common/storage.yaml + - ./config/common/logger.yaml secretGenerator: -- name: db-pass - literals: - - pass.txt="awesomesauce" + - name: db-pass + literals: + - pass.txt="awesomesauce" diff --git a/kustomize/base/single_cluster/headless/pass.txt b/kustomize/base/single_cluster/headless/pass.txt deleted file mode 100644 index f6b1b7073b..0000000000 --- a/kustomize/base/single_cluster/headless/pass.txt +++ /dev/null @@ -1 +0,0 @@ -awesomesauce diff --git a/kustomize/overlays/eks/config/admin/remote_data.yaml b/kustomize/overlays/eks/config/admin/remote_data.yaml deleted file mode 100644 index 5e56e0740e..0000000000 --- a/kustomize/overlays/eks/config/admin/remote_data.yaml +++ /dev/null @@ -1,6 +0,0 @@ -remoteData: - # TODO change this to match the region of the s3 bucket - region: "us-west-2" - scheme: aws - signedUrls: - durationMinutes: 3 diff --git a/kustomize/overlays/eks/datacatalog/kustomization.yaml b/kustomize/overlays/eks/datacatalog/kustomization.yaml deleted file mode 100644 index 96588e8106..0000000000 --- a/kustomize/overlays/eks/datacatalog/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -resources: - - service.yaml - -patches: - - deployment.yaml diff --git a/kustomize/overlays/eks/dependencies/alb_ingress/README.md b/kustomize/overlays/eks/dependencies/alb_ingress/README.md deleted file mode 100644 index 4d6403e0fe..0000000000 --- a/kustomize/overlays/eks/dependencies/alb_ingress/README.md +++ /dev/null @@ -1 +0,0 @@ -# :construction: Instructions to deploy ALB Ingress controller diff --git a/kustomize/overlays/eks/dependencies/alb_ingress/ingress.yaml b/kustomize/overlays/eks/dependencies/alb_ingress/ingress.yaml deleted file mode 100644 index 69ed75f223..0000000000 --- a/kustomize/overlays/eks/dependencies/alb_ingress/ingress.yaml +++ /dev/null @@ -1,47 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: Ingress -metadata: - name: "flytesystem" - namespace: "flyte" - annotations: - # TODO ALB can only be used for REST non grpc endpoints - kubernetes.io/ingress.class: alb - alb.ingress.kubernetes.io/tags: service_instance=production - alb.ingress.kubernetes.io/scheme: internet-facing - labels: - app: flyteadmin -spec: - rules: - - http: - paths: - - path: /console - backend: - serviceName: flyteconsole - servicePort: 80 - - path: /console/* - backend: - serviceName: flyteconsole - servicePort: 80 - - path: /api/* - backend: - serviceName: flyteadmin - servicePort: 80 - - path: /healthcheck - backend: - serviceName: flyteadmin - servicePort: 80 - - path: /v1/* - backend: - serviceName: flyteadmin - servicePort: 80 - # NOTE: Port 81 in flyteadmin is the GRPC server port for - # FlyteAdmin. - - path: /flyteidl.service.AdminService/* - backend: - serviceName: flyteadmin - servicePort: 81 - # Port 87 in FlyteAdmin maps to the redoc container. - - path: /openapi/* - backend: - serviceName: flyteadmin - servicePort: 87 diff --git a/kustomize/overlays/eks/admin/deployment.yaml b/kustomize/overlays/eks/flyte/admin/deployment.yaml similarity index 100% rename from kustomize/overlays/eks/admin/deployment.yaml rename to kustomize/overlays/eks/flyte/admin/deployment.yaml diff --git a/kustomize/overlays/eks/admin/service.yaml b/kustomize/overlays/eks/flyte/admin/service.yaml similarity index 100% rename from kustomize/overlays/eks/admin/service.yaml rename to kustomize/overlays/eks/flyte/admin/service.yaml diff --git a/kustomize/overlays/eks/admin/serviceaccount.yaml b/kustomize/overlays/eks/flyte/admin/serviceaccount.yaml similarity index 100% rename from kustomize/overlays/eks/admin/serviceaccount.yaml rename to kustomize/overlays/eks/flyte/admin/serviceaccount.yaml diff --git a/kustomize/overlays/eks/config/admin/db.yaml b/kustomize/overlays/eks/flyte/config/admin/db.yaml similarity index 100% rename from kustomize/overlays/eks/config/admin/db.yaml rename to kustomize/overlays/eks/flyte/config/admin/db.yaml diff --git a/kustomize/overlays/eks/config/admin/task_resource_defaults.yaml b/kustomize/overlays/eks/flyte/config/admin/task_resource_defaults.yaml similarity index 100% rename from kustomize/overlays/eks/config/admin/task_resource_defaults.yaml rename to kustomize/overlays/eks/flyte/config/admin/task_resource_defaults.yaml diff --git a/kustomize/overlays/eks/config/clusterresource-templates/ac_project-copilot-dataconfig.yaml b/kustomize/overlays/eks/flyte/config/clusterresource-templates/ac_project-copilot-dataconfig.yaml similarity index 100% rename from kustomize/overlays/eks/config/clusterresource-templates/ac_project-copilot-dataconfig.yaml rename to kustomize/overlays/eks/flyte/config/clusterresource-templates/ac_project-copilot-dataconfig.yaml diff --git a/kustomize/overlays/eks/config/clusterresource-templates/ad_spark-role.yaml b/kustomize/overlays/eks/flyte/config/clusterresource-templates/ad_spark-role.yaml similarity index 100% rename from kustomize/overlays/eks/config/clusterresource-templates/ad_spark-role.yaml rename to kustomize/overlays/eks/flyte/config/clusterresource-templates/ad_spark-role.yaml diff --git a/kustomize/overlays/eks/config/clusterresource-templates/ae_spark-service-account.yaml b/kustomize/overlays/eks/flyte/config/clusterresource-templates/ae_spark-service-account.yaml similarity index 100% rename from kustomize/overlays/eks/config/clusterresource-templates/ae_spark-service-account.yaml rename to kustomize/overlays/eks/flyte/config/clusterresource-templates/ae_spark-service-account.yaml diff --git a/kustomize/overlays/eks/config/clusterresource-templates/af_spark-role-binding.yaml b/kustomize/overlays/eks/flyte/config/clusterresource-templates/af_spark-role-binding.yaml similarity index 100% rename from kustomize/overlays/eks/config/clusterresource-templates/af_spark-role-binding.yaml rename to kustomize/overlays/eks/flyte/config/clusterresource-templates/af_spark-role-binding.yaml diff --git a/kustomize/overlays/gcp/config/common/storage.yaml b/kustomize/overlays/eks/flyte/config/common/storage.yaml similarity index 55% rename from kustomize/overlays/gcp/config/common/storage.yaml rename to kustomize/overlays/eks/flyte/config/common/storage.yaml index 6d678d0bbc..39dd393112 100644 --- a/kustomize/overlays/gcp/config/common/storage.yaml +++ b/kustomize/overlays/eks/flyte/config/common/storage.yaml @@ -1,14 +1,12 @@ storage: type: stow stow: - kind: google + kind: s3 config: - json: "" - # TODO: replace with the GCP project ID - project_id: - scopes: https://www.googleapis.com/auth/devstorage.read_write - # TODO replace with the container (bucket) in GCS used by Flyte as intermediate store - container: "flyte" + auth_type: iam + region: us-east-2 + # TODO replace with the container (bucket) in s3 used by Flyte as intermediate store + container: "flyte-demo" # NOTE this cache configuration is purely for propeller. But since we are having a common storage # config, we are configuring this value. In production create a separate storage config for # propeller and increase the cache size diff --git a/kustomize/overlays/eks/config/datacatalog/db.yaml b/kustomize/overlays/eks/flyte/config/datacatalog/db.yaml similarity index 100% rename from kustomize/overlays/eks/config/datacatalog/db.yaml rename to kustomize/overlays/eks/flyte/config/datacatalog/db.yaml diff --git a/kustomize/overlays/eks/config/propeller/core.yaml b/kustomize/overlays/eks/flyte/config/propeller/core.yaml similarity index 100% rename from kustomize/overlays/eks/config/propeller/core.yaml rename to kustomize/overlays/eks/flyte/config/propeller/core.yaml diff --git a/kustomize/overlays/eks/config/propeller/enabled_plugins.yaml b/kustomize/overlays/eks/flyte/config/propeller/enabled_plugins.yaml similarity index 100% rename from kustomize/overlays/eks/config/propeller/enabled_plugins.yaml rename to kustomize/overlays/eks/flyte/config/propeller/enabled_plugins.yaml diff --git a/kustomize/overlays/eks/config/propeller/plugins/catalog_cache.yaml b/kustomize/overlays/eks/flyte/config/propeller/plugins/catalog_cache.yaml similarity index 100% rename from kustomize/overlays/eks/config/propeller/plugins/catalog_cache.yaml rename to kustomize/overlays/eks/flyte/config/propeller/plugins/catalog_cache.yaml diff --git a/kustomize/overlays/eks/config/propeller/plugins/k8s.yaml b/kustomize/overlays/eks/flyte/config/propeller/plugins/k8s.yaml similarity index 100% rename from kustomize/overlays/eks/config/propeller/plugins/k8s.yaml rename to kustomize/overlays/eks/flyte/config/propeller/plugins/k8s.yaml diff --git a/kustomize/overlays/eks/config/propeller/plugins/qubole.yaml b/kustomize/overlays/eks/flyte/config/propeller/plugins/qubole.yaml similarity index 100% rename from kustomize/overlays/eks/config/propeller/plugins/qubole.yaml rename to kustomize/overlays/eks/flyte/config/propeller/plugins/qubole.yaml diff --git a/kustomize/overlays/eks/config/propeller/plugins/spark.yaml b/kustomize/overlays/eks/flyte/config/propeller/plugins/spark.yaml similarity index 100% rename from kustomize/overlays/eks/config/propeller/plugins/spark.yaml rename to kustomize/overlays/eks/flyte/config/propeller/plugins/spark.yaml diff --git a/kustomize/overlays/eks/config/propeller/plugins/task_logs.yaml b/kustomize/overlays/eks/flyte/config/propeller/plugins/task_logs.yaml similarity index 100% rename from kustomize/overlays/eks/config/propeller/plugins/task_logs.yaml rename to kustomize/overlays/eks/flyte/config/propeller/plugins/task_logs.yaml diff --git a/kustomize/overlays/eks/console/service.yaml b/kustomize/overlays/eks/flyte/console/service.yaml similarity index 100% rename from kustomize/overlays/eks/console/service.yaml rename to kustomize/overlays/eks/flyte/console/service.yaml diff --git a/kustomize/overlays/eks/datacatalog/deployment.yaml b/kustomize/overlays/eks/flyte/datacatalog/deployment.yaml similarity index 100% rename from kustomize/overlays/eks/datacatalog/deployment.yaml rename to kustomize/overlays/eks/flyte/datacatalog/deployment.yaml diff --git a/kustomize/overlays/eks/datacatalog/service.yaml b/kustomize/overlays/eks/flyte/datacatalog/service.yaml similarity index 100% rename from kustomize/overlays/eks/datacatalog/service.yaml rename to kustomize/overlays/eks/flyte/datacatalog/service.yaml diff --git a/kustomize/overlays/eks/flyte/ingress/README.md b/kustomize/overlays/eks/flyte/ingress/README.md new file mode 100644 index 0000000000..28fa058659 --- /dev/null +++ b/kustomize/overlays/eks/flyte/ingress/README.md @@ -0,0 +1,5 @@ +# :construction: Instructions to deploy ALB Ingress controller + +Follow instructions here to install ALB Ingress Controller: https://docs.aws.amazon.com/eks/latest/userguide/alb-ingress.html + +Replace `alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-east-2:111111111111:certificate/e92fefd8-6197-4249-a524-431d611c9af6` in ingress.yaml and ingress_grpc.yaml with your own SSL cert (that you will create by following ALB Instructions above) diff --git a/kustomize/overlays/eks/flyte/ingress/ingress.yaml b/kustomize/overlays/eks/flyte/ingress/ingress.yaml new file mode 100644 index 0000000000..6d2f06a235 --- /dev/null +++ b/kustomize/overlays/eks/flyte/ingress/ingress.yaml @@ -0,0 +1,150 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: flytesystem + namespace: flyte + annotations: + kubernetes.io/ingress.class: alb + alb.ingress.kubernetes.io/tags: service_instance=production + alb.ingress.kubernetes.io/scheme: internet-facing + alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS":443}]' + alb.ingress.kubernetes.io/actions.ssl-redirect: '{"Type": "redirect", "RedirectConfig": { "Protocol": "HTTPS", "Port": "443", "StatusCode": "HTTP_301"}}' + + # Instruct ALB Controller to not create multiple load balancers (and hence maintain a single endpoint for both GRPC and Http) + alb.ingress.kubernetes.io/group.name: flytesystem + + # Replace certificate Arn with one deployed to your EKS cluster. Follow instructions in README.md + alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-east-2:111111111111:certificate/e92fefd8-6197-4249-a524-431d611c9af6 + labels: + app: flyteadmin +spec: + rules: + - http: + paths: + - path: /* + pathType: ImplementationSpecific + backend: + service: + name: ssl-redirect + port: + name: use-annotation + - path: /console + pathType: ImplementationSpecific + backend: + service: + name: flyteconsole + port: + number: 80 + - path: /console/* + pathType: ImplementationSpecific + backend: + service: + name: flyteconsole + port: + number: 80 + - path: /api/* + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 80 + - path: /healthcheck + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 80 + - path: /v1/* + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 80 + # Port 87 in FlyteAdmin maps to the redoc container. + - path: /openapi/* + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 80 + - path: /.well-known/* + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 80 + - path: /.well-known + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 80 + - path: /login + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 80 + - path: /login/* + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 80 + - path: /logout + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 80 + - path: /logout/* + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 80 + - path: /callback + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 80 + - path: /callback/* + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 80 + - path: /me + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 80 + - path: /config + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 80 + - path: /config/* + pathType: ImplementationSpecific + backend: + service: + name: flyteadmin + port: + number: 80 diff --git a/kustomize/overlays/eks/flyte/ingress/ingress_grpc.yaml b/kustomize/overlays/eks/flyte/ingress/ingress_grpc.yaml new file mode 100644 index 0000000000..ce9c2e8892 --- /dev/null +++ b/kustomize/overlays/eks/flyte/ingress/ingress_grpc.yaml @@ -0,0 +1,45 @@ +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + alb.ingress.kubernetes.io/actions.ssl-redirect: '{"Type": "redirect", "RedirectConfig": { "Protocol": "HTTPS", "Port": "443", "StatusCode": "HTTP_301"}}' + alb.ingress.kubernetes.io/backend-protocol-version: GRPC + alb.ingress.kubernetes.io/listen-ports: '[{"HTTPS":443}]' + alb.ingress.kubernetes.io/scheme: internet-facing + alb.ingress.kubernetes.io/tags: service_instance=production + kubernetes.io/ingress.class: alb + nginx.ingress.kubernetes.io/ssl-redirect: "false" + alb.ingress.kubernetes.io/group.name: flytesystem + + # Replace certificate Arn with one deployed to your EKS cluster. Follow instructions in README.md + alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-east-2:111111111111:certificate/e92fefd8-6197-4249-a524-431d611c9af6 + labels: + app: flyteadmin + name: flytesystem-grpc + namespace: flyte +spec: + rules: + - http: + paths: + - backend: + service: + name: flyteadmin + port: + number: 81 + path: /flyteidl.service.AdminService/* + pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 81 + path: /flyteidl.service.AuthMetadataService/* + pathType: ImplementationSpecific + - backend: + service: + name: flyteadmin + port: + number: 81 + path: /flyteidl.service.IdentityService/* + pathType: ImplementationSpecific diff --git a/kustomize/overlays/eks/flyte/ingress/kustomization.yaml b/kustomize/overlays/eks/flyte/ingress/kustomization.yaml new file mode 100644 index 0000000000..2cb8f8d97a --- /dev/null +++ b/kustomize/overlays/eks/flyte/ingress/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ingress.yaml + - ingress_grpc.yaml diff --git a/kustomize/overlays/eks/flyte/kustomization.yaml b/kustomize/overlays/eks/flyte/kustomization.yaml new file mode 100644 index 0000000000..ebc66e3425 --- /dev/null +++ b/kustomize/overlays/eks/flyte/kustomization.yaml @@ -0,0 +1,63 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# Override the namespace +namespace: flyte + +bases: + - ../../../base/single_cluster/complete + - ingress + +resources: + - datacatalog/service.yaml + - propeller/service.yaml + +patchesStrategicMerge: + - admin/deployment.yaml + - admin/service.yaml + - admin/serviceaccount.yaml + - datacatalog/deployment.yaml + - propeller/deployment.yaml + - propeller/serviceaccount.yaml + - console/service.yaml + +# Files are read in alphabetical order. To ensure that we create the namespace first, prefix the file name with "aa". + +configMapGenerator: + - behavior: merge + files: + - ./config/admin/db.yaml + - ./config/admin/task_resource_defaults.yaml + - ./config/common/storage.yaml + name: flyte-admin-config + - behavior: merge + files: + - ./config/clusterresource-templates/ac_project-copilot-dataconfig.yaml + - ./config/clusterresource-templates/ad_spark-role.yaml + - ./config/clusterresource-templates/ae_spark-service-account.yaml + - ./config/clusterresource-templates/af_spark-role-binding.yaml + name: clusterresource-template + - behavior: merge + files: + - ./config/propeller/core.yaml + - ./config/propeller/enabled_plugins.yaml + - ./config/propeller/plugins/catalog_cache.yaml + - ./config/propeller/plugins/k8s.yaml + - ./config/propeller/plugins/qubole.yaml + - ./config/propeller/plugins/spark.yaml + - ./config/propeller/plugins/task_logs.yaml + - ./config/common/storage.yaml + name: flyte-propeller-config + - behavior: merge + files: + - ./config/common/storage.yaml + - ./config/datacatalog/db.yaml + name: datacatalog-config + +########### +# Use this to record the password for the Google CloudSQL Database +secretGenerator: + - name: db-pass + behavior: merge + literals: + - pass.txt="yourpassword" diff --git a/kustomize/overlays/eks/propeller/deployment.yaml b/kustomize/overlays/eks/flyte/propeller/deployment.yaml similarity index 100% rename from kustomize/overlays/eks/propeller/deployment.yaml rename to kustomize/overlays/eks/flyte/propeller/deployment.yaml diff --git a/kustomize/overlays/eks/propeller/service.yaml b/kustomize/overlays/eks/flyte/propeller/service.yaml similarity index 100% rename from kustomize/overlays/eks/propeller/service.yaml rename to kustomize/overlays/eks/flyte/propeller/service.yaml diff --git a/kustomize/overlays/eks/propeller/serviceaccount.yaml b/kustomize/overlays/eks/flyte/propeller/serviceaccount.yaml similarity index 100% rename from kustomize/overlays/eks/propeller/serviceaccount.yaml rename to kustomize/overlays/eks/flyte/propeller/serviceaccount.yaml diff --git a/kustomize/overlays/eks/ingress/ingress.yaml b/kustomize/overlays/eks/ingress/ingress.yaml deleted file mode 100644 index e6d36dfa8f..0000000000 --- a/kustomize/overlays/eks/ingress/ingress.yaml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: Ingress -metadata: - name: flytesystem - namespace: flyte - annotations: - # TODO ALB can only be used for REST non grpc endpoints - kubernetes.io/ingress.class: alb - alb.ingress.kubernetes.io/tags: service_instance=production - alb.ingress.kubernetes.io/scheme: internet-facing diff --git a/kustomize/overlays/eks/kustomization.yaml b/kustomize/overlays/eks/kustomization.yaml index 1d3a63bb9c..358eaefa80 100644 --- a/kustomize/overlays/eks/kustomization.yaml +++ b/kustomize/overlays/eks/kustomization.yaml @@ -1,13 +1,9 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -# Override the namespace -namespace: flyte - bases: -- ../../base/single_cluster/complete - -# All the resources that make up the deployment + # All the resources that make up the deployment + - flyte ######## # Operators to be enabled ############# @@ -16,81 +12,30 @@ bases: # This is used for Resource pooling. On cloud you can use hosted redis (e.g. AWS elasticache) # Contour is used to create ingress. On cloud service use the default provided ingress controllers or cloud LB's # Add node ports for ease of use locally -resources: -- ../../base/operators/spark -- ../../base/operators/kfoperators/pytorch - # TODO Fix deployment of alb_ingress controller - #- ./dependencies/alb_ingress + - ../../base/operators/spark + - ../../base/operators/kfoperators/pytorch # Optional dependency -- ./dependencies/redis - -patchesStrategicMerge: -- admin/deployment.yaml -- admin/service.yaml -- admin/serviceaccount.yaml -- datacatalog/deployment.yaml -- propeller/deployment.yaml -- propeller/serviceaccount.yaml -- ingress/ingress.yaml -- console/service.yaml - -# Files are read in alphabetical order. To ensure that we create the namespace first, prefix the file name with "aa". - -configMapGenerator: -- behavior: merge - files: - - ./config/admin/db.yaml - - ./config/admin/remote_data.yaml - - ./config/admin/task_resource_defaults.yaml - - ./config/common/storage.yaml - name: flyte-admin-config -- behavior: merge - files: - - ./config/clusterresource-templates/ac_project-copilot-dataconfig.yaml - - ./config/clusterresource-templates/ad_spark-role.yaml - - ./config/clusterresource-templates/ae_spark-service-account.yaml - - ./config/clusterresource-templates/af_spark-role-binding.yaml - name: clusterresource-template -- behavior: merge - files: - - ./config/propeller/core.yaml - - ./config/propeller/enabled_plugins.yaml - - ./config/propeller/plugins/catalog_cache.yaml - - ./config/propeller/plugins/k8s.yaml - - ./config/propeller/plugins/qubole.yaml - - ./config/propeller/plugins/spark.yaml - - ./config/propeller/plugins/task_logs.yaml - - ./config/common/storage.yaml - name: flyte-propeller-config -- behavior: merge - files: - - ./config/common/storage.yaml - - ./config/datacatalog/db.yaml - name: datacatalog-config - -########### -# Use this to record the password for the Google CloudSQL Database -secretGenerator: -- name: db-pass - behavior: merge - literals: - - pass.txt="yourpassword" + - ../../base/addons/redis # Images that should be used images: # FlyteAdmin - name: flyteadmin # match images with this name - newTag: v0.3.29 # override the tag - newName: ghcr.io/lyft/flyteadmin # override the name + newTag: v0.5.0 # FLYTEADMIN_TAG override the tag + newName: ghcr.io/flyteorg/flyteadmin # override the name # FlyteConsole - name: flyteconsole # match images with this name - newTag: v0.19.1 # override the tag - newName: ghcr.io/lyft/flyteconsole # override the namep + newTag: v0.19.7 # FLYTECONSOLE_TAG the tag + newName: ghcr.io/flyteorg/flyteconsole # override the namep # Flyte DataCatalog - name: datacatalog # match images with this name - newTag: v0.2.130 # override the tag - newName: ghcr.io/lyft/datacatalog # override the name + newTag: v0.3.2 # DATACATALOG_TAG override the tag + newName: ghcr.io/flyteorg/datacatalog # override the name # FlytePropeller - name: flytepropeller # match images with this name - newTag: v0.5.13 # override the tag + newTag: v0.10.6 # FLYTEPROPELLER_TAG override the tag + newName: ghcr.io/flyteorg/flytepropeller # override the name + # Webhook + - name: webhook # match images with this name + newTag: v0.10.6 # FLYTEPROPELLER_TAG override the tag newName: ghcr.io/flyteorg/flytepropeller # override the name diff --git a/kustomize/overlays/gcp/config/admin/remote_data.yaml b/kustomize/overlays/gcp/config/admin/remote_data.yaml deleted file mode 100644 index 8af409c9a4..0000000000 --- a/kustomize/overlays/gcp/config/admin/remote_data.yaml +++ /dev/null @@ -1,2 +0,0 @@ -remoteData: - scheme: "gcp" diff --git a/kustomize/overlays/gcp/datacatalog/kustomization.yaml b/kustomize/overlays/gcp/datacatalog/kustomization.yaml deleted file mode 100644 index 96588e8106..0000000000 --- a/kustomize/overlays/gcp/datacatalog/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -resources: - - service.yaml - -patches: - - deployment.yaml diff --git a/kustomize/overlays/gcp/dependencies/redis/deployment.yaml b/kustomize/overlays/gcp/dependencies/redis/deployment.yaml deleted file mode 100644 index 82067d702b..0000000000 --- a/kustomize/overlays/gcp/dependencies/redis/deployment.yaml +++ /dev/null @@ -1,59 +0,0 @@ -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: redis - namespace: flyte -spec: - replicas: 1 - selector: - matchLabels: - app: redis-resource-manager - serviceName: redis-resource-manager - template: - metadata: - labels: - app: redis-resource-manager - spec: - containers: - - env: - - name: REDIS_PASSWORD - value: mypassword - image: docker.io/bitnami/redis:4.0.2-r1 - imagePullPolicy: IfNotPresent - livenessProbe: - exec: - command: - - redis-cli - - ping - failureThreshold: 3 - initialDelaySeconds: 30 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 5 - name: redis-resource-manager - ports: - - containerPort: 6379 - name: redis - protocol: TCP - readinessProbe: - exec: - command: - - redis-cli - - ping - failureThreshold: 3 - initialDelaySeconds: 5 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - resources: - requests: - cpu: 200m - memory: 128Mi - volumeMounts: - - mountPath: /bitnami - name: redis-data - dnsPolicy: ClusterFirst - restartPolicy: Always - volumes: - - name: redis-data - emptyDir: {} diff --git a/kustomize/overlays/gcp/dependencies/redis/kustomization.yaml b/kustomize/overlays/gcp/dependencies/redis/kustomization.yaml deleted file mode 100644 index a944d005ca..0000000000 --- a/kustomize/overlays/gcp/dependencies/redis/kustomization.yaml +++ /dev/null @@ -1,3 +0,0 @@ -resources: -- deployment.yaml -- service.yaml diff --git a/kustomize/overlays/gcp/dependencies/redis/service.yaml b/kustomize/overlays/gcp/dependencies/redis/service.yaml deleted file mode 100644 index 8c86264ae3..0000000000 --- a/kustomize/overlays/gcp/dependencies/redis/service.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - labels: - app: redis-resource-manager - name: redis-resource-manager - namespace: flyte -spec: - ports: - - name: redis - port: 6379 - protocol: TCP - targetPort: redis - selector: - app: redis-resource-manager - type: ClusterIP diff --git a/kustomize/overlays/gcp/admin/deployment.yaml b/kustomize/overlays/gcp/flyte/admin/deployment.yaml similarity index 100% rename from kustomize/overlays/gcp/admin/deployment.yaml rename to kustomize/overlays/gcp/flyte/admin/deployment.yaml diff --git a/kustomize/overlays/gcp/admin/service.yaml b/kustomize/overlays/gcp/flyte/admin/service.yaml similarity index 100% rename from kustomize/overlays/gcp/admin/service.yaml rename to kustomize/overlays/gcp/flyte/admin/service.yaml diff --git a/kustomize/overlays/gcp/config/admin/db.yaml b/kustomize/overlays/gcp/flyte/config/admin/db.yaml similarity index 100% rename from kustomize/overlays/gcp/config/admin/db.yaml rename to kustomize/overlays/gcp/flyte/config/admin/db.yaml diff --git a/kustomize/overlays/gcp/config/admin/task_resource_defaults.yaml b/kustomize/overlays/gcp/flyte/config/admin/task_resource_defaults.yaml similarity index 100% rename from kustomize/overlays/gcp/config/admin/task_resource_defaults.yaml rename to kustomize/overlays/gcp/flyte/config/admin/task_resource_defaults.yaml diff --git a/kustomize/overlays/gcp/config/clusterresource-templates/ac_project-copilot-dataconfig.yaml b/kustomize/overlays/gcp/flyte/config/clusterresource-templates/ac_project-copilot-dataconfig.yaml similarity index 100% rename from kustomize/overlays/gcp/config/clusterresource-templates/ac_project-copilot-dataconfig.yaml rename to kustomize/overlays/gcp/flyte/config/clusterresource-templates/ac_project-copilot-dataconfig.yaml diff --git a/kustomize/overlays/gcp/config/clusterresource-templates/ad_spark-role.yaml b/kustomize/overlays/gcp/flyte/config/clusterresource-templates/ad_spark-role.yaml similarity index 100% rename from kustomize/overlays/gcp/config/clusterresource-templates/ad_spark-role.yaml rename to kustomize/overlays/gcp/flyte/config/clusterresource-templates/ad_spark-role.yaml diff --git a/kustomize/overlays/gcp/config/clusterresource-templates/ae_spark-service-account.yaml b/kustomize/overlays/gcp/flyte/config/clusterresource-templates/ae_spark-service-account.yaml similarity index 100% rename from kustomize/overlays/gcp/config/clusterresource-templates/ae_spark-service-account.yaml rename to kustomize/overlays/gcp/flyte/config/clusterresource-templates/ae_spark-service-account.yaml diff --git a/kustomize/overlays/gcp/config/clusterresource-templates/af_spark-role-binding.yaml b/kustomize/overlays/gcp/flyte/config/clusterresource-templates/af_spark-role-binding.yaml similarity index 100% rename from kustomize/overlays/gcp/config/clusterresource-templates/af_spark-role-binding.yaml rename to kustomize/overlays/gcp/flyte/config/clusterresource-templates/af_spark-role-binding.yaml diff --git a/kustomize/overlays/eks/config/common/storage.yaml b/kustomize/overlays/gcp/flyte/config/common/storage.yaml similarity index 100% rename from kustomize/overlays/eks/config/common/storage.yaml rename to kustomize/overlays/gcp/flyte/config/common/storage.yaml diff --git a/kustomize/overlays/gcp/config/datacatalog/db.yaml b/kustomize/overlays/gcp/flyte/config/datacatalog/db.yaml similarity index 100% rename from kustomize/overlays/gcp/config/datacatalog/db.yaml rename to kustomize/overlays/gcp/flyte/config/datacatalog/db.yaml diff --git a/kustomize/overlays/gcp/config/propeller/core.yaml b/kustomize/overlays/gcp/flyte/config/propeller/core.yaml similarity index 100% rename from kustomize/overlays/gcp/config/propeller/core.yaml rename to kustomize/overlays/gcp/flyte/config/propeller/core.yaml diff --git a/kustomize/overlays/gcp/config/propeller/enabled_plugins.yaml b/kustomize/overlays/gcp/flyte/config/propeller/enabled_plugins.yaml similarity index 100% rename from kustomize/overlays/gcp/config/propeller/enabled_plugins.yaml rename to kustomize/overlays/gcp/flyte/config/propeller/enabled_plugins.yaml diff --git a/kustomize/overlays/gcp/config/propeller/plugins/catalog_cache.yaml b/kustomize/overlays/gcp/flyte/config/propeller/plugins/catalog_cache.yaml similarity index 100% rename from kustomize/overlays/gcp/config/propeller/plugins/catalog_cache.yaml rename to kustomize/overlays/gcp/flyte/config/propeller/plugins/catalog_cache.yaml diff --git a/kustomize/overlays/gcp/config/propeller/plugins/k8s.yaml b/kustomize/overlays/gcp/flyte/config/propeller/plugins/k8s.yaml similarity index 100% rename from kustomize/overlays/gcp/config/propeller/plugins/k8s.yaml rename to kustomize/overlays/gcp/flyte/config/propeller/plugins/k8s.yaml diff --git a/kustomize/overlays/gcp/config/propeller/plugins/qubole.yaml b/kustomize/overlays/gcp/flyte/config/propeller/plugins/qubole.yaml similarity index 100% rename from kustomize/overlays/gcp/config/propeller/plugins/qubole.yaml rename to kustomize/overlays/gcp/flyte/config/propeller/plugins/qubole.yaml diff --git a/kustomize/overlays/gcp/config/propeller/plugins/spark.yaml b/kustomize/overlays/gcp/flyte/config/propeller/plugins/spark.yaml similarity index 100% rename from kustomize/overlays/gcp/config/propeller/plugins/spark.yaml rename to kustomize/overlays/gcp/flyte/config/propeller/plugins/spark.yaml diff --git a/kustomize/overlays/gcp/config/propeller/plugins/task_logs.yaml b/kustomize/overlays/gcp/flyte/config/propeller/plugins/task_logs.yaml similarity index 100% rename from kustomize/overlays/gcp/config/propeller/plugins/task_logs.yaml rename to kustomize/overlays/gcp/flyte/config/propeller/plugins/task_logs.yaml diff --git a/kustomize/overlays/gcp/console/service.yaml b/kustomize/overlays/gcp/flyte/console/service.yaml similarity index 100% rename from kustomize/overlays/gcp/console/service.yaml rename to kustomize/overlays/gcp/flyte/console/service.yaml diff --git a/kustomize/overlays/gcp/datacatalog/deployment.yaml b/kustomize/overlays/gcp/flyte/datacatalog/deployment.yaml similarity index 100% rename from kustomize/overlays/gcp/datacatalog/deployment.yaml rename to kustomize/overlays/gcp/flyte/datacatalog/deployment.yaml diff --git a/kustomize/overlays/gcp/datacatalog/service.yaml b/kustomize/overlays/gcp/flyte/datacatalog/service.yaml similarity index 100% rename from kustomize/overlays/gcp/datacatalog/service.yaml rename to kustomize/overlays/gcp/flyte/datacatalog/service.yaml diff --git a/kustomize/overlays/gcp/flyte/kustomization.yaml b/kustomize/overlays/gcp/flyte/kustomization.yaml new file mode 100644 index 0000000000..bb08dcf05a --- /dev/null +++ b/kustomize/overlays/gcp/flyte/kustomization.yaml @@ -0,0 +1,60 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# Override the namespace +namespace: flyte + +bases: +- ../../../base/single_cluster/complete + +resources: +- datacatalog/service.yaml +- propeller/service.yaml + +patchesStrategicMerge: +- admin/deployment.yaml +- admin/service.yaml +- datacatalog/deployment.yaml +- propeller/deployment.yaml +- console/service.yaml + +# Files are read in alphabetical order. To ensure that we create the namespace first, prefix the file name with "aa". + +configMapGenerator: +- behavior: merge + files: + - ./config/admin/db.yaml + - ./config/admin/task_resource_defaults.yaml + - ./config/common/storage.yaml + name: flyte-admin-config +- behavior: merge + files: + - ./config/clusterresource-templates/ac_project-copilot-dataconfig.yaml + - ./config/clusterresource-templates/ad_spark-role.yaml + - ./config/clusterresource-templates/ae_spark-service-account.yaml + - ./config/clusterresource-templates/af_spark-role-binding.yaml + name: clusterresource-template +- behavior: merge + files: + - ./config/propeller/core.yaml + - ./config/propeller/enabled_plugins.yaml + - ./config/propeller/plugins/catalog_cache.yaml + - ./config/propeller/plugins/k8s.yaml + - ./config/propeller/plugins/qubole.yaml + - ./config/propeller/plugins/spark.yaml + - ./config/propeller/plugins/task_logs.yaml + - ./config/common/storage.yaml + name: flyte-propeller-config +- behavior: merge + files: + - ./config/common/storage.yaml + - ./config/datacatalog/db.yaml + name: datacatalog-config + +########### +# Use this to record the password for the Google CloudSQL Database +secretGenerator: +- name: db-pass + behavior: merge + literals: + - pass.txt="yourpassword" diff --git a/kustomize/overlays/gcp/propeller/deployment.yaml b/kustomize/overlays/gcp/flyte/propeller/deployment.yaml similarity index 100% rename from kustomize/overlays/gcp/propeller/deployment.yaml rename to kustomize/overlays/gcp/flyte/propeller/deployment.yaml diff --git a/kustomize/overlays/gcp/propeller/service.yaml b/kustomize/overlays/gcp/flyte/propeller/service.yaml similarity index 100% rename from kustomize/overlays/gcp/propeller/service.yaml rename to kustomize/overlays/gcp/flyte/propeller/service.yaml diff --git a/kustomize/overlays/gcp/kustomization.yaml b/kustomize/overlays/gcp/kustomization.yaml index 86c77fe034..60d76f7a65 100644 --- a/kustomize/overlays/gcp/kustomization.yaml +++ b/kustomize/overlays/gcp/kustomization.yaml @@ -1,15 +1,9 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -# Override the namespace -namespace: flyte - bases: -- ../../base/single_cluster/complete - -# All the resources that make up the deployment -- datacatalog/service.yaml -- propeller/service.yaml + # All the resources that make up the deployment + - flyte ######## # Operators to be enabled ############# @@ -18,77 +12,32 @@ bases: # This is used for Resource pooling. On cloud you can use hosted redis (e.g. AWS elasticache) # Contour is used to create ingress. On cloud service use the default provided ingress controllers or cloud LB's # Add node ports for ease of use locally -resources: -- ../../base/operators/spark -- ../../base/operators/kfoperators/pytorch -- ./dependencies/cloudsqlproxy + - ../../base/ingress + - ../../base/operators/spark + - ../../base/operators/kfoperators/pytorch + - ../../base/addons/cloudsqlproxy # Optional dependency -- ./dependencies/redis - -patchesStrategicMerge: -- admin/deployment.yaml -- admin/service.yaml -- datacatalog/deployment.yaml -- propeller/deployment.yaml -- console/service.yaml - -# Files are read in alphabetical order. To ensure that we create the namespace first, prefix the file name with "aa". - -configMapGenerator: -- behavior: merge - files: - - ./config/admin/db.yaml - - ./config/admin/remote_data.yaml - - ./config/admin/task_resource_defaults.yaml - - ./config/common/storage.yaml - name: flyte-admin-config -- behavior: merge - files: - - ./config/clusterresource-templates/ac_project-copilot-dataconfig.yaml - - ./config/clusterresource-templates/ad_spark-role.yaml - - ./config/clusterresource-templates/ae_spark-service-account.yaml - - ./config/clusterresource-templates/af_spark-role-binding.yaml - name: clusterresource-template -- behavior: merge - files: - - ./config/propeller/core.yaml - - ./config/propeller/enabled_plugins.yaml - - ./config/propeller/plugins/catalog_cache.yaml - - ./config/propeller/plugins/k8s.yaml - - ./config/propeller/plugins/qubole.yaml - - ./config/propeller/plugins/spark.yaml - - ./config/propeller/plugins/task_logs.yaml - - ./config/common/storage.yaml - name: flyte-propeller-config -- behavior: merge - files: - - ./config/common/storage.yaml - - ./config/datacatalog/db.yaml - name: datacatalog-config - -########### -# Use this to record the password for the Google CloudSQL Database -secretGenerator: -- name: db-pass - behavior: merge - literals: - - pass.txt="yourpassword" + - ../../base/addons/redis # Images that should be used images: # FlyteAdmin - name: flyteadmin # match images with this name - newTag: v0.3.29 # override the tag - newName: ghcr.io/lyft/flyteadmin # override the name + newTag: v0.5.0 # FLYTEADMIN_TAG override the tag + newName: ghcr.io/flyteorg/flyteadmin # override the name # FlyteConsole - name: flyteconsole # match images with this name - newTag: v0.19.1 # override the tag - newName: ghcr.io/lyft/flyteconsole # override the namep + newTag: v0.19.7 # FLYTECONSOLE_TAG the tag + newName: ghcr.io/flyteorg/flyteconsole # override the namep # Flyte DataCatalog - name: datacatalog # match images with this name - newTag: v0.2.130 # override the tag - newName: ghcr.io/lyft/datacatalog # override the name + newTag: v0.3.2 # DATACATALOG_TAG override the tag + newName: ghcr.io/flyteorg/datacatalog # override the name # FlytePropeller - name: flytepropeller # match images with this name - newTag: v0.5.13 # override the tag + newTag: v0.10.6 # FLYTEPROPELLER_TAG override the tag + newName: ghcr.io/flyteorg/flytepropeller # override the name + # Webhook + - name: webhook # match images with this name + newTag: v0.10.6 # FLYTEPROPELLER_TAG override the tag newName: ghcr.io/flyteorg/flytepropeller # override the name diff --git a/kustomize/overlays/sandbox/config/propeller/plugins/task_logs.yaml b/kustomize/overlays/sandbox/config/propeller/plugins/task_logs.yaml deleted file mode 100644 index 0955b59387..0000000000 --- a/kustomize/overlays/sandbox/config/propeller/plugins/task_logs.yaml +++ /dev/null @@ -1,4 +0,0 @@ -plugins: - logs: - kubernetes-enabled: true - kubernetes-url: "http://localhost:30082" diff --git a/kustomize/overlays/sandbox/dependencies/contour_ingress_controller/contour.yaml b/kustomize/overlays/sandbox/dependencies/contour_ingress_controller/contour.yaml deleted file mode 100644 index 499f77f0b0..0000000000 --- a/kustomize/overlays/sandbox/dependencies/contour_ingress_controller/contour.yaml +++ /dev/null @@ -1,1880 +0,0 @@ -# This file is generated from the individual YAML files by generate-deployment.sh. Do not -# edit this file directly but instead edit the source files and re-render. -# -# Generated from: -# examples/contour/00-common.yaml -# examples/contour/01-contour-config.yaml -# examples/contour/01-crds.yaml -# examples/contour/02-job-certgen.yaml -# examples/contour/02-rbac.yaml -# examples/contour/02-role-contour.yaml -# examples/contour/02-service-contour.yaml -# examples/contour/02-service-envoy.yaml -# examples/contour/03-contour.yaml -# examples/contour/03-envoy.yaml -# - ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: contour - namespace: projectcontour ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: envoy - namespace: projectcontour - ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: contour - namespace: projectcontour -data: - contour.yaml: | - # - # server: - # determine which XDS Server implementation to utilize in Contour. - # xds-server-type: contour - # - # should contour expect to be running inside a k8s cluster - # incluster: true - # - # path to kubeconfig (if not running inside a k8s cluster) - # kubeconfig: /path/to/.kube/config - # - # disable HTTPProxy permitInsecure field - disablePermitInsecure: false - tls: - # minimum TLS version that Contour will negotiate - # minimum-protocol-version: "1.1" - # Defines the Kubernetes name/namespace matching a secret to use - # as the fallback certificate when requests which don't match the - # SNI defined for a vhost. - fallback-certificate: - # name: fallback-secret-name - # namespace: projectcontour - envoy-client-certificate: - # name: envoy-client-cert-secret-name - # namespace: projectcontour - # The following config shows the defaults for the leader election. - # leaderelection: - # configmap-name: leader-elect - # configmap-namespace: projectcontour - ### Logging options - # Default setting - accesslog-format: envoy - # To enable JSON logging in Envoy - # accesslog-format: json - # The default fields that will be logged are specified below. - # To customise this list, just add or remove entries. - # The canonical list is available at - # https://godoc.org/github.com/projectcontour/contour/internal/envoy#JSONFields - # json-fields: - # - "@timestamp" - # - "authority" - # - "bytes_received" - # - "bytes_sent" - # - "downstream_local_address" - # - "downstream_remote_address" - # - "duration" - # - "method" - # - "path" - # - "protocol" - # - "request_id" - # - "requested_server_name" - # - "response_code" - # - "response_flags" - # - "uber_trace_id" - # - "upstream_cluster" - # - "upstream_host" - # - "upstream_local_address" - # - "upstream_service_time" - # - "user_agent" - # - "x_forwarded_for" - # - # default-http-versions: - # - "HTTP/2" - # - "HTTP/1.1" - # - # The following shows the default proxy timeout settings. - # timeouts: - # request-timeout: infinity - # connection-idle-timeout: 60s - # stream-idle-timeout: 5m - # max-connection-duration: infinity - # connection-shutdown-grace-period: 5s - # - # Envoy cluster settings. - # cluster: - # configure the cluster dns lookup family - # valid options are: auto (default), v4, v6 - # dns-lookup-family: auto - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.2.9 - creationTimestamp: null - name: extensionservices.projectcontour.io -spec: - preserveUnknownFields: false - group: projectcontour.io - names: - kind: ExtensionService - listKind: ExtensionServiceList - plural: extensionservices - shortNames: - - extensionservice - - extensionservices - singular: extensionservice - scope: Namespaced - versions: - - name: v1alpha1 - schema: - openAPIV3Schema: - description: ExtensionService is the schema for the Contour extension services API. An ExtensionService resource binds a network service to the Contour API so that Contour API features can be implemented by collaborating components. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: ExtensionServiceSpec defines the desired state of an ExtensionService resource. - properties: - loadBalancerPolicy: - description: The policy for load balancing GRPC service requests. Note that the `Cookie` load balancing strategy cannot be used here. - properties: - strategy: - description: Strategy specifies the policy used to balance requests across the pool of backend pods. Valid policy names are `Random`, `RoundRobin`, `WeightedLeastRequest`, `Random` and `Cookie`. If an unknown strategy name is specified or no policy is supplied, the default `RoundRobin` policy is used. - type: string - type: object - protocol: - description: Protocol may be used to specify (or override) the protocol used to reach this Service. Values may be tls, h2, h2c. If omitted, protocol-selection falls back on Service annotations. - enum: - - h2 - - h2c - type: string - protocolVersion: - description: This field sets the version of the GRPC protocol that Envoy uses to send requests to the extension service. Since Contour always uses the v3 Envoy API, this is currently fixed at "v3". However, other protocol options will be available in future. - enum: - - v3 - type: string - services: - description: Services specifies the set of Kubernetes Service resources that receive GRPC extension API requests. If no weights are specified for any of the entries in this array, traffic will be spread evenly across all the services. Otherwise, traffic is balanced proportionally to the Weight field in each entry. - items: - description: ExtensionServiceTarget defines an Kubernetes Service to target with extension service traffic. - properties: - name: - description: Name is the name of Kubernetes service that will accept service traffic. - type: string - port: - description: Port (defined as Integer) to proxy traffic to since a service can have multiple defined. - exclusiveMaximum: true - maximum: 65536 - minimum: 1 - type: integer - weight: - description: Weight defines proportion of traffic to balance to the Kubernetes Service. - format: int32 - type: integer - required: - - name - - port - type: object - minItems: 1 - type: array - timeoutPolicy: - description: The timeout policy for requests to the services. - properties: - idle: - description: Timeout after which, if there are no active requests for this route, the connection between Envoy and the backend or Envoy and the external client will be closed. If not specified, there is no per-route idle timeout, though a connection manager-wide stream_idle_timeout default of 5m still applies. - pattern: ^(((\d*(\.\d*)?h)|(\d*(\.\d*)?m)|(\d*(\.\d*)?s)|(\d*(\.\d*)?ms)|(\d*(\.\d*)?us)|(\d*(\.\d*)?µs)|(\d*(\.\d*)?ns))+|infinity|infinite)$ - type: string - response: - description: Timeout for receiving a response from the server after processing a request from client. If not supplied, Envoy's default value of 15s applies. - pattern: ^(((\d*(\.\d*)?h)|(\d*(\.\d*)?m)|(\d*(\.\d*)?s)|(\d*(\.\d*)?ms)|(\d*(\.\d*)?us)|(\d*(\.\d*)?µs)|(\d*(\.\d*)?ns))+|infinity|infinite)$ - type: string - type: object - validation: - description: UpstreamValidation defines how to verify the backend service's certificate - properties: - caSecret: - description: Name of the Kubernetes secret be used to validate the certificate presented by the backend - type: string - subjectName: - description: Key which is expected to be present in the 'subjectAltName' of the presented certificate - type: string - required: - - caSecret - - subjectName - type: object - required: - - services - type: object - status: - description: ExtensionServiceStatus defines the observed state of an ExtensionService resource. - properties: - conditions: - description: "Conditions contains the current status of the ExtensionService resource. \n Contour will update a single condition, `Valid`, that is in normal-true polarity. \n Contour will not modify any other Conditions set in this block, in case some other controller wants to add a Condition." - items: - description: "DetailedCondition is an extension of the normal Kubernetes conditions, with two extra fields to hold sub-conditions, which provide more detailed reasons for the state (True or False) of the condition. \n `errors` holds information about sub-conditions which are fatal to that condition and render its state False. \n `warnings` holds information about sub-conditions which are not fatal to that condition and do not force the state to be False. \n Remember that Conditions have a type, a status, and a reason. \n The type is the type of the condition, the most important one in this CRD set is `Valid`. `Valid` is a positive-polarity condition: when it is `status: true` there are no problems. \n In more detail, `status: true` means that the object is has been ingested into Contour with no errors. `warnings` may still be present, and will be indicated in the Reason field. There must be zero entries in the `errors` slice in this case. \n `Valid`, `status: false` means that the object has had one or more fatal errors during processing into Contour. The details of the errors will be present under the `errors` field. There must be at least one error in the `errors` slice if `status` is `false`. \n For DetailedConditions of types other than `Valid`, the Condition must be in the negative polarity. When they have `status` `true`, there is an error. There must be at least one entry in the `errors` Subcondition slice. When they have `status` `false`, there are no serious errors, and there must be zero entries in the `errors` slice. In either case, there may be entries in the `warnings` slice. \n Regardless of the polarity, the `reason` and `message` fields must be updated with either the detail of the reason (if there is one and only one entry in total across both the `errors` and `warnings` slices), or `MultipleReasons` if there is more than one entry." - properties: - errors: - description: "Errors contains a slice of relevant error subconditions for this object. \n Subconditions are expected to appear when relevant (when there is a error), and disappear when not relevant. An empty slice here indicates no errors." - items: - description: "SubCondition is a Condition-like type intended for use as a subcondition inside a DetailedCondition. \n It contains a subset of the Condition fields. \n It is intended for warnings and errors, so `type` names should use abnormal-true polarity, that is, they should be of the form \"ErrorPresent: true\". \n The expected lifecycle for these errors is that they should only be present when the error or warning is, and should be removed when they are not relevant." - properties: - message: - description: "Message is a human readable message indicating details about the transition. \n This may be an empty string." - maxLength: 32768 - type: string - reason: - description: "Reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. \n The value should be a CamelCase string. \n This field may not be empty." - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: Status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: "Type of condition in `CamelCase` or in `foo.example.com/CamelCase`. \n This must be in abnormal-true polarity, that is, `ErrorFound` or `controller.io/ErrorFound`. \n The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)" - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - message - - reason - - status - - type - type: object - type: array - lastTransitionTime: - description: lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: message is a human readable message indicating details about the transition. This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: observedGeneration represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. --- Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be useful (see .node.status.conditions), the ability to deconflict is important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - warnings: - description: "Warnings contains a slice of relevant warning subconditions for this object. \n Subconditions are expected to appear when relevant (when there is a warning), and disappear when not relevant. An empty slice here indicates no warnings." - items: - description: "SubCondition is a Condition-like type intended for use as a subcondition inside a DetailedCondition. \n It contains a subset of the Condition fields. \n It is intended for warnings and errors, so `type` names should use abnormal-true polarity, that is, they should be of the form \"ErrorPresent: true\". \n The expected lifecycle for these errors is that they should only be present when the error or warning is, and should be removed when they are not relevant." - properties: - message: - description: "Message is a human readable message indicating details about the transition. \n This may be an empty string." - maxLength: 32768 - type: string - reason: - description: "Reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. \n The value should be a CamelCase string. \n This field may not be empty." - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: Status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: "Type of condition in `CamelCase` or in `foo.example.com/CamelCase`. \n This must be in abnormal-true polarity, that is, `ErrorFound` or `controller.io/ErrorFound`. \n The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)" - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - message - - reason - - status - - type - type: object - type: array - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - type: object - type: object - served: true - storage: true - subresources: - status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.2.9 - creationTimestamp: null - name: httpproxies.projectcontour.io -spec: - preserveUnknownFields: false - group: projectcontour.io - names: - kind: HTTPProxy - listKind: HTTPProxyList - plural: httpproxies - shortNames: - - proxy - - proxies - singular: httpproxy - scope: Namespaced - versions: - - additionalPrinterColumns: - - description: Fully qualified domain name - jsonPath: .spec.virtualhost.fqdn - name: FQDN - type: string - - description: Secret with TLS credentials - jsonPath: .spec.virtualhost.tls.secretName - name: TLS Secret - type: string - - description: The current status of the HTTPProxy - jsonPath: .status.currentStatus - name: Status - type: string - - description: Description of the current status - jsonPath: .status.description - name: Status Description - type: string - name: v1 - schema: - openAPIV3Schema: - description: HTTPProxy is an Ingress CRD specification. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: HTTPProxySpec defines the spec of the CRD. - properties: - includes: - description: Includes allow for specific routing configuration to be included from another HTTPProxy, possibly in another namespace. - items: - description: Include describes a set of policies that can be applied to an HTTPProxy in a namespace. - properties: - conditions: - description: 'Conditions are a set of rules that are applied to included HTTPProxies. In effect, they are added onto the Conditions of included HTTPProxy Route structs. When applied, they are merged using AND, with one exception: There can be only one Prefix MatchCondition per Conditions slice. More than one Prefix, or contradictory Conditions, will make the include invalid.' - items: - description: MatchCondition are a general holder for matching rules for HTTPProxies. One of Prefix or Header must be provided. - properties: - header: - description: Header specifies the header condition to match. - properties: - contains: - description: Contains specifies a substring that must be present in the header value. - type: string - exact: - description: Exact specifies a string that the header value must be equal to. - type: string - name: - description: Name is the name of the header to match against. Name is required. Header names are case insensitive. - type: string - notcontains: - description: NotContains specifies a substring that must not be present in the header value. - type: string - notexact: - description: NoExact specifies a string that the header value must not be equal to. The condition is true if the header has any other value. - type: string - present: - description: Present specifies that condition is true when the named header is present, regardless of its value. Note that setting Present to false does not make the condition true if the named header is absent. - type: boolean - required: - - name - type: object - prefix: - description: Prefix defines a prefix match for a request. - type: string - type: object - type: array - name: - description: Name of the HTTPProxy - type: string - namespace: - description: Namespace of the HTTPProxy to include. Defaults to the current namespace if not supplied. - type: string - required: - - name - type: object - type: array - routes: - description: Routes are the ingress routes. If TCPProxy is present, Routes is ignored. - items: - description: Route contains the set of routes for a virtual host. - properties: - authPolicy: - description: AuthPolicy updates the authorization policy that was set on the root HTTPProxy object for client requests that match this route. - properties: - context: - additionalProperties: - type: string - description: Context is a set of key/value pairs that are sent to the authentication server in the check request. If a context is provided at an enclosing scope, the entries are merged such that the inner scope overrides matching keys from the outer scope. - type: object - disabled: - description: When true, this field disables client request authentication for the scope of the policy. - type: boolean - type: object - conditions: - description: 'Conditions are a set of rules that are applied to a Route. When applied, they are merged using AND, with one exception: There can be only one Prefix MatchCondition per Conditions slice. More than one Prefix, or contradictory Conditions, will make the route invalid.' - items: - description: MatchCondition are a general holder for matching rules for HTTPProxies. One of Prefix or Header must be provided. - properties: - header: - description: Header specifies the header condition to match. - properties: - contains: - description: Contains specifies a substring that must be present in the header value. - type: string - exact: - description: Exact specifies a string that the header value must be equal to. - type: string - name: - description: Name is the name of the header to match against. Name is required. Header names are case insensitive. - type: string - notcontains: - description: NotContains specifies a substring that must not be present in the header value. - type: string - notexact: - description: NoExact specifies a string that the header value must not be equal to. The condition is true if the header has any other value. - type: string - present: - description: Present specifies that condition is true when the named header is present, regardless of its value. Note that setting Present to false does not make the condition true if the named header is absent. - type: boolean - required: - - name - type: object - prefix: - description: Prefix defines a prefix match for a request. - type: string - type: object - type: array - enableWebsockets: - description: Enables websocket support for the route. - type: boolean - healthCheckPolicy: - description: The health check policy for this route. - properties: - healthyThresholdCount: - description: The number of healthy health checks required before a host is marked healthy - format: int64 - minimum: 0 - type: integer - host: - description: The value of the host header in the HTTP health check request. If left empty (default value), the name "contour-envoy-healthcheck" will be used. - type: string - intervalSeconds: - description: The interval (seconds) between health checks - format: int64 - type: integer - path: - description: HTTP endpoint used to perform health checks on upstream service - type: string - timeoutSeconds: - description: The time to wait (seconds) for a health check response - format: int64 - type: integer - unhealthyThresholdCount: - description: The number of unhealthy health checks required before a host is marked unhealthy - format: int64 - minimum: 0 - type: integer - required: - - path - type: object - loadBalancerPolicy: - description: The load balancing policy for this route. - properties: - strategy: - description: Strategy specifies the policy used to balance requests across the pool of backend pods. Valid policy names are `Random`, `RoundRobin`, `WeightedLeastRequest`, `Random` and `Cookie`. If an unknown strategy name is specified or no policy is supplied, the default `RoundRobin` policy is used. - type: string - type: object - pathRewritePolicy: - description: The policy for rewriting the path of the request URL after the request has been routed to a Service. - properties: - replacePrefix: - description: ReplacePrefix describes how the path prefix should be replaced. - items: - description: ReplacePrefix describes a path prefix replacement. - properties: - prefix: - description: "Prefix specifies the URL path prefix to be replaced. \n If Prefix is specified, it must exactly match the MatchCondition prefix that is rendered by the chain of including HTTPProxies and only that path prefix will be replaced by Replacement. This allows HTTPProxies that are included through multiple roots to only replace specific path prefixes, leaving others unmodified. \n If Prefix is not specified, all routing prefixes rendered by the include chain will be replaced." - minLength: 1 - type: string - replacement: - description: Replacement is the string that the routing path prefix will be replaced with. This must not be empty. - minLength: 1 - type: string - required: - - replacement - type: object - type: array - type: object - permitInsecure: - description: Allow this path to respond to insecure requests over HTTP which are normally not permitted when a `virtualhost.tls` block is present. - type: boolean - requestHeadersPolicy: - description: The policy for managing request headers during proxying. - properties: - remove: - description: Remove specifies a list of HTTP header names to remove. - items: - type: string - type: array - set: - description: Set specifies a list of HTTP header values that will be set in the HTTP header. If the header does not exist it will be added, otherwise it will be overwritten with the new value. - items: - description: HeaderValue represents a header name/value pair - properties: - name: - description: Name represents a key of a header - minLength: 1 - type: string - value: - description: Value represents the value of a header specified by a key - minLength: 1 - type: string - required: - - name - - value - type: object - type: array - type: object - responseHeadersPolicy: - description: The policy for managing response headers during proxying. Rewriting the 'Host' header is not supported. - properties: - remove: - description: Remove specifies a list of HTTP header names to remove. - items: - type: string - type: array - set: - description: Set specifies a list of HTTP header values that will be set in the HTTP header. If the header does not exist it will be added, otherwise it will be overwritten with the new value. - items: - description: HeaderValue represents a header name/value pair - properties: - name: - description: Name represents a key of a header - minLength: 1 - type: string - value: - description: Value represents the value of a header specified by a key - minLength: 1 - type: string - required: - - name - - value - type: object - type: array - type: object - retryPolicy: - description: The retry policy for this route. - properties: - count: - description: NumRetries is maximum allowed number of retries. If not supplied, the number of retries is one. - format: int64 - minimum: 0 - type: integer - perTryTimeout: - description: PerTryTimeout specifies the timeout per retry attempt. Ignored if NumRetries is not supplied. - pattern: ^(((\d*(\.\d*)?h)|(\d*(\.\d*)?m)|(\d*(\.\d*)?s)|(\d*(\.\d*)?ms)|(\d*(\.\d*)?us)|(\d*(\.\d*)?µs)|(\d*(\.\d*)?ns))+|infinity|infinite)$ - type: string - retriableStatusCodes: - description: "RetriableStatusCodes specifies the HTTP status codes that should be retried. \n This field is only respected when you include `retriable-status-codes` in the `RetryOn` field." - items: - format: int32 - type: integer - type: array - retryOn: - description: "RetryOn specifies the conditions on which to retry a request. \n Supported [HTTP conditions](https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_filters/router_filter#x-envoy-retry-on): \n - `5xx` - `gateway-error` - `reset` - `connect-failure` - `retriable-4xx` - `refused-stream` - `retriable-status-codes` - `retriable-headers` \n Supported [gRPC conditions](https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_filters/router_filter#x-envoy-retry-grpc-on): \n - `cancelled` - `deadline-exceeded` - `internal` - `resource-exhausted` - `unavailable`" - items: - description: RetryOn is a string type alias with validation to ensure that the value is valid. - enum: - - 5xx - - gateway-error - - reset - - connect-failure - - retriable-4xx - - refused-stream - - retriable-status-codes - - retriable-headers - - cancelled - - deadline-exceeded - - internal - - resource-exhausted - - unavailable - type: string - type: array - type: object - services: - description: Services are the services to proxy traffic. - items: - description: Service defines an Kubernetes Service to proxy traffic. - properties: - mirror: - description: If Mirror is true the Service will receive a read only mirror of the traffic for this route. - type: boolean - name: - description: Name is the name of Kubernetes service to proxy traffic. Names defined here will be used to look up corresponding endpoints which contain the ips to route. - type: string - port: - description: Port (defined as Integer) to proxy traffic to since a service can have multiple defined. - exclusiveMaximum: true - maximum: 65536 - minimum: 1 - type: integer - protocol: - description: Protocol may be used to specify (or override) the protocol used to reach this Service. Values may be tls, h2, h2c. If omitted, protocol-selection falls back on Service annotations. - enum: - - h2 - - h2c - - tls - type: string - requestHeadersPolicy: - description: The policy for managing request headers during proxying. Rewriting the 'Host' header is not supported. - properties: - remove: - description: Remove specifies a list of HTTP header names to remove. - items: - type: string - type: array - set: - description: Set specifies a list of HTTP header values that will be set in the HTTP header. If the header does not exist it will be added, otherwise it will be overwritten with the new value. - items: - description: HeaderValue represents a header name/value pair - properties: - name: - description: Name represents a key of a header - minLength: 1 - type: string - value: - description: Value represents the value of a header specified by a key - minLength: 1 - type: string - required: - - name - - value - type: object - type: array - type: object - responseHeadersPolicy: - description: The policy for managing response headers during proxying. Rewriting the 'Host' header is not supported. - properties: - remove: - description: Remove specifies a list of HTTP header names to remove. - items: - type: string - type: array - set: - description: Set specifies a list of HTTP header values that will be set in the HTTP header. If the header does not exist it will be added, otherwise it will be overwritten with the new value. - items: - description: HeaderValue represents a header name/value pair - properties: - name: - description: Name represents a key of a header - minLength: 1 - type: string - value: - description: Value represents the value of a header specified by a key - minLength: 1 - type: string - required: - - name - - value - type: object - type: array - type: object - validation: - description: UpstreamValidation defines how to verify the backend service's certificate - properties: - caSecret: - description: Name of the Kubernetes secret be used to validate the certificate presented by the backend - type: string - subjectName: - description: Key which is expected to be present in the 'subjectAltName' of the presented certificate - type: string - required: - - caSecret - - subjectName - type: object - weight: - description: Weight defines percentage of traffic to balance traffic - format: int64 - minimum: 0 - type: integer - required: - - name - - port - type: object - minItems: 1 - type: array - timeoutPolicy: - description: The timeout policy for this route. - properties: - idle: - description: Timeout after which, if there are no active requests for this route, the connection between Envoy and the backend or Envoy and the external client will be closed. If not specified, there is no per-route idle timeout, though a connection manager-wide stream_idle_timeout default of 5m still applies. - pattern: ^(((\d*(\.\d*)?h)|(\d*(\.\d*)?m)|(\d*(\.\d*)?s)|(\d*(\.\d*)?ms)|(\d*(\.\d*)?us)|(\d*(\.\d*)?µs)|(\d*(\.\d*)?ns))+|infinity|infinite)$ - type: string - response: - description: Timeout for receiving a response from the server after processing a request from client. If not supplied, Envoy's default value of 15s applies. - pattern: ^(((\d*(\.\d*)?h)|(\d*(\.\d*)?m)|(\d*(\.\d*)?s)|(\d*(\.\d*)?ms)|(\d*(\.\d*)?us)|(\d*(\.\d*)?µs)|(\d*(\.\d*)?ns))+|infinity|infinite)$ - type: string - type: object - required: - - services - type: object - type: array - tcpproxy: - description: TCPProxy holds TCP proxy information. - properties: - healthCheckPolicy: - description: The health check policy for this tcp proxy - properties: - healthyThresholdCount: - description: The number of healthy health checks required before a host is marked healthy - format: int32 - type: integer - intervalSeconds: - description: The interval (seconds) between health checks - format: int64 - type: integer - timeoutSeconds: - description: The time to wait (seconds) for a health check response - format: int64 - type: integer - unhealthyThresholdCount: - description: The number of unhealthy health checks required before a host is marked unhealthy - format: int32 - type: integer - type: object - include: - description: Include specifies that this tcpproxy should be delegated to another HTTPProxy. - properties: - name: - description: Name of the child HTTPProxy - type: string - namespace: - description: Namespace of the HTTPProxy to include. Defaults to the current namespace if not supplied. - type: string - required: - - name - type: object - includes: - description: "IncludesDeprecated allow for specific routing configuration to be appended to another HTTPProxy in another namespace. \n Exists due to a mistake when developing HTTPProxy and the field was marked plural when it should have been singular. This field should stay to not break backwards compatibility to v1 users." - properties: - name: - description: Name of the child HTTPProxy - type: string - namespace: - description: Namespace of the HTTPProxy to include. Defaults to the current namespace if not supplied. - type: string - required: - - name - type: object - loadBalancerPolicy: - description: The load balancing policy for the backend services. - properties: - strategy: - description: Strategy specifies the policy used to balance requests across the pool of backend pods. Valid policy names are `Random`, `RoundRobin`, `WeightedLeastRequest`, `Random` and `Cookie`. If an unknown strategy name is specified or no policy is supplied, the default `RoundRobin` policy is used. - type: string - type: object - services: - description: Services are the services to proxy traffic - items: - description: Service defines an Kubernetes Service to proxy traffic. - properties: - mirror: - description: If Mirror is true the Service will receive a read only mirror of the traffic for this route. - type: boolean - name: - description: Name is the name of Kubernetes service to proxy traffic. Names defined here will be used to look up corresponding endpoints which contain the ips to route. - type: string - port: - description: Port (defined as Integer) to proxy traffic to since a service can have multiple defined. - exclusiveMaximum: true - maximum: 65536 - minimum: 1 - type: integer - protocol: - description: Protocol may be used to specify (or override) the protocol used to reach this Service. Values may be tls, h2, h2c. If omitted, protocol-selection falls back on Service annotations. - enum: - - h2 - - h2c - - tls - type: string - requestHeadersPolicy: - description: The policy for managing request headers during proxying. Rewriting the 'Host' header is not supported. - properties: - remove: - description: Remove specifies a list of HTTP header names to remove. - items: - type: string - type: array - set: - description: Set specifies a list of HTTP header values that will be set in the HTTP header. If the header does not exist it will be added, otherwise it will be overwritten with the new value. - items: - description: HeaderValue represents a header name/value pair - properties: - name: - description: Name represents a key of a header - minLength: 1 - type: string - value: - description: Value represents the value of a header specified by a key - minLength: 1 - type: string - required: - - name - - value - type: object - type: array - type: object - responseHeadersPolicy: - description: The policy for managing response headers during proxying. Rewriting the 'Host' header is not supported. - properties: - remove: - description: Remove specifies a list of HTTP header names to remove. - items: - type: string - type: array - set: - description: Set specifies a list of HTTP header values that will be set in the HTTP header. If the header does not exist it will be added, otherwise it will be overwritten with the new value. - items: - description: HeaderValue represents a header name/value pair - properties: - name: - description: Name represents a key of a header - minLength: 1 - type: string - value: - description: Value represents the value of a header specified by a key - minLength: 1 - type: string - required: - - name - - value - type: object - type: array - type: object - validation: - description: UpstreamValidation defines how to verify the backend service's certificate - properties: - caSecret: - description: Name of the Kubernetes secret be used to validate the certificate presented by the backend - type: string - subjectName: - description: Key which is expected to be present in the 'subjectAltName' of the presented certificate - type: string - required: - - caSecret - - subjectName - type: object - weight: - description: Weight defines percentage of traffic to balance traffic - format: int64 - minimum: 0 - type: integer - required: - - name - - port - type: object - type: array - type: object - virtualhost: - description: Virtualhost appears at most once. If it is present, the object is considered to be a "root" HTTPProxy. - properties: - authorization: - description: This field configures an extension service to perform authorization for this virtual host. Authorization can only be configured on virtual hosts that have TLS enabled. If the TLS configuration requires client certificate /validation, the client certificate is always included in the authentication check request. - properties: - authPolicy: - description: AuthPolicy sets a default authorization policy for client requests. This policy will be used unless overridden by individual routes. - properties: - context: - additionalProperties: - type: string - description: Context is a set of key/value pairs that are sent to the authentication server in the check request. If a context is provided at an enclosing scope, the entries are merged such that the inner scope overrides matching keys from the outer scope. - type: object - disabled: - description: When true, this field disables client request authentication for the scope of the policy. - type: boolean - type: object - extensionRef: - description: ExtensionServiceRef specifies the extension resource that will authorize client requests. - properties: - apiVersion: - description: API version of the referent. If this field is not specified, the default "projectcontour.io/v1alpha1" will be used - minLength: 1 - type: string - name: - description: "Name of the referent. \n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names" - minLength: 1 - type: string - namespace: - description: "Namespace of the referent. If this field is not specifies, the namespace of the resource that targets the referent will be used. \n More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/" - minLength: 1 - type: string - type: object - failOpen: - description: If FailOpen is true, the client request is forwarded to the upstream service even if the authorization server fails to respond. This field should not be set in most cases. It is intended for use only while migrating applications from internal authorization to Contour external authorization. - type: boolean - responseTimeout: - description: ResponseTimeout configures maximum time to wait for a check response from the authorization server. Timeout durations are expressed in the Go [Duration format](https://godoc.org/time#ParseDuration). Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". The string "infinity" is also a valid input and specifies no timeout. - pattern: ^(((\d*(\.\d*)?h)|(\d*(\.\d*)?m)|(\d*(\.\d*)?s)|(\d*(\.\d*)?ms)|(\d*(\.\d*)?us)|(\d*(\.\d*)?µs)|(\d*(\.\d*)?ns))+|infinity|infinite)$ - type: string - required: - - extensionRef - type: object - corsPolicy: - description: Specifies the cross-origin policy to apply to the VirtualHost. - properties: - allowCredentials: - description: Specifies whether the resource allows credentials. - type: boolean - allowHeaders: - description: AllowHeaders specifies the content for the *access-control-allow-headers* header. - items: - description: CORSHeaderValue specifies the value of the string headers returned by a cross-domain request. - pattern: ^[a-zA-Z0-9!#$%&'*+.^_`|~-]+$ - type: string - type: array - allowMethods: - description: AllowMethods specifies the content for the *access-control-allow-methods* header. - items: - description: CORSHeaderValue specifies the value of the string headers returned by a cross-domain request. - pattern: ^[a-zA-Z0-9!#$%&'*+.^_`|~-]+$ - type: string - type: array - allowOrigin: - description: AllowOrigin specifies the origins that will be allowed to do CORS requests. "*" means allow any origin. - items: - type: string - type: array - exposeHeaders: - description: ExposeHeaders Specifies the content for the *access-control-expose-headers* header. - items: - description: CORSHeaderValue specifies the value of the string headers returned by a cross-domain request. - pattern: ^[a-zA-Z0-9!#$%&'*+.^_`|~-]+$ - type: string - type: array - maxAge: - description: MaxAge indicates for how long the results of a preflight request can be cached. MaxAge durations are expressed in the Go [Duration format](https://godoc.org/time#ParseDuration). Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h". Only positive values are allowed while 0 disables the cache requiring a preflight OPTIONS check for all cross-origin requests. - type: string - required: - - allowMethods - - allowOrigin - type: object - fqdn: - description: The fully qualified domain name of the root of the ingress tree all leaves of the DAG rooted at this object relate to the fqdn. - type: string - tls: - description: If present the fields describes TLS properties of the virtual host. The SNI names that will be matched on are described in fqdn, the tls.secretName secret must contain a certificate that itself contains a name that matches the FQDN. - properties: - clientValidation: - description: "ClientValidation defines how to verify the client certificate when an external client establishes a TLS connection to Envoy. \n This setting: \n 1. Enables TLS client certificate validation. 2. Requires clients to present a TLS certificate (i.e. not optional validation). 3. Specifies how the client certificate will be validated." - properties: - caSecret: - description: Name of a Kubernetes secret that contains a CA certificate bundle. The client certificate must validate against the certificates in the bundle. - minLength: 1 - type: string - required: - - caSecret - type: object - enableFallbackCertificate: - description: EnableFallbackCertificate defines if the vhost should allow a default certificate to be applied which handles all requests which don't match the SNI defined in this vhost. - type: boolean - minimumProtocolVersion: - description: Minimum TLS version this vhost should negotiate - type: string - passthrough: - description: Passthrough defines whether the encrypted TLS handshake will be passed through to the backing cluster. Either Passthrough or SecretName must be specified, but not both. - type: boolean - secretName: - description: SecretName is the name of a TLS secret in the current namespace. Either SecretName or Passthrough must be specified, but not both. If specified, the named secret must contain a matching certificate for the virtual host's FQDN. - type: string - type: object - required: - - fqdn - type: object - type: object - status: - description: Status is a container for computed information about the HTTPProxy. - properties: - conditions: - description: "Conditions contains information about the current status of the HTTPProxy, in an upstream-friendly container. \n Contour will update a single condition, `Valid`, that is in normal-true polarity. That is, when `currentStatus` is `valid`, the `Valid` condition will be `status: true`, and vice versa. \n Contour will leave untouched any other Conditions set in this block, in case some other controller wants to add a Condition. \n If you are another controller owner and wish to add a condition, you *should* namespace your condition with a label, like `controller.domain.com/ConditionName`." - items: - description: "DetailedCondition is an extension of the normal Kubernetes conditions, with two extra fields to hold sub-conditions, which provide more detailed reasons for the state (True or False) of the condition. \n `errors` holds information about sub-conditions which are fatal to that condition and render its state False. \n `warnings` holds information about sub-conditions which are not fatal to that condition and do not force the state to be False. \n Remember that Conditions have a type, a status, and a reason. \n The type is the type of the condition, the most important one in this CRD set is `Valid`. `Valid` is a positive-polarity condition: when it is `status: true` there are no problems. \n In more detail, `status: true` means that the object is has been ingested into Contour with no errors. `warnings` may still be present, and will be indicated in the Reason field. There must be zero entries in the `errors` slice in this case. \n `Valid`, `status: false` means that the object has had one or more fatal errors during processing into Contour. The details of the errors will be present under the `errors` field. There must be at least one error in the `errors` slice if `status` is `false`. \n For DetailedConditions of types other than `Valid`, the Condition must be in the negative polarity. When they have `status` `true`, there is an error. There must be at least one entry in the `errors` Subcondition slice. When they have `status` `false`, there are no serious errors, and there must be zero entries in the `errors` slice. In either case, there may be entries in the `warnings` slice. \n Regardless of the polarity, the `reason` and `message` fields must be updated with either the detail of the reason (if there is one and only one entry in total across both the `errors` and `warnings` slices), or `MultipleReasons` if there is more than one entry." - properties: - errors: - description: "Errors contains a slice of relevant error subconditions for this object. \n Subconditions are expected to appear when relevant (when there is a error), and disappear when not relevant. An empty slice here indicates no errors." - items: - description: "SubCondition is a Condition-like type intended for use as a subcondition inside a DetailedCondition. \n It contains a subset of the Condition fields. \n It is intended for warnings and errors, so `type` names should use abnormal-true polarity, that is, they should be of the form \"ErrorPresent: true\". \n The expected lifecycle for these errors is that they should only be present when the error or warning is, and should be removed when they are not relevant." - properties: - message: - description: "Message is a human readable message indicating details about the transition. \n This may be an empty string." - maxLength: 32768 - type: string - reason: - description: "Reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. \n The value should be a CamelCase string. \n This field may not be empty." - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: Status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: "Type of condition in `CamelCase` or in `foo.example.com/CamelCase`. \n This must be in abnormal-true polarity, that is, `ErrorFound` or `controller.io/ErrorFound`. \n The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)" - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - message - - reason - - status - - type - type: object - type: array - lastTransitionTime: - description: lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: message is a human readable message indicating details about the transition. This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: observedGeneration represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. --- Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be useful (see .node.status.conditions), the ability to deconflict is important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - warnings: - description: "Warnings contains a slice of relevant warning subconditions for this object. \n Subconditions are expected to appear when relevant (when there is a warning), and disappear when not relevant. An empty slice here indicates no warnings." - items: - description: "SubCondition is a Condition-like type intended for use as a subcondition inside a DetailedCondition. \n It contains a subset of the Condition fields. \n It is intended for warnings and errors, so `type` names should use abnormal-true polarity, that is, they should be of the form \"ErrorPresent: true\". \n The expected lifecycle for these errors is that they should only be present when the error or warning is, and should be removed when they are not relevant." - properties: - message: - description: "Message is a human readable message indicating details about the transition. \n This may be an empty string." - maxLength: 32768 - type: string - reason: - description: "Reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. \n The value should be a CamelCase string. \n This field may not be empty." - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: Status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: "Type of condition in `CamelCase` or in `foo.example.com/CamelCase`. \n This must be in abnormal-true polarity, that is, `ErrorFound` or `controller.io/ErrorFound`. \n The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)" - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - message - - reason - - status - - type - type: object - type: array - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - currentStatus: - type: string - description: - type: string - loadBalancer: - description: LoadBalancer contains the current status of the load balancer. - properties: - ingress: - description: Ingress is a list containing ingress points for the load-balancer. Traffic intended for the service should be sent to these ingress points. - items: - description: 'LoadBalancerIngress represents the status of a load-balancer ingress point: traffic intended for the service should be sent to an ingress point.' - properties: - hostname: - description: Hostname is set for load-balancer ingress points that are DNS based (typically AWS load-balancers) - type: string - ip: - description: IP is set for load-balancer ingress points that are IP based (typically GCE or OpenStack load-balancers) - type: string - type: object - type: array - type: object - type: object - required: - - metadata - - spec - type: object - served: true - storage: true - subresources: - status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.2.9 - creationTimestamp: null - name: tlscertificatedelegations.projectcontour.io -spec: - preserveUnknownFields: false - group: projectcontour.io - names: - kind: TLSCertificateDelegation - listKind: TLSCertificateDelegationList - plural: tlscertificatedelegations - shortNames: - - tlscerts - singular: tlscertificatedelegation - scope: Namespaced - versions: - - name: v1 - schema: - openAPIV3Schema: - description: TLSCertificateDelegation is an TLS Certificate Delegation CRD specification. See design/tls-certificate-delegation.md for details. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: TLSCertificateDelegationSpec defines the spec of the CRD - properties: - delegations: - items: - description: CertificateDelegation maps the authority to reference a secret in the current namespace to a set of namespaces. - properties: - secretName: - description: required, the name of a secret in the current namespace. - type: string - targetNamespaces: - description: required, the namespaces the authority to reference the the secret will be delegated to. If TargetNamespaces is nil or empty, the CertificateDelegation' is ignored. If the TargetNamespace list contains the character, "*" the secret will be delegated to all namespaces. - items: - type: string - type: array - required: - - secretName - - targetNamespaces - type: object - type: array - required: - - delegations - type: object - status: - description: TLSCertificateDelegationStatus allows for the status of the delegation to be presented to the user. - properties: - conditions: - description: "Conditions contains information about the current status of the HTTPProxy, in an upstream-friendly container. \n Contour will update a single condition, `Valid`, that is in normal-true polarity. That is, when `currentStatus` is `valid`, the `Valid` condition will be `status: true`, and vice versa. \n Contour will leave untouched any other Conditions set in this block, in case some other controller wants to add a Condition. \n If you are another controller owner and wish to add a condition, you *should* namespace your condition with a label, like `controller.domain.com\\ConditionName`." - items: - description: "DetailedCondition is an extension of the normal Kubernetes conditions, with two extra fields to hold sub-conditions, which provide more detailed reasons for the state (True or False) of the condition. \n `errors` holds information about sub-conditions which are fatal to that condition and render its state False. \n `warnings` holds information about sub-conditions which are not fatal to that condition and do not force the state to be False. \n Remember that Conditions have a type, a status, and a reason. \n The type is the type of the condition, the most important one in this CRD set is `Valid`. `Valid` is a positive-polarity condition: when it is `status: true` there are no problems. \n In more detail, `status: true` means that the object is has been ingested into Contour with no errors. `warnings` may still be present, and will be indicated in the Reason field. There must be zero entries in the `errors` slice in this case. \n `Valid`, `status: false` means that the object has had one or more fatal errors during processing into Contour. The details of the errors will be present under the `errors` field. There must be at least one error in the `errors` slice if `status` is `false`. \n For DetailedConditions of types other than `Valid`, the Condition must be in the negative polarity. When they have `status` `true`, there is an error. There must be at least one entry in the `errors` Subcondition slice. When they have `status` `false`, there are no serious errors, and there must be zero entries in the `errors` slice. In either case, there may be entries in the `warnings` slice. \n Regardless of the polarity, the `reason` and `message` fields must be updated with either the detail of the reason (if there is one and only one entry in total across both the `errors` and `warnings` slices), or `MultipleReasons` if there is more than one entry." - properties: - errors: - description: "Errors contains a slice of relevant error subconditions for this object. \n Subconditions are expected to appear when relevant (when there is a error), and disappear when not relevant. An empty slice here indicates no errors." - items: - description: "SubCondition is a Condition-like type intended for use as a subcondition inside a DetailedCondition. \n It contains a subset of the Condition fields. \n It is intended for warnings and errors, so `type` names should use abnormal-true polarity, that is, they should be of the form \"ErrorPresent: true\". \n The expected lifecycle for these errors is that they should only be present when the error or warning is, and should be removed when they are not relevant." - properties: - message: - description: "Message is a human readable message indicating details about the transition. \n This may be an empty string." - maxLength: 32768 - type: string - reason: - description: "Reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. \n The value should be a CamelCase string. \n This field may not be empty." - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: Status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: "Type of condition in `CamelCase` or in `foo.example.com/CamelCase`. \n This must be in abnormal-true polarity, that is, `ErrorFound` or `controller.io/ErrorFound`. \n The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)" - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - message - - reason - - status - - type - type: object - type: array - lastTransitionTime: - description: lastTransitionTime is the last time the condition transitioned from one status to another. This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: message is a human readable message indicating details about the transition. This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: observedGeneration represents the .metadata.generation that the condition was set based upon. For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date with respect to the current state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. The value should be a CamelCase string. This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. --- Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be useful (see .node.status.conditions), the ability to deconflict is important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - warnings: - description: "Warnings contains a slice of relevant warning subconditions for this object. \n Subconditions are expected to appear when relevant (when there is a warning), and disappear when not relevant. An empty slice here indicates no warnings." - items: - description: "SubCondition is a Condition-like type intended for use as a subcondition inside a DetailedCondition. \n It contains a subset of the Condition fields. \n It is intended for warnings and errors, so `type` names should use abnormal-true polarity, that is, they should be of the form \"ErrorPresent: true\". \n The expected lifecycle for these errors is that they should only be present when the error or warning is, and should be removed when they are not relevant." - properties: - message: - description: "Message is a human readable message indicating details about the transition. \n This may be an empty string." - maxLength: 32768 - type: string - reason: - description: "Reason contains a programmatic identifier indicating the reason for the condition's last transition. Producers of specific condition types may define expected values and meanings for this field, and whether the values are considered a guaranteed API. \n The value should be a CamelCase string. \n This field may not be empty." - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: Status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: "Type of condition in `CamelCase` or in `foo.example.com/CamelCase`. \n This must be in abnormal-true polarity, that is, `ErrorFound` or `controller.io/ErrorFound`. \n The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)" - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - message - - reason - - status - - type - type: object - type: array - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - x-kubernetes-list-map-keys: - - type - x-kubernetes-list-type: map - type: object - required: - - metadata - - spec - type: object - served: true - storage: true - subresources: - status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] - ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: contour-certgen - namespace: projectcontour ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: contour - namespace: projectcontour -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: contour-certgen -subjects: -- kind: ServiceAccount - name: contour-certgen - namespace: projectcontour ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: contour-certgen - namespace: projectcontour -rules: -- apiGroups: - - "" - resources: - - secrets - verbs: - - create - - update ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: contour-certgen-v1.11.0 - namespace: projectcontour -spec: - ttlSecondsAfterFinished: 0 - template: - metadata: - labels: - app: "contour-certgen" - spec: - containers: - - name: contour - image: docker.io/projectcontour/contour:v1.11.0 - imagePullPolicy: Always - command: - - contour - - certgen - - --kube - - --incluster - - --overwrite - - --secrets-format=compact - - --namespace=$(CONTOUR_NAMESPACE) - env: - - name: CONTOUR_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - restartPolicy: Never - serviceAccountName: contour-certgen - securityContext: - runAsNonRoot: true - runAsUser: 65534 - runAsGroup: 65534 - parallelism: 1 - completions: 1 - backoffLimit: 1 - ---- - -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: contour -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: contour -subjects: -- kind: ServiceAccount - name: contour - namespace: projectcontour - -# The following ClusterRole is generated from kubebuilder RBAC tags by -# generate-rbac.sh. Do not edit this file directly but instead edit the source -# files and re-render. - ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - creationTimestamp: null - name: contour -rules: -- apiGroups: - - "" - resources: - - configmaps - verbs: - - create - - get - - update -- apiGroups: - - "" - resources: - - endpoints - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - secrets - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - services - verbs: - - get - - list - - watch -- apiGroups: - - apiextensions.k8s.io - resources: - - customresourcedefinitions - verbs: - - list -- apiGroups: - - networking.k8s.io - resources: - - gatewayclasses - - gateways - - httproutes - - tcproutes - verbs: - - get - - list - - watch -- apiGroups: - - networking.k8s.io - resources: - - ingresses - verbs: - - get - - list - - watch -- apiGroups: - - networking.k8s.io - resources: - - ingresses/status - verbs: - - create - - get - - update -- apiGroups: - - projectcontour.io - resources: - - extensionservices - verbs: - - get - - list - - watch -- apiGroups: - - projectcontour.io - resources: - - extensionservices/status - verbs: - - create - - get - - update -- apiGroups: - - projectcontour.io - resources: - - httpproxies - - tlscertificatedelegations - verbs: - - get - - list - - watch -- apiGroups: - - projectcontour.io - resources: - - httpproxies/status - verbs: - - create - - get - - update - ---- -apiVersion: v1 -kind: Service -metadata: - name: contour - namespace: projectcontour -spec: - ports: - - port: 8001 - name: xds - protocol: TCP - targetPort: 8001 - selector: - app: contour - type: ClusterIP - ---- -apiVersion: v1 -kind: Service -metadata: - name: envoy - namespace: projectcontour - annotations: - # This annotation puts the AWS ELB into "TCP" mode so that it does not - # do HTTP negotiation for HTTPS connections at the ELB edge. - # The downside of this is the remote IP address of all connections will - # appear to be the internal address of the ELB. See docs/proxy-proto.md - # for information about enabling the PROXY protocol on the ELB to recover - # the original remote IP address. - service.beta.kubernetes.io/aws-load-balancer-backend-protocol: tcp -spec: - externalTrafficPolicy: Local - ports: - - port: 80 - name: http - protocol: TCP - nodePort: 30081 - - port: 443 - name: https - protocol: TCP - selector: - app: envoy - type: LoadBalancer - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: contour - name: contour - namespace: projectcontour -spec: - replicas: 2 - strategy: - type: RollingUpdate - rollingUpdate: - # This value of maxSurge means that during a rolling update - # the new ReplicaSet will be created first. - maxSurge: 50% - selector: - matchLabels: - app: contour - template: - metadata: - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "8000" - labels: - app: contour - spec: - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - podAffinityTerm: - labelSelector: - matchLabels: - app: contour - topologyKey: kubernetes.io/hostname - weight: 100 - containers: - - args: - - serve - - --incluster - - --xds-address=0.0.0.0 - - --xds-port=8001 - - --envoy-service-http-port=80 - - --envoy-service-https-port=443 - - --contour-cafile=/certs/ca.crt - - --contour-cert-file=/certs/tls.crt - - --contour-key-file=/certs/tls.key - - --config-path=/config/contour.yaml - command: ["contour"] - image: docker.io/projectcontour/contour:v1.11.0 - imagePullPolicy: IfNotPresent - name: contour - ports: - - containerPort: 8001 - name: xds - protocol: TCP - - containerPort: 8000 - name: metrics - protocol: TCP - - containerPort: 6060 - name: debug - protocol: TCP - livenessProbe: - httpGet: - path: /healthz - port: 8000 - readinessProbe: - tcpSocket: - port: 8001 - initialDelaySeconds: 15 - periodSeconds: 10 - volumeMounts: - - name: contourcert - mountPath: /certs - readOnly: true - - name: contour-config - mountPath: /config - readOnly: true - env: - - name: CONTOUR_NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - - name: POD_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.name - dnsPolicy: ClusterFirst - serviceAccountName: contour - securityContext: - runAsNonRoot: true - runAsUser: 65534 - runAsGroup: 65534 - volumes: - - name: contourcert - secret: - secretName: contourcert - - name: contour-config - configMap: - name: contour - defaultMode: 0644 - items: - - key: contour.yaml - path: contour.yaml - ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - labels: - app: envoy - name: envoy - namespace: projectcontour -spec: - updateStrategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 10% - selector: - matchLabels: - app: envoy - template: - metadata: - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "8002" - prometheus.io/path: "/stats/prometheus" - labels: - app: envoy - spec: - containers: - - command: - - /bin/contour - args: - - envoy - - shutdown-manager - image: docker.io/projectcontour/contour:v1.11.0 - imagePullPolicy: IfNotPresent - lifecycle: - preStop: - exec: - command: - - /bin/contour - - envoy - - shutdown - livenessProbe: - httpGet: - path: /healthz - port: 8090 - initialDelaySeconds: 3 - periodSeconds: 10 - name: shutdown-manager - - args: - - -c - - /config/envoy.json - - --service-cluster $(CONTOUR_NAMESPACE) - - --service-node $(ENVOY_POD_NAME) - - --log-level info - command: - - envoy - image: docker.io/envoyproxy/envoy:v1.16.2 - imagePullPolicy: IfNotPresent - name: envoy - env: - - name: CONTOUR_NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - - name: ENVOY_POD_NAME - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.name - ports: - - containerPort: 80 - hostPort: 80 - name: http - protocol: TCP - - containerPort: 443 - hostPort: 443 - name: https - protocol: TCP - readinessProbe: - httpGet: - path: /ready - port: 8002 - initialDelaySeconds: 3 - periodSeconds: 4 - volumeMounts: - - name: envoy-config - mountPath: /config - - name: envoycert - mountPath: /certs - lifecycle: - preStop: - httpGet: - path: /shutdown - port: 8090 - scheme: HTTP - initContainers: - - args: - - bootstrap - - /config/envoy.json - - --xds-address=contour - - --xds-port=8001 - - --xds-resource-version=v3 - - --resources-dir=/config/resources - - --envoy-cafile=/certs/ca.crt - - --envoy-cert-file=/certs/tls.crt - - --envoy-key-file=/certs/tls.key - command: - - contour - image: docker.io/projectcontour/contour:v1.11.0 - imagePullPolicy: IfNotPresent - name: envoy-initconfig - volumeMounts: - - name: envoy-config - mountPath: /config - - name: envoycert - mountPath: /certs - readOnly: true - env: - - name: CONTOUR_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - automountServiceAccountToken: false - serviceAccountName: envoy - terminationGracePeriodSeconds: 300 - volumes: - - name: envoy-config - emptyDir: {} - - name: envoycert - secret: - secretName: envoycert - restartPolicy: Always diff --git a/kustomize/overlays/sandbox/dependencies/contour_ingress_controller/kustomization.yaml b/kustomize/overlays/sandbox/dependencies/contour_ingress_controller/kustomization.yaml deleted file mode 100644 index 5e16119d07..0000000000 --- a/kustomize/overlays/sandbox/dependencies/contour_ingress_controller/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -resources: - # We cannot just import from remote as namespace transformer causes conflicts - #- https://projectcontour.io/quickstart/v1.11.0/contour.yaml -- contour.yaml - diff --git a/kustomize/overlays/sandbox/dependencies/redis/deployment.yaml b/kustomize/overlays/sandbox/dependencies/redis/deployment.yaml deleted file mode 100644 index 82067d702b..0000000000 --- a/kustomize/overlays/sandbox/dependencies/redis/deployment.yaml +++ /dev/null @@ -1,59 +0,0 @@ -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: redis - namespace: flyte -spec: - replicas: 1 - selector: - matchLabels: - app: redis-resource-manager - serviceName: redis-resource-manager - template: - metadata: - labels: - app: redis-resource-manager - spec: - containers: - - env: - - name: REDIS_PASSWORD - value: mypassword - image: docker.io/bitnami/redis:4.0.2-r1 - imagePullPolicy: IfNotPresent - livenessProbe: - exec: - command: - - redis-cli - - ping - failureThreshold: 3 - initialDelaySeconds: 30 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 5 - name: redis-resource-manager - ports: - - containerPort: 6379 - name: redis - protocol: TCP - readinessProbe: - exec: - command: - - redis-cli - - ping - failureThreshold: 3 - initialDelaySeconds: 5 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 1 - resources: - requests: - cpu: 200m - memory: 128Mi - volumeMounts: - - mountPath: /bitnami - name: redis-data - dnsPolicy: ClusterFirst - restartPolicy: Always - volumes: - - name: redis-data - emptyDir: {} diff --git a/kustomize/overlays/sandbox/dependencies/redis/kustomization.yaml b/kustomize/overlays/sandbox/dependencies/redis/kustomization.yaml deleted file mode 100644 index a944d005ca..0000000000 --- a/kustomize/overlays/sandbox/dependencies/redis/kustomization.yaml +++ /dev/null @@ -1,3 +0,0 @@ -resources: -- deployment.yaml -- service.yaml diff --git a/kustomize/overlays/sandbox/dependencies/redis/service.yaml b/kustomize/overlays/sandbox/dependencies/redis/service.yaml deleted file mode 100644 index 8c86264ae3..0000000000 --- a/kustomize/overlays/sandbox/dependencies/redis/service.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - labels: - app: redis-resource-manager - name: redis-resource-manager - namespace: flyte -spec: - ports: - - name: redis - port: 6379 - protocol: TCP - targetPort: redis - selector: - app: redis-resource-manager - type: ClusterIP diff --git a/kustomize/overlays/sandbox/dependencies/storage/storage.yaml b/kustomize/overlays/sandbox/dependencies/storage/storage.yaml deleted file mode 100644 index 2f72d3c9a6..0000000000 --- a/kustomize/overlays/sandbox/dependencies/storage/storage.yaml +++ /dev/null @@ -1,63 +0,0 @@ ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: minio - namespace: flyte -spec: - selector: - matchLabels: - app: minio - template: - metadata: - labels: - app: minio - spec: - volumes: - - name: minio-storage - emptyDir: {} - containers: - - image: "minio/minio:RELEASE.2020-12-16T05-05-17Z" - name: minio - env: - - name: MINIO_ACCESS_KEY - value: minio - - name: MINIO_SECRET_KEY - value: miniostorage - args: - - server - - /data - ports: - - containerPort: 9000 - name: minio - volumeMounts: - - name: minio-storage - mountPath: /data ---- -apiVersion: v1 -kind: Service -metadata: - name: minio - namespace: flyte -spec: - externalName: minio - ports: - - port: 9000 - selector: - app: minio ---- -apiVersion: extensions/v1beta1 -kind: Ingress -metadata: - name: minio - namespace: flyte - annotations: - nginx.ingress.kubernetes.io/ssl-redirect: "false" -spec: - rules: - - http: - paths: - - path: /minio - backend: - serviceName: minio - servicePort: 9000 diff --git a/kustomize/overlays/sandbox/admin/deployment.yaml b/kustomize/overlays/sandbox/flyte/admin/deployment.yaml similarity index 91% rename from kustomize/overlays/sandbox/admin/deployment.yaml rename to kustomize/overlays/sandbox/flyte/admin/deployment.yaml index 2f02e350e7..5a30adeb68 100644 --- a/kustomize/overlays/sandbox/admin/deployment.yaml +++ b/kustomize/overlays/sandbox/flyte/admin/deployment.yaml @@ -20,12 +20,11 @@ spec: do echo waiting for database; sleep 2; done;", ] - name: run-migrations - image: flyteadmin:v0.3.29 + image: flyteadmin:v0.4.13 imagePullPolicy: IfNotPresent command: [ "flyteadmin", - "--logtostderr", "--config", "/etc/flyte/config/*.yaml", "migrate", @@ -35,12 +34,11 @@ spec: - name: config-volume mountPath: /etc/flyte/config - name: seed-projects - image: flyteadmin:v0.3.29 + image: flyteadmin:v0.4.13 imagePullPolicy: IfNotPresent command: [ "flyteadmin", - "--logtostderr", "--config", "/etc/flyte/config/*.yaml", "migrate", @@ -52,12 +50,11 @@ spec: - name: config-volume mountPath: /etc/flyte/config - name: sync-cluster-resources - image: flyteadmin:v0.3.29 + image: flyteadmin:v0.4.13 imagePullPolicy: IfNotPresent command: [ "flyteadmin", - "--logtostderr", "--config", "/etc/flyte/config/*.yaml", "clusterresource", diff --git a/kustomize/overlays/sandbox/config/admin/db.yaml b/kustomize/overlays/sandbox/flyte/config/admin/db.yaml similarity index 100% rename from kustomize/overlays/sandbox/config/admin/db.yaml rename to kustomize/overlays/sandbox/flyte/config/admin/db.yaml diff --git a/kustomize/overlays/sandbox/config/admin/task_resource_defaults.yaml b/kustomize/overlays/sandbox/flyte/config/admin/task_resource_defaults.yaml similarity index 100% rename from kustomize/overlays/sandbox/config/admin/task_resource_defaults.yaml rename to kustomize/overlays/sandbox/flyte/config/admin/task_resource_defaults.yaml diff --git a/kustomize/overlays/sandbox/config/clusterresource-templates/ac_project-copilot-dataconfig.yaml b/kustomize/overlays/sandbox/flyte/config/clusterresource-templates/ac_project-copilot-dataconfig.yaml similarity index 100% rename from kustomize/overlays/sandbox/config/clusterresource-templates/ac_project-copilot-dataconfig.yaml rename to kustomize/overlays/sandbox/flyte/config/clusterresource-templates/ac_project-copilot-dataconfig.yaml diff --git a/kustomize/overlays/sandbox/config/clusterresource-templates/ad_spark-role.yaml b/kustomize/overlays/sandbox/flyte/config/clusterresource-templates/ad_spark-role.yaml similarity index 100% rename from kustomize/overlays/sandbox/config/clusterresource-templates/ad_spark-role.yaml rename to kustomize/overlays/sandbox/flyte/config/clusterresource-templates/ad_spark-role.yaml diff --git a/kustomize/overlays/sandbox/config/clusterresource-templates/ae_spark-service-account.yaml b/kustomize/overlays/sandbox/flyte/config/clusterresource-templates/ae_spark-service-account.yaml similarity index 100% rename from kustomize/overlays/sandbox/config/clusterresource-templates/ae_spark-service-account.yaml rename to kustomize/overlays/sandbox/flyte/config/clusterresource-templates/ae_spark-service-account.yaml diff --git a/kustomize/overlays/sandbox/config/clusterresource-templates/af_spark-role-binding.yaml b/kustomize/overlays/sandbox/flyte/config/clusterresource-templates/af_spark-role-binding.yaml similarity index 100% rename from kustomize/overlays/sandbox/config/clusterresource-templates/af_spark-role-binding.yaml rename to kustomize/overlays/sandbox/flyte/config/clusterresource-templates/af_spark-role-binding.yaml diff --git a/kustomize/overlays/sandbox/config/common/logger.yaml b/kustomize/overlays/sandbox/flyte/config/common/logger.yaml similarity index 100% rename from kustomize/overlays/sandbox/config/common/logger.yaml rename to kustomize/overlays/sandbox/flyte/config/common/logger.yaml diff --git a/kustomize/overlays/sandbox/config/common/storage.yaml b/kustomize/overlays/sandbox/flyte/config/common/storage.yaml similarity index 100% rename from kustomize/overlays/sandbox/config/common/storage.yaml rename to kustomize/overlays/sandbox/flyte/config/common/storage.yaml diff --git a/kustomize/overlays/sandbox/config/datacatalog/db.yaml b/kustomize/overlays/sandbox/flyte/config/datacatalog/db.yaml similarity index 100% rename from kustomize/overlays/sandbox/config/datacatalog/db.yaml rename to kustomize/overlays/sandbox/flyte/config/datacatalog/db.yaml diff --git a/kustomize/overlays/sandbox/config/propeller/enabled_plugins.yaml b/kustomize/overlays/sandbox/flyte/config/propeller/enabled_plugins.yaml similarity index 100% rename from kustomize/overlays/sandbox/config/propeller/enabled_plugins.yaml rename to kustomize/overlays/sandbox/flyte/config/propeller/enabled_plugins.yaml diff --git a/kustomize/overlays/sandbox/config/propeller/plugins/k8s.yaml b/kustomize/overlays/sandbox/flyte/config/propeller/plugins/k8s.yaml similarity index 100% rename from kustomize/overlays/sandbox/config/propeller/plugins/k8s.yaml rename to kustomize/overlays/sandbox/flyte/config/propeller/plugins/k8s.yaml diff --git a/kustomize/overlays/sandbox/flyte/config/propeller/plugins/task_logs.yaml b/kustomize/overlays/sandbox/flyte/config/propeller/plugins/task_logs.yaml new file mode 100644 index 0000000000..2768f8c95d --- /dev/null +++ b/kustomize/overlays/sandbox/flyte/config/propeller/plugins/task_logs.yaml @@ -0,0 +1,4 @@ +plugins: + logs: + kubernetes-enabled: true + kubernetes-template-uri: "http://localhost:30082/#/log/{{ .namespace }}/{{ .podName }}/pod?namespace={{ .namespace }}" diff --git a/kustomize/overlays/sandbox/config/propeller/resource_manager.yaml b/kustomize/overlays/sandbox/flyte/config/propeller/resource_manager.yaml similarity index 100% rename from kustomize/overlays/sandbox/config/propeller/resource_manager.yaml rename to kustomize/overlays/sandbox/flyte/config/propeller/resource_manager.yaml diff --git a/kustomize/overlays/sandbox/flyte/kustomization.yaml b/kustomize/overlays/sandbox/flyte/kustomization.yaml new file mode 100644 index 0000000000..c0ad168558 --- /dev/null +++ b/kustomize/overlays/sandbox/flyte/kustomization.yaml @@ -0,0 +1,46 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# Override the namespace +namespace: flyte + +bases: + - ../../../base/single_cluster/complete + +resources: + - propeller/webhook-secret.yaml + +patchesStrategicMerge: + - admin/deployment.yaml + - propeller/webhook.yaml + +configMapGenerator: + - behavior: merge + files: + - ./config/admin/db.yaml + - ./config/admin/task_resource_defaults.yaml + - ./config/common/storage.yaml + - ./config/common/logger.yaml + name: flyte-admin-config + + - behavior: merge + files: + - ./config/clusterresource-templates/ac_project-copilot-dataconfig.yaml + name: clusterresource-template + + - behavior: merge + files: + - ./config/propeller/enabled_plugins.yaml + - ./config/propeller/resource_manager.yaml + - ./config/propeller/plugins/k8s.yaml + - ./config/propeller/plugins/task_logs.yaml + - ./config/common/storage.yaml + - ./config/common/logger.yaml + name: flyte-propeller-config + + - behavior: merge + files: + - ./config/common/storage.yaml + - ./config/common/logger.yaml + - ./config/datacatalog/db.yaml + name: datacatalog-config diff --git a/kustomize/overlays/sandbox/flyte/propeller/webhook-secret.yaml b/kustomize/overlays/sandbox/flyte/propeller/webhook-secret.yaml new file mode 100644 index 0000000000..4029be3716 --- /dev/null +++ b/kustomize/overlays/sandbox/flyte/propeller/webhook-secret.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +data: + password: bXl1c2Vy + user_secret: bXlzZWNyZXQ= + username: bXl1c2Vy +kind: Secret +metadata: + name: user-info + namespace: flyte +type: Opaque diff --git a/kustomize/overlays/sandbox/flyte/propeller/webhook.yaml b/kustomize/overlays/sandbox/flyte/propeller/webhook.yaml new file mode 100644 index 0000000000..a659e88141 --- /dev/null +++ b/kustomize/overlays/sandbox/flyte/propeller/webhook.yaml @@ -0,0 +1,18 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flyte-pod-webhook + namespace: flyte +spec: + template: + spec: + containers: + - name: webhook + volumeMounts: + - name: sample-secrets + mountPath: /etc/secrets/user-info + readOnly: true + volumes: + - name: sample-secrets + secret: + secretName: user-info diff --git a/kustomize/overlays/sandbox/kustomization.yaml b/kustomize/overlays/sandbox/kustomization.yaml index e60cd33045..67317484f3 100644 --- a/kustomize/overlays/sandbox/kustomization.yaml +++ b/kustomize/overlays/sandbox/kustomization.yaml @@ -1,77 +1,50 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -# Override the namespace -namespace: flyte - bases: - - ../../base/single_cluster/complete + # All the resources that make up the deployment + - flyte + ######## + # Operators to be enabled + ############# + # Dependencies to be configured + # In local we use minio, but in cloud environment use S3 / GCS / AFS / Oracle Blob store etc + # This is used for Resource pooling. On cloud you can use hosted redis (e.g. AWS elasticache) + # Contour is used to create ingress. On cloud service use the default provided ingress controllers or cloud LB's + # Add node ports for ease of use locally + - ../../base/ingress + - ../../base/addons/database + - ../../base/addons/kubernetes_dashboard + - ../../base/addons/storage + - ../../base/addons/contour_ingress_controller -# All the resources that make up the deployment -######## -# Operators to be enabled -############# -# Dependencies to be configured -# In local we use minio, but in cloud environment use S3 / GCS / AFS / Oracle Blob store etc -# This is used for Resource pooling. On cloud you can use hosted redis (e.g. AWS elasticache) -# Contour is used to create ingress. On cloud service use the default provided ingress controllers or cloud LB's -# Add node ports for ease of use locally resources: - - ./dependencies/database - - ./dependencies/storage - - ./dependencies/contour_ingress_controller - - ./dependencies/nodeport-services.yaml - - ../../base/namespace - -patchesStrategicMerge: - - admin/deployment.yaml - -configMapGenerator: - - behavior: merge - files: - - ./config/admin/db.yaml - - ./config/admin/task_resource_defaults.yaml - - ./config/common/storage.yaml - - ./config/common/logger.yaml - name: flyte-admin-config - - behavior: merge - files: - - ./config/clusterresource-templates/ac_project-copilot-dataconfig.yaml - name: clusterresource-template - - behavior: merge - files: - - ./config/propeller/enabled_plugins.yaml - - ./config/propeller/resource_manager.yaml - - ./config/propeller/plugins/k8s.yaml - - ./config/propeller/plugins/task_logs.yaml - - ./config/common/storage.yaml - - ./config/common/logger.yaml - name: flyte-propeller-config - - behavior: merge - files: - - ./config/common/storage.yaml - - ./config/common/logger.yaml - - ./config/datacatalog/db.yaml - name: datacatalog-config + - nodeport-services.yaml # Images that should be used images: # FlyteAdmin - name: flyteadmin # match images with this name - newTag: v0.3.29 # override the tag - newName: ghcr.io/lyft/flyteadmin # override the name + newTag: v0.5.0 # FLYTEADMIN_TAG override the tag + newName: ghcr.io/flyteorg/flyteadmin # override the name # FlyteConsole - name: flyteconsole # match images with this name - newTag: v0.19.1 # override the tag - newName: ghcr.io/lyft/flyteconsole # override the namep + newTag: v0.19.7 # FLYTECONSOLE_TAG the tag + newName: ghcr.io/flyteorg/flyteconsole # override the namep # Flyte DataCatalog - name: datacatalog # match images with this name - newTag: v0.2.130 # override the tag - newName: ghcr.io/lyft/datacatalog # override the name + newTag: v0.3.2 # DATACATALOG_TAG override the tag + newName: ghcr.io/flyteorg/datacatalog # override the name # FlytePropeller - name: flytepropeller # match images with this name - newTag: v0.5.13 # override the tag + newTag: v0.10.6 # FLYTEPROPELLER_TAG override the tag + newName: ghcr.io/flyteorg/flytepropeller # override the name + # Webhook + - name: webhook # match images with this name + newTag: v0.10.6 # FLYTEPROPELLER_TAG override the tag newName: ghcr.io/flyteorg/flytepropeller # override the name # Override postgres image to use alpine based (rather smaller) docker image - name: postgres newTag: 10.16-alpine + - name: minio/minio + newTag: RELEASE.2021-04-06T23-11-00Z diff --git a/kustomize/overlays/sandbox/dependencies/nodeport-services.yaml b/kustomize/overlays/sandbox/nodeport-services.yaml similarity index 100% rename from kustomize/overlays/sandbox/dependencies/nodeport-services.yaml rename to kustomize/overlays/sandbox/nodeport-services.yaml diff --git a/kustomize/overlays/test/dependencies/contour_ingress_controller/custom-config.yaml b/kustomize/overlays/test/dependencies/contour_ingress_controller/custom-config.yaml deleted file mode 100644 index c0fcfee56a..0000000000 --- a/kustomize/overlays/test/dependencies/contour_ingress_controller/custom-config.yaml +++ /dev/null @@ -1,134 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - name: ingressroutes.contour.heptio.com - labels: - component: ingressroute -spec: - group: contour.heptio.com - version: v1beta1 - scope: Namespaced - names: - plural: ingressroutes - kind: IngressRoute - additionalPrinterColumns: - - name: FQDN - type: string - description: Fully qualified domain name - JSONPath: .spec.virtualhost.fqdn - - name: TLS Secret - type: string - description: Secret with TLS credentials - JSONPath: .spec.virtualhost.tls.secretName - - name: First route - type: string - description: First routes defined - JSONPath: .spec.routes[0].match - - name: Status - type: string - description: The current status of the IngressRoute - JSONPath: .status.currentStatus - - name: Status Description - type: string - description: Description of the current status - JSONPath: .status.description - validation: - openAPIV3Schema: - properties: - spec: - required: - - routes - properties: - virtualhost: - properties: - fqdn: - type: string - pattern: ^([a-zA-Z0-9]+(-[a-zA-Z0-9]+)*\.)+[a-z]{2,}$ - aliases: - type: array - items: - type: string - pattern: ^([a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,}$ - strategy: - type: string - enum: - - RoundRobin - - LeastRequest - - Random - - RingHash - - Maglev - healthCheck: - type: object - required: - - path - properties: - path: - type: string - pattern: ^\/.*$ - intervalSeconds: - type: integer - timeoutSeconds: - type: integer - unhealthyThresholdCount: - type: integer - healthyThresholdCount: - type: integer - routes: - type: array - items: - required: - - match - properties: - match: - type: string - pattern: ^\/.*$ - delegate: - type: object - required: - - name - properties: - name: - type: string - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ # DNS-1123 - namespace: - type: string - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ # DNS-1123 - services: - type: array - items: - type: object - required: - - name - - port - properties: - name: - type: string - pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?$ # DNS-1123 - port: - type: integer - weight: - type: integer - strategy: - type: string - enum: - - RoundRobin - - WeightedLeastRequest - - Random - - RingHash - - Maglev - healthCheck: - type: object - required: - - path - properties: - path: - type: string - pattern: ^\/.*$ - intervalSeconds: - type: integer - timeoutSeconds: - type: integer - unhealthyThresholdCount: - type: integer - healthyThresholdCount: - type: integer diff --git a/kustomize/overlays/test/dependencies/contour_ingress_controller/deployment.yaml b/kustomize/overlays/test/dependencies/contour_ingress_controller/deployment.yaml deleted file mode 100644 index c24f2b504c..0000000000 --- a/kustomize/overlays/test/dependencies/contour_ingress_controller/deployment.yaml +++ /dev/null @@ -1,66 +0,0 @@ -# Create the actual deployment -apiVersion: apps/v1 -kind: Deployment -metadata: - name: contour - namespace: heptio-contour - labels: - app: contour -spec: - replicas: 1 - selector: - matchLabels: - app: contour - template: - metadata: - labels: - app: contour - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "8002" - prometheus.io/path: "/stats" - prometheus.io/format: "prometheus" - spec: - containers: - - image: docker.io/envoyproxy/envoy-alpine:v1.6.0 - name: envoy-envoyingressv1 - ports: - - containerPort: 80 - name: http - - containerPort: 8002 - name: statsd - command: ["envoy"] - args: ["-c", "/config/contour.yaml", "--service-cluster", "cluster0", "--service-node", "node0"] - volumeMounts: - - name: contour-config - mountPath: /config - resources: - requests: - cpu: 100m - memory: 100Mi - limits: - cpu: 100m - memory: 100Mi - - image: gcr.io/heptio-images/contour:v0.6.1 - imagePullPolicy: Always - ports: - - containerPort: 8000 - name: contour - name: contour-unknown - command: ["contour"] - args: ["serve", "--incluster", "--envoy-http-port=80", "--debug-http-port=6069"] - initContainers: - - image: gcr.io/heptio-images/contour:v0.6.1 - imagePullPolicy: Always - name: envoy-initconfig - command: ["contour"] - args: ["bootstrap", "/config/contour.yaml", "--statsd-enabled"] - volumeMounts: - - name: contour-config - mountPath: /config - volumes: - - name: contour-config - emptyDir: {} - dnsPolicy: ClusterFirst - serviceAccountName: contour - terminationGracePeriodSeconds: 30 diff --git a/kustomize/overlays/test/dependencies/contour_ingress_controller/kustomization.yaml b/kustomize/overlays/test/dependencies/contour_ingress_controller/kustomization.yaml deleted file mode 100644 index 83bbe44daf..0000000000 --- a/kustomize/overlays/test/dependencies/contour_ingress_controller/kustomization.yaml +++ /dev/null @@ -1,5 +0,0 @@ -resources: -- rbac.yaml -- custom-config.yaml -- service.yaml -- deployment.yaml diff --git a/kustomize/overlays/test/dependencies/contour_ingress_controller/rbac.yaml b/kustomize/overlays/test/dependencies/contour_ingress_controller/rbac.yaml deleted file mode 100644 index 4492a6f80f..0000000000 --- a/kustomize/overlays/test/dependencies/contour_ingress_controller/rbac.yaml +++ /dev/null @@ -1,66 +0,0 @@ -apiVersion: v1 -kind: ServiceAccount -metadata: - name: contour - namespace: heptio-contour ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRole -metadata: - name: contour -rules: -- apiGroups: - - "" - resources: - - configmaps - - endpoints - - nodes - - pods - - secrets - verbs: - - list - - watch -- apiGroups: - - "" - resources: - - nodes - verbs: - - get -- apiGroups: - - "" - resources: - - services - verbs: - - get - - list - - watch -- apiGroups: - - extensions - resources: - - ingresses - verbs: - - get - - list - - watch -- apiGroups: ["contour.heptio.com"] - resources: ["ingressroutes"] - verbs: - - get - - list - - watch - - put - - post - - patch ---- -apiVersion: rbac.authorization.k8s.io/v1beta1 -kind: ClusterRoleBinding -metadata: - name: contour -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: contour -subjects: -- kind: ServiceAccount - name: contour - namespace: heptio-contour diff --git a/kustomize/overlays/test/dependencies/contour_ingress_controller/service.yaml b/kustomize/overlays/test/dependencies/contour_ingress_controller/service.yaml deleted file mode 100644 index b7e10ab4af..0000000000 --- a/kustomize/overlays/test/dependencies/contour_ingress_controller/service.yaml +++ /dev/null @@ -1,17 +0,0 @@ -# Service -apiVersion: v1 -kind: Service -metadata: - name: contour - namespace: heptio-contour - labels: - app: contour -spec: - # use NodePort to make sure the service is accessible - type: NodePort - selector: - app: contour - ports: - - protocol: TCP - port: 80 - nodePort: 30081 diff --git a/kustomize/overlays/test/dependencies/database/database.yaml b/kustomize/overlays/test/dependencies/database/database.yaml deleted file mode 100644 index de17063c4c..0000000000 --- a/kustomize/overlays/test/dependencies/database/database.yaml +++ /dev/null @@ -1,41 +0,0 @@ ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: postgres - namespace: flyte -spec: - selector: - matchLabels: - app: postgres - template: - metadata: - labels: - app: postgres - spec: - volumes: - - name: postgres-storage - emptyDir: {} - containers: - - image: postgres - name: postgres - env: - - name: POSTGRES_HOST_AUTH_METHOD - value: trust - ports: - - containerPort: 5432 - name: postgres - volumeMounts: - - name: postgres-storage - mountPath: /var/lib/postgresql/data ---- -apiVersion: v1 -kind: Service -metadata: - name: postgres - namespace: flyte -spec: - ports: - - port: 5432 - selector: - app: postgres diff --git a/kustomize/overlays/test/dependencies/database/kustomization.yaml b/kustomize/overlays/test/dependencies/database/kustomization.yaml deleted file mode 100644 index ca2ede8d46..0000000000 --- a/kustomize/overlays/test/dependencies/database/kustomization.yaml +++ /dev/null @@ -1,2 +0,0 @@ -resources: -- database.yaml diff --git a/kustomize/overlays/test/dependencies/storage/kustomization.yaml b/kustomize/overlays/test/dependencies/storage/kustomization.yaml deleted file mode 100644 index 9a80610c4e..0000000000 --- a/kustomize/overlays/test/dependencies/storage/kustomization.yaml +++ /dev/null @@ -1,2 +0,0 @@ -resources: -- storage.yaml diff --git a/kustomize/overlays/test/dependencies/storage/storage.yaml b/kustomize/overlays/test/dependencies/storage/storage.yaml deleted file mode 100644 index 2f72d3c9a6..0000000000 --- a/kustomize/overlays/test/dependencies/storage/storage.yaml +++ /dev/null @@ -1,63 +0,0 @@ ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: minio - namespace: flyte -spec: - selector: - matchLabels: - app: minio - template: - metadata: - labels: - app: minio - spec: - volumes: - - name: minio-storage - emptyDir: {} - containers: - - image: "minio/minio:RELEASE.2020-12-16T05-05-17Z" - name: minio - env: - - name: MINIO_ACCESS_KEY - value: minio - - name: MINIO_SECRET_KEY - value: miniostorage - args: - - server - - /data - ports: - - containerPort: 9000 - name: minio - volumeMounts: - - name: minio-storage - mountPath: /data ---- -apiVersion: v1 -kind: Service -metadata: - name: minio - namespace: flyte -spec: - externalName: minio - ports: - - port: 9000 - selector: - app: minio ---- -apiVersion: extensions/v1beta1 -kind: Ingress -metadata: - name: minio - namespace: flyte - annotations: - nginx.ingress.kubernetes.io/ssl-redirect: "false" -spec: - rules: - - http: - paths: - - path: /minio - backend: - serviceName: minio - servicePort: 9000 diff --git a/kustomize/overlays/test/admin/deployment.yaml b/kustomize/overlays/test/flyte/admin/deployment.yaml similarity index 78% rename from kustomize/overlays/test/admin/deployment.yaml rename to kustomize/overlays/test/flyte/admin/deployment.yaml index c6f7e6cab4..70db6afa6c 100644 --- a/kustomize/overlays/test/admin/deployment.yaml +++ b/kustomize/overlays/test/flyte/admin/deployment.yaml @@ -17,25 +17,25 @@ spec: 'until pg_isready -h postgres -p 5432; do echo waiting for database; sleep 2; done;'] - name: run-migrations - image: flyteadmin:v0.3.5 + image: flyteadmin:v0.4.13 imagePullPolicy: IfNotPresent - command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/*.yaml", + command: ["flyteadmin", "--config", "/etc/flyte/config/*.yaml", "migrate", "run"] volumeMounts: - name: config-volume mountPath: /etc/flyte/config - name: seed-projects - image: flyteadmin:v0.3.5 + image: flyteadmin:v0.4.13 imagePullPolicy: IfNotPresent - command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/*.yaml", + command: ["flyteadmin", "--config", "/etc/flyte/config/*.yaml", "migrate", "seed-projects", "flytetester", "flytesnacks"] volumeMounts: - name: config-volume mountPath: /etc/flyte/config - name: sync-cluster-resources - image: flyteadmin:v0.3.5 + image: flyteadmin:v0.4.13 imagePullPolicy: IfNotPresent - command: ["flyteadmin", "--logtostderr", "--config", "/etc/flyte/config/*.yaml", "clusterresource", "sync"] + command: ["flyteadmin", "--config", "/etc/flyte/config/*.yaml", "clusterresource", "sync"] volumeMounts: - name: resource-templates mountPath: /etc/flyte/clusterresource/templates diff --git a/kustomize/overlays/test/config/admin/db.yaml b/kustomize/overlays/test/flyte/config/admin/db.yaml similarity index 100% rename from kustomize/overlays/test/config/admin/db.yaml rename to kustomize/overlays/test/flyte/config/admin/db.yaml diff --git a/kustomize/overlays/test/config/clusterresource-templates/ac_project-copilot-dataconfig.yaml b/kustomize/overlays/test/flyte/config/clusterresource-templates/ac_project-copilot-dataconfig.yaml similarity index 100% rename from kustomize/overlays/test/config/clusterresource-templates/ac_project-copilot-dataconfig.yaml rename to kustomize/overlays/test/flyte/config/clusterresource-templates/ac_project-copilot-dataconfig.yaml diff --git a/kustomize/overlays/test/config/common/logger.yaml b/kustomize/overlays/test/flyte/config/common/logger.yaml similarity index 100% rename from kustomize/overlays/test/config/common/logger.yaml rename to kustomize/overlays/test/flyte/config/common/logger.yaml diff --git a/kustomize/overlays/test/config/common/storage.yaml b/kustomize/overlays/test/flyte/config/common/storage.yaml similarity index 100% rename from kustomize/overlays/test/config/common/storage.yaml rename to kustomize/overlays/test/flyte/config/common/storage.yaml diff --git a/kustomize/overlays/test/config/datacatalog/db.yaml b/kustomize/overlays/test/flyte/config/datacatalog/db.yaml similarity index 100% rename from kustomize/overlays/test/config/datacatalog/db.yaml rename to kustomize/overlays/test/flyte/config/datacatalog/db.yaml diff --git a/kustomize/overlays/test/config/propeller/enabled_plugins.yaml b/kustomize/overlays/test/flyte/config/propeller/enabled_plugins.yaml similarity index 100% rename from kustomize/overlays/test/config/propeller/enabled_plugins.yaml rename to kustomize/overlays/test/flyte/config/propeller/enabled_plugins.yaml diff --git a/kustomize/overlays/test/config/propeller/plugins/k8s.yaml b/kustomize/overlays/test/flyte/config/propeller/plugins/k8s.yaml similarity index 100% rename from kustomize/overlays/test/config/propeller/plugins/k8s.yaml rename to kustomize/overlays/test/flyte/config/propeller/plugins/k8s.yaml diff --git a/kustomize/overlays/test/config/propeller/resource_manager.yaml b/kustomize/overlays/test/flyte/config/propeller/resource_manager.yaml similarity index 100% rename from kustomize/overlays/test/config/propeller/resource_manager.yaml rename to kustomize/overlays/test/flyte/config/propeller/resource_manager.yaml diff --git a/kustomize/overlays/test/flyte/kustomization.yaml b/kustomize/overlays/test/flyte/kustomization.yaml new file mode 100644 index 0000000000..60b964ed8c --- /dev/null +++ b/kustomize/overlays/test/flyte/kustomization.yaml @@ -0,0 +1,49 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +# Override the namespace +namespace: flyte + +bases: +- ../../../base/single_cluster/headless + +patchesStrategicMerge: +- admin/deployment.yaml + +configMapGenerator: +# the main admin configmap +- name: flyte-admin-config + behavior: merge + files: + - ./config/admin/db.yaml + - ./config/common/storage.yaml + - ./config/common/logger.yaml + +# cluster resource templates +- name: clusterresource-template + behavior: merge + files: + # Files are read in alphabetical order. To ensure that we create the namespace first, prefix the file name with "aa". + - ./config/clusterresource-templates/ac_project-copilot-dataconfig.yaml + +# Flyte Propeller Configuration +- name: flyte-propeller-config + behavior: merge + files: + - ./config/propeller/enabled_plugins.yaml + - ./config/propeller/resource_manager.yaml + - ./config/propeller/plugins/k8s.yaml + - ./config/common/storage.yaml + - ./config/common/logger.yaml + +# TODO Flyte Console Configuration +#- name: flyte-console-config +# files: +# - ./config/console.yaml + +- name: datacatalog-config + behavior: merge + files: + - ./config/datacatalog/db.yaml + - ./config/common/storage.yaml + - ./config/common/logger.yaml diff --git a/kustomize/overlays/test/kustomization.yaml b/kustomize/overlays/test/kustomization.yaml index 1c1832179b..644726e20f 100644 --- a/kustomize/overlays/test/kustomization.yaml +++ b/kustomize/overlays/test/kustomization.yaml @@ -1,81 +1,44 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -# Override the namespace -namespace: flyte - bases: - - ../../base/single_cluster/headless - -# All the resources that make up the deployment -resources: + # All the resources that make up the deployment + - flyte + ######## + # Operators to be enabled ############# # Dependencies to be configured - - ./dependencies/database # In local we use minio, but in cloud environment use S3 / GCS / AFS / Oracle Blob store etc - - ./dependencies/storage - # Contour is used to create ingress. On cloud service use the default provided ingress controllers or cloud LB's - - ./dependencies/contour_ingress_controller - - ../../base/namespace - -patches: - - ./admin/deployment.yaml - -configMapGenerator: - # the main admin configmap - - name: flyte-admin-config - behavior: merge - files: - - ./config/admin/db.yaml - - ./config/common/storage.yaml - - ./config/common/logger.yaml - # cluster resource templates - - name: clusterresource-template - behavior: merge - files: - # Files are read in alphabetical order. To ensure that we create the namespace first, prefix the file name with "aa". - - ./config/clusterresource-templates/ac_project-copilot-dataconfig.yaml - - # Flyte Propeller Configuration - - name: flyte-propeller-config - behavior: merge - files: - - ./config/propeller/enabled_plugins.yaml - - ./config/propeller/resource_manager.yaml - - ./config/propeller/plugins/k8s.yaml - - ./config/common/storage.yaml - - ./config/common/logger.yaml - - # TODO Flyte Console Configuration - #- name: flyte-console-config - # files: - # - ./config/console.yaml - - - name: datacatalog-config - behavior: merge - files: - - ./config/datacatalog/db.yaml - - ./config/common/storage.yaml - - ./config/common/logger.yaml + # This is used for Resource pooling. On cloud you can use hosted redis (e.g. AWS elasticache) + # Contour is used to create ingress. On cloud service use the default provided ingress controllers or cloud LB's + # Add node ports for ease of use locally + - ../../base/addons/database + - ../../base/addons/storage # Images that should be used images: # FlyteAdmin - name: flyteadmin # match images with this name - newTag: v0.3.29 # override the tag - newName: ghcr.io/lyft/flyteadmin # override the name + newTag: v0.5.0 # FLYTEADMIN_TAG override the tag + newName: ghcr.io/flyteorg/flyteadmin # override the name # FlyteConsole - name: flyteconsole # match images with this name - newTag: v0.19.1 # override the tag - newName: ghcr.io/lyft/flyteconsole # override the namep + newTag: v0.19.7 # FLYTECONSOLE_TAG the tag + newName: ghcr.io/flyteorg/flyteconsole # override the namep # Flyte DataCatalog - name: datacatalog # match images with this name - newTag: v0.2.130 # override the tag - newName: ghcr.io/lyft/datacatalog # override the name + newTag: v0.3.2 # DATACATALOG_TAG override the tag + newName: ghcr.io/flyteorg/datacatalog # override the name # FlytePropeller - name: flytepropeller # match images with this name - newTag: v0.5.13 # override the tag + newTag: v0.10.6 # FLYTEPROPELLER_TAG override the tag + newName: ghcr.io/flyteorg/flytepropeller # override the name + # Webhook + - name: webhook # match images with this name + newTag: v0.10.6 # FLYTEPROPELLER_TAG override the tag newName: ghcr.io/flyteorg/flytepropeller # override the name # Override postgres image to use alpine based (rather smaller) docker image - name: postgres newTag: 10.16-alpine + - name: minio/minio + newTag: RELEASE.2021-04-06T23-11-00Z diff --git a/requirements.in b/requirements.in new file mode 100644 index 0000000000..7c4e34693c --- /dev/null +++ b/requirements.in @@ -0,0 +1 @@ +grafanalib diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000..800331e6e7 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,10 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile requirements.in +# +attrs==20.3.0 + # via grafanalib +grafanalib==0.5.9 + # via -r requirements.in diff --git a/rsts/community/contribute.rst b/rsts/community/contribute.rst new file mode 100644 index 0000000000..bf1212adc1 --- /dev/null +++ b/rsts/community/contribute.rst @@ -0,0 +1,255 @@ +###################### +Contributing to Flyte +###################### + +Thank you for taking the time to contribute to Flyte! Here are some guidelines for you to follow, which will make your first and follow-up contributions easier. + +.. note:: + Please read our `Code of Conduct `_ before contributing to Flyte. + +Code +==== +An issue tagged with ``good first issue`` is the best place to start for first-time contributors. Look into them `here `_. + +To take a step ahead, check out the repositories available under `flyteorg `_. + +**Appetizer (for every repo): Fork and clone the concerned repository. Create a new branch on your fork and make the required changes. Create a pull request once your work is ready for review.** + +.. note:: + Note: To open a pull request, follow this `guide `_. + +*A piece of good news -- You can be added as a committer to any ``flyteorg`` repo as you become more involved with the project.* + +Example PR for your reference: `GitHub PR `_. A couple of checks are introduced to help in maintaining the robustness of the project. + +#. To get through DCO, sign off on every commit. (`Reference `_) +#. To improve code coverage, write unit tests to test your code. + +.. note:: + Format your Go code with ``golangci-lint`` followed by ``goimports`` (we used the same in the `Makefile `_), and Python code with ``black`` (use ``make fmt`` command which contains both black and isort). + +Environment Setup +***************** + +.. figure:: https://raw.githubusercontent.com/flyteorg/flyte/static-resources/img/contribution_guide/dependency_graph.png + :alt: Dependency Graph between various flyteorg repos + :align: center + :figclass: align-center + + The dependency graph between various flyteorg repos + +#. `flyte `_ + | Purpose: Deployment, Documentation, and Issues + | Languages: Kustomize & RST +#. `flyteidl `_ + | Purpose: The Flyte Workflow specification in `protocol buffers `_ which forms the core of Flyte + | Language: Protobuf + | Setup: Refer to the `README `_ +#. `flytepropeller `_ + | Purpose: Kubernetes native execution engine for Flyte Workflows and Tasks + | Language: Go + + Setup: + * Check for the Makefile in the root repo + * Run the following commands: + * ``make generate`` + * ``make test_unit`` + * ``make link`` + * To compile, run ``make compile`` +#. `flyteadmin `_ + | Purpose: Control Plane + | Language: Go + + Setup: + * Check for the Makefile in the root repo + * If the service code has to be tested, run it locally: + * ``make compile`` + * ``make server`` + * To seed data locally: + * ``make compile`` + * ``make seed_projects`` + * ``make migrate`` + * To run integration tests locally: + * ``make integration`` + * (or, to run in containerized dockernetes): ``make k8s_integration`` +#. `flytekit `_ + | Purpose: Python SDK & Tools + | Language: Python + | Setup: Refer to the `Flytekit Contribution Guide `_ +#. `flyteconsole `_ + | Purpose: Admin Console + | Language: Typescript + | Setup: Refer to the `README `_ +#. `datacatalog `_ + | Purpose: Manage Input & Output Artifacts + | Language: Go +#. `flyteplugins `_ + | Purpose: Flyte Plugins + | Language: Go + + Setup: + * Check for the Makefile in the root repo + * Run the following commands: + * ``make generate`` + * ``make test_unit`` + * ``make link`` +#. `flytestdlib `_ + | Purpose: Standard Library for Shared Components + | Language: Go +#. `flytesnacks `_ + | Purpose: Examples, Tips, and Tricks to use Flytekit SDKs + | Language: Python (In future, Java shall be added) + + Setup: + * If the Python code has to be tested, run it locally + * If the Python code has to be tested in a cluster: + * Run the ``make start`` command in the root directory of the flytesnacks repo + * Visit https://localhost:30081 to view the Flyte console consisting of the examples present in ``flytesnacks/cookbook/core`` directory + * To fetch the new dependencies and rebuild the image, run ``make register`` +#. `flytectl `_ + | Purpose: A Standalone Flyte CLI + | Language: Go + + Setup: + * Check for the Makefile in the root repo + * Run the following commands: + * ``make generate`` + * ``make test_unit`` + * ``make link`` + +Issues +====== +`GitHub Issues `_ is used for issue tracking. There are a variety of issue types available that you could use while filing an issue. + +* `Plugin Request `_ +* `Bug Report `_ +* `Documentation Bug/Update Request `_ +* `Core Feature Request `_ +* `Flytectl Feature Request `_ +* `Housekeeping `_ +* `UI Feature Request `_ + +If none of the above fits your requirements, file a `blank `_ issue. + +Documentation +============= +Flyte uses Sphinx for documentation and ``godocs`` for Golang. ``godocs`` is quite simple -- comment your code and you are good to go! + +Sphinx spans across multiple repositories under the `flyteorg `_ repository. It uses reStructured Text (rst) files to store the documentation content. For both the API and code-related content, it extracts docstrings from the code files. + +To get started, look into `reStructuredText reference `_. + +Environment Setup +***************** +Install all the requirements from the `docs-requirements.txt` file present in the root of a repository. + +.. code-block:: console + + pip install -r docs-requirements.txt + +From the ``docs`` directory present in the repository root (for ``flytesnacks``, ``docs`` is present in ``flytesnacks/cookbook``), run the command: + +.. code-block:: console + + make html + +.. note:: + For implicit targets, run ``make -C docs html``. + +You can then view the HTML pages in the ``docs/_build`` directory. + +.. note:: + For ``flyte`` repo, there is no ``docs`` directory. Instead, consider the ``rsts`` directory. To generate HTML files, run the following command in the root of the repo. + + .. code-block:: console + + make -C rsts html + +For minor edits that don’t require a local setup, you can edit the GitHub page in the documentation to propose the improvements. + +The edit option is found at the bottom of a page, as shown below. + +.. figure:: https://raw.githubusercontent.com/flyteorg/flyte/static-resources/img/contribution_guide/docs_edit.png + :alt: GitHub edit option for Documentation + :align: center + :figclass: align-center + +Intersphinx +*********** +`Intersphinx `_ can generate automatic links to the documentation of objects in other projects. + +To establish a reference to any other documentation from Flyte or within it, use intersphinx. + +To do so, create an ``intersphinx_mapping`` in the ``conf.py`` file present in the ``docs/source`` directory. + +For example: + +.. code-block:: python + + intersphinx_mapping = { + "python": ("https://docs.python.org/3", None), + "flytekit": ("https://flyte.readthedocs.io/projects/flytekit/en/master/", None), + } + +.. note:: + ``docs/source`` is present in the repository root. Click `here `_ to view the intersphinx configuration. + +The key refers to the name used to refer to the file (while referencing the documentation), and the URL denotes the precise location. + +Here are a couple of examples that you can refer to: + +.. code-block:: text + + Task: :std:doc:`generated/flytekit.task` + +Output: + +Task: :std:doc:`generated/flytekit.task` + +.. code-block:: text + + :std:doc:`Using custom words ` + +Output: + +:std:doc:`Using custom words ` + +| + +Linking to Python elements changes based on what you're linking to. Check out this `section `_ to learn more. + +| + +For instance, linking to the `task` decorator in flytekit uses the ``func`` role. + +.. code-block:: text + + Link to flytekit code :py:func:`flytekit:flytekit.task` + +Output: + +Link to flytekit code :py:func:`flytekit:flytekit.task` + +| + +Here are a couple more examples. + +.. code-block:: text + + :py:mod:`Module ` + :py:class:`Class ` + :py:data:`Data ` + :py:func:`Function ` + :py:meth:`Method ` + +Output: + +:py:mod:`Module ` + +:py:class:`Class ` + +:py:data:`Data ` + +:py:func:`Function ` + +:py:meth:`Method ` \ No newline at end of file diff --git a/rsts/community/docs.rst b/rsts/community/docs.rst deleted file mode 100644 index cef590e973..0000000000 --- a/rsts/community/docs.rst +++ /dev/null @@ -1,144 +0,0 @@ -.. _contribute-docs: - -###################### -Contributing to Docs -###################### - -************************ -Docs for various repos -************************ -Flyte is a large project and all the docs span multiple repositories. The core of the documention is in the `flyteorg/flyte `_ repository. -Flyte uses `Sphinx `_ to compile it docs. Docs are automatically pushed on merge to master and docs are hosted using `readthedocs.org `_ - -*************** -Sphinx and RST -*************** - -Style -========= - -Headers --------- -Typically, we try to follow these characters in this order for heading separation. - -.. code-block:: text - - # with overline - * with overline - = - - - ^ - -Intersphinx -============= -`Intersphinx `__ is a plugin that all Flyte repos that build Sphinx documentation use for cross-referencing with each other. There's some good background information on it on these `slides `__. - -Inventory File ----------------- -When Sphinx runs, an inventory file gets created and is available alongside each repo's HTML pages. For example at ``https://readthedocs.org/projects/flytecookbook/objects.inv``. This file is a compressed inventory of all the sections, tags, etc. in the flyte cookbook documentation. This inventory file is what allows the intersphinx plugin to cross-link between projects. - -There is an open-source tool called ``sphobjinv`` that has managed to `reverse engineer these files `__, and offers a CLI to help search for things inside them. - -Setup -------- -Installing ``sphobjinv`` and simple usage :: - - $ pip install sphobjinv - - # Using the CLI to query a hosted inventory file, note the -u switch - $ sphobjinv suggest https://flytekit.readthedocs.io/en/latest/ -u task - - No inventory at provided URL. - Attempting "https://flytekit.readthedocs.io/en/latest/objects.inv" ... - Remote inventory found. - - :py:function:`flytekit.task` - :std:doc:`tasks` - :std:doc:`tasks.extend` - :std:label:`tasks:tasks` - - # Using the CLI to query a local file, useful when iterating locally - $ sphobjinv suggest ~/go/src/github.com/flyteorg/flytekit/docs/build/html/objects.inv task - - :py:function:`flytekit.task` - :std:doc:`tasks` - :std:doc:`tasks.extend` - :std:label:`tasks:tasks` - -.. note:: - - Even though the ``sphobjinv`` CLI returns ``:py:function:...``, when actually creating a link you should just use ``:py:func:...``. See `this `__. Here is a quick list of mappings - - .. list-table:: Conversion table for - ``sphobjinv`` - :widths: 50 50 - :header-rows: 1 - - * - What the tool returns? - - What you should use instead? - * - :py:module: - - :py:mod: - * - :py:function: - - :py:func: - * - :std:label: - - :std:ref: - * - :py:method: - - :py:meth: - - - -Linking Examples ------------------- -In the ``conf.py`` file of each repo, there is an intersphinx mapping argument that looks something like this :: - - intersphinx_mapping = { - "python": ("https://docs.python.org/3", None), - "flytekit": ("https://flyte.readthedocs.io/projects/flytekit/en/master/", None), - ... - } - -This file is what tells the plugin where to look for these inventory files, and what project name to refer to each inventory file as. The project name is important because they're used when actually referencing something from the inventory. - -Here are some examples, first the code and then the link - -.. code-block:: text - - Task: :std:doc:`flytekit:tasks` - -Task: :std:doc:`flytekit:tasks` - ------ - -.. code-block:: text - - :std:doc:`Using custom words` - -:std:doc:`Using custom words` - -Python -^^^^^^^ -Linking to Python elements changes based on what you're linking to. Check out this `section `__. For instance linking to the ``task`` decorator in flytekit uses the ``func`` role. - -.. code-block:: text - - Link to flytekit code :py:func:`flytekit:flytekit.task` - -Link to flytekit code :py:func:`flytekit:flytekit.task` - -Other elements use different Sphinx roles, here are some examples using Python core docs. :: - - :py:mod:`Module ` - :py:class:`Class ` - :py:data:`Data ` - :py:func:`Function ` - :py:meth:`Method ` - - -:py:mod:`Module ` - -:py:class:`Class ` - -:py:data:`Data ` - -:py:func:`Function ` - -:py:meth:`Method ` diff --git a/rsts/community/index.rst b/rsts/community/index.rst index 6fbb8e96b5..65374e4d7d 100644 --- a/rsts/community/index.rst +++ b/rsts/community/index.rst @@ -12,10 +12,10 @@ so please join us on the - `Slack `_ - `Email `_ - `Twitter `_ +- Community Sync every other Tuesday, 9:00 AM PDT/PST please check out the `calendar `_ We love contributions, so please contribute to - docs - examples - new plugins or plugin ideas - just feedback in general - diff --git a/rsts/dive_deep/admin.rst b/rsts/concepts/admin.rst similarity index 75% rename from rsts/dive_deep/admin.rst rename to rsts/concepts/admin.rst index a6dcd7d879..a25f9e56a2 100644 --- a/rsts/dive_deep/admin.rst +++ b/rsts/concepts/admin.rst @@ -18,7 +18,7 @@ RPC FlyteAdmin uses the `grpc-gateway `__ library to serve incoming gRPC and HTTP requests with identical handlers. For a more detailed overview of the API, including request and response entities, see the admin -service `definition `__. The +service `definition `__. The RPC handlers are a thin shim that enforce some request structure validation and call out to appropriate :ref:`manager `. methods to process requests. @@ -59,14 +59,14 @@ The managers utilize additional components to process requests. These additional Repository ---------- Serialized entities (tasks, workflows, launch plans) and executions (workflow-, node- and task-) are stored as protos defined -`here `__. -We use the excellent `gorm `__ library to interface with our database, which currently supports a postgres +`here `__. +We use the excellent `gorm `__ library to interface with our database, which currently supports a postgres implementation. The actual code for issuing queries with gorm can be found in the -`gormimpl `__ directory. +`gormimpl `__ directory. Models ++++++ -Database models are defined in the `models `__ directory and correspond 1:1 with database tables [0]_. +Database models are defined in the `models `__ directory and correspond 1:1 with database tables [0]_. The full set of database tables includes: @@ -79,9 +79,9 @@ The full set of database tables includes: - task_executions - workflows -These database models inherit primary keys and indexes as defined in the corresponding `models `__ file. +These database models inherit primary keys and indexes as defined in the corresponding `models `__ file. -The repositories code also includes `transformers `__. +The repositories code also includes `transformers `__. These convert entities from the database format to a response format for the external API. If you change either of these structures, you will find you must change the corresponding transformers. @@ -114,7 +114,7 @@ As the name implies, ``common`` houses shared components used across different f Data ---- -Data interfaces are primarily handled by the `storage `__ library implemented in flytestdlib. However, neither this nor the underlying `stow `__ library expose `HEAD `__ support so the data package in admin exists as the layer responsible for additional, remote data operations. +Data interfaces are primarily handled by the `storage `__ library implemented in flytestdlib. However, neither this nor the underlying `stow `__ library expose `HEAD `__ support so the data package in admin exists as the layer responsible for additional, remote data operations. Errors ------ @@ -125,7 +125,7 @@ The errors directory contains centrally defined errors that are designed for com Runtime ------- -Values specific to the flyteadmin application as well as task and workflow registration and execution are configured in the `runtime `__ directory. These interfaces expose values configured in the ``flyteadmin`` top-level key in the application config. +Values specific to the flyteadmin application as well as task and workflow registration and execution are configured in the `runtime `__ directory. These interfaces expose values configured in the ``flyteadmin`` top-level key in the application config. .. _divedeep-admin-workflowengine: @@ -134,4 +134,4 @@ Workflowengine This directory contains interfaces to build and execute workflows leveraging flytepropeller compiler and client components. -.. [0] Unfortunately, given unique naming constraints, some models are redefined in `migration_models `__ to guarantee unique index values. +.. [0] Unfortunately, given unique naming constraints, some models are redefined in `migration_models `__ to guarantee unique index values. diff --git a/rsts/dive_deep/admin_service.rst b/rsts/concepts/admin_service.rst similarity index 79% rename from rsts/dive_deep/admin_service.rst rename to rsts/concepts/admin_service.rst index 1217bda081..005926e37e 100644 --- a/rsts/dive_deep/admin_service.rst +++ b/rsts/concepts/admin_service.rst @@ -6,7 +6,7 @@ FlyteAdmin Service Background Entities ======== -The `admin service definition `__ defines REST operations for the entities +The `admin service definition `__ defines REST operations for the entities flyteadmin administers. As a refresher, the primary :ref:`entities ` across Flyte map similarly to FlyteAdmin entities. @@ -26,20 +26,20 @@ Permitted operations: - Get - List -The above are designated by an `identifier `_ +The above are designated by an `identifier `_ which consists of a project, domain, name and version specification. These entities are for the most part immutable. To update one of these specific entities, the updated version must be reregistered with a unique and new version identifier attribute. -One caveat is that launch plan state can toggle between `ACTIVE or INACTIVE `_. +One caveat is that launch plan state can toggle between `ACTIVE or INACTIVE `_. At most one launch plan version across a shared project, domain and name specification can be active at a time. The state affects scheduled launch plans only. An inactive launch plan can still be used to launch individual executions. However, only an active launch plan runs on a schedule (if it has a schedule defined). Static entities metadata (Named Entities) +++++++++++++++++++++++++++++++++++++++++ -A `named entity `__ includes metadata for one of the above entities +A `named entity `__ includes metadata for one of the above entities (workflow, task or launch plan) across versions. A named entity includes a resource type (workflow, task or launch plan) and an -`id `__ which is composed of project, domain and name. +`id `__ which is composed of project, domain and name. A named entity also includes metadata, which are mutable attributes about the referenced entity. This metadata includes: @@ -72,7 +72,7 @@ Permitted operations: After an execution begins, flyte propeller monitors the execution and sends events which admin uses to update the above executions. -These `events `_ include +These `events `_ include - WorkflowExecutionEvent - NodeExecutionEvent @@ -174,11 +174,11 @@ Filterable fields vary based on entity types: - updated_at - workflows.{any workflow field above} (for example: workflow.domain) - state (you must use the integer enum e.g. 1) - - States are defined in `launch_plan.proto `_. + - States are defined in `launch_plan.proto `_. - Named Entity Metadata - state (you must use the integer enum e.g. 1) - - States are defined in `common.proto `_. + - States are defined in `common.proto `_. - Executions (Workflow executions) - project @@ -187,12 +187,12 @@ Filterable fields vary based on entity types: - workflow.{any workflow field above} (for example: workflow.domain) - launch_plan.{any launch plan field above} (for example: launch_plan.name) - phase (you must use the upper-cased string name e.g. RUNNING) - - Phases are defined in `execution.proto `__. + - Phases are defined in `execution.proto `__. - execution_created_at - execution_updated_at - duration (in seconds) - mode (you must use the integer enum e.g. 1) - - Modes are defined in `execution.proto `__. + - Modes are defined in `execution.proto `__. - user (authenticated user or role from flytekit config) - Node Executions @@ -200,7 +200,7 @@ Filterable fields vary based on entity types: - node_id - execution.{any execution field above} (for example: execution.domain) - phase (you must use the upper-cased string name e.g. QUEUED) - - Phases are defined in `execution.proto `__. + - Phases are defined in `execution.proto `__. - started_at - node_execution_created_at - node_execution_updated_at @@ -212,7 +212,7 @@ Filterable fields vary based on entity types: - execution.{any execution field above} (for example: execution.domain) - node_execution.{any node execution field above} (for example: node_execution.phase) - phase (you must use the upper-cased string name e.g. SUCCEEDED) - - Phases are defined in `execution.proto `__. + - Phases are defined in `execution.proto `__. - started_at - task_execution_created_at - task_execution_updated_at @@ -268,7 +268,7 @@ Only a subset of fields are supported for sorting list queries. The explicit lis - created_at - updated_at - state (you must use the integer enum e.g. 1) - - States are defined in `launch_plan.proto `_. + - States are defined in `launch_plan.proto `_. - ListWorkflowIds - project @@ -279,18 +279,18 @@ Only a subset of fields are supported for sorting list queries. The explicit lis - domain - name - phase (you must use the upper-cased string name e.g. RUNNING) - - Phases are defined in `execution.proto `__. + - Phases are defined in `execution.proto `__. - execution_created_at - execution_updated_at - duration (in seconds) - mode (you must use the integer enum e.g. 1) - - Modes are defined `execution.proto `__. + - Modes are defined `execution.proto `__. - ListNodeExecutions - node_id - retry_attempt - phase (you must use the upper-cased string name e.g. QUEUED) - - Phases are defined in `execution.proto `__. + - Phases are defined in `execution.proto `__. - started_at - node_execution_created_at - node_execution_updated_at @@ -299,7 +299,7 @@ Only a subset of fields are supported for sorting list queries. The explicit lis - retry_attempt - phase (you must use the upper-cased string name e.g. SUCCEEDED) - - Phases are defined in `execution.proto `__. + - Phases are defined in `execution.proto `__. - started_at - task_execution_created_at - task_execution_updated_at diff --git a/rsts/dive_deep/architecture.rst b/rsts/concepts/architecture.rst similarity index 98% rename from rsts/dive_deep/architecture.rst rename to rsts/concepts/architecture.rst index eeb8f8d64d..b8591e2671 100644 --- a/rsts/dive_deep/architecture.rst +++ b/rsts/concepts/architecture.rst @@ -11,7 +11,7 @@ FlyteIDL In Flyte, entities like "Workflows", "Tasks", "Launch Plans", and "Schedules" are recognized by multiple system components. In order for components to communicate effectively, they need a shared understanding about the structure of these entities. -The Flyte IDL (Interface Definition Language) is where shared Flyte entities are defined. This IDL also defines the RPC service definition for the `core Flyte API `_. +The Flyte IDL (Interface Definition Language) is where shared Flyte entities are defined. This IDL also defines the RPC service definition for the `core Flyte API `_. FlyteIDL uses the `protobuf `_ schema to describe entities. Clients are generated for Python, Golang, and JavaScript and imported by Flyte components. diff --git a/rsts/concepts/basics.rst b/rsts/concepts/basics.rst new file mode 100644 index 0000000000..8f970b2a5c --- /dev/null +++ b/rsts/concepts/basics.rst @@ -0,0 +1,18 @@ +.. _basics: + +###### +Basics +###### + +.. NOTE:: + + Coming soon 🛠 + +.. toctree:: + :maxdepth: 1 + :name: Basics + + flyte_ui + flyte_cli + deployment_options + glossary diff --git a/rsts/dive_deep/catalog.rst b/rsts/concepts/catalog.rst similarity index 100% rename from rsts/dive_deep/catalog.rst rename to rsts/concepts/catalog.rst diff --git a/rsts/dive_deep/console.rst b/rsts/concepts/console.rst similarity index 100% rename from rsts/dive_deep/console.rst rename to rsts/concepts/console.rst diff --git a/rsts/concepts/control_plane.rst b/rsts/concepts/control_plane.rst new file mode 100644 index 0000000000..83ebec2d36 --- /dev/null +++ b/rsts/concepts/control_plane.rst @@ -0,0 +1,13 @@ +################ +Control Plane +################ + +.. toctree:: + :maxdepth: 1 + + projects + domains + admin + admin_service + registration + console diff --git a/rsts/concepts/core.rst b/rsts/concepts/core.rst new file mode 100644 index 0000000000..d6eb0fb9c8 --- /dev/null +++ b/rsts/concepts/core.rst @@ -0,0 +1,15 @@ +.. _divedeep: + +############################ +Core Concepts & Architecture +############################ + +.. toctree:: + :maxdepth: 1 + :name: Concepts & Architecture + + overview + tasks + workflows_nodes + launchplans_schedules + architecture diff --git a/rsts/dive_deep/customizable_resources.rst b/rsts/concepts/customizable_resources.rst similarity index 73% rename from rsts/dive_deep/customizable_resources.rst rename to rsts/concepts/customizable_resources.rst index aa74435173..f3e1975edb 100644 --- a/rsts/dive_deep/customizable_resources.rst +++ b/rsts/concepts/customizable_resources.rst @@ -4,7 +4,7 @@ Adding customizable resources ############################# -For background on customizable resources, see :ref:`howto-managing-customizable-resources`. As a quick refresher, custom resources allow you to manage configurations for specific combinations of user projects, domains and workflows that override default values. Examples of such resources include execution clusters, task resource defaults, and `more `__. +For background on customizable resources, see :ref:`howto-managing-customizable-resources`. As a quick refresher, custom resources allow you to manage configurations for specific combinations of user projects, domains and workflows that override default values. Examples of such resources include execution clusters, task resource defaults, and `more `__. Example @@ -16,7 +16,7 @@ Now one of your user teams comes to you and wants to ensure that their critical Flyte IDL ^^^^^^^^^ -You'll want to introduce a new `matchable attribute `__, including a unique enum value and proto message definition. For example +You'll want to introduce a new `matchable attribute `__, including a unique enum value and proto message definition. For example :: @@ -37,13 +37,13 @@ You'll want to introduce a new `matchable attribute `__ for an example of what is required. +See the changes in this `file `__ for an example of what is required. Flyte admin ^^^^^^^^^^^ -Once your idl changes are released, update the logic of flyteadmin to `fetch `__ your new matchable priority resource and use it when creating executions or wherever makes sense for your use case. +Once your idl changes are released, update the logic of flyteadmin to `fetch `__ your new matchable priority resource and use it when creating executions or wherever makes sense for your use case. For example: @@ -70,6 +70,6 @@ For example: Flytekit ^^^^^^^^ -For convenience, add a flyte-cli wrapper for updating your new attribute. See `this PR `__ for the full required set of changes. +For convenience, add a flyte-cli wrapper for updating your new attribute. See `this PR `__ for the full required set of changes. That's it! You now have a new matchable attribute you can configure as the needs of your users evolve. diff --git a/rsts/community/compare.rst b/rsts/concepts/deployment_options.rst similarity index 52% rename from rsts/community/compare.rst rename to rsts/concepts/deployment_options.rst index 4323ecaee8..0ae2e5b1d1 100644 --- a/rsts/community/compare.rst +++ b/rsts/concepts/deployment_options.rst @@ -1,6 +1,7 @@ -.. _community_compare: - ################################### -Compare Flyte to other products +Deployment options (Local & Remote) ################################### +.. NOTE:: + + Coming soon 🛠 \ No newline at end of file diff --git a/rsts/dive_deep/domains.rst b/rsts/concepts/domains.rst similarity index 100% rename from rsts/dive_deep/domains.rst rename to rsts/concepts/domains.rst diff --git a/rsts/dive_deep/dynamic_spec.rst b/rsts/concepts/dynamic_spec.rst similarity index 100% rename from rsts/dive_deep/dynamic_spec.rst rename to rsts/concepts/dynamic_spec.rst diff --git a/rsts/concepts/execution_time.rst b/rsts/concepts/execution_time.rst new file mode 100644 index 0000000000..c19980680b --- /dev/null +++ b/rsts/concepts/execution_time.rst @@ -0,0 +1,14 @@ +###################### +Execution Time Details +###################### + +.. toctree:: + :maxdepth: 1 + + executions + state_machine + execution_timeline + observability + dynamic_spec + catalog + customizable_resources \ No newline at end of file diff --git a/rsts/dive_deep/execution_timeline.rst b/rsts/concepts/execution_timeline.rst similarity index 100% rename from rsts/dive_deep/execution_timeline.rst rename to rsts/concepts/execution_timeline.rst diff --git a/rsts/dive_deep/executions.rst b/rsts/concepts/executions.rst similarity index 100% rename from rsts/dive_deep/executions.rst rename to rsts/concepts/executions.rst diff --git a/rsts/concepts/flyte_cli.rst b/rsts/concepts/flyte_cli.rst new file mode 100644 index 0000000000..da251e189e --- /dev/null +++ b/rsts/concepts/flyte_cli.rst @@ -0,0 +1,7 @@ +############## +Flyte CLI +############## + +.. NOTE:: + + Coming soon 🛠 diff --git a/rsts/concepts/flyte_ui.rst b/rsts/concepts/flyte_ui.rst new file mode 100644 index 0000000000..e26edf382a --- /dev/null +++ b/rsts/concepts/flyte_ui.rst @@ -0,0 +1,7 @@ +################# +Flyte UI +################# + +.. NOTE:: + + Coming soon 🛠 \ No newline at end of file diff --git a/rsts/dive_deep/flyte_wf_tasks_high_level.png b/rsts/concepts/flyte_wf_tasks_high_level.png similarity index 100% rename from rsts/dive_deep/flyte_wf_tasks_high_level.png rename to rsts/concepts/flyte_wf_tasks_high_level.png diff --git a/rsts/concepts/glossary.rst b/rsts/concepts/glossary.rst new file mode 100644 index 0000000000..0be14da2ad --- /dev/null +++ b/rsts/concepts/glossary.rst @@ -0,0 +1,12 @@ +############ +Glossary +############ + +.. glossary:: + + Memoization + Memoization ensures that a method doesn't run for the same inputs more than once by keeping a record of the results for the given inputs. + +.. NOTE:: + + Coming soon 🛠 \ No newline at end of file diff --git a/rsts/dive_deep/launchplans_schedules.rst b/rsts/concepts/launchplans_schedules.rst similarity index 100% rename from rsts/dive_deep/launchplans_schedules.rst rename to rsts/concepts/launchplans_schedules.rst diff --git a/rsts/dive_deep/observability.rst b/rsts/concepts/observability.rst similarity index 97% rename from rsts/dive_deep/observability.rst rename to rsts/concepts/observability.rst index 4b7354095c..c9b4d69b0b 100644 --- a/rsts/dive_deep/observability.rst +++ b/rsts/concepts/observability.rst @@ -3,6 +3,8 @@ Metrics for your executions =========================== +.. tip:: Refer to :ref:`howto-monitoring` to see/use prebuilt dashboards published to Grafana Marketplace. The following section explains some other metrics that are very important. + Flyte-Provided Metrics ~~~~~~~~~~~~~~~~~~~~~~~ Whenever you run a workflow, Flyte Platform automatically emits high-level metrics. These metrics follow a consistent schema and @@ -57,14 +59,6 @@ wf This is the name of the workflow that was executing when this m ``{{project}}:{{domain}}:{{workflow_name}}`` =============== ================================================================================= -Flyte Task Container Metrics -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -To be built. - -Default Project Dashboard -^^^^^^^^^^^^^^^^^^^^^^^^^^^ -To be built. - User Stats With Flyte ~~~~~~~~~~~~~~~~~~~~~~ The workflow parameters object that the SDK injects into the various tasks has a statsd handle that users should call diff --git a/rsts/dive_deep/overview.rst b/rsts/concepts/overview.rst similarity index 100% rename from rsts/dive_deep/overview.rst rename to rsts/concepts/overview.rst diff --git a/rsts/dive_deep/projects.rst b/rsts/concepts/projects.rst similarity index 100% rename from rsts/dive_deep/projects.rst rename to rsts/concepts/projects.rst diff --git a/rsts/dive_deep/registration.rst b/rsts/concepts/registration.rst similarity index 100% rename from rsts/dive_deep/registration.rst rename to rsts/concepts/registration.rst diff --git a/rsts/concepts/state_machine.rst b/rsts/concepts/state_machine.rst new file mode 100644 index 0000000000..47457f4c15 --- /dev/null +++ b/rsts/concepts/state_machine.rst @@ -0,0 +1,78 @@ +.. _divedeep-state-machine: + +################################################# +Understanding the State Transition in a workflow +################################################# + +High Level Overview of how a Workflow progresses to Success +============================================================ + +.. image:: https://mermaid.ink/img/eyJjb2RlIjoic3RhdGVEaWFncmFtLXYyXG4gICAgWypdIC0tPiBSZWFkeVxuICAgIFJlYWR5IC0tPiBSdW5uaW5nXG4gICAgUnVubmluZyAtLT4gU3VjY2Vzc1xuXG4gICAgc3RhdGUgUnVubmluZyB7XG4gICAgICBbKl0gLS0-IE5vZGVRdWV1ZWRcbiAgICAgIE5vZGVRdWV1ZWQgLS0-IE5vZGVSdW5uaW5nXG4gICAgICBOb2RlUnVubmluZyAtLT4gTm9kZVN1Y2Nlc3NcblxuICAgICAgc3RhdGUgTm9kZVJ1bm5pbmcge1xuICAgICAgICBbKl0gLS0-IFRhc2tRdWV1ZWRcbiAgICAgICAgVGFza1F1ZXVlZCAtLT4gVGFza1J1bm5pbmdcbiAgICAgICAgVGFza1J1bm5pbmcgLS0-IFRhc2tTdWNjZXNzXG4gICAgICB9XG4gICAgfVxuXG4iLCJtZXJtYWlkIjp7fSwidXBkYXRlRWRpdG9yIjpmYWxzZX0 + :alt: Happy case for a workflow with one node and one task. + +This State diagram illustrates an extremely high level, simplistic view of the state transitions that a Workflow, with a single node and one task will go through as the observer observes success. + +The following section explains in detail the various observable (and some hidden) states for a workflow, node and tasks state transitions. + + +Workflow States +================ + +.. image:: https://mermaid.ink/img/eyJjb2RlIjoic3RhdGVEaWFncmFtLXYyXG4gICAgWypdIC0tPiBBYm9ydGVkIDogT24gc3lzdGVtIGVycm9ycyBtb3JlIHRoYW4gdGhyZXNob2xkXG4gICAgWypdIC0tPiBSZWFkeVxuICAgIFJlYWR5IC0tPiBSdW5uaW5nIDogV3JpdGUgaW5wdXRzIHRvIHdvcmtmbG93XG4gICAgUnVubmluZyAtLT4gUnVubmluZyA6IE9uIHN5c3RlbSBlcnJvclxuICAgIFJ1bm5pbmcgLS0-IFN1Y2NlZWRpbmcgOiBPbiBhbGwgTm9kZXMgU3VjY2Vzc1xuICAgIFN1Y2NlZWRpbmcgLS0-IFN1Y2NlZWRlZCA6IE9uIHN1Y2Nlc3NmdWwgZXZlbnQgc2VuZCB0byBBZG1pblxuICAgIFN1Y2NlZWRpbmcgLS0-IFN1Y2NlZWRpbmcgOiBPbiBzeXN0ZW0gZXJyb3JcbiAgICBSZWFkeSAtLT4gRmFpbGluZyA6IE9uIHByZWNvbmRpdGlvbiBmYWlsdXJlXG4gICAgUnVubmluZyAtLT4gRmFpbGluZyA6IE9uIGFueSBOb2RlIEZhaWx1cmVcbiAgICBSZWFkeSAtLT4gQWJvcnRlZCA6IE9uIHVzZXIgaW5pdGlhdGVkIGFib3J0XG4gICAgUnVubmluZyAtLT4gQWJvcnRlZCA6IE9uIHVzZXIgaW5pdGlhdGVkIGFib3J0XG4gICAgU3VjY2VlZGluZyAtLT4gQWJvcnRlZCA6IE9uIHVzZXIgaW5pdGlhdGVkIGFib3J0XG5cbiAgICBGYWlsaW5nIC0tPiBIYW5kbGVGYWlsdXJlTm9kZSA6IElmIEZhaWx1cmUgbm9kZSBleGlzdHNcbiAgICBGYWlsaW5nIC0tPiBBYm9ydGVkIDogT24gdXNlciBpbml0aWF0ZWQgYWJvcnRcbiAgICBIYW5kbGVGYWlsdXJlTm9kZSAtLT4gRmFpbGVkIDogT24gY29tcGxldGluZyBmYWlsdXJlIG5vZGVcbiAgICBIYW5kbGVGYWlsdXJlTm9kZSAtLT4gQWJvcnRlZCA6IE9uIHVzZXIgaW5pdGlhdGVkIGFib3J0XG4gICAgRmFpbGluZyAtLT4gRmFpbGVkIDogT24gc3VjY2Vzc2Z1bCBzZW5kIG9mIEZhaWx1cmUgbm9kZVxuICAgICIsIm1lcm1haWQiOnt9LCJ1cGRhdGVFZGl0b3IiOmZhbHNlfQ + :alt: The State diagram above illustrates the various states through which a workflow transitions. This is the core finite state machine (FSM) of a Workflow. + +The State diagram above illustrates the various states through which a Workflow transitions. This is the core Finite state machine of a Workflow. + +A Workflow always starts in the Ready State and ends either in Failed, Succeeded or Aborted state. +Any system error within a state causes a retry on that state. These retries are capped by system retries and will eventually lead to an Aborted state. + +Every transition between states is recorded in Flyteadmin using :std:ref:`gen/pb-protodoc/flyteidl/event/event.proto:flyteidl.event.workflowexecutionevent` + +The phases in the above state diagram are captured in the Admin database as specified here :std:ref:`api_enum_flyteidl.core.workflowexecution.phase` and are sent as part of the Execution Event. + +The state machine specification for the illustration can be found `here `_ + + +Node States +================ + +.. image:: https://mermaid.ink/img/eyJjb2RlIjoic3RhdGVEaWFncmFtLXYyXG4gICAgWypdIC0tPiBOb3RZZXRTdGFydGVkXG4gICAgWypdIC0tPiBBYm9ydGVkIDogV2lsbCBzdG9wIHRoZSBub2RlIGV4ZWN1dGlvblxuICAgIE5vdFlldFN0YXJ0ZWQgLS0-IFF1ZXVlZCA6IElmIGFsbCB1cHN0cmVhbSBub2RlcyBhcmUgcmVhZHkgaS5lLCBpbnB1dHMgYXJlIHJlYWR5XG4gICAgTm90WWV0U3RhcnRlZCAtLT4gU2tpcHBlZCA6IElmIHRoZSBicmFuY2ggd2FzIG5vdCB0YWtlblxuICAgIFF1ZXVlZCAtLT4gUnVubmluZyA6IFN0YXJ0IHRhc2sgZXhlY3V0aW9uIC0gYXR0ZW1wdCAwXG4gICAgUnVubmluZyAtLT4gVGltaW5nT3V0IDogSWYgdGFzayB0aW1lb3V0IGhhcyBlbGFwc2VkIGFuZCByZXRyeV9hdHRlbXB0cyA-PSBtYXhfcmV0cmllc1xuICAgIFRpbWluZ091dCAtLT4gVGltZWRPdXQgOiBJdCB0b3RhbCBub2RlIHRpbWVvdXQgaGFzIGVsYXBzZWRcbiAgICBSdW5uaW5nIC0tPiBSZXRyeWFibGVGYWlsdXJlIDogb24gcmV0cnlhYmxlIGZhaWx1cmVcbiAgICBSdW5uaW5nIC0tPiBEeW5hbWljUnVubmluZyA6IEZvciBkeW5hbWljIG5vZGVzIGdlbmVyYXRpbmcgd29ya2Zsb3dzXG4gICAgUmV0cnlhYmxlRmFpbHVyZSAtLT4gUnVubmluZyA6IGlmIHJldHJ5X2F0dGVtcHRzIDwgbWF4X3JldHJpZXNcbiAgICBSZXRyeWFibGVGYWlsdXJlIC0tPiBGYWlsaW5nIDogcmV0cnlfYXR0ZW1wdHMgPj0gbWF4X3JldHJpZXNcbiAgICBGYWlsaW5nIC0tPiBGYWlsZWRcbiAgICBSdW5uaW5nIC0tPiBTdWNjZWVkaW5nIDogSW50ZXJuYWwgc3RhdGVcbiAgICBEeW5hbWljUnVubmluZyAtLT4gU3VjY2VlZGluZ1xuICAgIER5bmFtaWNSdW5uaW5nIC0tPiBSZXRyeWFibGVGYWlsdXJlXG4gICAgRHluYW1pY1J1bm5pbmcgLS0-IFRpbWluZ091dFxuICAgIFN1Y2NlZWRpbmcgLS0-IFN1Y2NlZWRlZCA6IFVzZXIgb2JzZXJ2ZXMgdGhlIHRhc2sgYXMgc3VjY2VlZGVkXG4gICAgU3VjY2VlZGVkIC0tPiBbKl1cbiAgICBGYWlsZWQgLS0-IFsqXVxuIiwibWVybWFpZCI6e30sInVwZGF0ZUVkaXRvciI6ZmFsc2V9 + :alt: The State diagram above illustrates the various states through which a Node transitions. This is the core FSM for a Node. + +The state diagram above illustrates the various states through which a Node transitions. This is the core FSM for a Node. +From a user's point of view, a Workflow simply consists of a sequence of tasks. But to Flyte, a Workflow internally creates a meta entity called a + +Once a Workflow enters a ``Running`` state, it triggers the phantom ``start node`` of the workflow. The Start node is always the entry node of any workflow. The start node starts executing all its child-nodes using a modified DepthFirst Search algorithm recursively. + +Nodes can be of different types, as follows, but all the nodes traverse through the same transitions + +#. Start Node - Only exists during the execution and is not modeled in the core spec +#. :std:ref:`gen/pb-protodoc/flyteidl/core/workflow.proto:flyteidl.core.tasknode` +#. :std:ref:`gen/pb-protodoc/flyteidl/core/workflow.proto:flyteidl.core.branchnode` +#. :std:ref:`gen/pb-protodoc/flyteidl/core/workflow.proto:flyteidl.core.workflownode` +#. Dynamic node - which is just a task node that does not return outputs, but a dynamic workflow. When the task runs, it stays in a `RUNNING` state. Once the task completes and Flyte starts executing the dynamic workflow, the overarching node that contains both the original task and the dynamic workflow enters `DYNAMIC_RUNNING` state. +#. End Node - only exists during the execution and is not modeled in the core spec + +Every transition between states is recorded in Flyteadmin using :std:ref:`gen/pb-protodoc/flyteidl/event/event.proto:flyteidl.event.nodeexecutionevent` + +Every NodeExecutionEvent can have one of the :std:ref:`api_enum_flyteidl.core.nodeexecution.phase` + +.. note:: TODO add explanation for each phase + +The state machine specification for the illustration can be found `here `_ + +Task States +================ + +.. image:: https://mermaid.ink/img/eyJjb2RlIjoic3RhdGVEaWFncmFtLXYyXG4gICAgWypdIC0tPiBOb3RSZWFkeVxuICAgIFsqXSAtLT4gQWJvcnRlZCA6IEFib3J0ZWQgYnkgTm9kZUhhbmRsZXIgLSB0aW1lb3V0cywgZXh0cmVuYWwgYWJvcnQsIGV0Y1xuICAgIE5vdFJlYWR5IC0tPiBXYWl0aW5nRm9yUmVzb3VyY2VzIDogQmxvY2tlZCBvbiByZXNvdXJjZSBxdW90YSBvciByZXNvdXJjZSBwb29sIChvcHRpb25hbClcbiAgICBXYWl0aW5nRm9yUmVzb3VyY2VzIC0tPiBRdWV1ZWQgOiBIYXMgYmVlbiBzdWJtaXR0ZWQsIGJ1dCBoYXMgbm90IHN0YXJ0ZWQgKG9wdGlvbmFsKVxuICAgIFF1ZXVlZCAtLT4gSW5pdGlhbGl6aW5nIDogUHJlc3RhcnQgaW5pdGlhbGl6YXRpb24gKG9wdGlvbmFsKVxuICAgIEluaXRpYWxpemluZyAtLT4gUnVubmluZyA6IEFjdHVhbCBleGVjdXRpb24gb2YgdXNlciBjb2RlIGhhcyBzdGFydGVkXG4gICAgUnVubmluZyAtLT4gU3VjY2VzcyA6IFN1Y2Nlc3NmdWwgZXhlY3V0aW9uXG4gICAgUnVubmluZyAtLT4gUmV0cnlhYmxlRmFpbHVyZSA6IEZhaWxlZCB3aXRoIGEgcmV0cnlhYmxlIGVycm9yXG4gICAgUnVubmluZyAtLT4gUGVybWFuZW50RmFpbHVyZSA6IFVucmVjb3ZlcmFibGUgZmFpbHVyZSwgd2lsbCBzdG9wIGFsbCBleGVjdXRpb25cbiAgICBTdWNjZXNzIC0tPiBbKl1cbiAgICBSZXRyeWFibGVGYWlsdXJlIC0tPiBbKl1cbiAgICBQZXJtYW5lbnRGYWlsdXJlIC0tPiBbKl1cbiIsIm1lcm1haWQiOnt9LCJ1cGRhdGVFZGl0b3IiOmZhbHNlfQ + :alt: The State diagram above illustrates the various states through which a Task transitions. This is the core FSM for any Task in Flyte. + +The State diagram above illustrates the various states through which a Task transitions. + +Every transition between states is recorded in Flyteadmin using :std:ref:`gen/pb-protodoc/flyteidl/event/event.proto:flyteidl.event.taskexecutionevent` + +Every TaskExecutionEvent can have one of the :std:ref:`api_enum_flyteidl.core.taskexecution.phase` + +.. note:: TODO add explanation for each phase + +The state machine specification for the illustration can be found `here `_ diff --git a/rsts/dive_deep/tasks.rst b/rsts/concepts/tasks.rst similarity index 98% rename from rsts/dive_deep/tasks.rst rename to rsts/concepts/tasks.rst index 052ec5d380..5317b78003 100644 --- a/rsts/dive_deep/tasks.rst +++ b/rsts/concepts/tasks.rst @@ -59,7 +59,7 @@ Types ----- Since it's impossible to define the unit of execution of a task the same way for all kinds of tasks, Flyte allows different task types in the system. Flyte comes with a set of defined, battle tested task types but also allows for a very flexible model to -introducing new :ref:`howto-create-plugins`. +introducing new :ref:`plugins_extend_intro`. Fault tolerance --------------- diff --git a/rsts/dive_deep/workflows_nodes.rst b/rsts/concepts/workflows_nodes.rst similarity index 100% rename from rsts/dive_deep/workflows_nodes.rst rename to rsts/concepts/workflows_nodes.rst diff --git a/rsts/conf.py b/rsts/conf.py index bf7e909a38..feaf81a969 100644 --- a/rsts/conf.py +++ b/rsts/conf.py @@ -16,16 +16,20 @@ # import sys # sys.path.insert(0, os.path.abspath('.')) +import sphinx.application +import sphinx.errors +sphinx.application.ExtensionError = sphinx.errors.ExtensionError + # -- Project information ----------------------------------------------------- project = u'Flyte' -copyright = u'2020, Flyte Authors' +copyright = u'2021, Flyte Authors' author = u'Flyte' # The short X.Y version version = u'' # The full version, including alpha/beta/rc tags -release = u'0.10.0' +release = u'0.11.0' # -- General configuration --------------------------------------------------- @@ -55,6 +59,7 @@ "sphinx_tabs.tabs", "sphinxext.remoteliteralinclude", "sphinx_issues", + "sphinx_search.extension", ] extlinks = { @@ -97,36 +102,24 @@ # html_favicon = "images/flyte_circle_gradient_1_4x4.png" html_logo = "images/flyte_circle_gradient_1_4x4.png" -html_static_path = [] -html_theme = "sphinx_material" +html_theme = "furo" +html_title = "Flyte Docs" + +html_static_path = ["_static"] +templates_path = ["_templates"] + +pygments_style = "tango" +pygments_dark_style = "native" + html_theme_options = { - # Set the name of the project to appear in the navigation. - "nav_title": "Flyte", - # Set you GA account ID to enable tracking - "google_analytics_account": "G-YQL24L5CKY", - # Specify a base_url used to generate sitemap.xml. If not - # specified, then no sitemap will be built. - "base_url": "https://github.com/flyteorg/flyte", - # Set the color and the accent color - "color_primary": "deep-purple", - "color_accent": "blue", - # Set the repo location to get a badge with stats - "repo_url": "https://github.com/flyteorg/flyte/", - "repo_name": "flyte", - # Visible levels of the global TOC; -1 means unlimited - "globaltoc_depth": 1, - # If False, expand all TOC entries - "globaltoc_collapse": False, - # If True, show hidden TOC entries - "globaltoc_includehidden": False, - "nav_links": [ - {"href": "index", "internal": True, "title": "Flyte Docs"}, - { - "href": "https://flytecookbook.readthedocs.io", - "internal": False, - "title": "Flyte Tutorials", - }, - ], + "light_css_variables": { + "color-brand-primary": "#4300c9", + "color-brand-content": "#4300c9", + }, + "dark_css_variables": { + "color-brand-primary": "#9D68E4", + "color-brand-content": "#9D68E4", + }, } # Theme options are theme-specific and customize the look and feel of a theme @@ -148,7 +141,7 @@ # default: ``['localtoc.html', 'relations.html', 'sourcelink.html', # 'searchbox.html']``. # -html_sidebars = {"**": ["logo-text.html", "globaltoc.html", "localtoc.html", "searchbox.html"]} +# html_sidebars = {"**": ["logo-text.html", "globaltoc.html", "localtoc.html", "searchbox.html"]} # -- Options for HTMLHelp output --------------------------------------------- diff --git a/rsts/dive_deep/index.rst b/rsts/dive_deep/index.rst deleted file mode 100644 index 6e01c1dcff..0000000000 --- a/rsts/dive_deep/index.rst +++ /dev/null @@ -1,47 +0,0 @@ -.. _divedeep: - -########################################## -Architecture and Concepts -########################################## - -******************************** -Core Concepts & Architecture -******************************** - -.. toctree:: - :maxdepth: 1 - :name: Concepts & Architecture - - overview - tasks - workflows_nodes - launchplans_schedules - architecture - -******************************** -Control Plane Details -******************************** - -.. toctree:: - :maxdepth: 1 - - projects - domains - admin - admin_service - registration - console - -******************************** -Execution Time Details -******************************** - -.. toctree:: - :maxdepth: 1 - - executions - execution_timeline - observability - dynamic_spec - catalog - customizable_resources diff --git a/rsts/faq/gcp.rst b/rsts/faq/gcp.rst deleted file mode 100644 index 88202c1c59..0000000000 --- a/rsts/faq/gcp.rst +++ /dev/null @@ -1,22 +0,0 @@ -.. _faq_gcp: - -####################### -Google Cloud Platform -####################### - -I tried to run examples, but task fails with 401 error? -------------------------------------------------------- - Steps: - - Are you using Workload Identity, then you have to pass in the ServiceAccount when you create the launchplan. Docs here https://lyft.github.io/flyte/user/features/roles.html?highlight=serviceaccount#kubernetes-serviceaccount-examples More information about WorkloadIdentity at https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity - - If you are just using a simple Nodepool wide permissions then check the cluster's ServiceACcount for Storage permissions. Do they look fine? - - - If not, then start a dummy pod in the intended namespace and check for :: - - gcloud auth list - - -.. note:: - - FlytePropeller uses Google Application credentials, but gsutil does not use these credentials - - diff --git a/rsts/faq/index.rst b/rsts/faq/index.rst deleted file mode 100644 index 2c0141919e..0000000000 --- a/rsts/faq/index.rst +++ /dev/null @@ -1,12 +0,0 @@ -.. _faqs: - -###### -FAQs -###### - - -.. toctree:: - :maxdepth: 2 - :name: faqstoc - - gcp \ No newline at end of file diff --git a/rsts/getting_started.rst b/rsts/getting_started.rst new file mode 100644 index 0000000000..f21ddc9d7c --- /dev/null +++ b/rsts/getting_started.rst @@ -0,0 +1,103 @@ +.. _gettingstarted: + +Getting started +--------------- + +.. rubric:: Estimated time to complete: 3 minutes. + +Prerequisites +*************** + +Make sure you have `docker installed `__ and `git `__ installed, then install flytekit: + +Steps +***** + +1. First install the python Flytekit SDK and clone the ``flytesnacks`` repo: + +.. prompt:: bash + + pip install --pre flytekit + git clone git@github.com:flyteorg/flytesnacks.git flytesnacks + cd flytesnacks + +2. The repo comes with some useful Make targets to make your experimentation workflow easier. Run ``make help`` to get the supported commands. + Let's start a sandbox cluster: + +.. prompt:: bash + + make start + +3. Take a minute to explore Flyte Console through the provided URL. + +.. image:: https://github.com/flyteorg/flyte/raw/static-resources/img/first-run-console-2.gif + :alt: A quick visual tour for launching your first Workflow. + +4. Open ``hello_world.py`` in your favorite editor. + +.. code-block:: + + cookbook/core/basic/hello_world.py + +5. Add ``name: str`` as an argument to both ``my_wf`` and ``say_hello`` functions. Then update the body of ``say_hello`` to consume that argument. + +.. tip:: + + .. code-block:: python + + @task + def say_hello(name: str) -> str: + return f"hello world, {name}" + +.. tip:: + + .. code-block:: python + + @workflow + def my_wf(name: str) -> str: + res = say_hello(name=name) + return res + +6. Update the simple test at the bottom of the file to pass in a name. E.g. + +.. tip:: + + .. code-block:: python + + print(f"Running my_wf(name='adam') {my_wf(name='adam')}") + +7. When you run this file locally, it should output ``hello world, adam``. Run this command in your terminal: + +.. prompt:: bash + + python cookbook/core/basic/hello_world.py + +*Congratulations!* You have just run your first workflow. Now, let's run it on the sandbox cluster deployed earlier. + +8. Run: + +.. prompt:: bash + + REGISTRY=ghcr.io/flyteorg make fast_register + +9. Visit `the console `__, click launch, and enter your name as the input. + +10. Give it a minute and once it's done, check out "Inputs/Outputs" on the top right corner to see your updated greeting. + +.. image:: https://raw.githubusercontent.com/flyteorg/flyte/static-resources/img/flytesnacks/tutorial/exercise.gif + :alt: A quick visual tour for launching a workflow and checking the outputs when they're done. + +.. admonition:: Recap + + You have successfully: + + 1. Run a flyte sandbox cluster, + 2. Run a flyte workflow locally, + 3. Run a flyte workflow on a cluster. + + .. rubric:: 🎉 Congratulations, you just ran your first Flyte workflow 🎉 + + Next Steps: User Guide + ####################### + + To experience the full capabilities of Flyte, take a look at the `User Guide `__ 🛫 diff --git a/rsts/howto/authentication/index.rst b/rsts/howto/authentication/index.rst index eb4fe6d689..2c1e99fe64 100644 --- a/rsts/howto/authentication/index.rst +++ b/rsts/howto/authentication/index.rst @@ -1,92 +1,106 @@ .. _howto_authentication: -####################################### -How to setup Authentication? -####################################### +####################### +Authentication in Flyte +####################### -Flyte Admin ships with a canonical implementation of OAuth2, integrating seamlessly into an organization's existing identity provider. At Lyft, we use Okta as the IDP, but if you have issues integrating with another implementation of the OAuth server, please open an issue. +Flyte ships with a canonical implementation of OpenIDConnect client and OAuth2 Server, integrating seamlessly into an organization's existing identity provider. -*********************** -Components -*********************** +.. toctree:: + :maxdepth: 1 + :caption: Setting up Flyte Authentication + :name: howtoprovguidestoc -While the most obvious interaction with the Flyte control plane is through the web based UI, there are other critical components of Flyte that also need to be considered. These components should be thought of as third-party services even though the Flyte codebase provides them. + setup + migration -Flyte CLI -========= -Principal amongst these is the Flyte CLI. This is the command-line entrypoint to Flyte Admin and is used by both administrators and users more comfortable in the command line, or are running in a headless OS. +******** +Overview +******** -The IDP application corresponding to the CLI will need to support PKCE. +Flyte system consists of multiple components. For the purposes of this document, let's categorize them into server-side and client-side components: +- **Admin**: A server-side control plane component accessible from console, cli and other backends. +- **Catalog**: A server-side control plane component accessible from console, cli and other backends. +- **Console**: A client-side single page react app. +- **flyte-cli**: A python-based client-side command line interface that interacts with Admin and Catalog. +- **flytectl**: A go-based client-side command line interface that interacts with Admin and Catalog. +- **Propeller**: A server-side data plane component that interacts with both admin and catalog services. -Direct Client Access -==================== -The gRPC client provided by the Flyte IDL, or direct calls to the HTTP endpoints on Admin from within a running script are ways that we have seen users hit the control plane directly. We generally discourage this behavior as it leads to a possible self-imposed DOS vector, as they are generally made from within a running workflow itself. For instance, a Flyte task can fetch the definition for a launch plan associated with a completely different workflow, and then launch an execution of it. This is not the correct way to launch one workflow from another but for the time being remains possible. +************** +OpenID Connect +************** +Flyte supports OpenID Connect. A defacto standard for user authentication. After configuring OpenID Connect, users accessing flyte console or flytectl +(or other 3rd party apps) will be prompted to authenticate using the configured provider. -***************** -Swimlane Diagrams -***************** +.. image:: https://mermaid.ink/img/eyJjb2RlIjoic2VxdWVuY2VEaWFncmFtXG4lJXtjb25maWc6IHsgJ2ZvbnRGYW1pbHknOiAnTWVubG8nLCAnZm9udFNpemUnOiAxMCwgJ2ZvbnRXZWlnaHQnOiAxMDB9IH0lJVxuICAgIGF1dG9udW1iZXJcbiAgICBVc2VyLT4-K0Jyb3dzZXI6IC9ob21lXG4gICAgQnJvd3Nlci0-PitDb25zb2xlOiAvaG9tZVxuICAgIENvbnNvbGUtPj4tQnJvd3NlcjogMzAyIC9sb2dpblxuICAgIEJyb3dzZXItPj4rQWRtaW46IC9sb2dpblxuICAgIEFkbWluLT4-LUJyb3dzZXI6IElkcC5jb20vb2lkY1xuICAgIEJyb3dzZXItPj4rSWRwOiBJZHAuY29tL29pZGNcbiAgICBJZHAtPj4tQnJvd3NlcjogMzAyIC9sb2dpblxuICAgIEJyb3dzZXItPj4tVXNlcjogRW50ZXIgdXNlci9wYXNzXG4gICAgVXNlci0-PitCcm93c2VyOiBsb2dpblxuICAgIEJyb3dzZXItPj4rSWRwOiBTdWJtaXQgdXNlcm5hbWUvcGFzc1xuICAgIElkcC0-Pi1Ccm93c2VyOiBhZG1pbi8_YXV0aENvZGU9PGFiYz5cbiAgICBCcm93c2VyLT4-K0FkbWluOiBhZG1pbi9hdXRoQ29kZT08YWJjPlxuICAgIEFkbWluLT4-K0lkcDogRXhjaGFuZ2UgVG9rZW5zXG4gICAgSWRwLT4-LUFkbWluOiBpZHQsIGF0LCBydFxuICAgIEFkbWluLT4-K0Jyb3dzZXI6IFdyaXRlIENvb2tpZXMgJiBSZWRpcmVjdCB0byAvY29uc29sZVxuICAgIEJyb3dzZXItPj4rQ29uc29sZTogL2hvbWVcbiAgICBCcm93c2VyLT4-LVVzZXI6IFJlbmRlciAvaG9tZVxuIiwibWVybWFpZCI6eyJ0aGVtZSI6Im5ldXRyYWwifSwidXBkYXRlRWRpdG9yIjpmYWxzZX0 + :target: https://mermaid-js.github.io/mermaid-live-editor/#/edit/eyJjb2RlIjoic2VxdWVuY2VEaWFncmFtXG4lJXtjb25maWc6IHsgJ2ZvbnRGYW1pbHknOiAnTWVubG8nLCAnZm9udFNpemUnOiAxMCwgJ2ZvbnRXZWlnaHQnOiAxMDB9IH0lJVxuICAgIGF1dG9udW1iZXJcbiAgICBVc2VyLT4-K0Jyb3dzZXI6IC9ob21lXG4gICAgQnJvd3Nlci0-PitDb25zb2xlOiAvaG9tZVxuICAgIENvbnNvbGUtPj4tQnJvd3NlcjogMzAyIC9sb2dpblxuICAgIEJyb3dzZXItPj4rQWRtaW46IC9sb2dpblxuICAgIEFkbWluLT4-LUJyb3dzZXI6IElkcC5jb20vb2lkY1xuICAgIEJyb3dzZXItPj4rSWRwOiBJZHAuY29tL29pZGNcbiAgICBJZHAtPj4tQnJvd3NlcjogMzAyIC9sb2dpblxuICAgIEJyb3dzZXItPj4tVXNlcjogRW50ZXIgdXNlci9wYXNzXG4gICAgVXNlci0-PitCcm93c2VyOiBsb2dpblxuICAgIEJyb3dzZXItPj4rSWRwOiBTdWJtaXQgdXNlcm5hbWUvcGFzc1xuICAgIElkcC0-Pi1Ccm93c2VyOiBhZG1pbi8_YXV0aENvZGU9PGFiYz5cbiAgICBCcm93c2VyLT4-K0FkbWluOiBhZG1pbi9hdXRoQ29kZT08YWJjPlxuICAgIEFkbWluLT4-K0lkcDogRXhjaGFuZ2UgVG9rZW5zXG4gICAgSWRwLT4-LUFkbWluOiBpZHQsIGF0LCBydFxuICAgIEFkbWluLT4-K0Jyb3dzZXI6IFdyaXRlIENvb2tpZXMgJiBSZWRpcmVjdCB0byAvY29uc29sZVxuICAgIEJyb3dzZXItPj4rQ29uc29sZTogL2hvbWVcbiAgICBCcm93c2VyLT4-LVVzZXI6IFJlbmRlciAvaG9tZVxuIiwibWVybWFpZCI6eyJ0aGVtZSI6Im5ldXRyYWwifSwidXBkYXRlRWRpdG9yIjpmYWxzZX0 + :width: 600 + :alt: Flyte UI Swimlane -Flyte Admin authentication is implemented using the authorization code flow. +****** +OAuth2 +****** -Flyte UI Flow -============= -https://swimlanes.io/d/OmV4ybCkx +Flyte supports OAuth2 to control access to 3rd party and native apps. FlyteAdmin comes with a built in Authorization Server that can perform 3-legged +and 2-legged OAuth2 flows. It also supports delegating these responsibilities to an external Authorization Server. -.. image:: flyte_ui_flow.png - :width: 600 - :alt: Flyte UI Swimlane +Service Authentication using OAuth2 +=================================== +Propeller (and potentially other non-user facing services) can also authenticate using client_credentials to the Idp and be granted an +access_token valid to be used with admin and other backend services. -Flyte CLI Flow -============== -https://swimlanes.io/d/q64OxuoxT +Using FlyteAdmin's builtin Authorization Server: -.. image:: flyte_cli_flow.png - :width: 600 - :alt: Flyte CLI Swimlane +.. image:: https://mermaid.ink/img/eyJjb2RlIjoic2VxdWVuY2VEaWFncmFtXG4gICAgUHJvcGVsbGVyLT4-K0FkbWluOiAvdG9rZW4_Y2xpZW50X2NyZWRzJnNjb3BlPWh0dHBzOi8vYWRtaW4vXG4gICAgQWRtaW4tPj4tUHJvcGVsbGVyOiBhY2Nlc3NfdG9rZW5cbiAgICBQcm9wZWxsZXItPj4rQWRtaW46IC9saXN0X3Byb2plY3RzP3Rva2VuPWFjY2Vzc190b2tlbiIsIm1lcm1haWQiOnsidGhlbWUiOiJuZXV0cmFsIn0sInVwZGF0ZUVkaXRvciI6ZmFsc2V9 + :target: https://mermaid-js.github.io/mermaid-live-editor/#/edit/eyJjb2RlIjoic2VxdWVuY2VEaWFncmFtXG4gICAgUHJvcGVsbGVyLT4-K0FkbWluOiAvdG9rZW4_Y2xpZW50X2NyZWRzJnNjb3BlPWh0dHBzOi8vYWRtaW4vXG4gICAgQWRtaW4tPj4tUHJvcGVsbGVyOiBhY2Nlc3NfdG9rZW5cbiAgICBQcm9wZWxsZXItPj4rQWRtaW46IC9saXN0X3Byb2plY3RzP3Rva2VuPWFjY2Vzc190b2tlbiIsIm1lcm1haWQiOnsidGhlbWUiOiJuZXV0cmFsIn0sInVwZGF0ZUVkaXRvciI6ZmFsc2V9 + :width: 600 + :alt: Service Authentication Swimlane -************* -Configuration -************* +Using an External Authorization Server: -IDP Configuration -================= -Flyte Admin will require that the application in your identity provider be configured without PKCE and with a client secret. It should also be configured with a refresh token. +.. image:: https://mermaid.ink/img/eyJjb2RlIjoic2VxdWVuY2VEaWFncmFtXG4gICAgUHJvcGVsbGVyLT4-K0V4dGVybmFsIEF1dGhvcml6YXRpb24gU2VydmVyOiAvdG9rZW4_Y2xpZW50X2NyZWRzJnNjb3BlPWh0dHBzOi8vYWRtaW4vXG4gICAgRXh0ZXJuYWwgQXV0aG9yaXphdGlvbiBTZXJ2ZXItPj4tUHJvcGVsbGVyOiBhY2Nlc3NfdG9rZW5cbiAgICBQcm9wZWxsZXItPj4rQWRtaW46IC9saXN0X3Byb2plY3RzP3Rva2VuPWFjY2Vzc190b2tlbiIsIm1lcm1haWQiOnsidGhlbWUiOiJuZXV0cmFsIn0sInVwZGF0ZUVkaXRvciI6ZmFsc2V9 + :target: https://mermaid-js.github.io/mermaid-live-editor/#/edit/eyJjb2RlIjoic2VxdWVuY2VEaWFncmFtXG4gICAgUHJvcGVsbGVyLT4-K0V4dGVybmFsIEF1dGhvcml6YXRpb24gU2VydmVyOiAvdG9rZW4_Y2xpZW50X2NyZWRzJnNjb3BlPWh0dHBzOi8vYWRtaW4vXG4gICAgRXh0ZXJuYWwgQXV0aG9yaXphdGlvbiBTZXJ2ZXItPj4tUHJvcGVsbGVyOiBhY2Nlc3NfdG9rZW5cbiAgICBQcm9wZWxsZXItPj4rQWRtaW46IC9saXN0X3Byb2plY3RzP3Rva2VuPWFjY2Vzc190b2tlbiIsIm1lcm1haWQiOnsidGhlbWUiOiJuZXV0cmFsIn0sInVwZGF0ZUVkaXRvciI6ZmFsc2V9 + :width: 600 + :alt: Service Authentication Swimlane -Flyte Admin Configuration -========================= -Please refer to the `inline documentation `_ on the ``Config`` object in the ``auth`` package for a discussion on the settings required. +User Authentication in other clients (e.g. Cli) using OAuth2-Pkce +================================================================== -****** -CI -****** +Users accessing backend services through Cli should be able to use OAuth2-Pkce flow to authenticate (in a browser) to the Idp and be issued +an access_token valid to communicate with the intended backend service on behalf of the user. -If your organization does any automated registration, then you'll need to authenticate with the `basic authentication `_ flow (username and password effectively) as CI systems are generally not suitable OAuth resource owners. After retrieving an access token from the IDP, you can send it along to Flyte Admin as usual. +Using FlyteAdmin's builtin Authorization Server: -Flytekit configuration variables are automatically designed to look up values from relevant environment variables. To aid with continuous integration use-cases however, Flytekit configuration can also reference other environment variables. For instance, if your CI system is not capable of setting custom environment variables like ``FLYTE_CREDENTIALS_CLIENT_SECRET`` but does set the necessary settings under a different variable, you may use ``export FLYTE_CREDENTIALS_CLIENT_SECRET_FROM_ENV_VAR=OTHER_ENV_VARIABLE`` to redirect the lookup. A ``FLYTE_CREDENTIALS_CLIENT_SECRET_FROM_FILE`` redirect is available as well, where the value should be the full path to the file containing the value for the configuration setting, in this case, the client secret. We found this redirect behavior necessary when setting up registration within our own CI pipelines. +.. image:: https://mermaid.ink/img/eyJjb2RlIjoic2VxdWVuY2VEaWFncmFtXG4lJXtjb25maWc6IHsgJ2ZvbnRGYW1pbHknOiAnTWVubG8nLCAnZm9udFNpemUnOiAxMCwgJ2ZvbnRXZWlnaHQnOiAxMDB9IH0lJVxuICAgIGF1dG9udW1iZXJcbiAgICBVc2VyLT4-K0NsaTogZmx5dGVjdGwgbGlzdC1wcm9qZWN0c1xuICAgIENsaS0-PitBZG1pbjogYWRtaW4vbGlzdC1wcm9qZWN0cy9cbiAgICBBZG1pbi0-Pi1DbGk6IDQwMSBVbmF1dGhvcml6ZWRcbiAgICBDbGktPj4rQWRtaW46IGFkbWluL2NsaWVudC1jb25maWdcbiAgICBBZG1pbi0-Pi1DbGk6IENsaWVudF9pZD08YWJjPiwgLi4uXG4gICAgQ2xpLT4-K0Jyb3dzZXI6IC9hdXRoP3BrY2UmY29kZV9jaGFsbGVuZ2UsY2xpZW50X2lkLHNjb3BlXG4gICAgQnJvd3Nlci0-PitBZG1pbjogYWRtaW4vb2F1dGgyP3BrY2UuLi5cbiAgICBBZG1pbi0-Pi1Ccm93c2VyOiAzMDIgaWRwLmNvbS9sb2dpblxuICAgIE5vdGUgb3ZlciBCcm93c2VyLEFkbWluOiBUaGUgcHJpb3IgT3BlbklEIENvbm5lY3QgZmxvd1xuICAgIEJyb3dzZXItPj4rQWRtaW46IGFkbWluL2xvZ2dlZF9pblxuICAgIE5vdGUgb3ZlciBCcm93c2VyLEFkbWluOiBQb3RlbnRpYWxseSBzaG93IGN1c3RvbSBjb25zZW50IHNjcmVlblxuICAgIEFkbWluLT4-LUJyb3dzZXI6IGxvY2FsaG9zdC8_YXV0aENvZGU9PGFiYz5cbiAgICBCcm93c2VyLT4-K0NsaTogbG9jYWxob3N0L2F1dGhDb2RlPTxhYmM-XG4gICAgQ2xpLT4-K0FkbWluOiAvdG9rZW4_Y29kZSxjb2RlX3ZlcmlmaWVyXG4gICAgQWRtaW4tPj4tQ2xpOiBhY2Nlc3NfdG9rZW5cbiAgICBDbGktPj4rQWRtaW46IC9wcm9qZWN0cy8gKyBhY2Nlc3NfdG9rZW5cbiAgICBBZG1pbi0-Pi1DbGk6IHByb2plY3QxLCBwcm9qZWN0MlxuIiwibWVybWFpZCI6eyJ0aGVtZSI6Im5ldXRyYWwifSwidXBkYXRlRWRpdG9yIjpmYWxzZX0 + :target: https://mermaid-js.github.io/mermaid-live-editor/#/edit/eyJjb2RlIjoic2VxdWVuY2VEaWFncmFtXG4lJXtjb25maWc6IHsgJ2ZvbnRGYW1pbHknOiAnTWVubG8nLCAnZm9udFNpemUnOiAxMCwgJ2ZvbnRXZWlnaHQnOiAxMDB9IH0lJVxuICAgIGF1dG9udW1iZXJcbiAgICBVc2VyLT4-K0NsaTogZmx5dGVjdGwgbGlzdC1wcm9qZWN0c1xuICAgIENsaS0-PitBZG1pbjogYWRtaW4vbGlzdC1wcm9qZWN0cy9cbiAgICBBZG1pbi0-Pi1DbGk6IDQwMSBVbmF1dGhvcml6ZWRcbiAgICBDbGktPj4rQWRtaW46IGFkbWluL2NsaWVudC1jb25maWdcbiAgICBBZG1pbi0-Pi1DbGk6IENsaWVudF9pZD08YWJjPiwgLi4uXG4gICAgQ2xpLT4-K0Jyb3dzZXI6IC9hdXRoP3BrY2UmY29kZV9jaGFsbGVuZ2UsY2xpZW50X2lkLHNjb3BlXG4gICAgQnJvd3Nlci0-PitBZG1pbjogYWRtaW4vb2F1dGgyP3BrY2UuLi5cbiAgICBBZG1pbi0-Pi1Ccm93c2VyOiAzMDIgaWRwLmNvbS9sb2dpblxuICAgIE5vdGUgb3ZlciBCcm93c2VyLEFkbWluOiBUaGUgcHJpb3IgT3BlbklEIENvbm5lY3QgZmxvd1xuICAgIEJyb3dzZXItPj4rQWRtaW46IGFkbWluL2xvZ2dlZF9pblxuICAgIE5vdGUgb3ZlciBCcm93c2VyLEFkbWluOiBQb3RlbnRpYWxseSBzaG93IGN1c3RvbSBjb25zZW50IHNjcmVlblxuICAgIEFkbWluLT4-LUJyb3dzZXI6IGxvY2FsaG9zdC8_YXV0aENvZGU9PGFiYz5cbiAgICBCcm93c2VyLT4-K0NsaTogbG9jYWxob3N0L2F1dGhDb2RlPTxhYmM-XG4gICAgQ2xpLT4-K0FkbWluOiAvdG9rZW4_Y29kZSxjb2RlX3ZlcmlmaWVyXG4gICAgQWRtaW4tPj4tQ2xpOiBhY2Nlc3NfdG9rZW5cbiAgICBDbGktPj4rQWRtaW46IC9wcm9qZWN0cy8gKyBhY2Nlc3NfdG9rZW5cbiAgICBBZG1pbi0-Pi1DbGk6IHByb2plY3QxLCBwcm9qZWN0MlxuIiwibWVybWFpZCI6eyJ0aGVtZSI6Im5ldXRyYWwifSwidXBkYXRlRWRpdG9yIjpmYWxzZX0 + :width: 600 + :alt: CLI Authentication with Admin's own Authorization Server + +Using an External Authorization Server: -The following is a listing of the Flytekit configuration values we set in CI, along with a brief explanation where relevant. +.. image:: https://mermaid.ink/img/eyJjb2RlIjoic2VxdWVuY2VEaWFncmFtXG4lJXtjb25maWc6IHsgJ2ZvbnRGYW1pbHknOiAnTWVubG8nLCAnZm9udFNpemUnOiAxMCwgJ2ZvbnRXZWlnaHQnOiAxMDB9IH0lJVxuICAgIGF1dG9udW1iZXJcbiAgICBVc2VyLT4-K0NsaTogZmx5dGVjdGwgbGlzdC1wcm9qZWN0c1xuICAgIENsaS0-PitBZG1pbjogYWRtaW4vbGlzdC1wcm9qZWN0cy9cbiAgICBBZG1pbi0-Pi1DbGk6IDQwMSBVbmF1dGhvcml6ZWRcbiAgICBDbGktPj4rQWRtaW46IGFkbWluL2NsaWVudC1jb25maWdcbiAgICBBZG1pbi0-Pi1DbGk6IENsaWVudF9pZD08YWJjPiwgLi4uXG4gICAgQ2xpLT4-K0Jyb3dzZXI6IC9hdXRoP3BrY2UmY29kZV9jaGFsbGVuZ2UsY2xpZW50X2lkLHNjb3BlXG4gICAgQnJvd3Nlci0-PitFeHRlcm5hbElkcDogYWRtaW4vb2F1dGgyP3BrY2UuLi5cbiAgICBFeHRlcm5hbElkcC0-Pi1Ccm93c2VyOiAzMDIgaWRwLmNvbS9sb2dpblxuICAgIE5vdGUgb3ZlciBCcm93c2VyLEV4dGVybmFsSWRwOiBUaGUgcHJpb3IgT3BlbklEIENvbm5lY3QgZmxvd1xuICAgIEJyb3dzZXItPj4rRXh0ZXJuYWxJZHA6IC9sb2dnZWRfaW5cbiAgICBOb3RlIG92ZXIgQnJvd3NlcixFeHRlcm5hbElkcDogUG90ZW50aWFsbHkgc2hvdyBjdXN0b20gY29uc2VudCBzY3JlZW5cbiAgICBFeHRlcm5hbElkcC0-Pi1Ccm93c2VyOiBsb2NhbGhvc3QvP2F1dGhDb2RlPTxhYmM-XG4gICAgQnJvd3Nlci0-PitDbGk6IGxvY2FsaG9zdC9hdXRoQ29kZT08YWJjPlxuICAgIENsaS0-PitFeHRlcm5hbElkcDogL3Rva2VuP2NvZGUsY29kZV92ZXJpZmllclxuICAgIEV4dGVybmFsSWRwLT4-LUNsaTogYWNjZXNzX3Rva2VuXG4gICAgQ2xpLT4-K0FkbWluOiAvcHJvamVjdHMvICsgYWNjZXNzX3Rva2VuXG4gICAgQWRtaW4tPj4tQ2xpOiBwcm9qZWN0MSwgcHJvamVjdDJcbiIsIm1lcm1haWQiOnsidGhlbWUiOiJuZXV0cmFsIn0sInVwZGF0ZUVkaXRvciI6ZmFsc2V9 + :target: https://mermaid-js.github.io/mermaid-live-editor/#/edit/eyJjb2RlIjoic2VxdWVuY2VEaWFncmFtXG4lJXtjb25maWc6IHsgJ2ZvbnRGYW1pbHknOiAnTWVubG8nLCAnZm9udFNpemUnOiAxMCwgJ2ZvbnRXZWlnaHQnOiAxMDB9IH0lJVxuICAgIGF1dG9udW1iZXJcbiAgICBVc2VyLT4-K0NsaTogZmx5dGVjdGwgbGlzdC1wcm9qZWN0c1xuICAgIENsaS0-PitBZG1pbjogYWRtaW4vbGlzdC1wcm9qZWN0cy9cbiAgICBBZG1pbi0-Pi1DbGk6IDQwMSBVbmF1dGhvcml6ZWRcbiAgICBDbGktPj4rQWRtaW46IGFkbWluL2NsaWVudC1jb25maWdcbiAgICBBZG1pbi0-Pi1DbGk6IENsaWVudF9pZD08YWJjPiwgLi4uXG4gICAgQ2xpLT4-K0Jyb3dzZXI6IC9hdXRoP3BrY2UmY29kZV9jaGFsbGVuZ2UsY2xpZW50X2lkLHNjb3BlXG4gICAgQnJvd3Nlci0-PitFeHRlcm5hbElkcDogYWRtaW4vb2F1dGgyP3BrY2UuLi5cbiAgICBFeHRlcm5hbElkcC0-Pi1Ccm93c2VyOiAzMDIgaWRwLmNvbS9sb2dpblxuICAgIE5vdGUgb3ZlciBCcm93c2VyLEV4dGVybmFsSWRwOiBUaGUgcHJpb3IgT3BlbklEIENvbm5lY3QgZmxvd1xuICAgIEJyb3dzZXItPj4rRXh0ZXJuYWxJZHA6IC9sb2dnZWRfaW5cbiAgICBOb3RlIG92ZXIgQnJvd3NlcixFeHRlcm5hbElkcDogUG90ZW50aWFsbHkgc2hvdyBjdXN0b20gY29uc2VudCBzY3JlZW5cbiAgICBFeHRlcm5hbElkcC0-Pi1Ccm93c2VyOiBsb2NhbGhvc3QvP2F1dGhDb2RlPTxhYmM-XG4gICAgQnJvd3Nlci0-PitDbGk6IGxvY2FsaG9zdC9hdXRoQ29kZT08YWJjPlxuICAgIENsaS0-PitFeHRlcm5hbElkcDogL3Rva2VuP2NvZGUsY29kZV92ZXJpZmllclxuICAgIEV4dGVybmFsSWRwLT4-LUNsaTogYWNjZXNzX3Rva2VuXG4gICAgQ2xpLT4-K0FkbWluOiAvcHJvamVjdHMvICsgYWNjZXNzX3Rva2VuXG4gICAgQWRtaW4tPj4tQ2xpOiBwcm9qZWN0MSwgcHJvamVjdDJcbiIsIm1lcm1haWQiOnsidGhlbWUiOiJuZXV0cmFsIn0sInVwZGF0ZUVkaXRvciI6ZmFsc2V9 + :width: 600 + :alt: CLI Authentication with an external Authorization Server -* ``FLYTE_CREDENTIALS_CLIENT_ID`` and ``FLYTE_CREDENTIALS_CLIENT_SECRET`` - When using basic authentication, this is the username and password -* ``export FLYTE_CREDENTIALS_AUTH_MODE=basic`` - This tells the SDK to use basic authentication. If not set, Flytekit will assume you want to use the standard OAuth based three-legged flow. -* ``export FLYTE_CREDENTIALS_AUTHORIZATION_METADATA_KEY=text`` - At Lyft, we set this to conform to this `header config `_ on the Admin side. -* ``export FLYTE_CREDENTIALS_SCOPE=text`` - When using basic authentication, you'll need to specify a scope to the IDP (instead of ``openid``, as that's only for OAuth). Set that here. -* ``export FLYTE_PLATFORM_AUTH=True`` - Set this to force Flytekit to use authentication, even if not required by Admin. This is useful as you're rolling out the requirement. +Identity Providers Support +========================== ++-----------------+--------+-------------+---------------------+----------+-------+----------+--------+ +| Feature | Okta | Google free | GC Identity Service | Azure AD | Auth0 | KeyCloak | Github | ++=================+========+=============+=====================+==========+=======+==========+========+ +| OpenIdConnect | Yes | Yes | Yes | Yes | Yes | Yes | No | ++-----------------+--------+-------------+---------------------+----------+-------+----------+--------+ +| Custom RP | Yes | No | Yes | Yes | ? | Yes | No | ++-----------------+--------+-------------+---------------------+----------+-------+----------+--------+ ********** References ********** -RFCs -====== This collection of RFCs may be helpful to those who wish to investigate the implementation in more depth. * `OAuth2 RFC 6749 `_ diff --git a/rsts/howto/authentication/migration.rst b/rsts/howto/authentication/migration.rst new file mode 100644 index 0000000000..cc919c5040 --- /dev/null +++ b/rsts/howto/authentication/migration.rst @@ -0,0 +1,151 @@ +.. _howto_authentication_migrate: + +###################################################### +How to migrate your authentication config (pre 0.13.0) +###################################################### + +Using Okta as an example, previously, you would've had something like the following. + +On the Okta side, you'd have: + +* An Application (OpenID Connect Web) for Flyte Admin itself (e.g. **0oal5rch46pVhCGF45d6**). +* An Application (OpenID Native app) for flyte-cli/flytectl (e.g. **0oal62nxuD6OSFSRq5d6**). +* These two applications would be assigned to the relevant users. +* An Application (Web) for Flyte Propeller (e.g. **0abc5rch46pVhCGF9876**). +* These applications would either use the default Authorization server, or you'd create a new one. + +On the Admin side, you'd have the following configuration + +.. code-block:: yaml + + server: + # ... other settings + security: + secure: false + useAuth: true + allowCors: true + allowedOrigins: + - "*" + allowedHeaders: + - "Content-Type" + oauth: + baseUrl: https://dev-62129345.okta.com/oauth2/default/ + scopes: + - profile + - openid + - email + claims: + iss: https://dev-62129345.okta.com/oauth2/default + aud: 0oal5rch46pVhCGF45d6 + clientId: 0oal5rch46pVhCGF45d6 + clientSecretFile: "/Users/ytong/etc/secrets/oauth/secret" + authorizeUrl: "https://dev-62129345.okta.com/oauth2/default/v1/authorize" + tokenUrl: "https://dev-62129345.okta.com/oauth2/default/v1/token" + callbackUrl: "http://localhost:8088/callback" + cookieHashKeyFile: "/Users/ytong/etc/secrets/hashkey/hashkey" + cookieBlockKeyFile: "/Users/ytong/etc/secrets/blockkey/blockkey" + redirectUrl: "/api/v1/projects" + thirdPartyConfig: + flyteClient: + clientId: 0oal62nxuD6OSFSRq5d6 + redirectUri: http://localhost:12345/callback + +From the flyte-cli side, you needed these two settings + +.. code-block:: bash + + FLYTE_PLATFORM_HTTP_URL=http://localhost:8088 FLYTE_CREDENTIALS_CLIENT_ID=0oal62nxuD6OSFSRq5d6 flyte-cli ... + +**FLYTE_PLATFORM_HTTP_URL** is used because **flyte-cli** uses only gRPC to communicate with Admin. It needs to know the HTTP port (which Admin hosts on a different port because of limitations of the +grpc-gateway library). **flyte-cli** uses this setting to talk to **/.well-known/oauth-authorization-server** to retrieve information regarding the auth endpoints. Previously this redirected to the +Okta Authorization Server's metadata endpoint. With this change, Admin now hosts its own (even if still using the external Authorization Service). + +After version `0.13.0 `__ of the platfform, you can still use the IdP as the Authorization Server if you so choose. That configuration would become: + +.. code-block:: yaml + + server: + # ... other settings + security: + secure: false + useAuth: true + allowCors: true + allowedOrigins: + - "*" + allowedHeaders: + - "Content-Type" + auth: + authorizedUris: + # This should point at your public http Uri. + - https://flyte.mycompany.com + # This will be used by internal services in the same namespace as flyteadmin + - http://flyteadmin:80 + # This will be used by internal services in the same cluster but different namespaces + - http://flyteadmin.flyte.svc.cluster.local:80 + userAuth: + openId: + # Put the URL of the OpenID Connect provider. + baseUrl: https://dev-62129345.okta.com/oauth2/default # Okta with a custom Authorization Server + scopes: + - profile + - openid + - offline_access # Uncomment if OIdC supports issuing refresh tokens. + # Replace with the client id created for Flyte. + clientId: 0oal5rch46pVhCGF45d6 + appAuth: + # External delegates app auth responsibilities to an external authorization server, Internal means Flyte Admin does it itself + authServerType: External + thirdPartyConfig: + flyteClient: + clientId: 0oal62nxuD6OSFSRq5d6 + redirectUri: http://localhost:12345/callback + scopes: + - all + - offline + +Specifically, + +* The original **oauth** section has been moved two levels higher into its own section and renamed **auth** but enabling/disabling of authentication remains in the old place. +* Secrets by default will now be looked up in **/etc/secrets**. Use the following command to generate them. + +.. code-block:: bash + + flyteadmin secrets init -p /etc/secrets + +This will generate the new cookie hash/block keys, as well as other secrets Admin needs to run the Authorization server. + +* The **clientSecretFile** has been moved to **/etc/secrets/oidc_client_secret** so move that there. +* **claims** has been removed, just delete that. +* **authorizeUrl** and **tokenUrl** are no longer necessary. +* The **baseUrl** for the external Authorization Server is now in the **appAuth** section. +* The **thirdPartyConfig** has been moved to **appAuth** as well. +* **redirectUrl** has been defaulted to **/console**. If that's the value you want, then you no longer need this setting. + +From propeller side, you might have a configuration section that looks like this: + +.. code-block:: yaml + + admin: + endpoint: dns:///mycompany.domain.com + useAuth: true + clientId: flytepropeller + clientSecretLocation: /etc/secrets/client_secret + tokenUrl: https://demo.nuclyde.io/oauth2/token + scopes: + - all + +This can now be simplified to: + +.. code-block:: yaml + + admin: + endpoint: dns:///mycompany.domain.com + # If you are using the built-in authorization server, you can delete the following two lines: + clientId: flytepropeller + clientSecretLocation: /etc/secrets/client_secret + +Specifically, + +* **useAuth** is deprecated and will be removed in a future version. Auth requirement will be discovered through an anonymous admin discovery call. +* **tokenUrl** and **scopes** will also be discovered through a metadata call. +* **clientId** and **clientSecretLocation** have defaults that work out of the box with the built-in authorization server (e.g. if you setup Google OpenID Connect). \ No newline at end of file diff --git a/rsts/howto/authentication/setup.rst b/rsts/howto/authentication/setup.rst new file mode 100644 index 0000000000..4cab5e1b0f --- /dev/null +++ b/rsts/howto/authentication/setup.rst @@ -0,0 +1,257 @@ +.. _howto_authentication_setup: + +############################ +How to Set Up Authentication +############################ + +***************** +IdP Configuration +***************** +Flyte Admin requires that the application in your identity provider be configured as a web client (i.e. with a client secret). We recommend allowing the application to be issued a refresh token to avoid interrupting user's flow by redirecting to the IdP frequently. + +************************* +Flyte Admin Configuration +************************* +Please refer to the `inline documentation `_ on the ``Config`` object in the ``auth`` package for a discussion on the settings required. + +********************** +Example Configurations +********************** + +Here you can find canonical examples for how to set up some of the common IdPs to secure your flyte services. OpenID Connect enables users to authenticate, in the +browser, with an existing IdP. Flyte also allows connecting to an external OAuth2 Authorization Server to allow centerally managed 3rd party app access. + +OpenID Connect +=============== + +OpenID Connect allows users to authenticate to flyte in their browser using a familiar authentication provider (perhaps an organization-wide configured IdP). +Flyte supports connecting with external OIdC providers. Here are some examples for how to set these up: + +Google OpenID Connect +--------------------- + +Follow `Google Docs `__ on how to configure the IdP for OpenIDConnect. + +.. note:: + + Make sure to create an OAuth2 Client Credential. The `client_id` and `client_secret` will be needed in the following + steps. + +Okta OpenID Connect +------------------- + +Okta supports OpenID Connect protocol and the creation of custom OAuth2 Authorization Servers, allowing it to act as both the user and apps IdP. +It offers more detailed control on access policies, users' consent, and app management. + +1. If you don't already have an Okta account, sign up for one `here `__. +2. Create an app (choose Web for the platform) and OpenID Connect for the sign on method. +3. Add Login redirect URIs (e.g. http://localhost:30081/callback for sandbox or https:///callback) +4. OPTIONAL: Add logout redirect URIs (e.g. http://localhost:30081/logout for sandbox) +5. Note down the Client ID and Client Secret + +KeyCloak OpenID Connect +------------------------ + +`KeyCloak `__ is an open source solution for authentication, it supports both OpenID Connect and OAuth2 protocols (among others). +KeyCloak can be configured to be both the OpenID Connect and OAuth2 Authorization Server provider for flyte. + + + +Apply configuration +------------------- + +1. Store the `client_secret` in a k8s secrt as follows: + +.. prompt:: bash + + kubectl edit secret -n flyte flyte-admin-auth + +Add a new key under `stringData`: + +.. code-block:: yaml + + stringData: + oidc_client_secret: from the previous step + data: + ... + +Save and close your editor. + +2. Edit FlyteAdmin config to add `client_id` and configure auth as follows: + +.. prompt:: bash + + kubectl get deploy -n flyte flyteadmin -o yaml | grep "name: flyte-admin-config" + +This will output the name of the config map where the `client_id` needs to go. + +.. prompt:: bash + + kubectl edit configmap -n flyte + +Follow the inline comments to make the necessary changes: + +.. code-block:: yaml + + server: + ... + security: + secure: false + # 1. Enable Auth by turning useAuth to true + useAuth: true + ... + auth: + userAuth: + openId: + # 2. Put the URL of the OpenID Connect provider. + # baseUrl: https://accounts.google.com # Uncomment for Google + baseUrl: https://dev-14186422.okta.com/oauth2/default # Okta with a custom Authorization Server + scopes: + - profile + - openid + # - offline_access # Uncomment if OIdC supports issuing refresh tokens. + # 3. Replace with the client ID created for Flyte. + clientId: 0oakkheteNjCMERst5d6 + +Save and exit your editor. + +3. Restart `flyteadmin` for the changes to take effect: + +.. prompt:: bash + + kubectl rollout restart deployment/flyteadmin -n flyte + +OAuth2 Authorization Server +=========================== + +An OAuth2 Authorization Server allows external clients to request to authenticate and act on behalf of users (or as their own identities). Having +an OAuth2 Authorization Server enables flyte administrators control over which apps can be installed and what scopes (i.e. what privileges can they assume), +they are allowed to request or be granted. + +Flyte comes with a built-in authorization server that can be statically configured with a set of clients to request and act on behalf of the user. +The default clients are defined `here `__ +and the corresponding section can be modified through configs. + +To set up an external OAuth2 Authorization Server, please follow the instructions below: + +Okta IdP +-------- + +1. Under security -> API, click `Add Authorization Server`. Set the audience to the public URL of flyte admin (e.g. https://flyte.mycompany.io/). +2. Under `Access Policies`, click `Add New Access Policy` and walk through the wizard to allow access to the authorization server. +3. Under `Scopes`, click `Add Scope`. Set the name to `all` (required) and check `Require user consent for this scope` (recommended). +4. Create 2 apps (for fltyectl and flytepropeller) to enable these clients to communicate with the service. + Flytectl should be created as a `native client`. + FlytePropeller should be created as an `OAuth Service` and note the client ID and client Secrets provided. + +KeyCloak IdP +------------ + +`KeyCloak `__ is an open source solution for authentication, it supports both OpenID Connect and OAuth2 protocols (among others). +KeyCloak can be configured to be both the OpenID Connect and OAuth2 Authorization Server provider for flyte. + + + +Apply Configurations +-------------------- + +1. It is possible to direct flyte admin to use an external authorization server. To do so, edit the same config map once more and follow these changes: + +.. code-block:: yaml + + auth: + appAuth: + # 1. Choose External if you will use an external Authorization Server (e.g. a Custom Authorization server in Okta) + # Choose Self (or omit the value) to use Flyte Admin's internal (albeit limited) Authorization Server. + authServerType: External + + # 2. Optional: Set external auth server baseUrl if different from OpenId baseUrl. + externalAuthServer: + baseUrl: https://dev-14186422.okta.com/oauth2/auskngnn7uBViQq6b5d6 + thirdPartyConfig: + flyteClient: + # 3. Replace with a new Native Client ID provisioned in the custom authorization server + clientId: flytectl + + redirectUri: https://localhost:53593/callback + + # 4. "all" is a required scope and must be configured in the custom authorization server + scopes: + - offline + - all + userAuth: + openId: + baseUrl: https://dev-14186422.okta.com/oauth2/auskngnn7uBViQq6b5d6 # Okta with a custom Authorization Server + scopes: + - profile + - openid + # - offline_access # Uncomment if OIdC supports issuing refresh tokens. + clientId: 0oakkheteNjCMERst5d6 + +1. Store flyte propeller's `client_secret` in a k8s secrt as follows: + +.. prompt:: bash + + kubectl edit secret -n flyte flyte-propeller-auth + +Add a new key under `stringData`: + +.. code-block:: yaml + + stringData: + client_secret: from the previous step + data: + ... + +Save and close your editor. + +2. Edit FlytePropeller config to add `client_id` and configure auth as follows: + +.. prompt:: bash + + kubectl get deploy -n flyte flytepropeller -o yaml | grep "name: flyte-propeller-config" + +This will output the name of the config map where the `client_id` needs to go. + +.. prompt:: bash + + kubectl edit configmap -n flyte + +Follow the inline comments to make the necessary changes: + +.. code-block:: yaml + + admin: + # 1. Replace with the client_id provided by the OAuth2 Authorization Server above. + clientId: flytepropeller + +Close the editor + +3. Restart `flytepropeller` for the changes to take effect: + +.. prompt:: bash + + kubectl rollout restart deployment/flytepropeller -n flyte + +*************************** +Continuous Integration - CI +*************************** + +If your organization does any automated registration, then you'll need to authenticate with the `basic authentication `_ flow (username and password effectively). After retrieving an access token from the IDP, you can send it along to Flyte Admin as usual. + +Flytekit configuration variables are automatically designed to look up values from relevant environment variables. However, to aid with continuous integration use-cases, Flytekit configuration can also reference other environment variables. + +For instance, if your CI system is not capable of setting custom environment variables like ``FLYTE_CREDENTIALS_CLIENT_SECRET`` but does set the necessary settings under a different variable, you may use ``export FLYTE_CREDENTIALS_CLIENT_SECRET_FROM_ENV_VAR=OTHER_ENV_VARIABLE`` to redirect the lookup. A ``FLYTE_CREDENTIALS_CLIENT_SECRET_FROM_FILE`` redirect is available as well, where the value should be the full path to the file containing the value for the configuration setting, in this case, the client secret. We found this redirect behavior necessary when setting up registration within our own CI pipelines. + +The following is a listing of the Flytekit configuration values we set in CI, along with a brief explanation. + +* ``FLYTE_CREDENTIALS_CLIENT_ID`` and ``FLYTE_CREDENTIALS_CLIENT_SECRET`` + When using basic authentication, this is the username and password. +* ``export FLYTE_CREDENTIALS_AUTH_MODE=basic`` + This tells the SDK to use basic authentication. If not set, Flytekit will assume you want to use the standard OAuth based three-legged flow. +* ``export FLYTE_CREDENTIALS_AUTHORIZATION_METADATA_KEY=text`` + At Lyft, we set this to conform to this `header config `_ on the Admin side. +* ``export FLYTE_CREDENTIALS_SCOPE=text`` + When using basic authentication, you'll need to specify a scope to the IDP (instead of ``openid``, which is only for OAuth). Set that here. +* ``export FLYTE_PLATFORM_AUTH=True`` + Set this to force Flytekit to use authentication, even if not required by Admin. This is useful as you're rolling out the requirement. diff --git a/rsts/howto/enable_and_use_schedules.rst b/rsts/howto/enable_and_use_schedules.rst index 6bea7a0b65..9bc719282f 100644 --- a/rsts/howto/enable_and_use_schedules.rst +++ b/rsts/howto/enable_and_use_schedules.rst @@ -165,7 +165,7 @@ Workflow Executor ----------------- Scheduled events which trigger need to be handled by the workflow executor, which subscribes to triggered events from the SQS queue you've configured above. -.. CAUTION:: +.. NOTE:: Failure to configure a workflow executor will result in all your scheduled events piling up silently without ever kicking off workflow executions. diff --git a/rsts/howto/enable_backend_plugin.rst b/rsts/howto/enable_backend_plugin.rst index 00745e7a06..b5f1d054de 100644 --- a/rsts/howto/enable_backend_plugin.rst +++ b/rsts/howto/enable_backend_plugin.rst @@ -4,7 +4,7 @@ How do I enable backend plugins? ################################# -.. tip:: Flyte Backend plugins are awesome, but are not required to extend Flyte! You can always write a `flytekit-only plugin`. Refer to :std:ref:`sphx_glr_auto_core_advanced_custom_task_plugin.py`. Also refer to :ref:`howto-create-plugins`. +.. tip:: Flyte Backend plugins are awesome, but are not required to extend Flyte! You can always write a flytekit-only plugins. Refer to :ref:`plugins_extend_intro`. Flyte has a unique capability of adding backend plugins. Backend plugins enable Flyte platform to add new capabilities. This has several advantages, @@ -18,9 +18,9 @@ Ok, How do I enable the backend plugins? ========================================= To enable a backend plugin you have to add the ``ID`` of the plugin to the enabled plugins list. The ``enabled-plugins`` is available under the ``tasks > task-plugins`` section of FlytePropeller's configuration. -The `plugin configuration structure is defined here `_. An example of the config follows, +The `plugin configuration structure is defined here `_. An example of the config follows, -.. rli:: https://raw.githubusercontent.com/flyteorg/flyte/master/kustomize/overlays/sandbox/config/propeller/enabled_plugins.yaml +.. rli:: https://raw.githubusercontent.com/flyteorg/flyte/master/kustomize/overlays/sandbox/flyte/config/propeller/enabled_plugins.yaml :language: yaml How do I find the ``ID`` of the backend plugin? diff --git a/rsts/howto/execute_single_task.rst b/rsts/howto/execute_single_task.rst index ae7d788ec0..03b5c876f4 100644 --- a/rsts/howto/execute_single_task.rst +++ b/rsts/howto/execute_single_task.rst @@ -1,7 +1,7 @@ .. _howto_exec_single_task: #################################### -How Do I Execute a single task? +How do I execute a single task? #################################### @@ -32,7 +32,7 @@ After you've built an image with your updated task code, create an execution usi Just like workflow executions, you can optionally pass a user-defined name, labels, annotations, and/or notifications when launching a single task. -The type of ``my_single_task_execution`` is `SdkWorkflowExecution `_ +The type of ``my_single_task_execution`` is `SdkWorkflowExecution `_ and has the full set of methods and functionality available for conventional WorkflowExecutions. diff --git a/rsts/howto/execute_workflow.rst b/rsts/howto/execute_workflow.rst index b641192d36..3e4234831a 100644 --- a/rsts/howto/execute_workflow.rst +++ b/rsts/howto/execute_workflow.rst @@ -1,5 +1,9 @@ .. _howto_exec_workflow: #################################### -How Do I Execute a Workflow? +How do I execute a workflow? #################################### + +.. NOTE:: + + Coming soon 🛠 diff --git a/rsts/howto/fast_registration.rst b/rsts/howto/fast_registration.rst index cd3ef6e5e9..60b124dba4 100644 --- a/rsts/howto/fast_registration.rst +++ b/rsts/howto/fast_registration.rst @@ -4,7 +4,7 @@ How do I use Fast Registration? ******************************** -.. caution:: Experimental feature (beta) +.. NOTE:: Experimental feature (beta) Are you frustrated by having to wait for an image build in order to test out simple code changes to your Flyte workflows? If you're interested in reducing to your iteration cycle to mere seconds, read on below. diff --git a/rsts/howto/flytecli.rst b/rsts/howto/flytecli.rst index afc72779c6..043fc13f5e 100644 --- a/rsts/howto/flytecli.rst +++ b/rsts/howto/flytecli.rst @@ -38,7 +38,7 @@ the root directory of the flytecli package, and install the dependencies with ``pip install -e .``. -.. _doc: https://virtualenv.pypa.io/en/latest/installation/ +.. _doc: https://virtualenv.pypa.io/en/latest/installation.html Install from pypi [recommended] ------------------------------- diff --git a/rsts/howto/gcp.rst b/rsts/howto/gcp.rst new file mode 100644 index 0000000000..a8540fefa0 --- /dev/null +++ b/rsts/howto/gcp.rst @@ -0,0 +1,26 @@ +.. _faq_gcp: + +############################################## +How do I Use Flyte with Google Cloud Platform? +############################################## + +I tried to run examples, but task fails with 401 error? +------------------------------------------------------- + Steps: + + #. Are you using Workload Identity, then you have to pass in the ServiceAccount when you create the launchplan. + - Refer to docs :ref:`howto-serviceaccounts` + - More information about WorkloadIdentity at https://cloud.google.com/kubernetes-engine/docs/how-to/workload-identity + #. If you are just using a simple Nodepool wide permissions then check the cluster's ServiceAccount for Storage permissions. Do they look fine? + + #. If not, then start a dummy pod in the intended namespace and check for + +:: + + gcloud auth list + + +.. note:: + + FlytePropeller uses Google Application credentials, but gsutil does not use these credentials + diff --git a/rsts/howto/index.rst b/rsts/howto/index.rst index b66cdc53d0..1c0bbe5e1c 100644 --- a/rsts/howto/index.rst +++ b/rsts/howto/index.rst @@ -1,8 +1,8 @@ .. _howto: -######################## -How Do I...? -######################## +########################### +Frequently Asked Questions +########################### .. toctree:: :maxdepth: 1 @@ -31,12 +31,8 @@ How Do I...? labels_annotations notifications serviceaccount + gcp + secrets .. _howto_extend: - -======================= -How do I Extend flyte? -======================= - -Flyte was designed to be extensible. The section - :ref:`plugins`- dives into more details on extending Flyte and also provides available examples. \ No newline at end of file diff --git a/rsts/howto/install_sdk.rst b/rsts/howto/install_sdk.rst index 996aa0c9cb..0d2cb45a18 100644 --- a/rsts/howto/install_sdk.rst +++ b/rsts/howto/install_sdk.rst @@ -21,4 +21,6 @@ All Flytekiplugins are also published to pypi as independent libraries and can b How to install Flytekit Java? ################################# -.. todo: coming soon \ No newline at end of file +.. NOTE:: + + Coming soon 🛠 diff --git a/rsts/howto/interruptible.rst b/rsts/howto/interruptible.rst index 79b0589b5b..1abcb5f8ea 100644 --- a/rsts/howto/interruptible.rst +++ b/rsts/howto/interruptible.rst @@ -49,4 +49,6 @@ Most Flyte workloads should be good candidates for spot instances. If your task How to recover from interruptions? =================================== -.. todo: Intra-task checkpointing coming soon \ No newline at end of file +.. NOTE:: + + Coming soon 🛠 diff --git a/rsts/howto/managing_customizable_resources.rst b/rsts/howto/managing_customizable_resources.rst index 4bd0f28ee7..eb9c77828d 100644 --- a/rsts/howto/managing_customizable_resources.rst +++ b/rsts/howto/managing_customizable_resources.rst @@ -20,15 +20,17 @@ Flyte allows these custom settings along the following combination of dimensions Please see the :ref:`divedeep-projects` document for more information on projects and domains. Along these dimensions, the following settings are configurable. Note that not all three of the combinations above are valid for each of these settings. - Defaults for task resource requests and limits (when not specified by the author of the task). -- Settings for the cluster resource configuration that feeds into Admin's cluster resource manager. +- Settings for project-namespaced cluster resource configuration that feeds into Admin's cluster resource manager. - Execution queues that are used for Dynamic Tasks. Read more about execution queues here, but effectively they're meant to be used with constructs like AWS Batch. - Determining how workflow executions get assigned to clusters in a multi-cluster Flyte deployment. The proto definition is the definitive source of which -`matchable attributes `_ +`matchable attributes `_ can be customized. -Each of the four above settings are discussed below. Also, since the flyte-cli tool does not yet hit these endpoints, we are including some sample ``curl`` commands for administrators to reference. +Each of the four above settings are discussed below. Eventually all of these customizations will be overridable using +:std:ref:`flytectl`. Until then, flyte-cli command line options can be used to modify frequent use-cases, and barring +that we show examples using curl. Task Resources @@ -42,54 +44,93 @@ This includes setting default value for task resource requests and limits for th - storage In the absence of an override the global -`default values `__ +`default values `__ in the flyteadmin config are used. The override values from the database are assigned at execution time. +To update individual project-domain attributes, use the following as an example: + +.. prompt:: bash + + curl --request PUT 'https://flyte.company.net/api/v1/project_domain_attributes/projectname/staging' \ + --header 'Content-Type: application/json' --data-raw \ + '{"attributes":{"matchingAttributes":{"taskResourceAttributes":{"defaults":{"cpu": "1000", "memory": "5000Gi"}, "limits": {"cpu": "4000"}}}}' + + Cluster Resources ================= -These are free-form key-value pairs which are used when filling in the templates that Admin feeds into its cluster manager. The keys represent templatized variables in `clusterresource template yaml `__ and the values are what you want to see filled in. +These are free-form key-value pairs which are used when filling in the templates that Admin feeds into its cluster manager. The keys represent templatized variables in `clusterresource template yaml `__ and the values are what you want to see filled in. -In the absence of custom override values, templateData from the `flyteadmin config `__ is used as a default. +In the absence of custom override values, templateData from the `flyteadmin config `__ is used as a default. Note that these settings can only take on domain, or a project and domain specificity. Since Flyte has not tied in the notion of a workflow or a launch plan to any Kubernetes constructs, specifying a workflow or launch plan name doesn't make any sense. +Running the following, will make it so that when Admin fills in cluster resource templates, the K8s namespace ``flyteexamples-development`` will have a resource quota of 1000 CPU cores and 5TB of memory. + +.. prompt:: bash + + flyte-cli -h localhost:30081 -p flyteexamples -d development update-cluster-resource-attributes \ + --attributes projectQuotaCpu 1000 --attributes projectQuotaMemory 5000Gi + + +These values will in turn be used to fill in the template fields, for example: -Command -------- -Running the following, will make it so that when Admin fills in cluster resource templates, the K8s namespace ``projectname-staging`` will have a resource quota of 1000 CPU cores and 5TB of memory. +.. rli:: https://raw.githubusercontent.com/flyteorg/flyte/master/kustomize/base/single_cluster/headless/config/clusterresource-templates/ab_project-resource-quota.yaml -.. code-block:: console +from the base of this repository for the ``flyteexamples-development`` namespace and that namespace only. +For other namespaces, the `platform defaults `__ would still be applied. - curl --request PUT 'https://flyte.company.net/api/v1/project_domain_attributes/projectname/staging' --header 'Content-Type: application/json' --data-raw '{"attributes":{"matchingAttributes":{"clusterResourceAttributes":{"attributes":{"projectQuotaCpu": "1000", "projectQuotaMemory": "5000Gi"}}}}}' +======= + + flyte-cli -h localhost:30081 -p flyteexamples -d development update-cluster-resource-attributes \ + --attributes projectQuotaCpu 1000 --attributes projectQuotaMemory 5000Gi + + +These values will in turn be used to fill in the template fields, for example: + +.. rli:: https://raw.githubusercontent.com/flyteorg/flyte/master/kustomize/base/single_cluster/headless/config/clusterresource-templates/ab_project-resource-quota.yaml -These values will in turn be used to fill in the template fields in the file at ``kustomize/overlays/sandbox/admindeployment/clusterresource-templates/ab_project-resource-quota.yaml`` from the base of this repository for the ``projectname-staging`` namespace and that namespace only. For other namespaces, the defaults at the bottom of `this file `__ would still be applied. +from the base of this repository for the ``flyteexamples-development`` namespace and that namespace only. +For other namespaces, the `platform defaults `__ would still be applied. +.. note:: + + The template values, e.g. ``projectQuotaCpu`` or ``projectQuotaMemory`` are freeform strings. You must ensure that + they match the template placeholders in your `template file `__ + for your changes to take effect. Execution Queues ================ -Execution queues are use to determine where dynamic tasks run. +Execution queues are use to determine where tasks yielded by a dynamic :py:func:`flytekit:flytekit.maptask` run. Execution queues themselves are currently defined in the -`flyteadmin config `__. +`flyteadmin config `__. The **attributes** associated with an execution queue must match the **tags** for workflow executions. The tags are associated with configurable resources stored in the Admin database. .. prompt:: bash - curl --request PUT 'https://flyte.company.net/api/v1/workflow_attributes/project/domain/YourWorkflowName' --header 'Content-Type: application/json' --data-raw '{"attributes":{"matchingAttributes":{"executionQueueAttributes":{"tags":["my_queue"]}}}}' + flyte-cli -h localhost:30081 -p flyteexamples -d development update-execution-queue-attributes \ + --tags critical --tags gpu_intensive +You can view existing attributes for which tags can be assigned by visting `http://localhost:30081/api/v1/matchable_attributes?resource_type=3 `__. Execution Cluster Label ======================= This allows forcing a matching execution to always execute on a specific kubernetes cluster. +You can set this using flyte-cli: + +.. prompt:: bash + + flyte-cli -h localhost:30081 -p flyteexamples -d development update-execution-cluster-label --value mycluster + ********* Hierarchy @@ -131,11 +172,11 @@ All other inbound CreateExecution requests would use the default values specifie Debugging ********* -Use the `get `__ endpoint +Use the `get `__ endpoint to see if overrides exist for a specific resource. E.g. `https://example.com/api/v1/project_domain_attributes/widgetmodels/production?resource_type=2 `__ To get the global state of the world, use the list all endpoint, e.g. `https://example.com/api/v1/matchable_attributes?resource_type=2 `__. -The resource type enum (int) is defined in the `proto `__. +The resource type enum (int) is defined in the `proto `__. diff --git a/rsts/howto/monitoring/index.rst b/rsts/howto/monitoring/index.rst index 3e6bda1f15..d8fa308e18 100644 --- a/rsts/howto/monitoring/index.rst +++ b/rsts/howto/monitoring/index.rst @@ -1,5 +1,20 @@ -.. _howto_monitoring: +.. _howto-monitoring: ###################################### How do I monitor my Flyte deployment? ###################################### + +.. tip:: The flyte core team publishes a maintains Grafana dashboards built using prometheus data source and can be found `here `__. + +Flyte Backend is written in Golang and exposes stats using Prometheus. The Stats themselves are labeled with the Workflow, Task, Project & Domain whereever appropriate. + +The dashboards are divided into primarily 2 types + +- User facing dashboards. These are dashboards that a user can use to triage/investigate/observe performance and characterisitics for their Workflows and tasks. + The User facing dashboard is published under Grafana marketplace ID `13980 `_ + +- System Dashboards. These dashboards are useful for the system maintainer to maintain their Flyte deployments. These are further divided into + - DataPlane/FlytePropeller dashboards published @ `13979 `_ + - ControlPlane/Flyteadmin dashboards published @ `13981 `_ + +These are basic dashboards and do no include all the metrics that are exposed by Flyte. You can contribute to the dashboards and help us improve them - by referring to the build scripts `here `__. diff --git a/rsts/howto/multi_cluster/index.rst b/rsts/howto/multi_cluster/index.rst index 8df12ddf64..0bf2ff0105 100644 --- a/rsts/howto/multi_cluster/index.rst +++ b/rsts/howto/multi_cluster/index.rst @@ -1,8 +1,8 @@ .. _howto-multi-cluster: -######################################### -How Do I use Multiple Kubernetes Clusters -######################################### +########################################## +How do I use multiple Kubernetes clusters? +########################################## Scaling Beyond Kubernetes ------------------------- @@ -132,7 +132,7 @@ Deploy the user/control plane to one cluster (you could use one of 3 existing cl FlyteAdmin Remote Cluster Access ********************************* -Some deployments of Flyte may choose to run the control plane separate from the data plane. Flyte Admin is designed to create kubernetes resources in one or more Flyte data plane clusters. For Admin to access remote clusters, it needs credentials to each cluster. In kubernetes, scoped service credentials are created by configuring a “Role” resource in a Kubernetes cluster. When you attach that role to a “ServiceAccount”, Kubernetes generates a bearer token that permits access. We create a flyteadmin `ServiceAccount `_ in each data plane cluster to generate these tokens. +Some deployments of Flyte may choose to run the control plane separate from the data plane. Flyte Admin is designed to create kubernetes resources in one or more Flyte data plane clusters. For Admin to access remote clusters, it needs credentials to each cluster. In kubernetes, scoped service credentials are created by configuring a “Role” resource in a Kubernetes cluster. When you attach that role to a “ServiceAccount”, Kubernetes generates a bearer token that permits access. We create a flyteadmin `ServiceAccount `_ in each data plane cluster to generate these tokens. When you first create the Flyte Admin ServiceAccount in a new cluster, a bearer token is generated, and will continue to allow access unless the ServiceAccount is deleted. Once we create the Flyte Admin ServiceAccount on a cluster, we should never delete it. In order to feed the credentials to Flyte Admin, you must retrieve them from your new data plane cluster, and upload them to Admin somehow (within Lyft, we use Confidant for example). diff --git a/rsts/howto/new_project.rst b/rsts/howto/new_project.rst index 294eb21af6..9ccd0e32b2 100644 --- a/rsts/howto/new_project.rst +++ b/rsts/howto/new_project.rst @@ -7,7 +7,10 @@ How do I create/register a new project? Using flytectl --------------- -.. todo: link here +.. NOTE:: + + Coming soon 🛠 + Using flyte-cli @@ -24,4 +27,6 @@ If you refresh your console you'll see your new project appear! FlyteAdmin API reference ------------------------- -.. todo: link here \ No newline at end of file +.. NOTE:: + + Coming soon 🛠 diff --git a/rsts/howto/notifications.rst b/rsts/howto/notifications.rst index dc9b8aa058..d38d1702b9 100644 --- a/rsts/howto/notifications.rst +++ b/rsts/howto/notifications.rst @@ -107,7 +107,7 @@ Let's look at the following config section and go into what each value represent * **sender**: Your verified SES email sender * **body**: Configurable email body used in notifications -The full set of parameters which can be used for email templating are checked into `code `_. +The full set of parameters which can be used for email templating are checked into `code `_. .. _admin-config-example: diff --git a/rsts/howto/performance/index.rst b/rsts/howto/performance/index.rst index ad4f14bfe0..9c862102e5 100644 --- a/rsts/howto/performance/index.rst +++ b/rsts/howto/performance/index.rst @@ -3,3 +3,7 @@ ###################################################### How do I optimize performance of my Flyte Deployment? ###################################################### + +.. NOTE:: + + Coming soon 🛠 diff --git a/rsts/howto/productionize/index.rst b/rsts/howto/productionize/index.rst index 4104251076..7dbde0d04c 100644 --- a/rsts/howto/productionize/index.rst +++ b/rsts/howto/productionize/index.rst @@ -1,7 +1,7 @@ .. _howto_productionize: ############################################## -How Do I productionize my Flyte cluster +How do I productionize my Flyte cluster ############################################## .. toctree:: @@ -10,3 +10,4 @@ How Do I productionize my Flyte cluster :name: howtoprovguidestoc production + production_eks diff --git a/rsts/howto/productionize/production.rst b/rsts/howto/productionize/production.rst index 2627e51c33..e9c0d9ed98 100644 --- a/rsts/howto/productionize/production.rst +++ b/rsts/howto/productionize/production.rst @@ -12,14 +12,14 @@ A Flyte deployment contains around 50 kubernetes resources. The Flyte team has chosen to use the "kustomize" tool to manage these configs. Take a moment to read the `kustomize docs `__. Understanding kustomize will be important to modifying Flyte configurations. -The ``/kustomize`` directory in the `flyte repository `__ is designed for use with ``kustomize`` to tailor Flyte deployments to your needs. +The ``/kustomize`` directory in the `flyte repository `__ is designed for use with ``kustomize`` to tailor Flyte deployments to your needs. Important subdirectories are described below. base - The `base directory `__ contains minimal configurations for each Flyte component. + The `base directory `__ contains minimal configurations for each Flyte component. dependencies - The `dependencies directory `__ contains deploy configurations for components like ``PostgreSQL`` that Flyte depends on. + The `dependencies directory `__ contains deploy configurations for components like ``PostgreSQL`` that Flyte depends on. These directories were designed so that you can modify them using ``kustomize`` to generate a custom Flyte deployment. In fact, this is how we create the ``sandbox`` deployment. @@ -31,21 +31,21 @@ Understanding the Sandbox The sandbox deployment is managed by a set of kustomize `overlays `__ that alter the ``base`` configurations to compose the sandbox deployment. -The sandbox overlays live in the `kustomize/overlays/sandbox `__ directory. There are overlays for each component, and a "flyte" overlay that aggregates the components into a single deploy file. +The sandbox overlays live in the `kustomize/overlays/sandbox `__ directory. There are overlays for each component, and a "flyte" overlay that aggregates the components into a single deploy file. **Component Overlays** For each modified component, there is an kustomize overlay at ``kustomize/overlays/sandbox/{{ component }}``. The overlay will typically reference the ``base`` for that component, and modify it to the needs of the sandbox. - Using kustomize "patches", we add or override specific configs from the ``base`` resources. For example, in the "console" overlay, we specify a patch in the `kustomization.yaml `__. This patch adds memory and cpu limits to the console deployment config. + Using kustomize "patches", we add or override specific configs from the ``base`` resources. For example, in the "console" overlay, we specify a patch in the `kustomization.yaml `__. This patch adds memory and cpu limits to the console deployment config. - Each Flyte component requires at least one configuration file. The configuration files for each component live in the component overlay. For example, the FlyteAdmin config lives at `kustomize/overlays/sandbox/admindeployment/flyteadmin_config.yaml `__. These files get included as Kubernetes configmaps and mounted into pods. + Each Flyte component requires at least one configuration file. The configuration files for each component live in the component overlay. For example, the FlyteAdmin config lives at `kustomize/overlays/sandbox/admindeployment/flyteadmin_config.yaml `__. These files get included as Kubernetes configmaps and mounted into pods. **Flyte Overlay** The ``flyte`` overlay is meant to aggregate the components into a single deployment file. - The `kustomization.yaml overlay `__ in that directory lists the components to be included in the deploy. + The `kustomization.yaml overlay `__ in that directory lists the components to be included in the deploy. - We run ``kustomize build`` against the ``flyte`` directory to generate the complete `sandbox deployment yaml `__ we used earlier to deploy Flyte sandbox. + We run ``kustomize build`` against the ``flyte`` directory to generate the complete `sandbox deployment yaml `__ we used earlier to deploy Flyte sandbox. Creating Your Own Deployment **************************** @@ -59,7 +59,7 @@ NOTE: To do this, check out the ``flyte`` repo :: - git clone https://github.com/lyft/flyte.git + git clone https://github.com/flyteorg/flyte.git Copy the sandbox configuration to a new directory on your machine, and enter the new directory :: @@ -68,8 +68,8 @@ Copy the sandbox configuration to a new directory on your machine, and enter the Since the ``base`` files are not in your local copy, you'll need to make some slight modifications to reference the ``base`` files from our GitHub repository. :: - find . -name kustomization.yaml -print0 | xargs -0 sed -i.bak 's~../../../base~github.com/lyft/flyte/kustomize/base~' - find . -name kustomization.yaml -print0 | xargs -0 sed -i.bak 's~../../../dependencies~github.com/lyft/flyte/kustomize/dependencies~' + find . -name kustomization.yaml -print0 | xargs -0 sed -i.bak 's~../../../base~github.com/flyteorg/flyte/kustomize/base~' + find . -name kustomization.yaml -print0 | xargs -0 sed -i.bak 's~../../../dependencies~github.com/flyteorg/flyte/kustomize/dependencies~' find . -name '*.bak' | xargs rm You should now be able to run kustomize against the ``flyte`` directory :: @@ -94,7 +94,7 @@ First, you'll need to set up a reliable PostgreSQL database. The easiest way ach Next, remove old sandbox database by opening up the ``flyte/kustomization.yaml`` file and deleting database component. :: - - github.com/lyft/flyte/kustomize/dependencies/database + - github.com/flyteorg/flyte/kustomize/dependencies/database With this line removed, you can re-run ``kustomize build flyte > flyte_generated.yaml`` and see that the the postgres deployment has been removed from the ``flyte_generated.yaml`` file. @@ -119,7 +119,7 @@ Do the same thing in ``datacatalog/datacatalog_config.yaml``, but use the dbname Note: *You can mount the database password into the pod and use the "passwordPath" config to point to a file on disk instead of specifying the password here* -Next, remove the "check-db-ready" init container from `admindeployment/admindeployment.yaml `__. This check is no longer needed. +Next, remove the "check-db-ready" init container from `admindeployment/admindeployment.yaml `__. This check is no longer needed. Production Grade Object Store ***************************** @@ -133,7 +133,7 @@ To start, `create an s3 bucket flyte_generated.yaml`` and see that the sandbox object store has been removed from the ``flyte_generated.yaml`` file. @@ -152,7 +152,7 @@ Change the ``storage`` configuration in each of these configs to use your new s3 Note: *To use IAM roles for authentication, switch to the "iam" auth-type.* -Next, open ``propeller/plugins/config.yaml`` and remove the `default-env-vars `__ (no need to replace them, the default behavior is sufficient). +Next, open ``propeller/plugins/config.yaml`` and remove the `default-env-vars `__ (no need to replace them, the default behavior is sufficient). Now if you re-run ``kustomize build flyte > flyte_generated.yaml``, you should see that the configmaps have been updated. diff --git a/rsts/howto/productionize/production_eks.rst b/rsts/howto/productionize/production_eks.rst new file mode 100644 index 0000000000..56731f738c --- /dev/null +++ b/rsts/howto/productionize/production_eks.rst @@ -0,0 +1,21 @@ +.. _production-eks: + +Using AWS EKS to host Flyte +------------------------------ + +Illustration +************* + +.. note:: + + - Flyte needs a prefix in an AWS S3 bucket to store all its metadata. This is where the data about executions, workflows, tasks is stored + - this S3 bucket/prefix should be accessible to all FlytePropeller, FlyteAdmin, Datacatalog and running executions (user pods) + - FlyteAdmin can use any RDBMS database but we recommend Postgres. At scale we have used AWS Aurora + - Datacatalog also uses a postgres database similar to admin. They both could share the same physical instance, but prefer to have 2 logically separate databases + - If you want to use AWS IAM role for SeviceAccounts, then you have to manage the provisioning of the service account and providing it to Flyte at the time of execution + - For secrets, you can use Vault, Kube secrets etc, we are working on getting first class support for this + +.. image:: https://raw.githubusercontent.com/flyteorg/flyte/static-resources/img/core/flyte_single_cluster_eks.png + :alt: Illustration of setting up Flyte Cluster in a single AWS EKS (or any K8s cluster on AWS) + + diff --git a/rsts/howto/resource_manager/index.rst b/rsts/howto/resource_manager/index.rst index 87ec3faa2e..3ed54ad535 100644 --- a/rsts/howto/resource_manager/index.rst +++ b/rsts/howto/resource_manager/index.rst @@ -2,6 +2,9 @@ ################################################# -How Do I enable and configure resource manager? +How do I enable and configure resource manager? ################################################# +.. NOTE:: + + Coming soon 🛠 diff --git a/rsts/howto/resource_quota.rst b/rsts/howto/resource_quota.rst index 3d6e13894f..ac609818c2 100644 --- a/rsts/howto/resource_quota.rst +++ b/rsts/howto/resource_quota.rst @@ -3,3 +3,7 @@ ############################################### How do I limit resources per project/domain? ############################################### + +.. NOTE:: + + Coming soon 🛠 diff --git a/rsts/howto/sandbox.rst b/rsts/howto/sandbox.rst index dab5eedb8a..d8305fca72 100644 --- a/rsts/howto/sandbox.rst +++ b/rsts/howto/sandbox.rst @@ -1,4 +1,4 @@ -.. _howto_sandbox: +.. _howto-sandbox: ################################ How do I try out/install Flyte? @@ -17,18 +17,102 @@ Flyte can be run using a Kubernetes cluster only. This installs all the dependen #. The sandbox deployment is not suitable for production environments. For an in-depth overview of how to productionize your flyte deployment, checkout the :ref:`howto_productionize`. +.. image:: https://raw.githubusercontent.com/flyteorg/flyte/static-resources/img/core/flyte_sandbox_single_k8s_cluster.png + :alt: Architecture of Sandbox deployment of Flyte. Single K8s cluster + + ********************************************************* Deploy Flyte Sandbox environment locally - on your laptop ********************************************************* -Flyte Sandbox deployment can be run locally on your laptop. This will allow only some of the functionality, but is great to get started with. -Refer to :ref:`tutorials-getting-started-run-on-flyte`. +Ensure ``kubectl`` is installed. Follow `kubectl installation docs `_. On Mac:: + + brew install kubectl + + + +.. tabs:: + + .. tab:: Docker Image + + Refer to :ref:`getting-started-firstrun` + + .. tab:: k3d + + #. Install k3d Using ``curl``:: + + curl -s https://raw.githubusercontent.com/rancher/k3d/main/install.sh | bash + + Or Using ``wget`` :: + + wget -q -O - https://raw.githubusercontent.com/rancher/k3d/main/install.sh | bash + + #. Start a new K3s cluster called flyte:: + + k3d cluster create -p "30081:30081" --no-lb --k3s-server-arg '--no-deploy=traefik' --k3s-server-arg '--no-deploy=servicelb' flyte + + #. Ensure the context is set to the new cluster:: + + kubectl config set-context flyte + + #. Install Flyte:: + + kubectl create -f https://raw.githubusercontent.com/flyteorg/flyte/master/deployment/sandbox/flyte_generated.yaml + + + #. Connect to `FlyteConsole `__ + #. [Optional] You can delete the cluster once you are done with the tutorial using - :: + + k3d cluster delete flyte + + + .. note:: + + #. Sometimes Flyteconsole will not open up. This is probably because your docker networking is impacted. One solution is to restart docker and re-do the previous steps. + #. To debug you can try a simple excercise - run nginx as follows:: + + docker run -it --rm -p 8083:80 nginx + + Now connect to `locahost:8083 `__. If this does not work, then for sure the networking is impacted, please restart docker daemon. + + .. tab:: Docker-Mac + K8s + + #. `Install Docker for mac with Kubernetes as explained here `_ + #. Make sure Kubernetes is started and once started make sure your kubectx is set to the `docker-desktop` cluster, typically :: + + kubectl config set-context docker-desktop + + #. Install Flyte:: + + kubectl create -f https://raw.githubusercontent.com/flyteorg/flyte/master/deployment/sandbox/flyte_generated.yaml + + + #. Connect to `FlyteConsole `__ + + .. tab:: Using Minikube (Not recommended) + + #. Install `Minikube `_ + + #. Install Flyte:: + + kubectl create -f https://raw.githubusercontent.com/flyteorg/flyte/master/deployment/sandbox/flyte_generated.yaml + + + .. note:: + + - Minikube runs in a Virtual Machine on your host + - So if you try to access the flyte console on localhost, that will not work, because the Virtual Machine has a different IP address. + - Flyte runs within Kubernetes (minikube), thus to access FlyteConsole, you cannot just use https://localhost:30081/console, you need to use the IP address of the minikube VM instead of localhost + - Refer to https://kubernetes.io/docs/tutorials/hello-minikube/ to understand how to access a + also to register workflows, tasks etc or use the CLI to query Flyte service, you have to use the IP address. + - If you are building an image locally and want to execute on Minikube hosted Flyte environment, please push the image to docker registry running on the Minikube VM. + - Another alternative is to change the docker host, to build the docker image on the Minikube hosted docker daemon. https://minikube.sigs.k8s.io/docs/handbook/pushing/ provides more detailed information about this process. As a TL;DR, Flyte can only run images that are accessible to Kubernetes. To make an image accessible, you could either push it to a remote registry or to a regisry that is available to Kuberentes. In case on minikube this registry is the one thats running on the VM. .. _howto-sandbox-dedicated-k8s-cluster: ****************************************************************** -Deploy Flyte Sandbox environment to a dedicated kubernetes cluster +Deploy Flyte Sandbox environment to a Cloud Kubernetes cluster ****************************************************************** Cluster Requirements @@ -46,7 +130,7 @@ If you can access your cluster with ``kubectl cluster-info``, you're ready to de Deployment ========== -We'll proceed like with :ref:`locally hosted flyte ` with deploying the sandbox +We'll proceed like with :ref:`locally hosted flyte ` with deploying the sandbox Flyte configuration on your remote cluster. .. warning:: @@ -54,7 +138,7 @@ Flyte configuration on your remote cluster. #. The Flyte sandbox can be deployed with a single command :: - kubectl create -f https://raw.githubusercontent.com/lyft/flyte/master/deployment/sandbox/flyte_generated.yaml + kubectl create -f https://raw.githubusercontent.com/flyteorg/flyte/master/deployment/sandbox/flyte_generated.yaml #. You can now port-forward (or if you have load-balancer enabled then get an LB) to connect to remote FlyteConsole, as follows:: @@ -70,4 +154,4 @@ Deploy Flyte Sandbox environment to a shared kubernetes cluster The goal here is to deploy to an existing Kubernetes cluster - within one namespace only. This would allow multiple Flyte clusters to run within one K8s cluster. -.. caution:: coming soon! \ No newline at end of file +.. NOTE:: coming soon! diff --git a/rsts/howto/secrets.rst b/rsts/howto/secrets.rst new file mode 100644 index 0000000000..939c00f5d6 --- /dev/null +++ b/rsts/howto/secrets.rst @@ -0,0 +1,62 @@ +.. _howto-secrets: + +################################ +How to Inject Secrets Into Tasks +################################ + + +************************** +What Is Secrets Injection? +************************** + +Flyte supports running a wide variety of tasks; from containers to sql queries and service calls. In order for flyte-run +containers to request and access secrets, flyte now natively supports a Secret construct. + +For a simple task that launches a Pod, the flow will look something like this: + +.. image:: https://mermaid.ink/img/eyJjb2RlIjoic2VxdWVuY2VEaWFncmFtXG4gICAgUHJvcGVsbGVyLT4-K1BsdWdpbnM6IENyZWF0ZSBLOHMgUmVzb3VyY2VcbiAgICBQbHVnaW5zLT4-LVByb3BlbGxlcjogUmVzb3VyY2UgT2JqZWN0XG4gICAgUHJvcGVsbGVyLT4-K1Byb3BlbGxlcjogU2V0IExhYmVscyAmIEFubm90YXRpb25zXG4gICAgUHJvcGVsbGVyLT4-K0FwaVNlcnZlcjogQ3JlYXRlIE9iamVjdCAoZS5nLiBQb2QpXG4gICAgQXBpU2VydmVyLT4-K1BvZCBXZWJob29rOiAvbXV0YXRlXG4gICAgUG9kIFdlYmhvb2stPj4rUG9kIFdlYmhvb2s6IExvb2t1cCBnbG9iYWxzXG4gICAgUG9kIFdlYmhvb2stPj4rUG9kIFdlYmhvb2s6IEluamVjdCBTZWNyZXQgQW5ub3RhdGlvbnMgKGUuZy4gSzhzLCBWYXVsdC4uLiBldGMuKVxuICAgIFBvZCBXZWJob29rLT4-LUFwaVNlcnZlcjogTXV0YXRlZCBQb2RcbiAgICBcbiAgICAgICAgICAgICIsIm1lcm1haWQiOnt9LCJ1cGRhdGVFZGl0b3IiOmZhbHNlfQ + :target: https://mermaid.ink/img/eyJjb2RlIjoic2VxdWVuY2VEaWFncmFtXG4gICAgUHJvcGVsbGVyLT4-K1BsdWdpbnM6IENyZWF0ZSBLOHMgUmVzb3VyY2VcbiAgICBQbHVnaW5zLT4-LVByb3BlbGxlcjogUmVzb3VyY2UgT2JqZWN0XG4gICAgUHJvcGVsbGVyLT4-K1Byb3BlbGxlcjogU2V0IExhYmVscyAmIEFubm90YXRpb25zXG4gICAgUHJvcGVsbGVyLT4-K0FwaVNlcnZlcjogQ3JlYXRlIE9iamVjdCAoZS5nLiBQb2QpXG4gICAgQXBpU2VydmVyLT4-K1BvZCBXZWJob29rOiAvbXV0YXRlXG4gICAgUG9kIFdlYmhvb2stPj4rUG9kIFdlYmhvb2s6IExvb2t1cCBnbG9iYWxzXG4gICAgUG9kIFdlYmhvb2stPj4rUG9kIFdlYmhvb2s6IEluamVjdCBTZWNyZXQgQW5ub3RhdGlvbnMgKGUuZy4gSzhzLCBWYXVsdC4uLiBldGMuKVxuICAgIFBvZCBXZWJob29rLT4-LUFwaVNlcnZlcjogTXV0YXRlZCBQb2RcbiAgICBcbiAgICAgICAgICAgICIsIm1lcm1haWQiOnt9LCJ1cGRhdGVFZGl0b3IiOmZhbHNlfQ + +Where: + +1. Flyte invokes a plugin to create the K8s object. This can be a Pod or a more complex CRD (e.g. Spark, PyTorch, etc.) + + .. tip:: The plugin will ensure that labels and annotations are passed through to any Pod that will be spawned due to the creation of the CRD. + +3. Flyte will apply labels and annotations that are referenced to all secrets the task is requesting access to. +4. Flyte will send a POST request to ApiServer to create the object. +5. Before persisting the Pod, ApiServer will invoke all registered Pod Webhooks. Flyte's Pod Webhook will be called. +6. Flyte Pod Webhook will then lookup globally mounted secrets for each of the requested secrets. +7. If found, Pod Webhook will mount them directly in the Pod. If not found, it will inject the appropriate annotations to load the secrets for K8s (or Vault or Confidant or any other secret management system plugin configured) into the Pod. + +****************************** +How to Enable Secret Injection +****************************** + +This feature is available in Flytekit v0.17.0+. Here is an example of an annotated task: + +The webhook is included in all overlays in this repo. The deployment file creates (mainly) two things; a Job and a Deployment. + +1) flyte-pod-webhook-secrets Job: This job runs ``flytepropeller webhook init-certs`` command that issues self-signed + CA Certificate as well as a derived TLS certificate and its private key. It stores them into a new secret ``flyte-pod-webhook-secret``. +2) flyte-pod-webhook Deployment: This deployment creates the Webhook pod which creates a MutatingWebhookConfiguration + on startup. This serves as the registration contract with the ApiServer to know about the Webhook before it starts serving + traffic. + +******************* +Scaling the Webhook +******************* + +Vertical Scaling +================= + +To scale the Webhook to be able to process the number/rate of pods you need, you may need to configure a vertical `pod +autoscaler `_. + +Horizontal Scaling +================== + +The Webhook does not make any external API Requests in response to Pod mutation requests. It should be able to handle traffic +quickly, but a benchmark is needed. For horizontal scaling, adding additional replicas for the Pod in the +deployment should be sufficient. A single MutatingWebhookConfiguration object will be used, the same TLS certificate +will be shared across the pods and the Service created will automatically load balance traffic across the available pods. diff --git a/rsts/howto/serviceaccount.rst b/rsts/howto/serviceaccount.rst index 0c7ebad4de..bf720e9f0c 100644 --- a/rsts/howto/serviceaccount.rst +++ b/rsts/howto/serviceaccount.rst @@ -1,4 +1,4 @@ -. _howto_serviceaccounts: +.. _howto-serviceaccounts: ###################################################################### How do I use Kubernetes ServiceAccounts to access my cloud resources? diff --git a/rsts/images/flyte-and-lf.png b/rsts/images/flyte-and-lf.png new file mode 100644 index 0000000000..47c8958805 Binary files /dev/null and b/rsts/images/flyte-and-lf.png differ diff --git a/rsts/index.rst b/rsts/index.rst index 55a19ff5ea..b3b0f6af9e 100644 --- a/rsts/index.rst +++ b/rsts/index.rst @@ -1,45 +1,125 @@ -Flyte -===== -.. image:: images/flyte_lockup_gradient_on_light.png - -Flyte is a structured programming and distributed processing platform created at Lyft that enables highly concurrent, scalable and maintainable workflows for machine learning and data processing. -Welcome to the documentation hub for Flyte. .. toctree:: :maxdepth: 1 - :caption: Get started with Flyte - :name: tutorialstoc + :name: mainsections + :titlesonly: + :hidden: + + getting_started + User Guide + Tutorials + Concepts + API Reference + community/index + +.. toctree:: + :caption: Concepts + :maxdepth: -1 + :name: divedeeptoc + :hidden: + + concepts/basics + concepts/core + concepts/control_plane + concepts/execution_time + +.. toctree:: + :caption: Community + :maxdepth: -1 + :name: roadmaptoc + :hidden: - tutorials/first_run - tutorials/first_example - tutorials/run_on_flyte - tutorials/learn + Join the Community + community/contribute + community/roadmap .. toctree:: - :caption: How to Guides + :caption: API Reference + :maxdepth: -1 + :name: apireference + :hidden: + + References + +.. toctree:: + :caption: How-Tos :maxdepth: 1 :name: howtotoc + :hidden: plugins/index howto/index - faq/index + +Meet Flyte +========== -.. toctree:: - :caption: Diving Deep: Architecture, Concepts & References - :maxdepth: 1 - :name: divedeeptoc +.. raw:: html - dive_deep/index - reference/index +

The workflow automation platform for complex, mission-critical data and ML processes at scale

-.. toctree:: - :caption: Project Plan & Contributor Guide - :maxdepth: 1 - :name: roadmaptoc - community/index - community/docs - community/roadmap - community/compare +Flyte is an open-source, container-native, structured programming and distributed processing platform. It enables highly concurrent, scalable and maintainable workflows for machine learning and data processing. + +Created at `Lyft `__ in collaboration with Spotify, Freenome and many others, Flyte provides first class support for Python, Java, and Scala, and is built directly on Kubernetes for all the benefits containerization provides: portability, scalability, and reliability. + +The core unit of execution in Flyte is the ``task``, which you can easily write with the Flytekit Python SDK: + +.. code:: python + + @task + def greet(name: str) -> str: + return f"Welcome, {name}!" + +You can compose one or more tasks to create a ``workflow``: + +.. code:: python + + @task + def add_question(greeting: str) -> str: + return f"{greeting} How are you?" + + @workflow + def welcome(name: str) -> str: + greeting = greet(name=name) + return add_question(greeting=greeting) + + welcome("Traveler") + # Output: "Welcome, Traveler! How are you?" + + +Why Flyte? +========== + +Flyte's main purpose is to increase the development velocity for data processing and machine learning, enabling large-scale compute execution without the operational overhead. Teams can therefore focus on the business goals and not the infrastructure. + +Core Features +------------- + +* Container Native +* Extensible Backend & SDK’s +* Ergonomic SDK’s in Python, Java & Scala +* Versioned & Auditable - all actions are recorded +* Matches your workflow - Start with one task, convert to a pipeline, attach multiple schedules or trigger using a programmatic API or on-demand +* Battle-tested - millions of pipelines executed per month +* Vertically-Integrated Compute - serverless experience +* Deep understanding of data-lineage & provenance +* Operation Visibility - cost, performance, etc. +* Cross-Cloud Portable Pipelines + +Who's Using Flyte? +------------------ + +At `Lyft `__, Flyte has served production model training and data processing for over four years, becoming the de-facto platform for the Pricing, Locations, ETA, Mapping teams, Growth, Autonomous and other teams. + +For the most current list of Flyte's deployments, please click `here `_. + +Next Steps +---------- + +Whether you want to write Flyte workflows, deploy the Flyte platform to your k8 cluster, or extend and contribute to the architecture and design of Flyte, we have what you need. +* :ref:`Get Started ` +* :ref:`Main Concepts ` +* :ref:`Extend Flyte ` +* :ref:`Join the Community ` diff --git a/rsts/plugins/aws/athena.rst b/rsts/plugins/aws/athena.rst deleted file mode 100644 index 541b6313e6..0000000000 --- a/rsts/plugins/aws/athena.rst +++ /dev/null @@ -1,5 +0,0 @@ -.. _plugins-aws-athena: - -############### -AWS Athena -############### \ No newline at end of file diff --git a/rsts/plugins/aws/index.rst b/rsts/plugins/aws/index.rst deleted file mode 100644 index d9197ae8de..0000000000 --- a/rsts/plugins/aws/index.rst +++ /dev/null @@ -1,14 +0,0 @@ -.. _plugins-aws: - -############# -AWS Plugins -############# - -.. toctree:: - :maxdepth: 1 - :caption: Available Plugins - :name: pluginsawstoc - - athena - sagemaker - diff --git a/rsts/plugins/aws/sagemaker.rst b/rsts/plugins/aws/sagemaker.rst deleted file mode 100644 index 23ef769ad9..0000000000 --- a/rsts/plugins/aws/sagemaker.rst +++ /dev/null @@ -1,5 +0,0 @@ -.. _plugins-aws-sagemaker: - -############### -AWS Sagemaker -############### \ No newline at end of file diff --git a/rsts/plugins/extend/flyte_backend.rst b/rsts/plugins/extend/flyte_backend.rst deleted file mode 100644 index 680d54698c..0000000000 --- a/rsts/plugins/extend/flyte_backend.rst +++ /dev/null @@ -1,5 +0,0 @@ -.. _extend-plugin-flyte-backend: - -######################################## -Implement Backend Extensions (advanced) -######################################## \ No newline at end of file diff --git a/rsts/plugins/extend/flytekit_python.rst b/rsts/plugins/extend/flytekit_python.rst deleted file mode 100644 index e061d4ccb1..0000000000 --- a/rsts/plugins/extend/flytekit_python.rst +++ /dev/null @@ -1,5 +0,0 @@ -.. _extend-plugin-flytekit-python: - -################################## -Extend flytekit (python) -################################## \ No newline at end of file diff --git a/rsts/plugins/extend/intro.rst b/rsts/plugins/extend/intro.rst deleted file mode 100644 index 7651676b80..0000000000 --- a/rsts/plugins/extend/intro.rst +++ /dev/null @@ -1,126 +0,0 @@ -.. _plugins_extend_intro: - -########################### -When & How to Extend Flyte -########################### - -The Core of Flyte is a container execution engine, where you can write one or more tasks and string them together to form a data dependency DAG - called a ``workflow``. -If your work involves writing simple python or java tasks that can either perform operations on their own or can call out to external services - then there is **NO NEED to extend FLYTE**. - -But, in that case you can almost do everything using python / java or a container - So why should you even have to extend Flyte? - -================= -But First - Why? -================= - -Case 1: I want to use my special Types - e.g. my own DataFrame format -========================================================================== -Flyte, just like a programming language has a core type-system, but just like most languages, this type system can be extended by allowing users to add ``User defined Data types``. -A User defined data type can be something that Flyte does not really understand, but is extremely useful for a users specific needs. For example it can be a custom user structure or a grouping of images in a specific encoding. - -Flytekit natively supports handling of structured data like User defined structures like DataClasses using JSON as the representation format. An example of this is available in FlyteCookbook - :std:doc:`auto_core_intermediate/custom_objects`. - -For types that are not simply representable as JSON documents, Flytekit allows users to extends Flyte's type system and implement these types in Python. The user has to essentially implement a :py:class:`flytekit.extend.TypeTransformer` class to enable translation of the type from Users type to flyte understood types. As an example, -instead of using :py:class:`pandas.DataFrame` directly, you may want to use `Pandera `_ to perform validation of an input or output dataframe. an example can be found `here `_. - -To extend the type system in flytekit refer to an illustrative example found at - :std:ref:`advanced_custom_types`. - - -Case 2: Add a new Task Type - Flyte capability -=============================================== -So often times you want to interact with a service like, - - - a Database (Postgres, MySQL, etc) - - a DataWarehouse like (Snowflake, BigQuery, Redshift etc) - - a computation platform like (AWS EMR, Databricks etc) - -and you want this to be available like a template for all other users - open source or within your organization. This can be done by creating a task plugin. -A Task-plugin makes it possible for you or other users to use your idea natively within Flyte as this capability was built into the flyte platform. - -Thus for example, if you want users to write code simply using the ``@task`` decorator, but you want to provide a capability of running the function as a spark job or a sagemaker training job - then you can extend Flyte's task system - we will refer to this as the plugin and it could be possible to do the following - -.. code-block:: python - - @task(task_config=MyContainerExecutionTask( - plugin_specific_config_a=..., - plugin_specific_config_b=..., - ... - )) - def foo(...) -> ...: - ... - - -OR provide an interface like this - -.. code-block:: python - - query_task = SnowflakeQuery(query="Select * from x where x.time < {{.inputs.time}}", inputs=(time=datetime), results=pandas.DataFrame) - - @workflow - def my_wf(t: datetime) -> ...: - df = query_task(time=t) - return process(df=df) - - - -=========================================================== -I want to write a Task Plugin or add a new TaskType -=========================================================== - -Interestingly there are 2 options here. You can write a task plugin simply as an extension in flytekit, or you can go deeper and write a Plugin in the Flyte backend itself. - -Flytekit only plugin -====================== -An illustrative example of writing a flytekit plugin can be found at - :std:ref:`advanced_custom_task_plugin`. Flytekit plugins are simple to write and should invariably be -the first place you start at. Here - -**Pros** - -#. Simple to write, just implement in python. Flyte will treat it like a container execution and blindly pass control to the plugin -#. Simple to publish - flytekitplugins can be published as independent libraries and they follow a simple api. -#. Simple to perform testing - just test locally in flytekit - -**Cons** - -#. Limited ways of providing additional visibility in progress, or external links etc -#. Has to be implemented again in every language as these are SDK side plugins only -#. In case of side-effects, potentially of causing resource leaks. For example if the plugins runs a BigQuery Job, it is possible that the plugin may crash after running the Job and Flyte cannot guarantee that the BigQuery job wil be successfully terminated. -#. Potentially expensive - In cases where the plugin just runs a remote job - e.g how Airflow does, then running a new pod for every task execution causes severe strain on k8s and the task itself uses almost no CPUs. Also because of stateful natute, using spot-instances is not trivial. -#. A bug fix to the runtime, needs a new library version of the plugin -#. Not trivial to implement resource controls - e.g. throttling, resource pooling etc - -Backend Plugin -=============== - -Doc on how to writed a backend plugins is coming soon. A backend plugin essentially makes it possible for users to write extensions for FlytePropeller (Flytes scheduling engine). This enables complete control on the visualization and availability of the plugin. - -**Pros** - -#. Service oriented way of deploying new plugins - strong contracts. Maintainers can deploy new versions of the backend plugin, fix bugs, without needing the users to upgrade Libraries etc -#. Drastically cheaper and more efficient to execute. FlytePropeller is written in Golang and uses an event loop model. Each process of FlytePropeller can execute 1000's of tasks concurrently. -#. Flyte will guarantee resource cleanup -#. Flyteconsole plugins (capability coming soon) can be added to customize visualization and progress tracking of the execution -#. Resource controls and backpressure management is available -#. Implement once, use in any SDK or language - -**Cons** - -#. Need to be implemented in golang -#. Needs a FlytePropeller build - *currently* -#. Need to implement contract in some spec language like protobf, openAPI etc -#. Development cycle can be much slower than flytekit only plugins - - -=============================================== -How do I decide which path to take? -=============================================== - -.. image:: https://raw.githubusercontent.com/flyteorg/flyte/static-resources/img/core/extend_flyte_flowchart.png - :alt: Ok you want to add a plugin, but which type? Follow the flowchart and then select the right next steps. - - -Use the conclusion of the flow-chart to refer to the right doc -================================================================ - -- :ref:`extend-plugin-flytekit-python` -- :ref:`extend-plugin-flyte-backend` diff --git a/rsts/plugins/hive.rst b/rsts/plugins/hive.rst deleted file mode 100644 index 1d3d225793..0000000000 --- a/rsts/plugins/hive.rst +++ /dev/null @@ -1,5 +0,0 @@ -.. _plugins-hive: - -########## -BYOC Hive -########## \ No newline at end of file diff --git a/rsts/plugins/index.rst b/rsts/plugins/index.rst index 8a718e0258..d502c7a060 100644 --- a/rsts/plugins/index.rst +++ b/rsts/plugins/index.rst @@ -1,40 +1,12 @@ .. _plugins: -######################################## -Extend Flyte and Available Extenstions -######################################## - -.. _plugins_howto: - -==================== -How to extend Flyte -==================== -Flyte as platform was designed with extensibility as a core primitive. Flyte is essentially an integration framework and hence extensibility is possible through-out the system. -The following sections will guide you through writing your own extensions - either private or public (contribute back to the community). - -.. toctree:: - :maxdepth: 1 - :name: howtoextendtoc - - extend/intro - extend/flytekit_python - extend/flyte_backend - - ==================== Available Extensions ==================== -Following is a list of maintained plugins for Flyte and guides on how to install / use them. +The following is a list of maintained plugins for Flyte and guides on how to install / use them. .. toctree:: :maxdepth: 1 :name: pluginstoc - spark_k8s - pod - sqlite3 - pandera - papermill - hive - aws/index - kubeflow/index + spark_k8s \ No newline at end of file diff --git a/rsts/plugins/kubeflow/index.rst b/rsts/plugins/kubeflow/index.rst deleted file mode 100644 index c770acda03..0000000000 --- a/rsts/plugins/kubeflow/index.rst +++ /dev/null @@ -1,13 +0,0 @@ -.. _plugins-kubeflow: - -################ -Kubeflow Plugins -################ - -.. toctree:: - :maxdepth: 1 - :caption: Available Plugins - :name: pluginskftoc - - tensorflow_operator - pytorch_operator \ No newline at end of file diff --git a/rsts/plugins/kubeflow/pytorch_operator.rst b/rsts/plugins/kubeflow/pytorch_operator.rst deleted file mode 100644 index b2b3ba5f4f..0000000000 --- a/rsts/plugins/kubeflow/pytorch_operator.rst +++ /dev/null @@ -1,5 +0,0 @@ -.. _plugins-kf-pytorch-operator: - -################# -Pytorch Operator -################# \ No newline at end of file diff --git a/rsts/plugins/kubeflow/tensorflow_operator.rst b/rsts/plugins/kubeflow/tensorflow_operator.rst deleted file mode 100644 index 59db34e2fa..0000000000 --- a/rsts/plugins/kubeflow/tensorflow_operator.rst +++ /dev/null @@ -1,5 +0,0 @@ -.. _plugins-kf-tf-operator: - -############### -TF Operator -############### \ No newline at end of file diff --git a/rsts/plugins/pandera.rst b/rsts/plugins/pandera.rst deleted file mode 100644 index 687d803e27..0000000000 --- a/rsts/plugins/pandera.rst +++ /dev/null @@ -1,5 +0,0 @@ -.. _plugins-pandera: - -################################################## -Use Pandera to Enforce Type safety in DataFrames -################################################## \ No newline at end of file diff --git a/rsts/plugins/papermill.rst b/rsts/plugins/papermill.rst deleted file mode 100644 index 364490a686..0000000000 --- a/rsts/plugins/papermill.rst +++ /dev/null @@ -1,5 +0,0 @@ -.. _plugins-papermill: - -################################## -Use Papermill notebooks as tasks -################################## \ No newline at end of file diff --git a/rsts/plugins/pod.rst b/rsts/plugins/pod.rst deleted file mode 100644 index 9d03e56cb4..0000000000 --- a/rsts/plugins/pod.rst +++ /dev/null @@ -1,5 +0,0 @@ -.. _plugins-pod: - -######## -K8s Pods -######## \ No newline at end of file diff --git a/rsts/plugins/spark_k8s.rst b/rsts/plugins/spark_k8s.rst index 484650cbbe..772da4e598 100644 --- a/rsts/plugins/spark_k8s.rst +++ b/rsts/plugins/spark_k8s.rst @@ -11,7 +11,7 @@ Flyte has an optional plugin that makes it possible to run `Apache Spark `_ and a custom built `Flyte Spark Plugin `_. +Flyte Spark uses the `Spark On K8s Operator `_ and a custom built `Flyte Spark Plugin `_. The plugin is a backend plugin and you have to enable it in your deployment. To enable a plugin follow the steps in :ref:`howto-enable-backend-plugins`. -You can optionally configure the Plugin as per the - `backend Config Structure `_ and an example Config is defined +You can optionally configure the Plugin as per the - `backend Config Structure `_ and an example Config is defined `here `_, which looks like, .. rli:: https://raw.githubusercontent.com/flyteorg/flyte/master/kustomize/overlays/sandbox/config/propeller/plugins/spark.yaml diff --git a/rsts/plugins/sqlite3.rst b/rsts/plugins/sqlite3.rst deleted file mode 100644 index 91f778580b..0000000000 --- a/rsts/plugins/sqlite3.rst +++ /dev/null @@ -1,5 +0,0 @@ -.. _plugins-sqlite3: - -################################################## -Query SQLite3 databases -> DataFrames -################################################## \ No newline at end of file diff --git a/rsts/reference/index.rst b/rsts/reference/index.rst index 32281a70c4..c27aeaa579 100644 --- a/rsts/reference/index.rst +++ b/rsts/reference/index.rst @@ -1,8 +1,8 @@ .. _reference: -################################# -API, CLI and Component Reference -################################# +############# +API Reference +############# .. toctree:: @@ -10,10 +10,10 @@ API, CLI and Component Reference :caption: API Reference :name: apitoc - Flyte Core Language Specification (FlyteIDL) - Flytekit Python - Flytekit JAVA - Flytectl CLI reference + Flytekit Python + Flytekit Java + FlyteIDL + Flytectl .. toctree:: @@ -25,4 +25,3 @@ API, CLI and Component Reference FlyteAdmin FlytePlugins DataCatalog - Flytectl \ No newline at end of file diff --git a/rsts/tutorials/first_example.rst b/rsts/tutorials/first_example.rst deleted file mode 100644 index aa94e15713..0000000000 --- a/rsts/tutorials/first_example.rst +++ /dev/null @@ -1,53 +0,0 @@ -.. _tutorials-getting-started-first-example: - -###################################### -Write your first Flyte workflow -###################################### - -By the end of this getting started guide you'll be familiar with how easy it is to author a Flyte workflow and run it locally. - -.. rubric:: Estimated time to complete: <3 minutes. - - -Prerequisites -************* - -Ensure that you have `git `__ installed. - -First install the python Flytekit SDK and clone the ``flytekit-python-template`` repo :: - - pip install flytekit==0.16.0b6 - git clone git@github.com:flyteorg/flytekit-python-template.git flyteexamples - cd flyteexamples - rm -rf .git - git init - - -Flyte Tasks and Workflows -************************* - -Let's take a look at the example workflow found in `myapp/workflows/example.py `__ - -.. rli:: https://raw.githubusercontent.com/flyteorg/flytekit-python-template/main/myapp/workflows/example.py - :language: python - -The most basic Flyte primitive is a :std:doc:`task `. -Flyte tasks are units of work that can be composed in a :std:doc:`workflow `. - -You can call this task - -.. code-block:: python - - greet(name="world") - -and iterate locally before adding it to part of a larger overall workflow. - -Similarly, you can call this workflow - -.. code-block:: python - - hello_world(name=...) - -and iterate locally before moving on to register it with Flyte. - -.. tip:: Every invocation of a Flyte workflow requires specifying keyword args. diff --git a/rsts/tutorials/first_run.rst b/rsts/tutorials/first_run.rst deleted file mode 100644 index 3329311138..0000000000 --- a/rsts/tutorials/first_run.rst +++ /dev/null @@ -1,21 +0,0 @@ -.. _flyte-tutorials-firstrun: -.. currentmodule:: firstrun - -############################################ -Getting Started with Flyte -############################################ - -.. rubric:: Estimated time to complete: 2 minutes. - -Flyte enables scalable, reproducable and reliable orchestration of massively large workflows. In order to get a sense of the product, we have packaged a minimalist version of the Flyte system into a Docker image. - -With `docker installed `__, run this command: :: - - docker run --rm --privileged -p 30081:30081 ghcr.io/flyteorg/flyte-sandbox - -This creates a local Flyte sandbox. Once the sandbox is ready, you should see the following message: `Flyte is ready! Flyte UI is available at http://localhost:30081/console.`. Go ahead and visit that to check out the Flyte UI. - -A quick visual tour for launching your first Workflow: - -.. image:: https://github.com/flyteorg/flyte/raw/static-resources/img/first-run-console-2.gif - :alt: A quick visual tour for launching your first Workflow. diff --git a/rsts/tutorials/learn.rst b/rsts/tutorials/learn.rst deleted file mode 100644 index 1a87c3adff..0000000000 --- a/rsts/tutorials/learn.rst +++ /dev/null @@ -1,10 +0,0 @@ -.. _tutorials-learn: - -########################### -Learn Flytekit by Example -########################### - -Flytekit implements a programming model for Flyte in (python and Java). The best way to see the capabilities of Flyte is to try out Flytekit. - -`Learn by example - Flytekit Python `_ -========================================================================================= diff --git a/rsts/tutorials/run_on_flyte.rst b/rsts/tutorials/run_on_flyte.rst deleted file mode 100644 index 075a2d1e86..0000000000 --- a/rsts/tutorials/run_on_flyte.rst +++ /dev/null @@ -1,135 +0,0 @@ -.. _tutorials-getting-started-run-on-flyte: - -###################################### -Run Your Workflow on a Flyte Backend -###################################### - -************************ -Installing Flyte Locally -************************ - -This guide will walk you through a quick installation of Flyte on your laptop and then how to register and execute your -workflows against this deployment. (The tabs below have an option to install Flyte on a cloud provider as well) - -.. rubric:: Estimated time to complete: 5 minutes. - -Prerequisites -============= - -#. Ensure ``kubectl`` is installed. Follow `kubectl installation docs `_. On Mac:: - - brew install kubectl - -#. If running locally ensure you have docker installed - as explained `here `_ -#. If you prefer to run the Flyte test cluster on a cloud environment like `AWS EKS `_, `Google GKE `__, then follow the instructions in a tab below. The rest of it is mostly the same. - -Steps -====== - -.. tabs:: - - .. tab:: Using k3d - - #. Install k3d Using ``curl``:: - - curl -s https://raw.githubusercontent.com/rancher/k3d/main/install.sh | bash - - Or Using ``wget`` :: - - wget -q -O - https://raw.githubusercontent.com/rancher/k3d/main/install.sh | bash - - #. Start a new K3s cluster called flyte:: - - k3d cluster create -p "30081:30081" --no-lb --k3s-server-arg '--no-deploy=traefik' --k3s-server-arg '--no-deploy=servicelb' flyte - - #. Ensure the context is set to the new cluster:: - - kubectl config set-context flyte - - #. Install Flyte:: - - kubectl create -f https://raw.githubusercontent.com/flyteorg/flyte/master/deployment/sandbox/flyte_generated.yaml - - - #. Connect to `FlyteConsole `__ - #. [Optional] You can delete the cluster once you are done with the tutorial using - :: - - k3d cluster delete flyte - - - .. note:: - - #. Sometimes Flyteconsole will not open up. This is probably because your docker networking is impacted. One solution is to restart docker and re-do the previous steps. - #. To debug you can try a simple excercise - run nginx as follows:: - - docker run -it --rm -p 8083:80 nginx - - Now connect to `locahost:8083 `__. If this does not work, then for sure the networking is impacted, please restart docker daemon. - - .. tab:: Using Docker for Mac - - Coming soon - - .. tab:: Install Flyte in EKS/GKE etc - - Refer to the :ref:`howto-sandbox-dedicated-k8s-cluster` guide. - - Once you've deployed flyte sandbox to a cloud provider, read on below to run your first workflow remotely. - - -.. _tutorials-run-flyte-laptop: - -**************************** -Running your Flyte Workflows -**************************** - -Registration -============ - -Set-up ------- - -In a separate process, port-forward the flyte minio service. If you're using a locally deployed Flyte sandbox, simply: :: - - kubectl -n flyte port-forward service/minio 9000 - - -Register your workflows ------------------------ - -From within root directory of ``flyteexamples`` you created :ref:`previously ` -feel free to make any changes and then register :: - - FLYTE_AWS_ENDPOINT=http://localhost:9000 FLYTE_AWS_ACCESS_KEY_ID=minio \ - FLYTE_AWS_SECRET_ACCESS_KEY=miniostorage make fast_register - -If you're port-forwarding minio somewhere else, substitute the ``FLYTE_AWS_ENDPOINT`` accordingly. - -.. rubric:: Boom! It's that simple. - -Run your workflows ------------------- - -Visit the page housing workflows registered for your project (method if you used k3d): -`http://localhost:30081/console/projects/flyteexamples/workflows `__ -else if you used docker-desktop or something else, then copy paste this URL into the browser and fill in the ````:: - - http:///console/projects/flyteexamples/workflows - - -Select your workflow, click the bright purple "Launch Workflow" button in the upper right, update the "name" input -argument as you please, proceed to launch and you'll have triggered an execution! - -.. note:: - - After registration Flyte Workflows exist in the FlyteAdmin service and can be triggered using the - - console - - Command line - - directly invoking the REST API - - on a schedule - - More on this later - -Optionally you can create a new project ----------------------------------------- -Refer to :ref:`howto_new_project`. \ No newline at end of file diff --git a/script/prepare_artifacts.sh b/script/prepare_artifacts.sh new file mode 100644 index 0000000000..e2d60e7f8d --- /dev/null +++ b/script/prepare_artifacts.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -ex + +for file in ./deployment/**/flyte_generated.yaml; do + if [ -f "$file" ]; then + result=${file/#"./deployment/"} + result=${result/%"/flyte_generated.yaml"} + cp $file "./release/flyte_${result}_manifest.yaml" + fi +done diff --git a/script/release.sh b/script/release.sh new file mode 100755 index 0000000000..b605cd9b0d --- /dev/null +++ b/script/release.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +set -ex + +FLYTEADMIN_TAG=$(curl --silent "https://api.github.com/repos/flyteorg/flyteadmin/releases/latest" | jq -r .tag_name) +DATACATALOG_TAG=$(curl --silent "https://api.github.com/repos/flyteorg/datacatalog/releases/latest" | jq -r .tag_name) +FLYTECONSOLE_TAG=$(curl --silent "https://api.github.com/repos/flyteorg/flyteconsole/releases/latest" | jq -r .tag_name) +FLYTEPROPELLER_TAG=$(curl --silent "https://api.github.com/repos/flyteorg/flytepropeller/releases/latest" | jq -r .tag_name) + +grep -rlZ "newTag:[^P]*# FLYTEADMIN_TAG" ./kustomize/overlays | xargs -0 sed -i "s/newTag:[^P]*# FLYTEADMIN_TAG/newTag: ${FLYTEADMIN_TAG} # FLYTEADMIN_TAG/g" +grep -rlZ "newTag:[^P]*# DATACATALOG_TAG" ./kustomize/overlays | xargs -0 sed -i "s/newTag:[^P]*# DATACATALOG_TAG/newTag: ${DATACATALOG_TAG} # DATACATALOG_TAG/g" +grep -rlZ "newTag:[^P]*# FLYTECONSOLE_TAG" ./kustomize/overlays | xargs -0 sed -i "s/newTag:[^P]*# FLYTECONSOLE_TAG/newTag: ${FLYTECONSOLE_TAG} # FLYTECONSOLE_TAG/g" +grep -rlZ "newTag:[^P]*# FLYTEPROPELLER_TAG" ./kustomize/overlays | xargs -0 sed -i "s/newTag:[^P]*# FLYTEPROPELLER_TAG/newTag: ${FLYTEPROPELLER_TAG} # FLYTEPROPELLER_TAG/g" diff --git a/stats/README.md b/stats/README.md new file mode 100644 index 0000000000..2ecea73bd5 --- /dev/null +++ b/stats/README.md @@ -0,0 +1,19 @@ +## Developing stats + +These scripts generate dashboards by running the following make target in the +flyteorg/flyte root folder. + +```bash +make stats +``` + +Ensure that you have installed the requirements.txt + +```bash +pip install -r requirements.txt +``` + +Refer to [Grafanalib](https://github.com/weaveworks/grafanalib) to understand +how to write the dashboards. + +Currently the dashboards are manually uploaded to [Grafana marketplace](https://grafana.com/grafana/dashboards?search=flyte) diff --git a/stats/flyteadmin_dashboard.py b/stats/flyteadmin_dashboard.py new file mode 100644 index 0000000000..c8e8aa819d --- /dev/null +++ b/stats/flyteadmin_dashboard.py @@ -0,0 +1,237 @@ +import typing +from grafanalib.core import ( + Alert, AlertCondition, Dashboard, Graph, + GreaterThan, OP_AND, OPS_FORMAT, Row, RTYPE_SUM, SECONDS_FORMAT, + SHORT_FORMAT, single_y_axis, Target, TimeRange, YAxes, YAxis, DataSourceInput, MILLISECONDS_FORMAT +) + +# ------------------------------ +# For Gostats we recommend using +# Grafana dashboard ID 10826 - https://grafana.com/grafana/dashboards/10826 +# + +DATASOURCE_NAME = "DS_PROM" +DATASOURCE = "${%s}" % DATASOURCE_NAME + + +class FlyteAdmin(object): + APIS = [ + "create_execution", + "create_launch_plan", + "create_task", + "create_workflow", + "create_node_execution_event", + "create_task_execution_event", + "get_execution", + "get_launch_plan", + "get_task", + "get_workflow", + "get_node_execution", + "get_task_execution", + "get_active_launch_plan", + "list_execution", + "list_launch_plan", + "list_task", + "list_workflow", + "list_node_execution", + "list_task_execution", + "list_active_launch_plan", + ] + + ENTITIES = [ + "executions", + "task_executions", + "node_executions", + "workflows", + "launch_plans", + "project", + ] + + DB_OPS = [ + "get", + "list", + "create", + "update", + "list", + "list_identifiers", + "delete", + "exists", + ] + + @staticmethod + def error_codes(api: str, interval: int = 1) -> Graph: + return Graph( + title=f"{api} return codes", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(irate(flyte:admin:{api}:codes:OK[{interval}m]))', + legendFormat="ok", + refId='A', + ), + Target( + expr=f'sum(irate(flyte:admin:{api}:codes:InvalidArgument[{interval}m]))', + legendFormat="invalid-args", + refId='B', + ), + Target( + expr=f'sum(irate(flyte:admin:{api}:codes:AlreadyExists[{interval}m]))', + legendFormat="already-exists", + refId='C', + ), + Target( + expr=f'sum(irate(flyte:admin:{api}:codes:FailedPrecondition[{interval}m]))', + legendFormat="failed-precondition", + refId='D', + ), + ], + yAxes=YAxes( + YAxis(format=OPS_FORMAT), + YAxis(format=SHORT_FORMAT), + ), + ) + + @staticmethod + def error_vs_success(api: str, interval: int = 1) -> Graph: + return Graph( + title=f"{api} success vs errors", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(irate(flyte:admin:{api}:errors[{interval}m]))', + legendFormat="errors", + refId='A', + ), + Target( + expr=f'sum(irate(flyte:admin:{api}:success[{interval}m]))', + legendFormat="success", + refId='B', + ), + ], + yAxes=YAxes( + YAxis(format=OPS_FORMAT), + YAxis(format=SHORT_FORMAT), + ), + ) + + @staticmethod + def api_latency(api: str, interval: int = 1) -> Graph: + return Graph( + title=f"{api} Latency", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(flyte:admin:{api}:duration_ms) by (quantile)', + refId='A', + ), + ], + yAxes=single_y_axis(format=MILLISECONDS_FORMAT), + ) + + @staticmethod + def create_api_row(api: str, collapse: bool, interval: int = 1) -> Row: + return Row( + title=f"{api} stats", + collapse=collapse, + panels=[ + FlyteAdmin.error_codes(api, interval), + FlyteAdmin.error_vs_success(api, interval), + FlyteAdmin.api_latency(api, interval), + ] + ) + + @staticmethod + def db_latency(entity: str, op: str, interval: int = 1) -> Graph: + return Graph( + title=f"{op} Latency", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(flyte:admin:database:postgres:repositories:{entity}:{op}_ms) by (quantile)', + refId='A', + ), + ], + yAxes=single_y_axis(format=MILLISECONDS_FORMAT), + ) + + @staticmethod + def create_entity_db_row_latency(entity: str, collapse: bool, interval: int = 1) -> Row: + r = Row( + title=f"DB {entity} ops stats", + collapse=collapse, + panels=[], + ) + for op in FlyteAdmin.DB_OPS: + r.panels.append(FlyteAdmin.db_latency(entity, op=op, interval=interval)) + return r + + @staticmethod + def db_count(entity: str, op: str, interval: int = 1) -> Graph: + return Graph( + title=f"{op} Count Ops", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(rate(flyte:admin:database:postgres:repositories:{entity}:{op}_ms_count[{interval}m]))', + refId='A', + ), + ], + yAxes=YAxes( + YAxis(format=OPS_FORMAT), + YAxis(format=SHORT_FORMAT), + ), + ) + + @staticmethod + def create_entity_db_count(entity: str, collapse: bool, interval: int = 1) -> Row: + r = Row( + title=f"DB {entity} ops stats", + collapse=collapse, + panels=[], + ) + for op in FlyteAdmin.DB_OPS: + r.panels.append(FlyteAdmin.db_count(entity, op=op, interval=interval)) + return r + + @staticmethod + def create_all_entity_db_rows(collapse: bool, interval: int = 1) -> typing.List[Row]: + rows = [] + for entity in FlyteAdmin.ENTITIES: + rows.append(FlyteAdmin.create_entity_db_row_latency(entity=entity, collapse=collapse, interval=interval)) + rows.append(FlyteAdmin.create_entity_db_count(entity=entity, collapse=collapse, interval=interval)) + return rows + + @staticmethod + def create_all_apis(interval: int = 5) -> typing.List[Row]: + rows = [] + for api in FlyteAdmin.APIS: + rows.append(FlyteAdmin.create_api_row(api, collapse=True, interval=interval)) + return rows + + @staticmethod + def create_all_rows(interval: int = 5) -> typing.List[Row]: + rows = [] + rows.extend(FlyteAdmin.create_all_entity_db_rows(collapse=True, interval=interval)) + rows.extend(FlyteAdmin.create_all_apis(interval)) + return rows + + +dashboard = Dashboard( + tags=["flyte", "prometheus", "flyteadmin", "flyte-controlplane"], + inputs=[ + DataSourceInput( + name=DATASOURCE_NAME, + label="Prometheus", + description="Prometheus server that connects to Flyte", + pluginId="prometheus", + pluginName="Prometheus", + ), + ], + editable=False, + title="Flyte Admin Dashboard (via Prometheus)", + rows=FlyteAdmin.create_all_rows(), + description="Flyte Admin/Control Plane Dashboard. This is great for monitoring FlyteAdmin and the Service API.", +).auto_panel_ids() + +if __name__ == "__main__": + print(dashboard.to_json_data()) diff --git a/stats/flytepropeller_dashboard.py b/stats/flytepropeller_dashboard.py new file mode 100644 index 0000000000..bc306fee94 --- /dev/null +++ b/stats/flytepropeller_dashboard.py @@ -0,0 +1,732 @@ +import typing + +from grafanalib.core import ( + Dashboard, Graph, Gauge, Stat, + OPS_FORMAT, Row, SHORT_FORMAT, single_y_axis, Target, YAxes, YAxis, MILLISECONDS_FORMAT, DataSourceInput, + PERCENT_FORMAT, NO_FORMAT, SECONDS_FORMAT +) + +# ------------------------------ +# For Gostats we recommend using +# Grafana dashboard ID 10826 - https://grafana.com/grafana/dashboards/10826 +# + +DATASOURCE_NAME = "DS_PROM" +DATASOURCE = "${%s}" % DATASOURCE_NAME + + +class FlytePropeller(object): + + @staticmethod + def create_free_workers() -> Graph: + return Graph( + title="Free workers count", + dataSource=DATASOURCE, + targets=[ + Target( + expr='sum(flyte:propeller:all:free_workers_count) by (kubernetes_pod_name)', + refId='A', + ), + ], + yAxes=YAxes( + YAxis(format=OPS_FORMAT), + YAxis(format=SHORT_FORMAT), + ), + ) + + @staticmethod + def round_latency_per_wf(interval: int = 1) -> Graph: + return Graph( + title=f"round Latency per workflow", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(rate(flyte:propeller:all:round:raw_ms[{interval}m])) by (wf)', + refId='A', + ), + ], + yAxes=single_y_axis(format=MILLISECONDS_FORMAT), + ) + + @staticmethod + def round_latency(interval: int = 1) -> Graph: + return Graph( + title=f"round Latency by quantile", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(rate(flyte:propeller:all:round:raw_unlabeled_ms[{interval}m])) by (quantile)', + refId='A', + ), + ], + yAxes=single_y_axis(format=MILLISECONDS_FORMAT), + ) + + @staticmethod + def round_panic() -> Graph: + return Graph( + title="Round panic", + dataSource=DATASOURCE, + targets=[ + Target( + expr='sum(rate(flyte:propeller:all:round:panic_unlabeled[5m]))', + refId='A', + ), + ], + yAxes=YAxes( + YAxis(format=OPS_FORMAT), + YAxis(format=SHORT_FORMAT), + ), + ) + + @staticmethod + def streak_length() -> Graph: + return Graph( + title="Avg streak length", + dataSource=DATASOURCE, + targets=[ + Target( + expr='avg(flyte:propeller:all:round:streak_length_unlabeled)', + refId='A', + ), + ], + yAxes=YAxes( + YAxis(format=OPS_FORMAT), + YAxis(format=SHORT_FORMAT), + ), + ) + + @staticmethod + def system_errors() -> Graph: + return Graph( + title="System errors", + dataSource=DATASOURCE, + targets=[ + Target( + expr='sum(deriv(flyte:propeller:all:round:system_error_unlabeled[5m]))', + refId='A', + ), + ], + yAxes=YAxes( + YAxis(format=OPS_FORMAT), + YAxis(format=SHORT_FORMAT), + ), + ) + + @staticmethod + def abort_errors() -> Graph: + return Graph( + title="System errors", + dataSource=DATASOURCE, + targets=[ + Target( + expr='sum(rate(flyte:propeller:all:round:abort_error[5m]))', + refId='A', + ), + ], + yAxes=YAxes( + YAxis(format=OPS_FORMAT), + YAxis(format=SHORT_FORMAT), + ), + ) + + @staticmethod + def workflows_per_project() -> Graph: + return Graph( + title=f"Running Workflows per project", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(flyte:propeller:all:collector:flyteworkflow) by (project)', + refId='A', + ), + ], + yAxes=YAxes( + YAxis(format=OPS_FORMAT), + YAxis(format=SHORT_FORMAT), + ), + ) + + @staticmethod + def plugin_success_vs_failures() -> Graph: + """ + TODO We need to convert the plugin names to be labels, so that prometheus can perform queries correctly + """ + return Graph( + title=f"Plugin Failures", + dataSource=DATASOURCE, + targets=[ + Target( + expr='{__name__=~"flyte:propeller:all:node:plugin:.*_failure_unlabeled"}', + refId='A', + ), + Target( + expr='{__name__=~"flyte:propeller:all:node:plugin:.*_success_unlabeled"}', + refId='A', + ), + ], + yAxes=YAxes( + YAxis(format=OPS_FORMAT), + YAxis(format=SHORT_FORMAT), + ), + ) + + @staticmethod + def node_exec_latency() -> Graph: + return Graph( + title=f"Node Exec latency quantile and workflow", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(flyte:propeller:all:node:node_exec_latency_us) by (quantile, wf) / 1000', + refId='A', + ), + ], + yAxes=single_y_axis(format=MILLISECONDS_FORMAT), + ) + + @staticmethod + def node_event_recording_latency() -> Graph: + return Graph( + title=f"Node Event event recording latency quantile and workflow", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(flyte:propeller:all:node:event_recording:success_duration_ms) by (quantile, wf)', + refId='A', + ), + Target( + expr=f'sum(flyte:propeller:all:node:event_recording:failure_duration_ms) by (quantile, wf)', + refId='B', + ), + ], + yAxes=single_y_axis(format=MILLISECONDS_FORMAT), + ) + + @staticmethod + def node_input_latency() -> Graph: + return Graph( + title=f"Node Input latency quantile and workflow", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(flyte:propeller:all:node:node_input_latency_ms) by (quantile, wf)', + refId='A', + ), + ], + yAxes=single_y_axis(format=MILLISECONDS_FORMAT), + ) + + @staticmethod + def metastore_failures(): + # Copy counts sum(rate(flyte:propeller:all:metastore:copy:overall_unlabeled_ms_count[5m])) + return Graph( + title=f"Failures from metastore", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(rate(flyte:propeller:all:metastore:head_failure_unlabeled[5m]))', + legendFormat="head-failure", + refId='A', + ), + Target( + expr=f'sum(rate(flyte:propeller:all:metastore:bad_container_unlabeled[5m]))', + legendFormat="bad-container", + refId='A', + ), + Target( + expr=f'sum(rate(flyte:propeller:all:metastore:bad_key_unlabeled[5m]))', + legendFormat="bad-key", + refId='A', + ), + Target( + expr=f'sum(rate(flyte:propeller:all:metastore:read_failure_unlabeled[5m]))', + legendFormat="read-failure", + refId='A', + ), + Target( + expr=f'sum(rate(flyte:propeller:all:metastore:write_failure_unlabeled[5m]))', + legendFormat="write-failure", + refId='A', + ), + ], + yAxes=YAxes( + YAxis(format=OPS_FORMAT), + YAxis(format=SHORT_FORMAT), + ), + ) + + @staticmethod + def metastore_cache_hit_percentage(interval: int) -> Graph: + """ + TODO replace with metric math maybe? + """ + return Graph( + title="cache hit percentage", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'(sum(rate(flyte:propeller:all:metastore:cache_hit[{interval}m])) * 100) / (sum(rate(flyte:propeller:all:metastore:cache_miss[{interval}m])) + sum(rate(flyte:propeller:all:metastore:cache_hit[{interval}m])))', + refId='A', + ), + ], + yAxes=single_y_axis(format=PERCENT_FORMAT), + ) + + @staticmethod + def metastore_latencies(collapse: bool) -> Row: + return Row( + title=f"Metastore latencies", + collapse=collapse, + panels=[ + Graph( + title=f"Metastore copy latency", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(flyte:propeller:all:metastore:copy:overall_unlabeled_ms) by (quantile)', + refId='A', + ), + ], + yAxes=single_y_axis(format=MILLISECONDS_FORMAT), + ), + Graph( + title=f"Metastore write latency by workflow", + dataSource=DATASOURCE, + targets=[ + Target( + expr='sum(flyte:propeller:all:metastore:write_ms) by (quantile, wf)', + refId='A', + ), + ], + yAxes=single_y_axis(format=MILLISECONDS_FORMAT), + ), + Graph( + title=f"Metastore read open latency by workflow", + dataSource=DATASOURCE, + targets=[ + Target( + expr='sum(flyte:propeller:all:metastore:read_open_ms) by (quantile, wf)', + refId='A', + ), + ], + yAxes=single_y_axis(format=MILLISECONDS_FORMAT), + ), + Graph( + title=f"Metastore head latency by workflow", + dataSource=DATASOURCE, + targets=[ + Target( + expr='sum(flyte:propeller:all:metastore:head_ms) by (quantile, wf)', + refId='A', + ), + ], + yAxes=single_y_axis(format=MILLISECONDS_FORMAT), + ), + Graph( + title=f"Metastore fetch latency by workflow", + dataSource=DATASOURCE, + targets=[ + Target( + expr='sum(flyte:propeller:all:metastore:proto_fetch_ms) by (quantile, wf)', + legendFormat="proto-fetch", + refId='A', + ), + + Target( + expr='sum(flyte:propeller:all:metastore:remote_fetch_ms) by (quantile, wf)', + legendFormat="remote-fetch", + refId='B', + ), + ], + yAxes=single_y_axis(format=MILLISECONDS_FORMAT), + ), + ] + ) + + @staticmethod + def admin_launcher_cache() -> Graph: + return Graph( + title="Admin Launcher cache", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(rate(flyte:propeller:all:admin_launcher:cache_hit[5m]))', + legendFormat="hit", + refId='A', + ), + + Target( + expr=f'sum(rate(flyte:propeller:all:admin_launcher:cache_miss[5m]))', + legendFormat="miss", + refId='B', + ), + ], + yAxes=single_y_axis(format=MILLISECONDS_FORMAT), + ) + + @staticmethod + def dynamic_wf_build() -> typing.List[Graph]: + return [ + Graph( + title="Dynamic workflow build latency", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(flyte:propeller:all:node:build_dynamic_workflow_us) by (quantile, wf) / 1000', + refId='A', + ), + ], + yAxes=single_y_axis(format=MILLISECONDS_FORMAT), + ), + Graph( + title="Dynamic workflow build count", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(rate(flyte:propeller:all:node:build_dynamic_workflow_us_count[5m])) by (wf)', + refId='A', + ), + ], + yAxes=single_y_axis(format=NO_FORMAT), + ), + ] + + @staticmethod + def task_event_recording() -> typing.List[Graph]: + return [ + Graph( + title="task event recording latency", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(flyte:propeller:all:task:event_recording:success_duration_ms) by (quantile, wf)', + refId='A', + ), + ], + yAxes=single_y_axis(format=MILLISECONDS_FORMAT), + ), + Graph( + title="task event recording count", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(rate(flyte:propeller:all:task:event_recording:success_duration_ms_count[5m])) by (wf)', + legendFormat="success wf", + refId='A', + ), + Target( + expr=f'sum(rate(flyte:propeller:all:task:event_recording:failure_duration_ms_count[5m])) by (wf)', + legendFormat="failure", + refId='A', + ), + ], + yAxes=single_y_axis(format=NO_FORMAT), + ), + ] + + @staticmethod + def node_event_recording() -> typing.List[Graph]: + return [ + Graph( + title="node event recording latency success", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(flyte:propeller:all:node:event_recording:success_duration_ms) by (quantile, wf)', + refId='A', + ), + ], + yAxes=single_y_axis(format=MILLISECONDS_FORMAT), + ), + Graph( + title="node event recording count", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(rate(flyte:propeller:all:node:event_recording:success_duration_ms_count[5m])) by (wf)', + legendFormat="success", + refId='A', + ), + Target( + expr=f'sum(rate(flyte:propeller:all:node:event_recording:failure_duration_ms_count[5m])) by (wf)', + legendFormat="failure", + refId='A', + ), + ], + yAxes=single_y_axis(format=NO_FORMAT), + ), + ] + + @staticmethod + def wf_event_recording() -> typing.List[Graph]: + return [ + Graph( + title="wf event recording latency success", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(flyte:propeller:all:workflow:event_recording:success_duration_ms) by (quantile, wf)', + refId='A', + ), + ], + yAxes=single_y_axis(format=MILLISECONDS_FORMAT), + ), + Graph( + title="wf event recording count", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(rate(flyte:propeller:all:workflow:event_recording:success_duration_ms_count[5m])) by (wf)', + legendFormat="success", + refId='A', + ), + Target( + expr=f'sum(rate(flyte:propeller:all:workflow:event_recording:failure_duration_ms_count[5m])) by (wf)', + legendFormat="failure", + refId='A', + ), + ], + yAxes=single_y_axis(format=NO_FORMAT), + ), + ] + + @staticmethod + def wf_store_latency(collapse: bool) -> Row: + return Row( + title="etcD write metrics", + collapse=collapse, + panels=[ + Graph( + title="wf update etcD latency", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(flyte:propeller:all:wf_update_latency_ms) by (quantile)', + refId='A', + ), + ], + yAxes=single_y_axis(format=MILLISECONDS_FORMAT), + ), + Graph( + title="etcD writes", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(rate(flyte:propeller:all:wf_update_latency_ms_count[5m]))', + refId='A', + ), + ], + yAxes=single_y_axis(format=NO_FORMAT), + ), + Graph( + title="etcD write conflicts", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(rate(flyte:propeller:all:wf_update_conflict[5m]))', + refId='A', + ), + ], + yAxes=single_y_axis(format=NO_FORMAT), + ), + Graph( + title="etcD write fail", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(rate(flyte:propeller:all:wf_update_failed[5m]))', + refId='A', + ), + ], + yAxes=single_y_axis(format=NO_FORMAT), + ), + ]) + + @staticmethod + def perf_metrics(collapse: bool) -> Row: + r = Row( + title="Perf metrics", + collapse=collapse, + panels=[], + ) + r.panels.extend(FlytePropeller.wf_event_recording()) + r.panels.extend(FlytePropeller.node_event_recording()) + r.panels.extend(FlytePropeller.task_event_recording()) + r.panels.extend(FlytePropeller.dynamic_wf_build()) + r.panels.append(FlytePropeller.admin_launcher_cache()) + return r + + @staticmethod + def metastore_metrics(interval: int, collapse: bool) -> Row: + return Row( + title="Metastore failures and cache", + collapse=collapse, + panels=[ + FlytePropeller.metastore_cache_hit_percentage(interval), + FlytePropeller.metastore_failures(), + ], + ) + + @staticmethod + def node_errors() -> Graph: + return Graph( + title="node event recording count", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(rate(flyte:propeller:all:node:perma_system_error_duration_unlabeled_ms_count[5m]))', + legendFormat="system error", + refId='A', + ), + Target( + expr=f'sum(rate(flyte:propeller:all:node:perma_user_error_duration_unlabeled_ms[5m]))', + legendFormat="user error", + refId='A', + ), + Target( + expr=f'sum(rate(flyte:propeller:all:node:perma_unknown_error_duration_unlabeled_ms[5m]))', + legendFormat="user error", + refId='A', + ), + ], + yAxes=single_y_axis(format=NO_FORMAT), + ) + + @staticmethod + def queue_metrics(collapse: bool) -> Row: + return Row( + title="FlytePropeller Queue metrics", + collapse=collapse, + panels=[ + Graph( + title="WF Adds to main queue", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(rate(flyte:propeller:all:main_adds[5m]))', + refId='A', + ), + ], + yAxes=single_y_axis(format=SHORT_FORMAT), + ), + Graph( + title="Unprocessed Queue depth", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(rate(flyte:propeller:all:main_depth[5m]))', + refId='A', + ), + ], + yAxes=single_y_axis(format=SHORT_FORMAT), + ), + Graph( + title="Item retries", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(rate(flyte:propeller:all:main_retries[5m]))', + refId='A', + ), + ], + yAxes=single_y_axis(format=SHORT_FORMAT), + ), + Graph( + title="Seconds of unfinished work in progress", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'flyte:propeller:all:main_unfinished_work_s', + refId='A', + ), + ], + yAxes=single_y_axis(format=SECONDS_FORMAT), + ), + Graph( + title="Workqueue work average duration", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(rate(flyte:propeller:all:main_work_duration_us_sum[5m]) / rate(flyte:propeller:all:main_work_duration_us_count[5m]))', + refId='A', + ), + ], + yAxes=single_y_axis(format=SECONDS_FORMAT), + ), + Graph( + title="Duration for which an item stays in queue - avg", + dataSource=DATASOURCE, + targets=[ + Target( + expr=f'sum(rate(flyte:propeller:all:main_queue_latency_us_sum[5m]) / rate(flyte:propeller:all:main_queue_latency_us_count[5m]))', + refId='A', + ), + ], + yAxes=single_y_axis(format=SECONDS_FORMAT), + ), + ], + ) + + @staticmethod + def node_metrics(collapse: bool) -> Row: + return Row( + title="Node Metrics", + collapse=collapse, + panels=[ + FlytePropeller.node_exec_latency(), + FlytePropeller.node_input_latency(), + FlytePropeller.node_event_recording_latency(), + FlytePropeller.node_errors(), + ], + ) + + @staticmethod + def core_metrics(interval: int, collapse: bool) -> Row: + return Row( + title="Core metrics", + collapse=collapse, + panels=[ + FlytePropeller.create_free_workers(), + FlytePropeller.abort_errors(), + FlytePropeller.system_errors(), + FlytePropeller.plugin_success_vs_failures(), + FlytePropeller.round_latency(interval), + FlytePropeller.round_latency_per_wf(interval), + FlytePropeller.round_panic(), + FlytePropeller.workflows_per_project(), + ], + ) + + @staticmethod + def create_all_rows(interval: int = 5) -> typing.List[Row]: + return [ + FlytePropeller.core_metrics(interval, False), + FlytePropeller.metastore_metrics(interval, True), + FlytePropeller.metastore_latencies(True), + FlytePropeller.node_metrics(True), + FlytePropeller.perf_metrics(True), + FlytePropeller.wf_store_latency(False), + FlytePropeller.queue_metrics(True), + ] + + +dashboard = Dashboard( + tags=["flyte", "prometheus", "flytepropeller", "flyte-dataplane"], + inputs=[ + DataSourceInput( + name=DATASOURCE_NAME, + label="Prometheus", + description="Prometheus server that connects to Flyte", + pluginId="prometheus", + pluginName="Prometheus", + ), + ], + editable=False, + title="Flyte Propeller Dashboard (via Prometheus)", + rows=FlytePropeller.create_all_rows(interval=5), + description="Flyte Propeller Dashboard. This is great for monitoring FlytePropeller / Flyte data plane deployments. This is mostly useful for the Flyte deployment maintainer", +).auto_panel_ids() + +if __name__ == "__main__": + print(dashboard.to_json_data()) diff --git a/stats/flyteuser_dashboard.py b/stats/flyteuser_dashboard.py new file mode 100644 index 0000000000..a094108955 --- /dev/null +++ b/stats/flyteuser_dashboard.py @@ -0,0 +1,287 @@ +import typing +from grafanalib.core import ( + Alert, AlertCondition, Dashboard, Graph, + GreaterThan, OP_AND, OPS_FORMAT, Row, RTYPE_SUM, SECONDS_FORMAT, + SHORT_FORMAT, single_y_axis, Target, TimeRange, YAxes, YAxis, MILLISECONDS_FORMAT, Templating, Template, + DataSourceInput +) + +DATASOURCE_NAME = "DS_PROM" +DATASOURCE = "${%s}" % DATASOURCE_NAME + + +class FlyteUserDashboard(object): + + @staticmethod + def workflow_stats(collapse: bool) -> Row: + return Row( + title="Workflow Stats", + collapse=collapse, + panels=[ + Graph( + title="Accepted Workflow", + dataSource=DATASOURCE, + targets=[ + Target( + expr='sum(rate(flyte:propeller:all:workflow:accepted{project=~"$project", domain=~"$domain", wf=~"$workflow"}[5m]))', + refId='A', + ), + ], + yAxes=YAxes( + YAxis(format=OPS_FORMAT), + YAxis(format=SHORT_FORMAT), + ), + ), + Graph( + title="Successful Workflow", + dataSource=DATASOURCE, + targets=[ + Target( + expr='sum(rate(flyte:propeller:all:workflow:success_duration_ms_count{project=~"$project", domain=~"$domain", wf=~"$workflow"}[5m]))', + refId='A', + ), + ], + yAxes=YAxes( + YAxis(format=OPS_FORMAT), + YAxis(format=SHORT_FORMAT), + ), + ), + Graph( + title="Failed Workflow", + dataSource=DATASOURCE, + targets=[ + Target( + expr='sum(rate(flyte:propeller:all:workflow:failure_duration_ms_count{project=~"$project", domain=~"$domain", wf=~"$workflow"}[5m]))', + refId='A', + ), + ], + yAxes=YAxes( + YAxis(format=OPS_FORMAT), + YAxis(format=SHORT_FORMAT), + ), + ), + Graph( + title="Aborted Workflow", + dataSource=DATASOURCE, + targets=[ + Target( + expr='sum(rate(flyte:propeller:all:workflow:workflow_aborted{project=~"$project", domain=~"$domain", wf=~"$workflow"}[5m]))', + refId='A', + ), + ], + yAxes=YAxes( + YAxis(format=OPS_FORMAT), + YAxis(format=SHORT_FORMAT), + ), + ), + Graph( + title="Successful workflow execution time by Quantile", + dataSource=DATASOURCE, + targets=[ + Target( + expr='sum(flyte:propeller:all:workflow:success_duration_ms{project=~"$project", domain=~"$domain", wf=~"$workflow"}) by (quantile)', + refId='A', + ), + ], + yAxes=single_y_axis(format=MILLISECONDS_FORMAT), + ), + Graph( + title="Failed workflow execution time by Quantile", + dataSource=DATASOURCE, + targets=[ + Target( + expr='sum(flyte:propeller:all:workflow:failed_duration_ms{project=~"$project", domain=~"$domain", wf=~"$workflow"}) by (quantile)', + refId='A', + ), + ], + yAxes=single_y_axis(format=MILLISECONDS_FORMAT), + ), + Graph( + title="Node queuing latency by Quantile", + dataSource=DATASOURCE, + targets=[ + Target( + expr='sum(flyte:propeller:all:node:queueing_latency_ms{project=~"$project", domain=~"$domain", wf=~"$workflow"}) by (quantile)', + refId='A', + ), + ], + yAxes=single_y_axis(format=MILLISECONDS_FORMAT), + ), + ]) + + @staticmethod + def quota_stats(collapse: bool) -> Row: + return Row( + title="Kubernetes Quota Usage stats", + collapse=collapse, + panels=[ + Graph( + title="CPU Limits vs usage", + dataSource=DATASOURCE, + targets=[ + Target( + expr='kube_resourcequota{resource="limits.cpu", namespace="$project-$domain", type="hard"}', + refId='A', + legendFormat="max cpu", + ), + Target( + expr='kube_resourcequota{resource="limits.cpu", namespace="$project-$domain", type="used"}', + refId='B', + legendFormat="used cpu", + ), + ], + yAxes=YAxes( + YAxis(format=OPS_FORMAT), + YAxis(format=SHORT_FORMAT), + ), + ), + Graph( + title="Mem Limits vs usage", + dataSource=DATASOURCE, + targets=[ + Target( + expr='kube_resourcequota{resource="limits.memory", namespace="$project-$domain", type="hard"}', + refId='A', + legendFormat="max mem", + ), + Target( + expr='kube_resourcequota{resource="limits.memory", namespace="$project-$domain", type="used"}', + refId='B', + legendFormat="used mem", + ), + ], + yAxes=YAxes( + YAxis(format=OPS_FORMAT), + YAxis(format=SHORT_FORMAT), + ), + ), + ]) + + @staticmethod + def resource_stats(collapse: bool) -> Row: + return Row( + title="Task stats", + collapse=collapse, + panels=[ + Graph( + title="Pending tasks", + dataSource=DATASOURCE, + targets=[ + Target( + expr='sum(kube_pod_container_status_waiting * on(pod) group_left(label_execution_id, label_task_name, label_node_id, label_workflow_name) kube_pod_labels{label_execution_id !="",namespace=~"$project-$domain",label_workflow_name=~"$workflow"}) by (namespace, label_execution_id, label_task_name, label_node_id, label_workflow_name) > 0', + refId='A', + ), + ], + yAxes=single_y_axis(format=SHORT_FORMAT), + ), + Graph( + title="Memory Usage Percentage", + dataSource=DATASOURCE, + targets=[ + Target( + expr='(max(container_memory_rss{image!=""} * on(pod) group_left(label_execution_id, label_task_name, label_node_id, label_workflow_name) kube_pod_labels{label_execution_id !="",namespace=~"$project-$domain",label_workflow_name=~"$workflow"} * on(pod) group_left(phase) kube_pod_status_phase{phase="Running"}) by (namespace, pod, label_execution_id, label_task_name, label_node_id, label_workflow_name) / max(kube_pod_container_resource_limits_memory_bytes{container!=""} * on(pod) group_left(label_execution_id, label_task_name, label_node_id, label_workflow_name) kube_pod_labels{label_execution_id !=""} * on(pod) group_left(phase) kube_pod_status_phase{phase="Running"}) by (namespace, pod, label_execution_id, label_task_name, label_node_id, label_workflow_name)) > 0', + refId='A', + ), + ], + yAxes=single_y_axis(format=SHORT_FORMAT), + ), + Graph( + title="CPU Usage Percentage", + dataSource=DATASOURCE, + targets=[ + Target( + expr='(sum(rate(container_cpu_usage_seconds_total{image!=""}[2m]) * on(pod) group_left(label_execution_id, label_task_name, label_node_id, label_workflow_name) kube_pod_labels{label_execution_id !="",namespace=~"$project-$domain",label_workflow_name=~"$workflow"} * on(pod) group_left(phase) kube_pod_status_phase{phase="Running"}) by (namespace, pod, label_execution_id, label_task_name, label_node_id, label_workflow_name) / sum(kube_pod_container_resource_limits_cpu_cores{container!=""} * on(pod) group_left(label_execution_id, label_task_name, label_node_id, label_workflow_name) kube_pod_labels{label_execution_id !=""} * on(pod) group_left(phase) kube_pod_status_phase{phase="Running"}) by (namespace, pod, label_execution_id, label_task_name, label_node_id, label_workflow_name)) > 0', + refId='A', + ), + ], + yAxes=single_y_axis(format=SHORT_FORMAT), + ), + ]) + + @staticmethod + def errors(collapse: bool) -> Row: + return Row( + title="Error (System vs user)", + collapse=collapse, + panels=[ + Graph( + title="User errors", + dataSource=DATASOURCE, + targets=[ + Target( + expr='sum(rate(flyte:propeller:all:node:user_error_duration_ms_count{project=~"$project",domain=~"$domain",wf=~"$project:$domain:$workflow"}[5m]))', + refId='A', + ), + ], + yAxes=single_y_axis(format=SHORT_FORMAT), + ), + Graph( + title="System errors", + dataSource=DATASOURCE, + targets=[ + Target( + expr='sum(rate(flyte:propeller:all:node:system_error_duration_ms_count{project=~"$project",domain=~"$domain",wf=~"$project:$domain:$workflow"}[5m]))', + refId='A', + ), + ], + yAxes=single_y_axis(format=SHORT_FORMAT), + ), + ]) + + @staticmethod + def create_all_rows(interval: int) -> typing.List[Row]: + return [ + FlyteUserDashboard.workflow_stats(False), + FlyteUserDashboard.quota_stats(True), + FlyteUserDashboard.resource_stats(True), + FlyteUserDashboard.errors(True), + ] + + +project_template = Template( + name="project", + dataSource=DATASOURCE, + query="label_values(flyte:propeller:all:collector:flyteworkflow, project)", + sort=True, +) + +domain_template = Template( + name="domain", + dataSource=DATASOURCE, + query="label_values(flyte:propeller:all:collector:flyteworkflow, domain)", + sort=True, +) + +wf_template = Template( + name="workflow", + dataSource=DATASOURCE, + query="label_values(flyte:propeller:all:collector:flyteworkflow, wf)", + sort=True, + allValue=".*", + includeAll=True, +) + +dashboard = Dashboard( + tags=["flyte", "prometheus", "flyteuser", "flyte-user"], + inputs=[ + DataSourceInput( + name=DATASOURCE_NAME, + label="Prometheus", + description="Prometheus server that connects to Flyte", + pluginId="prometheus", + pluginName="Prometheus", + ), + ], + editable=False, + title="Flyte User Dashboard (via Prometheus)", + rows=FlyteUserDashboard.create_all_rows(interval=5), + templating=Templating(list=[ + project_template, + domain_template, + wf_template, + ]), + description="Flyte User Dashboard. This is great to get a birds-eye and drill down view of executions in your Flyte cluster. Useful for the user.", +).auto_panel_ids() + +if __name__ == "__main__": + print(dashboard.to_json_data()) diff --git a/stats/prom.yaml b/stats/prom.yaml new file mode 100644 index 0000000000..6268f856a3 --- /dev/null +++ b/stats/prom.yaml @@ -0,0 +1,20 @@ +- name: flyterunningtask_metrics + rules: + - record: flyterunningtask:pending + expr: sum(kube_pod_container_status_waiting * on(pod) group_left(label_execution_id, label_task_name, label_node_id, label_workflow_name) kube_pod_labels{label_execution_id !=""}) by (namespace, label_execution_id, label_task_name, label_node_id, label_workflow_name) > 0 + - record: flyterunningtask:cpu:limit + expr: sum(kube_pod_container_resource_limits_cpu_cores{container!=""} * on(pod) group_left(label_execution_id, label_task_name, label_node_id, label_workflow_name) kube_pod_labels{label_execution_id !=""} * on(pod) group_left(phase) kube_pod_status_phase{phase="Running"}) by (namespace, label_execution_id, label_task_name, label_node_id, label_workflow_name) > 0 + - record: flyterunningtask:cpu:request + expr: sum(kube_pod_container_resource_requests_cpu_cores{container!=""} * on(pod) group_left(label_execution_id, label_task_name, label_node_id, label_workflow_name) kube_pod_labels{label_execution_id !=""} * on(pod) group_left(phase) kube_pod_status_phase{phase="Running"}) by (namespace, label_execution_id, label_task_name, label_node_id, label_workflow_name) > 0 + - record: flyterunningtask:cpu:usage:percentage + expr: (sum(rate(container_cpu_usage_seconds_total{image!=""}[2m]) * on(pod) group_left(label_execution_id, label_task_name, label_node_id, label_workflow_name) kube_pod_labels{label_execution_id !=""} * on(pod) group_left(phase) kube_pod_status_phase{phase="Running"}) by (namespace, label_execution_id, label_task_name, label_node_id, label_workflow_name) / sum(kube_pod_container_resource_limits_cpu_cores{container!=""} * on(pod) group_left(label_execution_id, label_task_name, label_node_id, label_workflow_name) kube_pod_labels{label_execution_id !=""} * on(pod) group_left(phase) kube_pod_status_phase{phase="Running"}) by (namespace, label_execution_id, label_task_name, label_node_id, label_workflow_name)) > 0 + - record: flyterunningtask:pods:cpu:usage:percentage + expr: (sum(rate(container_cpu_usage_seconds_total{image!=""}[2m]) * on(pod) group_left(label_execution_id, label_task_name, label_node_id, label_workflow_name) kube_pod_labels{label_execution_id !=""} * on(pod) group_left(phase) kube_pod_status_phase{phase="Running"}) by (namespace, pod, label_execution_id, label_task_name, label_node_id, label_workflow_name) / sum(kube_pod_container_resource_limits_cpu_cores{container!=""} * on(pod) group_left(label_execution_id, label_task_name, label_node_id, label_workflow_name) kube_pod_labels{label_execution_id !=""} * on(pod) group_left(phase) kube_pod_status_phase{phase="Running"}) by (namespace, pod, label_execution_id, label_task_name, label_node_id, label_workflow_name)) > 0 + - record: flyterunningtask:memory:limit + expr: sum(kube_pod_container_resource_limits_memory_bytes{container!=""} * on(pod) group_left(label_execution_id, label_task_name, label_node_id, label_workflow_name) kube_pod_labels{label_execution_id !=""} * on(pod) group_left(phase) kube_pod_status_phase{phase="Running"}) by (namespace, label_execution_id, label_task_name, label_node_id, label_workflow_name) > 0 + - record: flyterunningtask:memory:request + expr: sum(kube_pod_container_resource_requests_memory_bytes{container!=""} * on(pod) group_left(label_execution_id, label_task_name, label_node_id, label_workflow_name) kube_pod_labels{label_execution_id !=""} * on(pod) group_left(phase) kube_pod_status_phase{phase="Running"}) by (namespace, label_execution_id, label_task_name, label_node_id, label_workflow_name) > 0 + - record: flyterunningtask:memory:usage:percentage + expr: (max(container_memory_rss{image!=""} * on(pod) group_left(label_execution_id, label_task_name, label_node_id, label_workflow_name) kube_pod_labels{label_execution_id !=""} * on(pod) group_left(phase) kube_pod_status_phase{phase="Running"}) by (namespace, label_execution_id, label_task_name, label_node_id, label_workflow_name) / max(kube_pod_container_resource_limits_memory_bytes{container!=""} * on(pod) group_left(label_execution_id, label_task_name, label_node_id, label_workflow_name) kube_pod_labels{label_execution_id !=""} * on(pod) group_left(phase) kube_pod_status_phase{phase="Running"}) by (namespace, label_execution_id, label_task_name, label_node_id, label_workflow_name)) > 0 + - record: flyterunningtask:pods:memory:usage:percentage + expr: (max(container_memory_rss{image!=""} * on(pod) group_left(label_execution_id, label_task_name, label_node_id, label_workflow_name) kube_pod_labels{label_execution_id !=""} * on(pod) group_left(phase) kube_pod_status_phase{phase="Running"}) by (namespace, pod, label_execution_id, label_task_name, label_node_id, label_workflow_name) / max(kube_pod_container_resource_limits_memory_bytes{container!=""} * on(pod) group_left(label_execution_id, label_task_name, label_node_id, label_workflow_name) kube_pod_labels{label_execution_id !=""} * on(pod) group_left(phase) kube_pod_status_phase{phase="Running"}) by (namespace, pod, label_execution_id, label_task_name, label_node_id, label_workflow_name)) > 0