From 7c4965b68f3e6598fa423004015ba680afdbebb7 Mon Sep 17 00:00:00 2001 From: Tom Coufal <7453394+tumido@users.noreply.github.com> Date: Tue, 29 Mar 2022 14:22:23 +0200 Subject: [PATCH] [byon] Implement backend (#156) * feat(byon): List all notebooks Signed-off-by: Tomas Coufal * feat(byon): Get single notebook Signed-off-by: Tomas Coufal * fix(byon): Update api spec to include software and id Signed-off-by: Tomas Coufal * feat(byon): Schedule new notebook import Signed-off-by: Tomas Coufal * feat(byon): Delete notebook Signed-off-by: Tomas Coufal * feat(byon): Update notebook Signed-off-by: Tomas Coufal --- .../src/routes/api/notebook/notebooksUtils.ts | 200 ++++++++++++++++-- backend/src/types.ts | 78 ++++++- 2 files changed, 253 insertions(+), 25 deletions(-) diff --git a/backend/src/routes/api/notebook/notebooksUtils.ts b/backend/src/routes/api/notebook/notebooksUtils.ts index e4c64243c4..c38b2f6d6c 100644 --- a/backend/src/routes/api/notebook/notebooksUtils.ts +++ b/backend/src/routes/api/notebook/notebooksUtils.ts @@ -1,13 +1,60 @@ import { FastifyRequest } from 'fastify'; -import { KubeFastifyInstance, Notebook } from '../../../types'; +import createError from 'http-errors'; +import { KubeFastifyInstance, Notebook, ImageStreamListKind, ImageStreamKind, NotebookStatus, PipelineRunListKind, PipelineRunKind, NotebookCreateRequest, NotebookUpdateRequest } from '../../../types'; + +const mapImageStreamToNotebook = (is: ImageStreamKind): Notebook => ({ + id: is.metadata.name, + name: is.metadata.annotations["opendatahub.io/notebook-image-name"], + description: is.metadata.annotations["opendatahub.io/notebook-image-name"], + phase: is.metadata.annotations["opendatahub.io/notebook-image-phase"] as NotebookStatus, + visible: is.metadata.annotations["opendatahub.io/notebook-image-visible"] === "true", + error: Boolean(is.metadata.annotations["opendatahub.io/notebook-image-messages"]) + ? JSON.parse(is.metadata.annotations["opendatahub.io/notebook-image-messages"]) + : [], + packages: is.spec.tags && JSON.parse(is.spec.tags[0].annotations["opendatahub.io/notebook-python-dependencies"]), + software: is.spec.tags && JSON.parse(is.spec.tags[0].annotations["opendatahub.io/notebook-software"]), + uploaded: is.metadata.creationTimestamp, + url: is.metadata.annotations["opendatahub.io/notebook-image-url"], +}) + +const mapPipelineRunToNotebook = (plr: PipelineRunKind): Notebook => ({ + id: plr.metadata.name, + name: plr.spec.params.find(p => p.name === "name")?.value, + description: plr.spec.params.find(p => p.name === "desc")?.value, + url: plr.spec.params.find(p => p.name === "url")?.value, + phase: "Importing", +}) export const getNotebooks = async ( fastify: KubeFastifyInstance, ): Promise<{ notebooks: Notebook[]; error: string }> => { - const notebooks: Notebook[] = []; - // const coreV1Api = fastify.kube.coreV1Api; - // const namespace = fastify.kube.namespace; + const customObjectsApi = fastify.kube.customObjectsApi; + const namespace = fastify.kube.namespace; + try { + const imageStreams = await customObjectsApi.listNamespacedCustomObject( + "image.openshift.io", + "v1", + namespace, + "imagestreams", + undefined, undefined, undefined, + "app.kubernetes.io/created-by=byon" + ).then(r => r.body as ImageStreamListKind) + const pipelineRuns = await customObjectsApi.listNamespacedCustomObject( + "tekton.dev", + "v1beta1", + namespace, + "pipelineruns", + undefined, undefined, undefined, + "app.kubernetes.io/created-by=byon" + ).then(r => r.body as PipelineRunListKind) + + const imageStreamNames = imageStreams.items.map(is => is.metadata.name) + const notebooks: Notebook[] = [ + ...imageStreams.items.map(is => mapImageStreamToNotebook(is)), + ...pipelineRuns.items.filter(plr => !imageStreamNames.includes(plr.metadata.name)).map(plr => mapPipelineRunToNotebook(plr)), + ] + return { notebooks: notebooks, error: null }; } catch (e) { if (e.response?.statusCode !== 404) { @@ -21,14 +68,36 @@ export const getNotebook = async ( fastify: KubeFastifyInstance, request: FastifyRequest, ): Promise<{ notebooks: Notebook; error: string }> => { - const notebook: Notebook = { - name: '', - repo: '', - }; - // const coreV1Api = fastify.kube.coreV1Api; - // const namespace = fastify.kube.namespace; + const customObjectsApi = fastify.kube.customObjectsApi; + const namespace = fastify.kube.namespace; + const params = request.params as { notebook: string }; + try { - return { notebooks: notebook, error: null }; + const imageStream = await customObjectsApi.getNamespacedCustomObject( + "image.openshift.io", + "v1", + namespace, + "imagestreams", + params.notebook + ).then(r => r.body as ImageStreamKind).catch(r => null) + + if (imageStream) { + return { notebooks: mapImageStreamToNotebook(imageStream), error: null }; + } + + const pipelineRun = await customObjectsApi.getNamespacedCustomObject( + "tekton.dev", + "v1beta1", + namespace, + "pipelineruns", + params.notebook + ).then(r => r.body as PipelineRunKind).catch(r => null) + + if (pipelineRun) { + return { notebooks: mapPipelineRunToNotebook(pipelineRun), error: null }; + } + + throw new createError.NotFound(`Notebook ${params.notebook} does not exist.`) } catch (e) { if (e.response?.statusCode !== 404) { fastify.log.error('Unable to retrieve notebook image(s): ' + e.toString()); @@ -41,9 +110,51 @@ export const addNotebook = async ( fastify: KubeFastifyInstance, request: FastifyRequest, ): Promise<{ success: boolean; error: string }> => { - // const coreV1Api = fastify.kube.coreV1Api; - // const namespace = fastify.kube.namespace; + const customObjectsApi = fastify.kube.customObjectsApi; + const namespace = fastify.kube.namespace; + const body = request.body as NotebookCreateRequest; + + const payload: PipelineRunKind = { + apiVersion: "tekton.dev/v1beta1", + kind: "PipelineRun", + metadata: { + generateName: "byon-import-jupyterhub-image-run-" + }, + spec: { + params: [ + { name: "desc", value: body.description}, + { name: "name", value: body.name}, + { name: "url", value: body.url}, + ], + pipelineRef: { + name: "byon-import-jupyterhub-image" + }, + workspaces: [ + { + name: "data", + volumeClaimTemplate: { + spec: { + accessModes: ["ReadWriteOnce"], + resources: { + requests: { + storage: "10Mi" + } + } + } + } + } + ] + } + } + try { + await customObjectsApi.createNamespacedCustomObject( + "tekton.dev", + "v1beta1", + namespace, + "pipelineruns", + payload + ) return { success: true, error: null }; } catch (e) { if (e.response?.statusCode !== 404) { @@ -57,14 +168,31 @@ export const deleteNotebook = async ( fastify: KubeFastifyInstance, request: FastifyRequest, ): Promise<{ success: boolean; error: string }> => { - // const coreV1Api = fastify.kube.coreV1Api; - // const namespace = fastify.kube.namespace; + const customObjectsApi = fastify.kube.customObjectsApi; + const namespace = fastify.kube.namespace; + const params = request.params as { notebook: string }; + try { + await customObjectsApi.deleteNamespacedCustomObject( + "image.openshift.io", + "v1", + namespace, + "imagestreams", + params.notebook + ).catch(e => {throw createError(e.statusCode, e?.body?.message)}) + // Cleanup in case the pipelinerun is still present and was not garbage collected yet + await customObjectsApi.deleteNamespacedCustomObject( + "tekton.dev", + "v1beta1", + namespace, + "pipelineruns", + params.notebook + ).catch(() => {}) return { success: true, error: null }; } catch (e) { if (e.response?.statusCode !== 404) { - fastify.log.error('Unable to update notebook image: ' + e.toString()); - return { success: false, error: 'Unable to update notebook image: ' + e.message }; + fastify.log.error('Unable to delete notebook image: ' + e.toString()); + return { success: false, error: 'Unable to delete notebook image: ' + e.message }; } } }; @@ -73,9 +201,43 @@ export const updateNotebook = async ( fastify: KubeFastifyInstance, request: FastifyRequest, ): Promise<{ success: boolean; error: string }> => { - // const coreV1Api = fastify.kube.coreV1Api; - // const namespace = fastify.kube.namespace; + const customObjectsApi = fastify.kube.customObjectsApi; + const namespace = fastify.kube.namespace; + const params = request.params as { notebook: string }; + const body = request.body as NotebookUpdateRequest; + try { + const imageStream = await customObjectsApi.getNamespacedCustomObject( + "image.openshift.io", + "v1", + namespace, + "imagestreams", + params.notebook + ).then(r => r.body as ImageStreamKind).catch(e => {throw createError(e.statusCode, e?.body?.message)}) + + if (body.packages && imageStream.spec.tags) { + imageStream.spec.tags[0].annotations["opendatahub.io/notebook-python-dependencies"] = JSON.stringify(body.packages) + } + if (body.software && imageStream.spec.tags) { + imageStream.spec.tags[0].annotations["opendatahub.io/notebook-software"] = JSON.stringify(body.software) + } + if (typeof body.visible !== "undefined") { + imageStream.metadata.annotations["opendatahub.io/notebook-image-visible"] = body.visible.toString() + } + + await customObjectsApi.patchNamespacedCustomObject( + "image.openshift.io", + "v1", + namespace, + "imagestreams", + params.notebook, + imageStream, + undefined, undefined, undefined, + { + headers: { "Content-Type": "application/merge-patch+json" } + } + ).catch(e => console.log(e)) + return { success: true, error: null }; } catch (e) { if (e.response?.statusCode !== 404) { diff --git a/backend/src/types.ts b/backend/src/types.ts index 7a95698a52..ef1b6a6548 100644 --- a/backend/src/types.ts +++ b/backend/src/types.ts @@ -31,17 +31,23 @@ export declare type QuickStart = { }; // Properties common to (almost) all Kubernetes resources. -export type K8sResourceCommon = { +export type K8sResourceBase = { apiVersion?: string; kind?: string; +} + +export type K8sResourceCommon = { metadata?: { name?: string; namespace?: string; + generateName?: string; uid?: string; labels?: { [key: string]: string }; annotations?: { [key: string]: string }; + creationTimestamp?: Date; }; -}; +} & K8sResourceBase; + export enum BUILD_PHASE { none = 'Not started', @@ -228,18 +234,78 @@ export type NotebookError = { export type NotebookStatus = "Importing" | "Validating" | "Succeeded" | "Failed"; export type Notebook = { - name: string; - url: string; - description?: string; + id: string; phase?: NotebookStatus; user?: string; uploaded?: Date; + error?: NotebookError; +} & NotebookCreateRequest & NotebookUpdateRequest; + +export type NotebookCreateRequest = { + name: string; + url: string; + description?: string; +} + +export type NotebookUpdateRequest = { + id: string; visible?: boolean; packages?: NotebookPackage[]; - error?: NotebookError; + software?: NotebookPackage[]; } export type NotebookPackage = { name: string; version: string; } + + +export type ImageStreamTagSpec = { + name: string; + annotations?: { [key: string]: string }; + from?: { + kind: string; + name: string; + } +} +export type ImageStreamKind = { + spec?: { + tags: ImageStreamTagSpec[]; + } + status?: any +} & K8sResourceCommon; + +export type ImageStreamListKind = { + items: ImageStreamKind[]; +} & K8sResourceBase; + +export type PipelineRunKind = { + spec: { + params: { + name: string; + value: string; + }[] + pipelineRef: { + name: string; + } + workspaces?: [ + { + name: string + volumeClaimTemplate: { + spec: { + accessModes: string[] + resources: { + requests: { + storage: string + } + } + } + } + } + ] + } +} & K8sResourceCommon; + +export type PipelineRunListKind = { + items: PipelineRunKind[]; +} & K8sResourceBase;