From 2290b8fa3f9da7b4dd10ba7ff774fface0328f07 Mon Sep 17 00:00:00 2001 From: bjwswang Date: Fri, 17 Nov 2023 09:24:09 +0000 Subject: [PATCH] feat: implement datasource management for arctl Signed-off-by: bjwswang --- .gitignore | 3 +- README.md | 1 + arctl/README.md | 21 ++- arctl/datasource.go | 158 +++++++++++++++++- arctl/main.go | 6 +- arctl/printer/printer.go | 14 ++ go.mod | 2 +- .../go-server/pkg/datasource/datasource.go | 11 +- 8 files changed, 203 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index c7eafeddb..c5c190106 100644 --- a/.gitignore +++ b/.gitignore @@ -112,4 +112,5 @@ Temporary Items .apdisk .vagrant/ _output/ -*.bkp \ No newline at end of file +*.bkp +tmp_role.yaml \ No newline at end of file diff --git a/README.md b/README.md index b1f4d484d..f41a76cb7 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,7 @@ If you don't have a kubernetes cluster, you can schedule a [kind cluster](https: We provide a Command Line Tool `arctl` to interact with `arcadia`. See [here](./arctl/README.md) for more details. +- ✅ datasource management - ✅ local dataset management ## Pure Go Toolchains diff --git a/arctl/README.md b/arctl/README.md index 8ab61d4f9..f3a1c896e 100644 --- a/arctl/README.md +++ b/arctl/README.md @@ -25,11 +25,13 @@ Available Commands: chat Do LLM chat with similarity search(optional) completion Generate the autocompletion script for the specified shell dataset Manage dataset locally + datasource Manage datasources help Help about any command Flags: - -h, --help help for arctl - --home string home directory to use (default "/Users/bjwswang/.arcadia") + -h, --help help for arctl + --home string home directory to use (default "/home/bjwswang/.arcadia") + -n, --namespace string namespace to use (default "default") Use "arctl [command] --help" for more information about a command. ``` @@ -61,16 +63,27 @@ Available Commands: chat Do LLM chat with similarity search(optional) completion Generate the autocompletion script for the specified shell dataset Manage dataset locally + datasource Manage datasources help Help about any command Flags: - -h, --help help for arctl - --home string home directory to use (default "/Users/bjwswang/.arcadia") + -h, --help help for arctl + --home string home directory to use (default "/home/bjwswang/.arcadia") + -n, --namespace string namespace to use (default "default") Use "arctl [command] --help" for more information about a command. ``` ## Usage +### Datasource management + +You can use `arctl` to manage your datasources with the following commands: + +- `arctl datasource list`: list your datasources +- `arctl datasource create`: create a new datasource +- `arctl datasource delete`: delete datasource along with the auth secret(if set) +- `arctl datasource get`: get a datasource info + ### Local dataset management You can use `arctl` to manage your dataset locally with the following commands: diff --git a/arctl/datasource.go b/arctl/datasource.go index 70e39de08..d112a47e1 100644 --- a/arctl/datasource.go +++ b/arctl/datasource.go @@ -17,14 +17,24 @@ limitations under the License. package main import ( + "errors" + "fmt" + "os" + "github.com/spf13/cobra" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/klog/v2" "github.com/kubeagi/arcadia/arctl/printer" "github.com/kubeagi/arcadia/graphql-server/go-server/pkg/datasource" ) var ( - datasourcePrintHeaders = []string{"name", "displayName", "creator", "endpoint"} + datasourcePrintHeaders = []string{"name", "displayName", "creator", "endpoint", "oss"} ) func NewDatasourceCmd() *cobra.Command { @@ -33,15 +43,122 @@ func NewDatasourceCmd() *cobra.Command { Short: "Manage datasources", } + cmd.AddCommand(DatasourceCreateCmd()) + cmd.AddCommand(DatasourceGetCmd()) + cmd.AddCommand(DatasourceDeleteCmd()) cmd.AddCommand(DatasourceListCmd()) return cmd } +func DatasourceCreateCmd() *cobra.Command { + var empytDatasource bool + // endpoint flags + var endpointURL, endpointAuthUser, endpointAuthPwd string + var endpointInsecure bool + var ossBucket string + + // command definition + cmd := &cobra.Command{ + Use: "create [name]", + Short: "Create a datasource", + Long: "Create a datasource", + RunE: func(cmd *cobra.Command, args []string) error { + if len(os.Args) < 4 { + return errors.New("missing datasource name") + } + name := os.Args[3] + + if endpointURL == "" && !empytDatasource { + return errors.New("set --empty if you want to create a empty datasource") + } + + var endpointAuthSecret string + if endpointURL != "" { + // create auth secret for datasource + if endpointAuthUser != "" && endpointAuthPwd != "" { + endpointAuthSecret = fmt.Sprintf("datasource-%s-authsecret", name) + secret := corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: corev1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{Name: endpointAuthSecret, Namespace: namespace}, + Data: map[string][]byte{ + "rootUser": []byte(endpointAuthUser), + "rootPassword": []byte(endpointAuthPwd), + }, + } + unstructuredDatasource, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&secret) + if err != nil { + return fmt.Errorf("failed to convert auth secret: %w", err) + } + _, err = kubeClient.Resource(schema.GroupVersionResource{ + Group: corev1.SchemeGroupVersion.Group, + Version: corev1.SchemeGroupVersion.Version, + Resource: "secrets", + }).Namespace(namespace).Create(cmd.Context(), &unstructured.Unstructured{Object: unstructuredDatasource}, metav1.CreateOptions{}) + if err != nil { + return fmt.Errorf("failed to create auth secret: %w", err) + } + klog.Infof("Successfully created authsecret %s\n", endpointAuthSecret) + } + } + + _, err = datasource.CreateDatasource(cmd.Context(), kubeClient, name, namespace, endpointURL, endpointAuthSecret, ossBucket, displayName, endpointInsecure) + if err != nil { + return err + } + klog.Infof("Successfully created datasource %s\n", name) + + return nil + }, + } + // Common flags + cmd.Flags().StringVar(&displayName, "display-name", "", "The displayname for datasource") + cmd.Flags().StringVar(&description, "description", "A datasource for LLMOps", "The description for datasource") + + // Empyt datasource (means using system-datasource to provide its data) + cmd.Flags().BoolVar(&empytDatasource, "empty", false, "Whether to create a empty datasource") + + // Endpoint flags + cmd.Flags().StringVar(&endpointURL, "endpoint-url", "", "The endpoint url to access datasource.If not provided,a empty datasource will be created") + cmd.Flags().StringVar(&endpointAuthUser, "endpoint-auth-user", "", "The endpoint's user for datasource authentication") + cmd.Flags().StringVar(&endpointAuthPwd, "endpoint-auth-password", "", "The endpoint's user password for datasource authentication") + cmd.Flags().BoolVar(&endpointInsecure, "endpoint-insecure", true, "Whether to access datasource without secure check.Default is yes") + + // Object storage service flags + cmd.Flags().StringVar(&ossBucket, "oss-bucket", "", "The object storage service bucket name") + + return cmd +} + +func DatasourceGetCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "get [name]", + Short: "Get datasource", + RunE: func(cmd *cobra.Command, args []string) error { + if len(os.Args) < 4 { + return errors.New("missing datasource name") + } + name := os.Args[3] + + ds, err := datasource.ReadDatasource(cmd.Context(), kubeClient, name, namespace) + if err != nil { + return fmt.Errorf("failed to find datasource: %w", err) + } + + return printer.PrintYaml(ds) + }, + } + + return cmd +} + func DatasourceListCmd() *cobra.Command { cmd := &cobra.Command{ Use: "list [usage]", - Short: "List many datasources", + Short: "List datasources", RunE: func(cmd *cobra.Command, args []string) error { list, err := datasource.ListDatasources(cmd.Context(), kubeClient, namespace, "", "") if err != nil { @@ -58,3 +175,40 @@ func DatasourceListCmd() *cobra.Command { return cmd } + +func DatasourceDeleteCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "delete [name]", + Short: "Delete a datasource", + RunE: func(cmd *cobra.Command, args []string) error { + if len(os.Args) < 4 { + return errors.New("missing datasource name") + } + name := os.Args[3] + ds, err := datasource.ReadDatasource(cmd.Context(), kubeClient, name, namespace) + if err != nil { + return fmt.Errorf("failed to get datasource: %w", err) + } + // delete secrets + if ds.Endpoint != nil && ds.Endpoint.AuthSecret != nil { + err = kubeClient.Resource(schema.GroupVersionResource{ + Group: corev1.SchemeGroupVersion.Group, + Version: corev1.SchemeGroupVersion.Version, + Resource: "secrets", + }).Namespace(namespace).Delete(cmd.Context(), ds.Endpoint.AuthSecret.Name, metav1.DeleteOptions{}) + if err != nil { + return fmt.Errorf("failed to delete auth secret: %w", err) + } + klog.Infof("Successfully deleted authsecret %s\n", ds.Endpoint.AuthSecret.Name) + } + _, err = datasource.DeleteDatasource(cmd.Context(), kubeClient, name, namespace, "", "") + if err != nil { + return fmt.Errorf("faield to delete datasource: %w", err) + } + klog.Infof("Successfully deleted datasource %s\n", name) + return err + }, + } + + return cmd +} diff --git a/arctl/main.go b/arctl/main.go index d144e7d44..e56b24094 100644 --- a/arctl/main.go +++ b/arctl/main.go @@ -33,6 +33,10 @@ var ( namespace string kubeClient dynamic.Interface + + // common spec to all resources + displayName string + description string ) func NewCLI() *cobra.Command { @@ -61,7 +65,7 @@ func NewCLI() *cobra.Command { arctl.AddCommand(NewChatCmd()) arctl.PersistentFlags().StringVar(&home, "home", filepath.Join(os.Getenv("HOME"), ".arcadia"), "home directory to use") - arctl.PersistentFlags().StringVar(&namespace, "namespace", "default", "namespace to use") + arctl.PersistentFlags().StringVarP(&namespace, "namespace", "n", "default", "namespace to use") return arctl } diff --git a/arctl/printer/printer.go b/arctl/printer/printer.go index 43d868e3a..6e8dca5a3 100644 --- a/arctl/printer/printer.go +++ b/arctl/printer/printer.go @@ -23,6 +23,8 @@ import ( "reflect" "strings" "text/tabwriter" + + "gopkg.in/yaml.v2" ) func Print(headers []string, objs []any) { @@ -64,3 +66,15 @@ func GetByHeader(obj any, s string) string { return "" } + +func PrintYaml(obj any) error { + data, err := yaml.Marshal(obj) + if err != nil { + return err + } + _, err = os.Stdout.Write(data) + if err != nil { + return err + } + return nil +} diff --git a/go.mod b/go.mod index 3232f99a8..d9816fd39 100644 --- a/go.mod +++ b/go.mod @@ -126,7 +126,7 @@ require ( gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.24.2 // indirect k8s.io/component-base v0.24.2 // indirect diff --git a/graphql-server/go-server/pkg/datasource/datasource.go b/graphql-server/go-server/pkg/datasource/datasource.go index 5c6021d5a..91fe6590f 100644 --- a/graphql-server/go-server/pkg/datasource/datasource.go +++ b/graphql-server/go-server/pkg/datasource/datasource.go @@ -41,6 +41,7 @@ func datasource2model(obj *unstructured.Unstructured) *model.Datasource { } url, _, _ := unstructured.NestedString(obj.Object, "spec", "endpoint", "url") authsecret, _, _ := unstructured.NestedString(obj.Object, "spec", "endpoint", "authSecret", "name") + authsecretNamespace, _, _ := unstructured.NestedString(obj.Object, "spec", "endpoint", "authSecret", "namespace") displayName, _, _ := unstructured.NestedString(obj.Object, "spec", "displayName") bucket, _, _ := unstructured.NestedString(obj.Object, "spec", "oss", "bucket") insecure, _, _ := unstructured.NestedBool(obj.Object, "spec", "endpoint", "insecure") @@ -56,8 +57,9 @@ func datasource2model(obj *unstructured.Unstructured) *model.Datasource { endpoint := model.Endpoint{ URL: &url, AuthSecret: &model.TypedObjectReference{ - Kind: "Secret", - Name: authsecret, + Kind: "Secret", + Name: authsecret, + Namespace: &authsecretNamespace, }, Insecure: &insecure, } @@ -96,8 +98,9 @@ func CreateDatasource(ctx context.Context, c dynamic.Interface, name, namespace, Enpoint: &v1alpha1.Endpoint{ URL: url, AuthSecret: &v1alpha1.TypedObjectReference{ - Kind: "Secret", - Name: authsecret, + Kind: "Secret", + Name: authsecret, + Namespace: &namespace, }, Insecure: insecure, },