feat: implement datasource management for arctl

Signed-off-by: bjwswang <bjwswang@gmail.com>

.gitignore

@@ -112,4 +112,5 @@ Temporary Items
 .apdisk
 .vagrant/
 _output/
-*.bkp
+*.bkp
+tmp_role.yaml

README.md

@@ -54,6 +54,7 @@ If you don't have a kubernetes cluster, you can schedule a [kind cluster](https:
 
 We provide a Command Line Tool `arctl` to interact with `arcadia`. See [here](./arctl/README.md) for more details.
 
+- ✅ datasource management
 - ✅ local dataset management
 
 ## Pure Go Toolchains

arctl/README.md

@@ -25,11 +25,13 @@ Available Commands:
   chat        Do LLM chat with similarity search(optional)
   completion  Generate the autocompletion script for the specified shell
   dataset     Manage dataset locally
+  datasource  Manage datasources
   help        Help about any command
 
 Flags:
-  -h, --help          help for arctl
-      --home string   home directory to use (default "/Users/bjwswang/.arcadia")
+  -h, --help               help for arctl
+      --home string        home directory to use (default "/home/bjwswang/.arcadia")
+  -n, --namespace string   namespace to use (default "default")
 
 Use "arctl [command] --help" for more information about a command.
 ```
@@ -61,16 +63,27 @@ Available Commands:
   chat        Do LLM chat with similarity search(optional)
   completion  Generate the autocompletion script for the specified shell
   dataset     Manage dataset locally
+  datasource  Manage datasources
   help        Help about any command
 
 Flags:
-  -h, --help          help for arctl
-      --home string   home directory to use (default "/Users/bjwswang/.arcadia")
+  -h, --help               help for arctl
+      --home string        home directory to use (default "/home/bjwswang/.arcadia")
+  -n, --namespace string   namespace to use (default "default")
 
 Use "arctl [command] --help" for more information about a command.
 ```
 
 ## Usage
+### Datasource management
+
+You can use `arctl` to manage your datasources with the following commands:
+
+- `arctl datasource list`: list your datasources
+- `arctl datasource create`: create a new datasource
+- `arctl datasource delete`: delete datasource along with the auth secret(if set)
+- `arctl datasource get`: get a datasource info
+
 ### Local dataset management
 
 You can use `arctl` to manage your dataset locally with the following commands:

arctl/datasource.go

@@ -17,14 +17,24 @@ limitations under the License.
 package main
 
 import (
+	"errors"
+	"fmt"
+	"os"
+
 	"github.com/spf13/cobra"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/klog/v2"
 
 	"github.com/kubeagi/arcadia/arctl/printer"
 	"github.com/kubeagi/arcadia/graphql-server/go-server/pkg/datasource"
 )
 
 var (
-	datasourcePrintHeaders = []string{"name", "displayName", "creator", "endpoint"}
+	datasourcePrintHeaders = []string{"name", "displayName", "creator", "endpoint", "oss"}
 )
 
 func NewDatasourceCmd() *cobra.Command {
@@ -33,15 +43,122 @@ func NewDatasourceCmd() *cobra.Command {
 		Short: "Manage datasources",
 	}
 
+	cmd.AddCommand(DatasourceCreateCmd())
+	cmd.AddCommand(DatasourceGetCmd())
+	cmd.AddCommand(DatasourceDeleteCmd())
 	cmd.AddCommand(DatasourceListCmd())
 
 	return cmd
 }
 
+func DatasourceCreateCmd() *cobra.Command {
+	var empytDatasource bool
+	// endpoint flags
+	var endpointURL, endpointAuthUser, endpointAuthPwd string
+	var endpointInsecure bool
+	var ossBucket string
+
+	// command definition
+	cmd := &cobra.Command{
+		Use:   "create [name]",
+		Short: "Create a datasource",
+		Long:  "Create a datasource",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if len(os.Args) < 4 {
+				return errors.New("missing datasource name")
+			}
+			name := os.Args[3]
+
+			if endpointURL == "" && !empytDatasource {
+				return errors.New("set --empty if you want to create a empty datasource")
+			}
+
+			var endpointAuthSecret string
+			if endpointURL != "" {
+				// create auth secret for datasource
+				if endpointAuthUser != "" && endpointAuthPwd != "" {
+					endpointAuthSecret = fmt.Sprintf("datasource-%s-authsecret", name)
+					secret := corev1.Secret{
+						TypeMeta: metav1.TypeMeta{
+							Kind:       "Secret",
+							APIVersion: corev1.SchemeGroupVersion.String(),
+						},
+						ObjectMeta: metav1.ObjectMeta{Name: endpointAuthSecret, Namespace: namespace},
+						Data: map[string][]byte{
+							"rootUser":     []byte(endpointAuthUser),
+							"rootPassword": []byte(endpointAuthPwd),
+						},
+					}
+					unstructuredDatasource, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&secret)
+					if err != nil {
+						return fmt.Errorf("failed to convert auth secret: %w", err)
+					}
+					_, err = kubeClient.Resource(schema.GroupVersionResource{
+						Group:    corev1.SchemeGroupVersion.Group,
+						Version:  corev1.SchemeGroupVersion.Version,
+						Resource: "secrets",
+					}).Namespace(namespace).Create(cmd.Context(), &unstructured.Unstructured{Object: unstructuredDatasource}, metav1.CreateOptions{})
+					if err != nil {
+						return fmt.Errorf("failed to create auth secret: %w", err)
+					}
+					klog.Infof("Successfully created authsecret %s\n", endpointAuthSecret)
+				}
+			}
+
+			_, err = datasource.CreateDatasource(cmd.Context(), kubeClient, name, namespace, endpointURL, endpointAuthSecret, ossBucket, displayName, endpointInsecure)
+			if err != nil {
+				return err
+			}
+			klog.Infof("Successfully created datasource %s\n", name)
+
+			return nil
+		},
+	}
+	// Common flags
+	cmd.Flags().StringVar(&displayName, "display-name", "", "The displayname for datasource")
+	cmd.Flags().StringVar(&description, "description", "A datasource for LLMOps", "The description for datasource")
+
+	// Empyt datasource (means using system-datasource to provide its data)
+	cmd.Flags().BoolVar(&empytDatasource, "empty", false, "Whether to create a empty datasource")
+
+	// Endpoint flags
+	cmd.Flags().StringVar(&endpointURL, "endpoint-url", "", "The endpoint url to access datasource.If not provided,a empty datasource will be created")
+	cmd.Flags().StringVar(&endpointAuthUser, "endpoint-auth-user", "", "The endpoint's user for datasource authentication")
+	cmd.Flags().StringVar(&endpointAuthPwd, "endpoint-auth-password", "", "The endpoint's user password for datasource authentication")
+	cmd.Flags().BoolVar(&endpointInsecure, "endpoint-insecure", true, "Whether to access datasource without secure check.Default is yes")
+
+	// Object storage service flags
+	cmd.Flags().StringVar(&ossBucket, "oss-bucket", "", "The object storage service bucket name")
+
+	return cmd
+}
+
+func DatasourceGetCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "get [name]",
+		Short: "Get datasource",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if len(os.Args) < 4 {
+				return errors.New("missing datasource name")
+			}
+			name := os.Args[3]
+
+			ds, err := datasource.ReadDatasource(cmd.Context(), kubeClient, name, namespace)
+			if err != nil {
+				return fmt.Errorf("failed to find datasource: %w", err)
+			}
+
+			return printer.PrintYaml(ds)
+		},
+	}
+
+	return cmd
+}
+
 func DatasourceListCmd() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "list [usage]",
-		Short: "List many datasources",
+		Short: "List datasources",
 		RunE: func(cmd *cobra.Command, args []string) error {
 			list, err := datasource.ListDatasources(cmd.Context(), kubeClient, namespace, "", "")
 			if err != nil {
@@ -58,3 +175,40 @@ func DatasourceListCmd() *cobra.Command {
 
 	return cmd
 }
+
+func DatasourceDeleteCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "delete [name]",
+		Short: "Delete a datasource",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if len(os.Args) < 4 {
+				return errors.New("missing datasource name")
+			}
+			name := os.Args[3]
+			ds, err := datasource.ReadDatasource(cmd.Context(), kubeClient, name, namespace)
+			if err != nil {
+				return fmt.Errorf("failed to get datasource: %w", err)
+			}
+			// delete secrets
+			if ds.Endpoint != nil && ds.Endpoint.AuthSecret != nil {
+				err = kubeClient.Resource(schema.GroupVersionResource{
+					Group:    corev1.SchemeGroupVersion.Group,
+					Version:  corev1.SchemeGroupVersion.Version,
+					Resource: "secrets",
+				}).Namespace(namespace).Delete(cmd.Context(), ds.Endpoint.AuthSecret.Name, metav1.DeleteOptions{})
+				if err != nil {
+					return fmt.Errorf("failed to delete auth secret: %w", err)
+				}
+				klog.Infof("Successfully deleted authsecret %s\n", ds.Endpoint.AuthSecret.Name)
+			}
+			_, err = datasource.DeleteDatasource(cmd.Context(), kubeClient, name, namespace, "", "")
+			if err != nil {
+				return fmt.Errorf("faield to delete datasource: %w", err)
+			}
+			klog.Infof("Successfully deleted datasource %s\n", name)
+			return err
+		},
+	}
+
+	return cmd
+}

arctl/main.go

@@ -33,6 +33,10 @@ var (
 	namespace string
 
 	kubeClient dynamic.Interface
+
+	// common spec to all resources
+	displayName string
+	description string
 )
 
 func NewCLI() *cobra.Command {
@@ -61,7 +65,7 @@ func NewCLI() *cobra.Command {
 	arctl.AddCommand(NewChatCmd())
 
 	arctl.PersistentFlags().StringVar(&home, "home", filepath.Join(os.Getenv("HOME"), ".arcadia"), "home directory to use")
-	arctl.PersistentFlags().StringVar(&namespace, "namespace", "default", "namespace to use")
+	arctl.PersistentFlags().StringVarP(&namespace, "namespace", "n", "default", "namespace to use")
 
 	return arctl
 }

arctl/printer/printer.go

@@ -23,6 +23,8 @@ import (
 	"reflect"
 	"strings"
 	"text/tabwriter"
+
+	"gopkg.in/yaml.v2"
 )
 
 func Print(headers []string, objs []any) {
@@ -64,3 +66,15 @@ func GetByHeader(obj any, s string) string {
 
 	return "<none>"
 }
+
+func PrintYaml(obj any) error {
+	data, err := yaml.Marshal(obj)
+	if err != nil {
+		return err
+	}
+	_, err = os.Stdout.Write(data)
+	if err != nil {
+		return err
+	}
+	return nil
+}

go.mod

@@ -126,7 +126,7 @@ require (
 	gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/apiextensions-apiserver v0.24.2 // indirect
 	k8s.io/component-base v0.24.2 // indirect

graphql-server/go-server/pkg/datasource/datasource.go

@@ -41,6 +41,7 @@ func datasource2model(obj *unstructured.Unstructured) *model.Datasource {
 	}
 	url, _, _ := unstructured.NestedString(obj.Object, "spec", "endpoint", "url")
 	authsecret, _, _ := unstructured.NestedString(obj.Object, "spec", "endpoint", "authSecret", "name")
+	authsecretNamespace, _, _ := unstructured.NestedString(obj.Object, "spec", "endpoint", "authSecret", "namespace")
 	displayName, _, _ := unstructured.NestedString(obj.Object, "spec", "displayName")
 	bucket, _, _ := unstructured.NestedString(obj.Object, "spec", "oss", "bucket")
 	insecure, _, _ := unstructured.NestedBool(obj.Object, "spec", "endpoint", "insecure")
@@ -56,8 +57,9 @@ func datasource2model(obj *unstructured.Unstructured) *model.Datasource {
 	endpoint := model.Endpoint{
 		URL: &url,
 		AuthSecret: &model.TypedObjectReference{
-			Kind: "Secret",
-			Name: authsecret,
+			Kind:      "Secret",
+			Name:      authsecret,
+			Namespace: &authsecretNamespace,
 		},
 		Insecure: &insecure,
 	}
@@ -96,8 +98,9 @@ func CreateDatasource(ctx context.Context, c dynamic.Interface, name, namespace,
 				Enpoint: &v1alpha1.Endpoint{
 					URL: url,
 					AuthSecret: &v1alpha1.TypedObjectReference{
-						Kind: "Secret",
-						Name: authsecret,
+						Kind:      "Secret",
+						Name:      authsecret,
+						Namespace: &namespace,
 					},
 					Insecure: insecure,
 				},