Merge pull request #32 from ganganxiaojiu/main

support install tools and nineinfra with localpath's charts

cmd/common.go

@@ -430,6 +430,10 @@ func GenThriftSvcName(name string) string {
 	return name + DefaultNineSuffix + "-kyuubi"
 }
 
+func GenDorisSvcName(name string) string {
+	return name + DefaultNineSuffix + "-doris-fe-service"
+}
+
 func GenThriftServiceAccountName(name string) string {
 	return name + DefaultNineSuffix + "-kyuubi"
 }
@@ -446,6 +450,10 @@ func GetThriftIpAndPort(name string, ns string) (string, int32) {
 	return GetSvcAccessInfo(GenThriftSvcName(name), DefaultThriftPortName, ns)
 }
 
+func GetDorisIpAndPort(name string, ns string) (string, int32) {
+	return GetSvcAccessInfo(GenDorisSvcName(name), DefaultDorisPortName, ns)
+}
+
 func GetThriftPodName(name string, ns string) (string, error) {
 	path, _ := rootCmd.Flags().GetString(kubeconfig)
 	client, err := GetKubeClient(path)

cmd/constants.go

@@ -46,7 +46,11 @@ const (
 	DefaultSparkUIName       = "spark-ui"
 )
 const (
-	FeaturesOlapKey = "olap"
+	FeaturesOlapKey           = "olap"
+	DefaultDorisPortName      = "query-port"
+	DefaultDorisAdminUser     = "root"
+	DefaultDorisAdminPassword = ""
+	DefaultDorisDatabaseName  = "nineinfra"
 )
 
 var (
@@ -99,13 +103,17 @@ var NineClusterProjectNameSuffix = map[string]string{
 	"metastore":  "-nine-metastore",
 	"minio":      "-nine-ss-0",
 	"postgresql": "-nine-pg",
+	"doris-fe":   "-nine-doris-fe",
+	"doris-be":   "-nine-doris-fe",
 }
 
 var NineClusterProjectWorkloadList = map[string]string{
 	"kyuubi":     "statefulset",
 	"metastore":  "statefulset",
 	"minio":      "statefulset",
 	"postgresql": "cluster",
+	"doris-fe":   "statefulset",
+	"doris-be":   "statefulset",
 }
 
 var NineToolList = map[string]interface{}{

cmd/helm.go

@@ -70,11 +70,15 @@ func InitHelm() error {
 	return nil
 }
 
-func HelmInstall(name string, repoName string, chart string, version string, namespace string, flags string) error {
+func HelmInstall(name string, repoName string, chartPath string, chart string, version string, namespace string, flags string) error {
 	if repoName == "" {
-		repoName = DefaultHelmRepoName
+		if chartPath != "" {
+			chart = chartPath + "/" + ChartName2TarName(chart)
+		} else {
+			repoName = DefaultHelmRepoName
+			chart = repoName + "/" + chart
+		}
 	}
-	chart = repoName + "/" + chart
 	if flags == "" {
 		_, errput, err := runCommand("helm", "install", name, chart, "--version", version, "-n", namespace)
 		if err != nil && !strings.Contains(errput, "in use") {
@@ -90,11 +94,23 @@ func HelmInstall(name string, repoName string, chart string, version string, nam
 	return nil
 }
 
-func HelmInstallWithParameters(name string, repoName string, chart string, version string, namespace string, parameters ...string) error {
+func ChartName2TarName(chart string) string {
+	if chartVersion, ok := DefaultToolsChartList[chart]; ok {
+		return fmt.Sprintf("%s-v%s.tar.gz", chart, chartVersion)
+	}
+	return ""
+}
+
+func HelmInstallWithParameters(name string, repoName string, chartPath string, chart string, version string, namespace string, parameters ...string) error {
 	if repoName == "" {
-		repoName = DefaultHelmRepoName
+		if chartPath != "" {
+			chart = chartPath + "/" + ChartName2TarName(chart)
+		} else {
+			repoName = DefaultHelmRepoName
+			chart = repoName + "/" + chart
+		}
 	}
-	chart = repoName + "/" + chart
+
 	args := []string{"install", name, chart, "--version", version, "-n", namespace}
 	args = append(args, parameters...)
 	_, errput, err := runCommand("helm", args...)

cmd/install.go

@@ -17,9 +17,10 @@ const (
 )
 
 type operatorInstallCmd struct {
-	out    io.Writer
-	errOut io.Writer
-	output bool
+	out       io.Writer
+	errOut    io.Writer
+	output    bool
+	chartPath string
 }
 
 func newInstallCmd(out io.Writer, errOut io.Writer) *cobra.Command {
@@ -43,6 +44,7 @@ func newInstallCmd(out io.Writer, errOut io.Writer) *cobra.Command {
 	cmd = DisableHelp(cmd)
 	f := cmd.Flags()
 	f.BoolVar(&DEBUG, "debug", false, "print debug information")
+	f.StringVarP(&o.chartPath, "chart-path", "p", "", "local path of the charts")
 	return cmd
 }
 
@@ -68,7 +70,7 @@ func (o *operatorInstallCmd) run() error {
 	}
 
 	for c, v := range DefaultChartList {
-		err := HelmInstall(c, "", c, v, DefaultNamespace, flags)
+		err := HelmInstall(c, "", o.chartPath, c, v, DefaultNamespace, flags)
 		if err != nil {
 			fmt.Printf("Error: %v \n", err)
 			os.Exit(1)

cmd/tools.go

@@ -5,6 +5,7 @@ import (
 	"database/sql"
 	"errors"
 	"fmt"
+	_ "github.com/go-sql-driver/mysql"
 	"github.com/spf13/cobra"
 	"gopkg.in/yaml.v2"
 	"io"
@@ -45,6 +46,7 @@ type toolsCmd struct {
 	nineName    string
 	toolkitArgs []string // --nodes flag
 	deletePVC   bool
+	chartPath   string
 }
 
 type DatabasesConnection struct {
@@ -89,6 +91,7 @@ func newToolsCmd(out io.Writer, errOut io.Writer) *cobra.Command {
 	f.StringVar(&DefaultToolAirflowTag, "airflow-tag", "2.7.3", "airflow image tag")
 	f.StringVarP(&DefaultStorageClass, "storage-pool", "s", "nineinfra-default", "storage pool fo tools")
 	f.BoolVar(&c.deletePVC, "delete-pvc", false, "delete the ninecluster tools pvcs")
+	f.StringVarP(&c.chartPath, "chart-path", "p", "", "local path of the charts")
 	f.StringVarP(&c.ns, "namespace", "n", "", "k8s namespace for tools")
 	f.BoolVar(&DEBUG, "debug", false, "print debug information")
 	return cmd
@@ -151,6 +154,23 @@ func (t *toolsCmd) genSupersetSecretFile() error {
 	return nil
 }
 
+func (t *toolsCmd) createDorisDatabase(ip string, port int32, user string, password string) error {
+	connStr := fmt.Sprintf("%s@tcp(%s:%d)/", user, ip, port)
+	db, err := sql.Open("mysql", connStr)
+	if err != nil {
+		fmt.Printf("Error:%v\n", err)
+		return err
+	}
+	defer db.Close()
+
+	_, err = db.Exec("CREATE DATABASE IF NOT EXISTS " + DefaultDorisDatabaseName)
+	if err != nil {
+		fmt.Printf("Error:%v\n", err)
+		return err
+	}
+	return nil
+}
+
 func (t *toolsCmd) genSupersetDataSourcesFile() error {
 	thriftIP, thriftPort := GetThriftIpAndPort(t.nineName, t.ns)
 	if thriftIP == "" || thriftPort == 0 {
@@ -170,6 +190,21 @@ func (t *toolsCmd) genSupersetDataSourcesFile() error {
 			},
 		},
 	}
+	dorisIP, dorisPort := GetDorisIpAndPort(t.nineName, t.ns)
+	if dorisIP != "" && dorisPort != 0 {
+		if err := t.createDorisDatabase(dorisIP, dorisPort, DefaultDorisAdminUser, DefaultDorisAdminPassword); err != nil {
+			return err
+		}
+		data["databases"] = append(data["databases"], DatabasesConnection{
+			AllowFileUpload: true,
+			AllowCTAS:       true,
+			AllowCVAS:       true,
+			DatabaseName:    DefaultDorisDatabaseName,
+			Extra:           "{\r\n    \"metadata_params\": {},\r\n    \"engine_params\": {},\r\n    \"metadata_cache_timeout\": {},\r\n    \"schemas_allowed_for_file_upload\": []\r\n}",
+			SqlAlchemyURI:   fmt.Sprintf("mysql://%s@%s:%d", DefaultDorisAdminUser, dorisIP, dorisPort),
+			Tables:          []string{},
+		})
+	}
 	yamlData, err := yaml.Marshal(&data)
 	if err != nil {
 		return err
@@ -318,7 +353,7 @@ func (t *toolsCmd) createDatabase(tool string) error {
 
 func (t *toolsCmd) installRedis(parameters []string) error {
 	relName := DefaultToolsNamePrefix + DefaultToolRedisName
-	err := HelmInstallWithParameters(relName, "", DefaultToolRedisName, DefaultToolsChartList[DefaultToolRedisName], t.ns, t.genRedisParameters(relName, parameters)...)
+	err := HelmInstallWithParameters(relName, "", t.chartPath, DefaultToolRedisName, DefaultToolsChartList[DefaultToolRedisName], t.ns, t.genRedisParameters(relName, parameters)...)
 	if err != nil {
 		return err
 	}
@@ -335,7 +370,7 @@ func (t *toolsCmd) installAirflow(parameters []string) error {
 		return err
 	}
 	relName := DefaultToolsNamePrefix + DefaultToolAirflowName
-	err = HelmInstallWithParameters(relName, "", DefaultToolAirflowName, DefaultToolsChartList[DefaultToolAirflowName], t.ns, t.genAirflowParameters(relName, parameters)...)
+	err = HelmInstallWithParameters(relName, "", t.chartPath, DefaultToolAirflowName, DefaultToolsChartList[DefaultToolAirflowName], t.ns, t.genAirflowParameters(relName, parameters)...)
 	if err != nil {
 		return err
 	}
@@ -352,7 +387,7 @@ func (t *toolsCmd) installSuperset(parameters []string) error {
 		return err
 	}
 	relName := DefaultToolsNamePrefix + DefaultToolSupersetName
-	err = HelmInstallWithParameters(relName, "", DefaultToolSupersetName, DefaultToolsChartList[DefaultToolSupersetName], t.ns, t.genSupersetParameters(relName, parameters)...)
+	err = HelmInstallWithParameters(relName, "", t.chartPath, DefaultToolSupersetName, DefaultToolsChartList[DefaultToolSupersetName], t.ns, t.genSupersetParameters(relName, parameters)...)
 	if err != nil {
 		return err
 	}
@@ -361,7 +396,7 @@ func (t *toolsCmd) installSuperset(parameters []string) error {
 
 func (t *toolsCmd) installZookeeper(parameters []string) error {
 	relName := DefaultToolsNamePrefix + DefaultToolZookeeperName
-	err := HelmInstallWithParameters(relName, "", DefaultToolZookeeperName, DefaultToolsChartList[DefaultToolZookeeperName], t.ns, t.genZookeeperParameters(relName, parameters)...)
+	err := HelmInstallWithParameters(relName, "", t.chartPath, DefaultToolZookeeperName, DefaultToolsChartList[DefaultToolZookeeperName], t.ns, t.genZookeeperParameters(relName, parameters)...)
 	if err != nil {
 		return err
 	}
@@ -374,7 +409,7 @@ func (t *toolsCmd) installNifi(parameters []string) error {
 		return err
 	}
 	relName := DefaultToolsNamePrefix + DefaultToolNifiName
-	err = HelmInstallWithParameters(relName, "", DefaultToolNifiName, DefaultToolsChartList[DefaultToolNifiName], t.ns, t.genNifiParameters(relName, parameters)...)
+	err = HelmInstallWithParameters(relName, "", t.chartPath, DefaultToolNifiName, DefaultToolsChartList[DefaultToolNifiName], t.ns, t.genNifiParameters(relName, parameters)...)
 	if err != nil {
 		return err
 	}

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/beltran/gohive v1.6.0
 	github.com/cloudnative-pg/client v0.0.0-00010101000000-000000000000
 	github.com/cloudnative-pg/cloudnative-pg v1.21.1
+	github.com/go-sql-driver/mysql v1.7.1
 	github.com/manifoldco/promptui v0.9.0
 	github.com/minio/directpv/apis/directpv.min.io/v1beta1 v0.0.0-00010101000000-000000000000
 	github.com/nineinfra/nineinfra v0.5.0

go.sum

@@ -47,6 +47,8 @@ github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2Kv
 github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k=
 github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g=
 github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14=
+github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI=
+github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls=
 github.com/go-zookeeper/zk v1.0.1 h1:LmXNmSnkNsNKai+aDu6sHRr8ZJzIrHJo8z8Z4sm8cT8=