diff --git a/cmd/common.go b/cmd/common.go index 91b5e3f..f99ff22 100755 --- a/cmd/common.go +++ b/cmd/common.go @@ -430,6 +430,10 @@ func GenThriftSvcName(name string) string { return name + DefaultNineSuffix + "-kyuubi" } +func GenDorisSvcName(name string) string { + return name + DefaultNineSuffix + "-doris-fe-service" +} + func GenThriftServiceAccountName(name string) string { return name + DefaultNineSuffix + "-kyuubi" } @@ -446,6 +450,10 @@ func GetThriftIpAndPort(name string, ns string) (string, int32) { return GetSvcAccessInfo(GenThriftSvcName(name), DefaultThriftPortName, ns) } +func GetDorisIpAndPort(name string, ns string) (string, int32) { + return GetSvcAccessInfo(GenDorisSvcName(name), DefaultDorisPortName, ns) +} + func GetThriftPodName(name string, ns string) (string, error) { path, _ := rootCmd.Flags().GetString(kubeconfig) client, err := GetKubeClient(path) diff --git a/cmd/constants.go b/cmd/constants.go index 311c7a2..dda8f3c 100755 --- a/cmd/constants.go +++ b/cmd/constants.go @@ -46,7 +46,11 @@ const ( DefaultSparkUIName = "spark-ui" ) const ( - FeaturesOlapKey = "olap" + FeaturesOlapKey = "olap" + DefaultDorisPortName = "query-port" + DefaultDorisAdminUser = "root" + DefaultDorisAdminPassword = "" + DefaultDorisDatabaseName = "nineinfra" ) var ( @@ -99,6 +103,8 @@ var NineClusterProjectNameSuffix = map[string]string{ "metastore": "-nine-metastore", "minio": "-nine-ss-0", "postgresql": "-nine-pg", + "doris-fe": "-nine-doris-fe", + "doris-be": "-nine-doris-fe", } var NineClusterProjectWorkloadList = map[string]string{ @@ -106,6 +112,8 @@ var NineClusterProjectWorkloadList = map[string]string{ "metastore": "statefulset", "minio": "statefulset", "postgresql": "cluster", + "doris-fe": "statefulset", + "doris-be": "statefulset", } var NineToolList = map[string]interface{}{ diff --git a/cmd/helm.go b/cmd/helm.go index 4f3f954..9138fac 100755 --- a/cmd/helm.go +++ b/cmd/helm.go @@ -70,11 +70,15 @@ func InitHelm() error { return nil } -func HelmInstall(name string, repoName string, chart string, version string, namespace string, flags string) error { +func HelmInstall(name string, repoName string, chartPath string, chart string, version string, namespace string, flags string) error { if repoName == "" { - repoName = DefaultHelmRepoName + if chartPath != "" { + chart = chartPath + "/" + ChartName2TarName(chart) + } else { + repoName = DefaultHelmRepoName + chart = repoName + "/" + chart + } } - chart = repoName + "/" + chart if flags == "" { _, errput, err := runCommand("helm", "install", name, chart, "--version", version, "-n", namespace) if err != nil && !strings.Contains(errput, "in use") { @@ -90,11 +94,23 @@ func HelmInstall(name string, repoName string, chart string, version string, nam return nil } -func HelmInstallWithParameters(name string, repoName string, chart string, version string, namespace string, parameters ...string) error { +func ChartName2TarName(chart string) string { + if chartVersion, ok := DefaultToolsChartList[chart]; ok { + return fmt.Sprintf("%s-v%s.tar.gz", chart, chartVersion) + } + return "" +} + +func HelmInstallWithParameters(name string, repoName string, chartPath string, chart string, version string, namespace string, parameters ...string) error { if repoName == "" { - repoName = DefaultHelmRepoName + if chartPath != "" { + chart = chartPath + "/" + ChartName2TarName(chart) + } else { + repoName = DefaultHelmRepoName + chart = repoName + "/" + chart + } } - chart = repoName + "/" + chart + args := []string{"install", name, chart, "--version", version, "-n", namespace} args = append(args, parameters...) _, errput, err := runCommand("helm", args...) diff --git a/cmd/install.go b/cmd/install.go index 9fd119d..3bef059 100755 --- a/cmd/install.go +++ b/cmd/install.go @@ -17,9 +17,10 @@ const ( ) type operatorInstallCmd struct { - out io.Writer - errOut io.Writer - output bool + out io.Writer + errOut io.Writer + output bool + chartPath string } func newInstallCmd(out io.Writer, errOut io.Writer) *cobra.Command { @@ -43,6 +44,7 @@ func newInstallCmd(out io.Writer, errOut io.Writer) *cobra.Command { cmd = DisableHelp(cmd) f := cmd.Flags() f.BoolVar(&DEBUG, "debug", false, "print debug information") + f.StringVarP(&o.chartPath, "chart-path", "p", "", "local path of the charts") return cmd } @@ -68,7 +70,7 @@ func (o *operatorInstallCmd) run() error { } for c, v := range DefaultChartList { - err := HelmInstall(c, "", c, v, DefaultNamespace, flags) + err := HelmInstall(c, "", o.chartPath, c, v, DefaultNamespace, flags) if err != nil { fmt.Printf("Error: %v \n", err) os.Exit(1) diff --git a/cmd/tools.go b/cmd/tools.go index 2d23f6d..294138e 100755 --- a/cmd/tools.go +++ b/cmd/tools.go @@ -5,6 +5,7 @@ import ( "database/sql" "errors" "fmt" + _ "github.com/go-sql-driver/mysql" "github.com/spf13/cobra" "gopkg.in/yaml.v2" "io" @@ -45,6 +46,7 @@ type toolsCmd struct { nineName string toolkitArgs []string // --nodes flag deletePVC bool + chartPath string } type DatabasesConnection struct { @@ -89,6 +91,7 @@ func newToolsCmd(out io.Writer, errOut io.Writer) *cobra.Command { f.StringVar(&DefaultToolAirflowTag, "airflow-tag", "2.7.3", "airflow image tag") f.StringVarP(&DefaultStorageClass, "storage-pool", "s", "nineinfra-default", "storage pool fo tools") f.BoolVar(&c.deletePVC, "delete-pvc", false, "delete the ninecluster tools pvcs") + f.StringVarP(&c.chartPath, "chart-path", "p", "", "local path of the charts") f.StringVarP(&c.ns, "namespace", "n", "", "k8s namespace for tools") f.BoolVar(&DEBUG, "debug", false, "print debug information") return cmd @@ -151,6 +154,23 @@ func (t *toolsCmd) genSupersetSecretFile() error { return nil } +func (t *toolsCmd) createDorisDatabase(ip string, port int32, user string, password string) error { + connStr := fmt.Sprintf("%s@tcp(%s:%d)/", user, ip, port) + db, err := sql.Open("mysql", connStr) + if err != nil { + fmt.Printf("Error:%v\n", err) + return err + } + defer db.Close() + + _, err = db.Exec("CREATE DATABASE IF NOT EXISTS " + DefaultDorisDatabaseName) + if err != nil { + fmt.Printf("Error:%v\n", err) + return err + } + return nil +} + func (t *toolsCmd) genSupersetDataSourcesFile() error { thriftIP, thriftPort := GetThriftIpAndPort(t.nineName, t.ns) if thriftIP == "" || thriftPort == 0 { @@ -170,6 +190,21 @@ func (t *toolsCmd) genSupersetDataSourcesFile() error { }, }, } + dorisIP, dorisPort := GetDorisIpAndPort(t.nineName, t.ns) + if dorisIP != "" && dorisPort != 0 { + if err := t.createDorisDatabase(dorisIP, dorisPort, DefaultDorisAdminUser, DefaultDorisAdminPassword); err != nil { + return err + } + data["databases"] = append(data["databases"], DatabasesConnection{ + AllowFileUpload: true, + AllowCTAS: true, + AllowCVAS: true, + DatabaseName: DefaultDorisDatabaseName, + Extra: "{\r\n \"metadata_params\": {},\r\n \"engine_params\": {},\r\n \"metadata_cache_timeout\": {},\r\n \"schemas_allowed_for_file_upload\": []\r\n}", + SqlAlchemyURI: fmt.Sprintf("mysql://%s@%s:%d", DefaultDorisAdminUser, dorisIP, dorisPort), + Tables: []string{}, + }) + } yamlData, err := yaml.Marshal(&data) if err != nil { return err @@ -318,7 +353,7 @@ func (t *toolsCmd) createDatabase(tool string) error { func (t *toolsCmd) installRedis(parameters []string) error { relName := DefaultToolsNamePrefix + DefaultToolRedisName - err := HelmInstallWithParameters(relName, "", DefaultToolRedisName, DefaultToolsChartList[DefaultToolRedisName], t.ns, t.genRedisParameters(relName, parameters)...) + err := HelmInstallWithParameters(relName, "", t.chartPath, DefaultToolRedisName, DefaultToolsChartList[DefaultToolRedisName], t.ns, t.genRedisParameters(relName, parameters)...) if err != nil { return err } @@ -335,7 +370,7 @@ func (t *toolsCmd) installAirflow(parameters []string) error { return err } relName := DefaultToolsNamePrefix + DefaultToolAirflowName - err = HelmInstallWithParameters(relName, "", DefaultToolAirflowName, DefaultToolsChartList[DefaultToolAirflowName], t.ns, t.genAirflowParameters(relName, parameters)...) + err = HelmInstallWithParameters(relName, "", t.chartPath, DefaultToolAirflowName, DefaultToolsChartList[DefaultToolAirflowName], t.ns, t.genAirflowParameters(relName, parameters)...) if err != nil { return err } @@ -352,7 +387,7 @@ func (t *toolsCmd) installSuperset(parameters []string) error { return err } relName := DefaultToolsNamePrefix + DefaultToolSupersetName - err = HelmInstallWithParameters(relName, "", DefaultToolSupersetName, DefaultToolsChartList[DefaultToolSupersetName], t.ns, t.genSupersetParameters(relName, parameters)...) + err = HelmInstallWithParameters(relName, "", t.chartPath, DefaultToolSupersetName, DefaultToolsChartList[DefaultToolSupersetName], t.ns, t.genSupersetParameters(relName, parameters)...) if err != nil { return err } @@ -361,7 +396,7 @@ func (t *toolsCmd) installSuperset(parameters []string) error { func (t *toolsCmd) installZookeeper(parameters []string) error { relName := DefaultToolsNamePrefix + DefaultToolZookeeperName - err := HelmInstallWithParameters(relName, "", DefaultToolZookeeperName, DefaultToolsChartList[DefaultToolZookeeperName], t.ns, t.genZookeeperParameters(relName, parameters)...) + err := HelmInstallWithParameters(relName, "", t.chartPath, DefaultToolZookeeperName, DefaultToolsChartList[DefaultToolZookeeperName], t.ns, t.genZookeeperParameters(relName, parameters)...) if err != nil { return err } @@ -374,7 +409,7 @@ func (t *toolsCmd) installNifi(parameters []string) error { return err } relName := DefaultToolsNamePrefix + DefaultToolNifiName - err = HelmInstallWithParameters(relName, "", DefaultToolNifiName, DefaultToolsChartList[DefaultToolNifiName], t.ns, t.genNifiParameters(relName, parameters)...) + err = HelmInstallWithParameters(relName, "", t.chartPath, DefaultToolNifiName, DefaultToolsChartList[DefaultToolNifiName], t.ns, t.genNifiParameters(relName, parameters)...) if err != nil { return err } diff --git a/go.mod b/go.mod index 6cb2c99..c2378f2 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/beltran/gohive v1.6.0 github.com/cloudnative-pg/client v0.0.0-00010101000000-000000000000 github.com/cloudnative-pg/cloudnative-pg v1.21.1 + github.com/go-sql-driver/mysql v1.7.1 github.com/manifoldco/promptui v0.9.0 github.com/minio/directpv/apis/directpv.min.io/v1beta1 v0.0.0-00010101000000-000000000000 github.com/nineinfra/nineinfra v0.5.0 diff --git a/go.sum b/go.sum index 78b5778..4de825c 100644 --- a/go.sum +++ b/go.sum @@ -47,6 +47,8 @@ github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2Kv github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= +github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/go-zookeeper/zk v1.0.1 h1:LmXNmSnkNsNKai+aDu6sHRr8ZJzIrHJo8z8Z4sm8cT8=