diff --git a/bin/alluxio b/bin/alluxio index d6b596610a63..0806b5d221b4 100755 --- a/bin/alluxio +++ b/bin/alluxio @@ -10,8 +10,20 @@ # See the NOTICE file distributed with this work for information regarding copyright ownership. # +set -eu -BIN=$(cd "$( dirname "$( readlink "$0" || echo "$0" )" )"; pwd) +BIN_DIR=$(cd "$( dirname "$( readlink "$0" || echo "$0" )" )"; pwd) +ROOT_PATH="${BIN_DIR}/.." +# run the compiled go binary according to the system OS and arch +CLI_PATH="${ROOT_PATH}/cli/src/alluxio.org/cli/bin/alluxioCli-$(uname)-$(uname -m)" -# temporary placeholder that redirects to alluxio-bash to prepare for golangCli branch merge -"${BIN}/alluxio-bash" "$@" +# if pom.xml exists, assume development environment and compile before running +if [[ -f "${ROOT_PATH}/pom.xml" ]]; then + "${ROOT_PATH}/build/cli/build-cli.sh" + "${CLI_PATH}" --rootPath="${ROOT_PATH}" "$@" + exit +fi + +# otherwise, assume deployed environment which will have binaries precompiled +# run with the flag to point to assembly jars in their deployed locations +"${CLI_PATH}" --rootPath="${ROOT_PATH}" --deployed-env "$@" diff --git a/bin/alluxio-start.sh b/bin/alluxio-start.sh index 02a83db8c237..7af95c94e071 100755 --- a/bin/alluxio-start.sh +++ b/bin/alluxio-start.sh @@ -12,5 +12,16 @@ BIN=$(cd "$( dirname "$( readlink "$0" || echo "$0" )" )"; pwd) -# temporary placeholder that redirects to alluxio-start-bash.sh to prepare for golangCli branch merge -"${BIN}/alluxio-start-bash.sh" "$@" +echo "The alluxio-start.sh script is deprecated. Use the \"bin/alluxio process start\" command to start processes." + +# while the leading argument starts with a -, continue to parse flags +# this will skip the mount related arguments that are no longer relevant and would cause an error to the new start command +startArgs=() +for arg in "$@"; do + startArgs+=("${arg}") + if [[ "${arg}" != -* ]]; then + break + fi +done + +"${BIN}"/alluxio process start "${startArgs[@]}" diff --git a/bin/alluxio-stop.sh b/bin/alluxio-stop.sh index 99d41babba15..53ff541e87ff 100755 --- a/bin/alluxio-stop.sh +++ b/bin/alluxio-stop.sh @@ -12,5 +12,6 @@ BIN=$(cd "$( dirname "$( readlink "$0" || echo "$0" )" )"; pwd) -# temporary placeholder that redirects to alluxio-stop-bash.sh to prepare for golangCli branch merge -"${BIN}/alluxio-stop-bash.sh" "$@" +echo "The alluxio-stop.sh script is deprecated. Use the \"bin/alluxio process stop\" command to stop processes." + +"${BIN}"/alluxio process stop "$@" diff --git a/build/cli/build-cli.sh b/build/cli/build-cli.sh new file mode 100755 index 000000000000..b0ba852c6622 --- /dev/null +++ b/build/cli/build-cli.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# +# The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 +# (the "License"). You may not use this work except in compliance with the License, which is +# available at www.apache.org/licenses/LICENSE-2.0 +# +# This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, +# either express or implied, as more fully set forth in the License. +# +# See the NOTICE file distributed with this work for information regarding copyright ownership. +# + +set -eu + +CLI_DIR=$(cd "$( dirname "$( readlink "$0" || echo "$0" )" )"; pwd) + +# mapping of GOOS/GOARCH value to uname value +declare -A ALL_OS +ALL_OS["linux"]="Linux" +ALL_OS["darwin"]="Darwin" +declare -A ALL_ARCH +ALL_ARCH["amd64"]="x86_64" +ALL_ARCH["arm64"]="aarch64" + +MAIN_PATH="cli/main.go" +USAGE="Usage: build-cli.sh [-a] +-a Build executables for all OS and architecture combinations +" + +main() { + build_all="false" + while getopts "a" opt; do + case "${opt}" in + a) + build_all="true" + ;; + *) + echo -e "${USAGE}" >&2 + exit 1 + ;; + esac + done + + cliBinDir="${CLI_DIR}/../../cli/src/alluxio.org/cli/bin" + mkdir -p "${cliBinDir}" + + cd "${CLI_DIR}/../../cli/src/alluxio.org/" + + if [[ ${build_all} == "false" ]]; then + GO111MODULE=on go build -o "${cliBinDir}/alluxioCli-$(uname)-$(uname -m)" "${MAIN_PATH}" + else + for osKey in "${!ALL_OS[@]}"; do + for archKey in "${!ALL_ARCH[@]}"; do + echo "Building executable for ${osKey} ${archKey}" + GO111MODULE=on GOOS="${osKey}" GOARCH="${archKey}" go build -o "${cliBinDir}/alluxioCli-${ALL_OS[$osKey]}-${ALL_ARCH[$archKey]}" "${MAIN_PATH}" + done + done + fi +} + +main "$@" diff --git a/cli/README.md b/cli/README.md new file mode 100644 index 000000000000..78c2354be1e6 --- /dev/null +++ b/cli/README.md @@ -0,0 +1,58 @@ +# Alluxio CLI + +The Alluxio command line interface is the single entrypoint for users to: +- Initialize and configure the Alluxio cluster +- Start and stop processes +- Expose information about the running cluster +- Interact with the filesystem, running commands such as `ls` and `cp` +- Perform administrator level actions such as format or backup + +The CLI is invoked through the shell script at `bin/cli.sh`. +Commands follow the format of: +```console +bin/cli.sh [--[=]] [] +``` + +Add the `-h` flag to view more details regarding a service or operation. + +## Layout and naming conventions + +The choice of names for services, operations, and flags should be succinct: short and unambiguous. +Use of a single word is strongly preferred, but otherwise the name parts should be delimited by a dash such as `foo-bar`. + +For example, let's assume there is a `mark` operation as part of a `item` service that can be `set` or `unset` on an item `name`. +The recommended format for a command is +```console +bin/cli.sh item mark --set name +bin/cli.sh item mark --unset name +``` +where it is expected that either `--set` or `--unset` are specified. +This is preferred over the alternative of two separate commands with `setMark` and `unsetMark` as the operations. + +## User input + +### Flags and arguments +After selecting the desired command, additional user input can be parsed, as a mix of arguments and/or flags: +- Arguments: `bin/cli.sh command arg1 arg2 ...` +- Flags: `bin/cli.sh command --flag1 --flag2 val2 ...` + +Flags are strictly preferred over arguments. +- The flag name conveys context; an argument does not +- Arguments must be ordered; flags can be declared arbitrarily +- Flags can be designated as required to ensure user input. +- Repeated flags can be defined to capture an ordered list of inputs, ex. `--target target1 --target target2` + +### Input validation + +User inputs should be validated by the CLI command as much as possible as opposed to the resulting invocation. + +## Output conventions and java invocation + +A majority of commands result in invoking a java class with arguments to execute the expected operation and possibly return some output. +The output returned from the java invocation should tend towards being plain or machine parseable, such as a JSON formatted string, +rather than terminal friendly or human readable format. +When appropriate, the CLI command will default to formatting this output to be terminal friendly, with an option to output in a machine parseable format. + +## References + +Github CLI guidelines: https://primer.style/design/native/cli/foundations diff --git a/cli/src/alluxio.org/cli/bin/.gitignore b/cli/src/alluxio.org/cli/bin/.gitignore new file mode 100644 index 000000000000..33a525764e24 --- /dev/null +++ b/cli/src/alluxio.org/cli/bin/.gitignore @@ -0,0 +1 @@ +alluxioCli* diff --git a/cli/src/alluxio.org/cli/cmd/cache/cache.go b/cli/src/alluxio.org/cli/cmd/cache/cache.go new file mode 100644 index 000000000000..de2e8b3916ce --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/cache/cache.go @@ -0,0 +1,25 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package cache + +import ( + "alluxio.org/cli/env" +) + +var Service = &env.Service{ + Name: "cache", + Description: "Worker-related file system and format operations.", + Commands: []env.Command{ + Format, + Free, + }, +} diff --git a/cli/src/alluxio.org/cli/cmd/cache/format.go b/cli/src/alluxio.org/cli/cmd/cache/format.go new file mode 100644 index 000000000000..7a090dbd36c8 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/cache/format.go @@ -0,0 +1,53 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package cache + +import ( + "fmt" + + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var Format = &FormatCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "format", + JavaClassName: "alluxio.cli.Format", + ShellJavaOpts: fmt.Sprintf(env.JavaOptFormat, env.ConfAlluxioLoggerType, "Console"), + }, +} + +type FormatCommand struct { + *env.BaseJavaCommand +} + +func (c *FormatCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *FormatCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: Format.CommandName, + Short: "Format Alluxio worker nodes.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + return cmd +} + +func (c *FormatCommand) Run(args []string) error { + javaArgs := []string{"worker"} + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/cache/free.go b/cli/src/alluxio.org/cli/cmd/cache/free.go new file mode 100644 index 000000000000..d3c0c37376fe --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/cache/free.go @@ -0,0 +1,80 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package cache + +import ( + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + + "alluxio.org/cli/cmd/names" + "alluxio.org/cli/env" +) + +var Free = &FreeCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "free", + JavaClassName: names.FileSystemShellJavaClass, + }, +} + +type FreeCommand struct { + *env.BaseJavaCommand + worker string + path string + force bool +} + +func (c *FreeCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *FreeCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: Free.CommandName, + Short: "Synchronously free all blocks and directories of specific worker, " + + "or free the space occupied by a file or a directory in Alluxio", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + cmd.Flags().StringVar(&c.worker, "worker", "", "The worker to free") + cmd.Flags().StringVar(&c.path, "path", "", "The file or directory to free") + cmd.Flags().BoolVarP(&c.force, "force", "f", false, + "Force freeing pinned files in the directory") + cmd.MarkFlagsMutuallyExclusive("worker", "path") + return cmd +} + +func (c *FreeCommand) Run(args []string) error { + var javaArgs []string + if c.worker == "" { + if c.path != "" { + // free directory + javaArgs = append(javaArgs, "free") + if c.force { + javaArgs = append(javaArgs, "-f") + } + javaArgs = append(javaArgs, c.path) + } else { + return stacktrace.NewError("neither worker nor path to free specified") + } + } else { + if c.path == "" { + // free workers + javaArgs = append(javaArgs, "freeWorker", c.worker) + } else { + return stacktrace.NewError("both worker and path to free specified") + } + } + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/conf/conf.go b/cli/src/alluxio.org/cli/cmd/conf/conf.go new file mode 100644 index 000000000000..1c42c088d4e4 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/conf/conf.go @@ -0,0 +1,25 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package conf + +import ( + "alluxio.org/cli/env" +) + +var Service = &env.Service{ + Name: "conf", + Description: "Get, set, and validate configuration settings, primarily those defined in conf/alluxio-site.properties", + Commands: []env.Command{ + Get, + Log, + }, +} diff --git a/cli/src/alluxio.org/cli/cmd/conf/get.go b/cli/src/alluxio.org/cli/cmd/conf/get.go new file mode 100644 index 000000000000..717f75a447c9 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/conf/get.go @@ -0,0 +1,91 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package conf + +import ( + "bytes" + "fmt" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + + "alluxio.org/cli/env" + "alluxio.org/log" +) + +var Get = &GetCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "get", + JavaClassName: "alluxio.cli.GetConf", + ShellJavaOpts: fmt.Sprintf(env.JavaOptFormat, env.ConfAlluxioConfValidationEnabled, false), + }, +} + +type GetCommand struct { + *env.BaseJavaCommand + + ShowMaster bool + ShowSource bool + Unit string +} + +func (c *GetCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *GetCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: fmt.Sprintf("%v [key]", Get.CommandName), + Short: "Look up a configuration value by its property key or print all configuration if no key is provided", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + cmd.Flags().BoolVar(&c.ShowMaster, "master", false, "Show configuration properties used by the master") + cmd.Flags().BoolVar(&c.ShowSource, "source", false, "Show source of the configuration property instead of the value") + cmd.Flags().StringVar(&c.Unit, "unit", "", + `Unit of the value to return, converted to correspond to the given unit. +E.g., with "--unit KB", a configuration value of "4096B" will return 4 +Possible options include B, KB, MB, GB, TP, PB, MS, S, M, H, D`) + return cmd +} + +func (c *GetCommand) Run(args []string) error { + var javaArgs []string + if c.ShowMaster { + javaArgs = append(javaArgs, "--master") + } + if c.ShowSource { + javaArgs = append(javaArgs, "--source") + } + if c.Unit != "" { + javaArgs = append(javaArgs, "--unit", c.Unit) + } + javaArgs = append(javaArgs, args...) + + return c.Base().Run(javaArgs) +} + +func (c *GetCommand) FetchValue(key string) (string, error) { + cmd := c.RunJavaClassCmd([]string{key}) + + errBuf := &bytes.Buffer{} + cmd.Stderr = errBuf + + log.Logger.Debugln(cmd.String()) + out, err := cmd.Output() + if err != nil { + return "", stacktrace.Propagate(err, "error getting conf for %v\nstderr: %v", key, errBuf.String()) + } + return string(out), nil +} diff --git a/cli/src/alluxio.org/cli/cmd/conf/log.go b/cli/src/alluxio.org/cli/cmd/conf/log.go new file mode 100644 index 000000000000..7f5521850bd0 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/conf/log.go @@ -0,0 +1,68 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package conf + +import ( + "strings" + + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var Log = &LogCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "log", + JavaClassName: "alluxio.cli.LogLevel", + }, +} + +type LogCommand struct { + *env.BaseJavaCommand + + LogName string + Level string + Targets []string +} + +func (c *LogCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *LogCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: Log.CommandName, + Short: "Get or set the log level for the specified logger", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + const name = "name" + cmd.Flags().StringVar(&c.LogName, name, "", "Logger name (ex. alluxio.master.file.DefaultFileSystemMaster)") + cmd.MarkFlagRequired(name) + cmd.Flags().StringVar(&c.Level, "level", "", "If specified, sets the specified logger at the given level") + cmd.Flags().StringSliceVar(&c.Targets, "target", nil, "A target name among . Defaults to master,workers,job_master,job_workers") + return cmd +} + +func (c *LogCommand) Run(_ []string) error { + javaArgs := []string{"--logName", c.LogName} + if c.Level != "" { + javaArgs = append(javaArgs, "--level", c.Level) + } + if len(c.Targets) > 0 { + javaArgs = append(javaArgs, "--target", strings.Join(c.Targets, ",")) + } + + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/exec/class.go b/cli/src/alluxio.org/cli/cmd/exec/class.go new file mode 100644 index 000000000000..c4d4edfeb892 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/exec/class.go @@ -0,0 +1,70 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package exec + +import ( + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var Class = &ClassCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "class", + }, +} + +type ClassCommand struct { + *env.BaseJavaCommand + mainClass string + jarFile string + module string +} + +func (c *ClassCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *ClassCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: "class", + Short: "Run the main method of an Alluxio class.", + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + cmd.Flags().StringVar(&c.jarFile, "jar", "", + "Determine a JAR file to run.") + cmd.Flags().StringVar(&c.module, "m", "", + "Determine a module to run.") + cmd.MarkFlagsMutuallyExclusive("jar", "m") + return cmd +} + +func (c *ClassCommand) Run(args []string) error { + var javaArgs []string + if c.jarFile != "" { + javaArgs = append(javaArgs, "-jar", c.jarFile) + } else if c.module != "" { + javaArgs = append(javaArgs, "-m", c.module) + } else if len(args) != 0 { + c.JavaClassName = args[0] + } else { + return stacktrace.Propagate(nil, "None of JAR, module, nor a java class is specified") + } + + if len(args) > 1 { + javaArgs = append(javaArgs, args[1:]...) + } + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/exec/exec.go b/cli/src/alluxio.org/cli/cmd/exec/exec.go new file mode 100644 index 000000000000..ae661da7df37 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/exec/exec.go @@ -0,0 +1,27 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package exec + +import ( + "alluxio.org/cli/env" +) + +var Service = &env.Service{ + Name: "exec", + Description: "Run the main method of an Alluxio class, or end-to-end tests on an Alluxio cluster.", + Commands: []env.Command{ + Class, + TestHdfsMount, + TestRun, + TestUfs, + }, +} diff --git a/cli/src/alluxio.org/cli/cmd/exec/test_basic_io.go b/cli/src/alluxio.org/cli/cmd/exec/test_basic_io.go new file mode 100644 index 000000000000..6899c21fac8c --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/exec/test_basic_io.go @@ -0,0 +1,88 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package exec + +import ( + "fmt" + + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var TestRun = &TestRunCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "basicIOTest", + JavaClassName: "alluxio.cli.TestRunner", + ShellJavaOpts: fmt.Sprintf(env.JavaOptFormat, env.ConfAlluxioLoggerType, "Console"), + }, +} + +type TestRunCommand struct { + *env.BaseJavaCommand + directory string + operation string + readType string + workers string + writeType string +} + +func (c *TestRunCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *TestRunCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: "basicIOTest", + Args: cobra.NoArgs, + Short: "Run all end-to-end tests, or a specific test, on an Alluxio cluster.", + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + cmd.Flags().StringVar(&c.directory, "directory", "/", + "Alluxio path for the tests working directory. Default: /") + cmd.Flags().StringVar(&c.operation, "operation", "", + "The operation to test, either BASIC or BASIC_NON_BYTE_BUFFER. \n"+ + "By default both operations are tested.") + cmd.Flags().StringVar(&c.readType, "readType", "", + "The read type to use, one of NO_CACHE, CACHE, CACHE_PROMOTE. \n"+ + "By default all readTypes are tested.") + cmd.Flags().StringVar(&c.workers, "workers", "", + "Alluxio worker addresses to run tests on. \n"+ + "If not specified, random ones will be used.") + cmd.Flags().StringVar(&c.writeType, "writeType", "", + "The write type to use, one of MUST_CACHE, CACHE_THROUGH, THROUGH, ASYNC_THROUGH. \n"+ + "By default all writeTypes are tested.") + return cmd +} + +func (c *TestRunCommand) Run(args []string) error { + var javaArgs []string + if c.directory != "" { + javaArgs = append(javaArgs, "--directory", c.directory) + } + if c.operation != "" { + javaArgs = append(javaArgs, "--operation", c.operation) + } + if c.readType != "" { + javaArgs = append(javaArgs, "--readType", c.readType) + } + if c.workers != "" { + javaArgs = append(javaArgs, "--workers", c.workers) + } + if c.writeType != "" { + javaArgs = append(javaArgs, "--writeType", c.writeType) + } + + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/exec/test_hdfs_mount.go b/cli/src/alluxio.org/cli/cmd/exec/test_hdfs_mount.go new file mode 100644 index 000000000000..470e96372efe --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/exec/test_hdfs_mount.go @@ -0,0 +1,74 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package exec + +import ( + "fmt" + + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var TestHdfsMount = &TestHdfsMountCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "hdfsMountTest", + JavaClassName: "alluxio.cli.ValidateHdfsMount", + ShellJavaOpts: fmt.Sprintf(env.JavaOptFormat, env.ConfAlluxioLoggerType, "Console"), + }, +} + +type TestHdfsMountCommand struct { + *env.BaseJavaCommand + path string + readonly bool + shared bool + option string +} + +func (c *TestHdfsMountCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *TestHdfsMountCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: "hdfsMountTest", + Args: cobra.NoArgs, + Short: "Tests runs a set of validations against the given hdfs path.", + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + const path = "path" + cmd.Flags().StringVar(&c.path, path, "", "specifies the HDFS path you want to validate.") + cmd.MarkFlagRequired(path) + cmd.Flags().BoolVar(&c.readonly, "readonly", false, "mount point is readonly in Alluxio.") + cmd.Flags().BoolVar(&c.shared, "shared", false, "mount point is shared.") + cmd.Flags().StringVar(&c.option, "option", "", "options associated with this mount point.") + return cmd +} + +func (c *TestHdfsMountCommand) Run(args []string) error { + var javaArgs []string + javaArgs = append(javaArgs, c.path) + if c.readonly { + javaArgs = append(javaArgs, "--readonly") + } + if c.shared { + javaArgs = append(javaArgs, "--shared") + } + if c.option != "" { + javaArgs = append(javaArgs, "--option", c.option) + } + + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/exec/test_ufs.go b/cli/src/alluxio.org/cli/cmd/exec/test_ufs.go new file mode 100644 index 000000000000..bf14b18013fb --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/exec/test_ufs.go @@ -0,0 +1,61 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package exec + +import ( + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var TestUfs = &TestUfsCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "ufsTest", + JavaClassName: "alluxio.cli.UnderFileSystemContractTest", + }, +} + +type TestUfsCommand struct { + *env.BaseJavaCommand + path string + test []string +} + +func (c *TestUfsCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *TestUfsCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: "ufsTest", + Args: cobra.NoArgs, + Short: "Test the integration between Alluxio and the given UFS to validate UFS semantics", + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + const path = "path" + cmd.Flags().StringVar(&c.path, path, "", "the full UFS path to run tests against.") + cmd.MarkFlagRequired(path) + cmd.Flags().StringSliceVar(&c.test, "test", nil, "Test name, this option can be passed multiple times to indicate multipleZ tests") + + return cmd +} + +func (c *TestUfsCommand) Run(args []string) error { + javaArgs := []string{"--path", c.path} + for _, singleTest := range c.test { + javaArgs = append(javaArgs, "--test", singleTest) + } + + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/fs/cat.go b/cli/src/alluxio.org/cli/cmd/fs/cat.go new file mode 100644 index 000000000000..c47f8aa6a81e --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/fs/cat.go @@ -0,0 +1,52 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package fs + +import ( + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +func Cat(className string) env.Command { + return &CatCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "cat", + JavaClassName: className, + Parameters: []string{"cat"}, + }, + } +} + +type CatCommand struct { + *env.BaseJavaCommand +} + +func (c *CatCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *CatCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: "cat [path]", + Short: "Print specified file's content", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + return cmd +} + +func (c *CatCommand) Run(args []string) error { + return c.Base().Run(args) +} diff --git a/cli/src/alluxio.org/cli/cmd/fs/checksum.go b/cli/src/alluxio.org/cli/cmd/fs/checksum.go new file mode 100644 index 000000000000..2e674bac0da9 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/fs/checksum.go @@ -0,0 +1,52 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package fs + +import ( + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +func Checksum(className string) env.Command { + return &ChecksumCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "checksum", + JavaClassName: className, + Parameters: []string{"checksum"}, + }, + } +} + +type ChecksumCommand struct { + *env.BaseJavaCommand +} + +func (c *ChecksumCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *ChecksumCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: "checksum [path]", + Short: "Calculates the md5 checksum of a specified file", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + return cmd +} + +func (c *ChecksumCommand) Run(args []string) error { + return c.Base().Run(args) +} diff --git a/cli/src/alluxio.org/cli/cmd/fs/cp.go b/cli/src/alluxio.org/cli/cmd/fs/cp.go new file mode 100644 index 000000000000..8a1120c9dbc2 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/fs/cp.go @@ -0,0 +1,119 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package fs + +import ( + "strconv" + "strings" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +func Cp(className string) env.Command { + return &CopyCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "cp", + JavaClassName: className, + }, + } +} + +type CopyCommand struct { + *env.BaseJavaCommand + + bufferSize string + isRecursive bool + preserve bool + threads int +} + +func (c *CopyCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *CopyCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: "cp [srcPath] [dstPath]", + Short: "Copy a file or directory", + Long: `Copies a file or directory in the Alluxio filesystem or between local and Alluxio filesystems +Use the file:// schema to indicate a local filesystem path (ex. file:///absolute/path/to/file) and +use the recursive flag to copy directories`, + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + cmd.Flags().StringVar(&c.bufferSize, "buffer-size", "", "Read buffer size when coping to or from local, with defaults of 64MB and 8MB respectively") + cmd.Flags().BoolVarP(&c.isRecursive, "recursive", "R", false, "True to copy the directory subtree to the destination directory") + cmd.Flags().BoolVarP(&c.preserve, "preserve", "p", false, "Preserve file permission attributes when copying files; all ownership, permissions, and ACLs will be preserved") + cmd.Flags().IntVar(&c.threads, "thread", 0, "Number of threads used to copy files in parallel, defaults to 2 * CPU cores") + return cmd +} + +const ( + localScheme = "file://" +) + +func (c *CopyCommand) Run(args []string) error { + src, dst := args[0], args[1] + if strings.HasPrefix(src, localScheme) && strings.HasPrefix(dst, localScheme) { + return stacktrace.NewError("both src and dst paths are local file paths") + } + var cmd string + if strings.HasPrefix(src, localScheme) { + cmd = "copyFromLocal" + if c.preserve { + return stacktrace.NewError("cannot set preserve flag when copying from local file path") + } + if c.isRecursive { + return stacktrace.NewError("cannot set recursive flag when copying from local file path") + } + } else if strings.HasPrefix(dst, localScheme) { + cmd = "copyToLocal" + if c.preserve { + return stacktrace.NewError("cannot set preserve flag when copying to local file path") + } + if c.isRecursive { + return stacktrace.NewError("cannot set recursive flag when copying to local file path") + } + if c.threads != 0 { + return stacktrace.NewError("cannot set thread flag when copying to local file path") + } + } else { + cmd = "cp" + if c.bufferSize != "" { + return stacktrace.NewError("can only set buffer-size flag when copying to or from a local file path") + } + } + if c.threads < 0 { + return stacktrace.NewError("thread value must be positive but was %v", c.threads) + } + + javaArgs := []string{cmd} + if c.bufferSize != "" { + javaArgs = append(javaArgs, "--buffersize", c.bufferSize) + } + if c.isRecursive { + javaArgs = append(javaArgs, "--recursive") + } + if c.preserve { + javaArgs = append(javaArgs, "--preserve") + } + if c.threads != 0 { + javaArgs = append(javaArgs, "--thread", strconv.Itoa(c.threads)) + } + javaArgs = append(javaArgs, args...) + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/fs/fs.go b/cli/src/alluxio.org/cli/cmd/fs/fs.go new file mode 100644 index 000000000000..d2d067cff7d0 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/fs/fs.go @@ -0,0 +1,45 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package fs + +import ( + "alluxio.org/cli/cmd/names" + "alluxio.org/cli/env" +) + +var Service = &env.Service{ + Name: "fs", + Description: "Operations to interface with the Alluxio filesystem", + Commands: Cmds(names.FileSystemShellJavaClass), +} + +func Cmds(className string) []env.Command { + var ret []env.Command + for _, c := range []func(string) env.Command{ + Cat, + Checksum, + Cp, + Head, + Ls, + Mkdir, + Mv, + Rm, + Stat, + Tail, + Test, + Touch, + } { + ret = append(ret, c(className)) + } + + return ret +} diff --git a/cli/src/alluxio.org/cli/cmd/fs/head.go b/cli/src/alluxio.org/cli/cmd/fs/head.go new file mode 100644 index 000000000000..a96af645c108 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/fs/head.go @@ -0,0 +1,60 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package fs + +import ( + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +func Head(className string) env.Command { + return &HeadCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "head", + JavaClassName: className, + Parameters: []string{"head"}, + }, + } +} + +type HeadCommand struct { + *env.BaseJavaCommand + + bytes string +} + +func (c *HeadCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *HeadCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: "head [path]", + Short: "Print the leading bytes from the specified file", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + cmd.Flags().StringVar(&c.bytes, "bytes", "", "Byte size to print") + return cmd +} + +func (c *HeadCommand) Run(args []string) error { + var javaArgs []string + if c.bytes != "" { + javaArgs = append(javaArgs, "-c", c.bytes) + } + javaArgs = append(javaArgs, args...) + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/fs/ls.go b/cli/src/alluxio.org/cli/cmd/fs/ls.go new file mode 100644 index 000000000000..7966107cdd4a --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/fs/ls.go @@ -0,0 +1,140 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package fs + +import ( + "fmt" + "strings" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +func Ls(className string) env.Command { + return &LsCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "ls", + JavaClassName: className, + Parameters: []string{"ls"}, + }, + } +} + +type LsCommand struct { + *env.BaseJavaCommand + + listDirAsFile bool + forceLoadMetadata bool + isHumanReadable bool + omitMountInfo bool + pinnedFileOnly bool + isRecursive bool + isReverse bool + sortBy string + timestamp string +} + +func (c *LsCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +var ( + sortOptions = []string{ + "creationTime", + "inMemoryPercentage", + "lastAccessTime", + "lastModificationTime", + "name", + "path", + "size", + } + timestampOptions = []string{ + "createdTime", + "lastAccessTime", + "lastModifiedTime", + } +) + +func (c *LsCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: "ls [path]", + Short: "Prints information for files and directories at the given path", + Long: `Displays information for all files and directories directly under the specified paths, including permission, owner, group, size (bytes for files or the number of children for directories), persistence state, last modified time, the percentage of content already in Alluxio, and the path`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + // special case to overwrite the -h shorthand for --help flag for --human-readable + cmd.PersistentFlags().BoolP("help", "", false, "help for this command") + + cmd.Flags().BoolVarP(&c.listDirAsFile, "list-dir-as-file", "d", false, "List directories as files") + cmd.Flags().BoolVarP(&c.forceLoadMetadata, "load-metadata", "f", false, "Force load metadata for immediate children in a directory") + cmd.Flags().BoolVarP(&c.isHumanReadable, "human-readable", "h", false, "Print sizes in human readable format") + cmd.Flags().BoolVarP(&c.omitMountInfo, "omit-mount-info", "m", false, "Omit mount point related information such as the UFS path") + cmd.Flags().BoolVarP(&c.pinnedFileOnly, "pinned-files", "p", false, "Only show pinned files") + cmd.Flags().BoolVarP(&c.isRecursive, "recursive", "R", false, "List subdirectories recursively") + cmd.Flags().BoolVarP(&c.isReverse, "reverse", "r", false, "Reverse sorted order") + cmd.Flags().StringVar(&c.sortBy, "sort", "", fmt.Sprintf("Sort entries by column, one of {%v}", strings.Join(sortOptions, "|"))) + cmd.Flags().StringVar(&c.timestamp, "timestamp", "", fmt.Sprintf("Display specified timestamp of entry, one of {%v}", strings.Join(timestampOptions, "|"))) + return cmd +} + +func (c *LsCommand) Run(args []string) error { + var javaArgs []string + if c.listDirAsFile { + javaArgs = append(javaArgs, "-d") + } + if c.forceLoadMetadata { + javaArgs = append(javaArgs, "-f") + } + if c.isHumanReadable { + javaArgs = append(javaArgs, "-h") + } + if c.omitMountInfo { + javaArgs = append(javaArgs, "-m") + } + if c.pinnedFileOnly { + javaArgs = append(javaArgs, "-p") + } + if c.isRecursive { + javaArgs = append(javaArgs, "-R") + } + if c.isReverse { + javaArgs = append(javaArgs, "-r") + } + if c.sortBy != "" { + if err := checkAllowed(c.sortBy, sortOptions...); err != nil { + return stacktrace.Propagate(err, "error validating sort options") + } + javaArgs = append(javaArgs, "--sort", c.sortBy) + } + if c.timestamp != "" { + if err := checkAllowed(c.timestamp, timestampOptions...); err != nil { + return stacktrace.Propagate(err, "error validating timestamp options") + } + javaArgs = append(javaArgs, "--timestamp", c.timestamp) + } + javaArgs = append(javaArgs, args...) + return c.Base().Run(javaArgs) +} + +func checkAllowed(val string, allowed ...string) error { + for _, i := range allowed { + if val == i { + return nil + } + } + return stacktrace.NewError("value %v must be one of %v", val, strings.Join(allowed, ",")) +} diff --git a/cli/src/alluxio.org/cli/cmd/fs/mkdir.go b/cli/src/alluxio.org/cli/cmd/fs/mkdir.go new file mode 100644 index 000000000000..8a7ab4da9769 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/fs/mkdir.go @@ -0,0 +1,52 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package fs + +import ( + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +func Mkdir(className string) env.Command { + return &MkdirCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "mkdir", + JavaClassName: className, + Parameters: []string{"mkdir"}, + }, + } +} + +type MkdirCommand struct { + *env.BaseJavaCommand +} + +func (c *MkdirCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *MkdirCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: "mkdir [path1 path2 ...]", + Short: "Create directories at the specified paths, creating the parent directory if not exists", + Args: cobra.MinimumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + return cmd +} + +func (c *MkdirCommand) Run(args []string) error { + return c.Base().Run(args) +} diff --git a/cli/src/alluxio.org/cli/cmd/fs/mv.go b/cli/src/alluxio.org/cli/cmd/fs/mv.go new file mode 100644 index 000000000000..3bd7d6ca8141 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/fs/mv.go @@ -0,0 +1,52 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package fs + +import ( + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +func Mv(className string) env.Command { + return &MoveCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "mv", + JavaClassName: className, + Parameters: []string{"mv"}, + }, + } +} + +type MoveCommand struct { + *env.BaseJavaCommand +} + +func (c *MoveCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *MoveCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: "mv [srcPath] [dstPath]", + Short: "Rename a file or directory", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + return cmd +} + +func (c *MoveCommand) Run(args []string) error { + return c.Base().Run(args) +} diff --git a/cli/src/alluxio.org/cli/cmd/fs/rm.go b/cli/src/alluxio.org/cli/cmd/fs/rm.go new file mode 100644 index 000000000000..3b10974ad8d3 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/fs/rm.go @@ -0,0 +1,75 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package fs + +import ( + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +func Rm(className string) env.Command { + return &RmCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "rm", + JavaClassName: className, + Parameters: []string{"rm"}, + }, + } +} + +type RmCommand struct { + *env.BaseJavaCommand + + alluxioOnly bool + deleteMount bool + isRecursive bool + skipUfsCheck bool +} + +func (c *RmCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *RmCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: "rm [path]", + Short: "Remove the specified file", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + cmd.Flags().BoolVar(&c.alluxioOnly, "alluxio-only", false, "True to only remove data and metadata from Alluxio cache") + cmd.Flags().BoolVarP(&c.deleteMount, "delete-mount", "m", false, "True to remove mount points within the specified directory subtree, which would otherwise cause failures") + cmd.Flags().BoolVarP(&c.isRecursive, "recursive", "R", false, "True to recursively remove files within the specified directory subtree") + cmd.Flags().BoolVarP(&c.skipUfsCheck, "skip-ufs-check", "U", false, "True to skip checking if corresponding UFS contents are in sync") + return cmd +} + +func (c *RmCommand) Run(args []string) error { + var javaArgs []string + if c.alluxioOnly { + javaArgs = append(javaArgs, "--alluxioOnly") + } + if c.deleteMount { + javaArgs = append(javaArgs, "--deleteMountPoint") + } + if c.isRecursive { + javaArgs = append(javaArgs, "-R") + } + if c.skipUfsCheck { + javaArgs = append(javaArgs, "-U") + } + javaArgs = append(javaArgs, args...) + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/fs/stat.go b/cli/src/alluxio.org/cli/cmd/fs/stat.go new file mode 100644 index 000000000000..1248fc2601b2 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/fs/stat.go @@ -0,0 +1,84 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package fs + +import ( + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +func Stat(className string) env.Command { + return &StatCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "stat", + JavaClassName: className, + Parameters: []string{"stat"}, + }, + } +} + +type StatCommand struct { + *env.BaseJavaCommand + + Path string + FileId string + Format string +} + +func (c *StatCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *StatCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: "stat", + Short: "Displays info for the specified file or directory", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(nil) + }, + }) + const path, fileId = "path", "file-id" + cmd.Flags().StringVar(&c.Path, path, "", "Path to file or directory") + cmd.Flags().StringVar(&c.FileId, fileId, "", "File id of file") + cmd.Flags().StringVarP(&c.Format, "format", "f", "", `Display info in the given format: + "%N": name of the file + "%z": size of file in bytes + "%u": owner + "%g": group name of owner + "%i": file id of the file + "%y": modification time in UTC in 'yyyy-MM-dd HH:mm:ss' format + "%Y": modification time as Unix timestamp in milliseconds + "%b": Number of blocks allocated for file +`) + cmd.MarkFlagsMutuallyExclusive(path, fileId) + return cmd +} + +func (c *StatCommand) Run(_ []string) error { + if c.Path == "" && c.FileId == "" { + return stacktrace.NewError("must set one of --path or --file-id flags") + } + + var javaArgs []string + if c.Format != "" { + javaArgs = append(javaArgs, "-f", c.Format) + } + if c.Path != "" { + javaArgs = append(javaArgs, c.Path) + } else if c.FileId != "" { + javaArgs = append(javaArgs, "--file-id", c.FileId) + } + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/fs/tail.go b/cli/src/alluxio.org/cli/cmd/fs/tail.go new file mode 100644 index 000000000000..5affc193fd13 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/fs/tail.go @@ -0,0 +1,60 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package fs + +import ( + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +func Tail(className string) env.Command { + return &TailCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "tail", + JavaClassName: className, + Parameters: []string{"tail"}, + }, + } +} + +type TailCommand struct { + *env.BaseJavaCommand + + bytes string +} + +func (c *TailCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *TailCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: "tail [path]", + Short: "Print the trailing bytes from the specified file", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + cmd.Flags().StringVar(&c.bytes, "bytes", "", "Byte size to print") + return cmd +} + +func (c *TailCommand) Run(args []string) error { + var javaArgs []string + if c.bytes != "" { + javaArgs = append(javaArgs, "-c", c.bytes) + } + javaArgs = append(javaArgs, args...) + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/fs/test.go b/cli/src/alluxio.org/cli/cmd/fs/test.go new file mode 100644 index 000000000000..6c9146ae7456 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/fs/test.go @@ -0,0 +1,80 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package fs + +import ( + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +func Test(className string) env.Command { + return &TestCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "test", + JavaClassName: className, + Parameters: []string{"test"}, + }, + } +} + +type TestCommand struct { + *env.BaseJavaCommand + + isDirectory bool + isExists bool + isFile bool + isNotEmpty bool + isZeroLength bool +} + +func (c *TestCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *TestCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: "test [path]", + Short: "Test a property of a path, returning 0 if the property is true, or 1 otherwise", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + cmd.Flags().BoolVarP(&c.isDirectory, "dir", "d", false, "Test if path is a directory") + cmd.Flags().BoolVarP(&c.isExists, "exists", "e", false, "Test if path exists") + cmd.Flags().BoolVarP(&c.isFile, "file", "f", false, "Test if path is a file") + cmd.Flags().BoolVarP(&c.isFile, "not-empty", "s", false, "Test if path is not empty") + cmd.Flags().BoolVarP(&c.isFile, "zero", "z", false, "Test if path is zero length") + return cmd +} + +func (c *TestCommand) Run(args []string) error { + var javaArgs []string + if c.isDirectory { + javaArgs = append(javaArgs, "-d") + } + if c.isExists { + javaArgs = append(javaArgs, "-e") + } + if c.isFile { + javaArgs = append(javaArgs, "-f") + } + if c.isNotEmpty { + javaArgs = append(javaArgs, "-s") + } + if c.isZeroLength { + javaArgs = append(javaArgs, "-z") + } + javaArgs = append(javaArgs, args...) + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/fs/touch.go b/cli/src/alluxio.org/cli/cmd/fs/touch.go new file mode 100644 index 000000000000..c827cc57c3ca --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/fs/touch.go @@ -0,0 +1,52 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package fs + +import ( + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +func Touch(className string) env.Command { + return &TouchCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "touch", + JavaClassName: className, + Parameters: []string{"touch"}, + }, + } +} + +type TouchCommand struct { + *env.BaseJavaCommand +} + +func (c *TouchCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *TouchCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: "touch [path]", + Short: "Create a 0 byte file at the specified path, which will also be created in the under file system", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + return cmd +} + +func (c *TouchCommand) Run(args []string) error { + return c.Base().Run(args) +} diff --git a/cli/src/alluxio.org/cli/cmd/generate/docs.go b/cli/src/alluxio.org/cli/cmd/generate/docs.go new file mode 100644 index 000000000000..92e457647bb9 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/generate/docs.go @@ -0,0 +1,52 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package generate + +import ( + "fmt" + + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var Docs = &DocsCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "docs", + JavaClassName: "alluxio.cli.DocGenerator", + ShellJavaOpts: fmt.Sprintf(env.JavaOptFormat, env.ConfAlluxioLoggerType, "Console"), + }, +} + +type DocsCommand struct { + *env.BaseJavaCommand +} + +func (c *DocsCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *DocsCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: Docs.CommandName, + Short: "Generate docs automatically.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + return cmd +} + +func (c *DocsCommand) Run(args []string) error { + return c.Base().Run(args) +} diff --git a/cli/src/alluxio.org/cli/cmd/generate/generate.go b/cli/src/alluxio.org/cli/cmd/generate/generate.go new file mode 100644 index 000000000000..3a26a653570e --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/generate/generate.go @@ -0,0 +1,24 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package generate + +import ( + "alluxio.org/cli/env" +) + +var Service = &env.Service{ + Name: "generate", + Description: "Generate docs automatically if one doesn't exist.", + Commands: []env.Command{ + Docs, + }, +} diff --git a/cli/src/alluxio.org/cli/cmd/info/cache.go b/cli/src/alluxio.org/cli/cmd/info/cache.go new file mode 100644 index 000000000000..12f84e30bbd3 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/info/cache.go @@ -0,0 +1,71 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package info + +import ( + "strings" + + "github.com/spf13/cobra" + + "alluxio.org/cli/cmd/names" + "alluxio.org/cli/env" +) + +var Cache = &CacheCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "cache", + JavaClassName: names.FileSystemAdminShellJavaClass, + Parameters: []string{"report", "capacity"}, + }, +} + +type CacheCommand struct { + *env.BaseJavaCommand + + liveWorkers bool + lostWorkers bool + workersList []string // list of worker hostnames or ip addresses +} + +func (c *CacheCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *CacheCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: Cache.CommandName, + Short: "Reports worker capacity information", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(nil) + }, + }) + const live, lost, worker = "live", "lost", "worker" + cmd.Flags().BoolVar(&c.liveWorkers, live, false, "Only show live workers for capacity report") + cmd.Flags().BoolVar(&c.lostWorkers, lost, false, "Only show lost workers for capacity report") + cmd.Flags().StringSliceVar(&c.workersList, worker, nil, "Only show specified workers for capacity report, labeled by hostname or IP address") + cmd.MarkFlagsMutuallyExclusive(live, lost, worker) + return cmd +} + +func (c *CacheCommand) Run(_ []string) error { + // TODO: output all in a serializable format and filter/trim as specified by flags + var args []string + if c.liveWorkers { + args = append(args, "-live") + } else if c.lostWorkers { + args = append(args, "-lost") + } else if len(c.workersList) > 0 { + args = append(args, "-workers", strings.Join(c.workersList, ",")) + } + return c.Base().Run(args) +} diff --git a/cli/src/alluxio.org/cli/cmd/info/collect.go b/cli/src/alluxio.org/cli/cmd/info/collect.go new file mode 100644 index 000000000000..9894aace45e8 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/info/collect.go @@ -0,0 +1,151 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package info + +import ( + "fmt" + "strconv" + "strings" + "time" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var Collect = &CollectCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "collect", + JavaClassName: "alluxio.cli.bundler.CollectInfo", + }, +} + +type CollectCommand struct { + *env.BaseJavaCommand + + additionalLogs []string + endTime string + excludeLogs []string + excludeWorkerMetrics bool + includeLogs []string + local bool + maxThreads int + outputPath string + startTime string +} + +func (c *CollectCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +const dateFormat = "2006-01-02T15:04:05" + +func (c *CollectCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: fmt.Sprintf("%v [command]", Collect.CommandName), + Short: "Collects information such as logs, config, metrics, and more from the running Alluxio cluster and bundle into a single tarball", + Long: `Collects information such as logs, config, metrics, and more from the running Alluxio cluster and bundle into a single tarball +[command] must be one of the following values: + all runs all the commands below + cluster: runs a set of Alluxio commands to collect information about the Alluxio cluster + conf: collects the configuration files under ${ALLUXIO_HOME}/config/ + env: runs a set of linux commands to collect information about the cluster + jvm: collects jstack from the JVMs + log: collects the log files under ${ALLUXIO_HOME}/logs/ + metrics: collects Alluxio system metrics + +WARNING: This command MAY bundle credentials. To understand the risks refer to the docs here. +https://docs.alluxio.io/os/user/edge/en/operation/Troubleshooting.html#collect-alluxio-cluster-information +`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + cmd.Flags().StringSliceVar(&c.additionalLogs, "additional-logs", nil, "Additional file name prefixes from ${ALLUXIO_HOME}/logs to include in the tarball, inclusive of the default log files") + cmd.Flags().StringVar(&c.endTime, "end-time", "", "Logs that do not contain entries before this time will be ignored, format must be like "+dateFormat) + cmd.Flags().StringSliceVar(&c.excludeLogs, "exclude-logs", nil, "File name prefixes from ${ALLUXIO_HOME}/logs to exclude; this is evaluated after adding files from --additional-logs") + cmd.Flags().BoolVar(&c.excludeWorkerMetrics, "exclude-worker-metrics", false, "True to skip worker metrics collection") + cmd.Flags().StringSliceVar(&c.includeLogs, "include-logs", nil, "File name prefixes from ${ALLUXIO_HOME}/logs to include in the tarball, ignoring the default log files; cannot be used with --exclude-logs or --additional-logs") + cmd.Flags().BoolVar(&c.local, "local", false, "True to only collect information from the local machine") + cmd.Flags().IntVar(&c.maxThreads, "max-threads", 1, "Parallelism of the command; use a smaller value to limit network I/O when transferring tarballs") + const outputPath = "output-path" + cmd.Flags().StringVar(&c.outputPath, outputPath, "", "Output directory to write collect info tarball to") + cmd.MarkFlagRequired(outputPath) + cmd.Flags().StringVar(&c.startTime, "start-time", "", "Logs that do not contain entries after this time will be ignored, format must be like "+dateFormat) + return cmd +} + +func (c *CollectCommand) Run(args []string) error { + // TODO: use flags instead of arguments to parse user input + commands := map[string]string{ + "all": "all", + "cluster": "collectAlluxioInfo", + "conf": "collectConfig", + "env": "collectEnv", + "jvm": "collectJvmInfo", + "log": "collectLog", + "metrics": "collectMetrics", + } + commandArg, ok := commands[args[0]] + if !ok { + var cmds []string + for c := range commands { + cmds = append(cmds, c) + } + return stacktrace.NewError("first argument must be one of %v", strings.Join(cmds, ", ")) + } + + var javaArgs []string + if c.additionalLogs != nil { + if c.includeLogs != nil { + return stacktrace.NewError("cannot set both --include-logs and --additional-logs") + } + javaArgs = append(javaArgs, "--additional-logs", strings.Join(c.additionalLogs, ",")) + } + if c.endTime != "" { + if _, err := time.Parse(dateFormat, c.endTime); err != nil { + return stacktrace.Propagate(err, "could not parse end time %v", c.endTime) + } + javaArgs = append(javaArgs, "--end-time", c.endTime) + } + if c.excludeLogs != nil { + if c.includeLogs != nil { + return stacktrace.NewError("cannot set both --include-logs and --exclude-logs") + } + javaArgs = append(javaArgs, "--exclude-logs", strings.Join(c.excludeLogs, ",")) + } + if c.excludeWorkerMetrics { + javaArgs = append(javaArgs, "--exclude-worker-metrics") + } + if c.includeLogs != nil { + // already checked exclusivity with --additional-logs and --exclude-logs + javaArgs = append(javaArgs, "--include-logs", strings.Join(c.includeLogs, ",")) + } + if c.local { + javaArgs = append(javaArgs, "--local") + } + if c.maxThreads > 1 { + javaArgs = append(javaArgs, "--max-threads", strconv.Itoa(c.maxThreads)) + } + if c.startTime != "" { + if _, err := time.Parse(dateFormat, c.startTime); err != nil { + return stacktrace.Propagate(err, "could not parse start time %v", c.startTime) + } + javaArgs = append(javaArgs, "--start-time", c.startTime) + } + + javaArgs = append(javaArgs, commandArg, c.outputPath) + + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/info/info.go b/cli/src/alluxio.org/cli/cmd/info/info.go new file mode 100644 index 000000000000..4b3bc79338d4 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/info/info.go @@ -0,0 +1,28 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package info + +import ( + "alluxio.org/cli/env" +) + +var Service = &env.Service{ + Name: "info", + Description: "Retrieve and/or display info about the running Alluxio cluster", + Commands: []env.Command{ + Cache, + Collect, + Master, + Report, + Version, + }, +} diff --git a/cli/src/alluxio.org/cli/cmd/info/master.go b/cli/src/alluxio.org/cli/cmd/info/master.go new file mode 100644 index 000000000000..2c371f066723 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/info/master.go @@ -0,0 +1,50 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package info + +import ( + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var Master = &MasterCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "master", + JavaClassName: "alluxio.cli.fs.FileSystemShell", + Parameters: []string{"masterInfo"}, + }, +} + +type MasterCommand struct { + *env.BaseJavaCommand +} + +func (c *MasterCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *MasterCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: Master.CommandName, + Short: "Prints information regarding master fault tolerance such as leader address and list of master addresses", + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(nil) + }, + }) + return cmd +} + +func (c *MasterCommand) Run(_ []string) error { + // TODO: output in a serializable format + return c.Base().Run(nil) +} diff --git a/cli/src/alluxio.org/cli/cmd/info/report.go b/cli/src/alluxio.org/cli/cmd/info/report.go new file mode 100644 index 000000000000..afc53450ed76 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/info/report.go @@ -0,0 +1,82 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package info + +import ( + "fmt" + "strings" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + + "alluxio.org/cli/cmd/names" + "alluxio.org/cli/env" +) + +var Report = &ReportCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "report", + JavaClassName: names.FileSystemAdminShellJavaClass, + Parameters: []string{"report"}, + }, +} + +type ReportCommand struct { + *env.BaseJavaCommand +} + +func (c *ReportCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *ReportCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: fmt.Sprintf("%v [arg]", Report.CommandName), + Short: "Reports Alluxio running cluster information", + Long: `Reports Alluxio running cluster information +[arg] can be one of the following values: + jobservice: job service metrics information + metrics: metrics information + summary: cluster summary + ufs: under storage system information + +Defaults to summary if no arg is provided +`, + Args: cobra.RangeArgs(0, 1), + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + return cmd +} + +func (c *ReportCommand) Run(args []string) error { + reportArg := "summary" + if len(args) == 1 { + options := map[string]struct{}{ + "jobservice": {}, + "metrics": {}, + "summary": {}, + "ufs": {}, + } + if _, ok := options[args[0]]; !ok { + var cmds []string + for c := range options { + cmds = append(cmds, c) + } + return stacktrace.NewError("first argument must be one of %v", strings.Join(cmds, ", ")) + } + reportArg = args[0] + } + // TODO: output all in a serializable format and filter/trim as specified by flags + return c.Base().Run([]string{reportArg}) +} diff --git a/cli/src/alluxio.org/cli/cmd/info/version.go b/cli/src/alluxio.org/cli/cmd/info/version.go new file mode 100644 index 000000000000..d37171f08245 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/info/version.go @@ -0,0 +1,49 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package info + +import ( + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var Version = &VersionCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "version", + JavaClassName: "alluxio.cli.Version", + }, +} + +type VersionCommand struct { + *env.BaseJavaCommand +} + +func (c *VersionCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *VersionCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: Version.CommandName, + Short: "Print Alluxio version and exit.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + return cmd +} + +func (c *VersionCommand) Run(args []string) error { + return c.Base().Run(args) +} diff --git a/cli/src/alluxio.org/cli/cmd/job/cancel.go b/cli/src/alluxio.org/cli/cmd/job/cancel.go new file mode 100644 index 000000000000..7fb5ab5ef955 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/job/cancel.go @@ -0,0 +1,61 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package job + +import ( + "strconv" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + + "alluxio.org/cli/cmd/names" + "alluxio.org/cli/env" +) + +var Cancel = &CancelCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "cancel", + JavaClassName: names.JobShellJavaClass, + }, +} + +type CancelCommand struct { + *env.BaseJavaCommand + jobId int +} + +func (c *CancelCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *CancelCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: Cancel.CommandName, + Short: "Cancels a job asynchronously.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + const id = "id" + cmd.Flags().IntVar(&c.jobId, id, 0, "Determine a job ID to cancel") + cmd.MarkFlagRequired(id) + return cmd +} + +func (c *CancelCommand) Run(args []string) error { + if c.jobId <= 0 { + return stacktrace.NewError("Flag --id should be a positive integer") + } + javaArgs := []string{"cancel", strconv.Itoa(c.jobId)} + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/job/cmd_status.go b/cli/src/alluxio.org/cli/cmd/job/cmd_status.go new file mode 100644 index 000000000000..5a7001752e25 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/job/cmd_status.go @@ -0,0 +1,61 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package job + +import ( + "strconv" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + + "alluxio.org/cli/cmd/names" + "alluxio.org/cli/env" +) + +var CmdStatus = &CmdStatusCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "cmdStatus", + JavaClassName: names.JobShellJavaClass, + }, +} + +type CmdStatusCommand struct { + *env.BaseJavaCommand + jobControlId int +} + +func (c *CmdStatusCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *CmdStatusCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: CmdStatus.CommandName, + Short: "Get the status information for a distributed command.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + const id = "id" + cmd.Flags().IntVar(&c.jobControlId, id, 0, "Determine the job control ID to get the status information") + cmd.MarkFlagRequired(id) + return cmd +} + +func (c *CmdStatusCommand) Run(args []string) error { + if c.jobControlId <= 0 { + return stacktrace.NewError("Flag --id should be a positive integer") + } + javaArgs := []string{"getCmdStatus", strconv.Itoa(c.jobControlId)} + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/job/job.go b/cli/src/alluxio.org/cli/cmd/job/job.go new file mode 100644 index 000000000000..97327b36858d --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/job/job.go @@ -0,0 +1,27 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package job + +import "alluxio.org/cli/env" + +var Service = &env.Service{ + Name: "job", + Description: "Command line tool for interacting with the job service.", + Commands: []env.Command{ + Cancel, + CmdStatus, + JobStatus, + Leader, + List, + Load, + }, +} diff --git a/cli/src/alluxio.org/cli/cmd/job/job_status.go b/cli/src/alluxio.org/cli/cmd/job/job_status.go new file mode 100644 index 000000000000..d9d2b94d967e --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/job/job_status.go @@ -0,0 +1,67 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package job + +import ( + "strconv" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + + "alluxio.org/cli/cmd/names" + "alluxio.org/cli/env" +) + +var JobStatus = &JobStatusCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "jobStatus", + JavaClassName: names.JobShellJavaClass, + }, +} + +type JobStatusCommand struct { + *env.BaseJavaCommand + jobId int + everyTask bool +} + +func (c *JobStatusCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *JobStatusCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: JobStatus.CommandName, + Short: "Displays the status info for the specific job.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + const id = "id" + cmd.Flags().IntVar(&c.jobId, id, 0, "Determine the job ID to get status info") + cmd.MarkFlagRequired(id) + cmd.Flags().BoolVarP(&c.everyTask, "every-task", "v", false, "Determine display the status of every task") + return cmd +} + +func (c *JobStatusCommand) Run(args []string) error { + if c.jobId <= 0 { + return stacktrace.NewError("Flag --id should be a positive integer") + } + javaArgs := []string{"stat"} + if c.everyTask { + javaArgs = append(javaArgs, "-v") + } + javaArgs = append(javaArgs, strconv.Itoa(c.jobId)) + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/job/leader.go b/cli/src/alluxio.org/cli/cmd/job/leader.go new file mode 100644 index 000000000000..e095bbb7292f --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/job/leader.go @@ -0,0 +1,51 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package job + +import ( + "github.com/spf13/cobra" + + "alluxio.org/cli/cmd/names" + "alluxio.org/cli/env" +) + +var Leader = &LeaderCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "leader", + JavaClassName: names.JobShellJavaClass, + }, +} + +type LeaderCommand struct { + *env.BaseJavaCommand +} + +func (c *LeaderCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *LeaderCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: Leader.CommandName, + Short: "Prints the hostname of the job master service leader.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + return cmd +} + +func (c *LeaderCommand) Run(args []string) error { + javaArgs := []string{"leader"} + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/job/list.go b/cli/src/alluxio.org/cli/cmd/job/list.go new file mode 100644 index 000000000000..57dc1f070180 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/job/list.go @@ -0,0 +1,52 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package job + +import ( + "github.com/spf13/cobra" + + "alluxio.org/cli/cmd/names" + "alluxio.org/cli/env" +) + +var List = &ListCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "list", + JavaClassName: names.JobShellJavaClass, + }, +} + +type ListCommand struct { + *env.BaseJavaCommand +} + +func (c *ListCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *ListCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: List.CommandName, + Short: "Prints the IDs of the most recent jobs, running and finished, " + + "in the history up to the capacity set in alluxio.job.master.job.capacity", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + return cmd +} + +func (c *ListCommand) Run(args []string) error { + javaArgs := []string{"ls"} + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/job/load.go b/cli/src/alluxio.org/cli/cmd/job/load.go new file mode 100644 index 000000000000..88734c91cb97 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/job/load.go @@ -0,0 +1,55 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package job + +import ( + "github.com/spf13/cobra" + + "alluxio.org/cli/cmd/names" + "alluxio.org/cli/env" +) + +var Load = &LoadCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "load", + JavaClassName: names.FileSystemShellJavaClass, + }, +} + +type LoadCommand struct { + *env.BaseJavaCommand + path string +} + +func (c *LoadCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *LoadCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: Load.CommandName, + Short: "Submit load job to Alluxio master, update job options if already exists", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + const path = "path" + cmd.Flags().StringVar(&c.path, path, "", "Determine the path of the load job to submit") + cmd.MarkFlagRequired(path) + return cmd +} + +func (c *LoadCommand) Run(args []string) error { + javaArgs := []string{"load", c.path, "--submit"} + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/journal/format.go b/cli/src/alluxio.org/cli/cmd/journal/format.go new file mode 100644 index 000000000000..47b508e22053 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/journal/format.go @@ -0,0 +1,69 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package journal + +import ( + "bytes" + "fmt" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + + "alluxio.org/cli/env" + "alluxio.org/log" +) + +var Format = &FormatCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "format", + JavaClassName: "alluxio.cli.Format", + Parameters: []string{"master"}, + UseServerClasspath: true, + ShellJavaOpts: fmt.Sprintf(env.JavaOptFormat, env.ConfAlluxioLoggerType, "Console"), + }, +} + +type FormatCommand struct { + *env.BaseJavaCommand +} + +func (c *FormatCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *FormatCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: Format.CommandName, + Short: "Format the local Alluxio master journal", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(args) + }, + }) + return cmd +} + +func (c *FormatCommand) Run(_ []string) error { + return c.Base().Run(nil) +} + +func (c *FormatCommand) Format() error { + cmd := c.RunJavaClassCmd(nil) + errBuf := &bytes.Buffer{} + cmd.Stderr = errBuf + + log.Logger.Debugln(cmd.String()) + if err := cmd.Run(); err != nil { + return stacktrace.Propagate(err, "error running %v\nstderr: %v", c.CommandName, errBuf.String()) + } + return nil +} diff --git a/cli/src/alluxio.org/cli/cmd/journal/journal.go b/cli/src/alluxio.org/cli/cmd/journal/journal.go new file mode 100644 index 000000000000..69d751ea476d --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/journal/journal.go @@ -0,0 +1,25 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package journal + +import ( + "alluxio.org/cli/env" +) + +var Service = &env.Service{ + Name: "journal", + Description: "Journal related operations", + Commands: []env.Command{ + Format, + Read, + }, +} diff --git a/cli/src/alluxio.org/cli/cmd/journal/read.go b/cli/src/alluxio.org/cli/cmd/journal/read.go new file mode 100644 index 000000000000..69c29fd02c0a --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/journal/read.go @@ -0,0 +1,88 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package journal + +import ( + "fmt" + "strconv" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + + "alluxio.org/cli/env" +) + +var Read = &ReadCommand{ + BaseJavaCommand: &env.BaseJavaCommand{ + CommandName: "read", + JavaClassName: "alluxio.master.journal.tool.JournalTool", + UseServerClasspath: true, + ShellJavaOpts: fmt.Sprintf(env.JavaOptFormat, env.ConfAlluxioLoggerType, "Console"), + }, +} + +type ReadCommand struct { + *env.BaseJavaCommand + + end int + inputDir string + master string + outputDir string + start int +} + +func (c *ReadCommand) Base() *env.BaseJavaCommand { + return c.BaseJavaCommand +} + +func (c *ReadCommand) ToCommand() *cobra.Command { + cmd := c.Base().InitRunJavaClassCmd(&cobra.Command{ + Use: Read.CommandName, + Short: "Read an Alluxio journal file to a human-readable version", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return c.Run(nil) + }, + }) + cmd.Flags().IntVar(&c.end, "end", -1, "end log sequence number (exclusive)") + cmd.Flags().StringVar(&c.inputDir, "input-dir", "", "input directory on-disk to read the journal content from") + cmd.Flags().StringVar(&c.master, "master", "FileSystemMaster", "name of the master class") + cmd.Flags().StringVar(&c.outputDir, "output-dir", "", "output directory to write journal content to") + cmd.Flags().IntVar(&c.start, "start", 0, "start log sequence number (inclusive)") + return cmd +} + +func (c *ReadCommand) Run(_ []string) error { + var javaArgs []string + if c.start < 0 { + return stacktrace.NewError("start log sequence number must be non-negative but was %v", c.start) + } + if c.end < -1 { + return stacktrace.NewError("end log sequence number must be non-negative but was %v", c.end) + } + if c.end != -1 { + javaArgs = append(javaArgs, "-end", strconv.Itoa(c.end)) + } + if c.inputDir != "" { + javaArgs = append(javaArgs, "-inputDir", c.inputDir) + } + if c.master != "FileSystemMaster" { + javaArgs = append(javaArgs, "-master", c.master) + } + if c.outputDir != "" { + javaArgs = append(javaArgs, "-outputDir", c.outputDir) + } + if c.start > 0 { + javaArgs = append(javaArgs, "-start", strconv.Itoa(c.start)) + } + return c.Base().Run(javaArgs) +} diff --git a/cli/src/alluxio.org/cli/cmd/names/names.go b/cli/src/alluxio.org/cli/cmd/names/names.go new file mode 100644 index 000000000000..51fff704dade --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/names/names.go @@ -0,0 +1,19 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package names + +const ( + BinAlluxio = "bin/alluxio" + FileSystemAdminShellJavaClass = "alluxio.cli.fsadmin.FileSystemAdminShell" + FileSystemShellJavaClass = "alluxio.cli.fs.FileSystemShell" + JobShellJavaClass = "alluxio.cli.job.JobShell" +) diff --git a/cli/src/alluxio.org/cli/cmd/process/process.go b/cli/src/alluxio.org/cli/cmd/process/process.go new file mode 100644 index 000000000000..65983a2714e6 --- /dev/null +++ b/cli/src/alluxio.org/cli/cmd/process/process.go @@ -0,0 +1,25 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package process + +import ( + "alluxio.org/cli/env" +) + +var Service = &env.Service{ + Name: "process", + Description: "Start, stop, and other operations related to the cluster processes", + Commands: []env.Command{ + &env.StartProcessCommand{}, + &env.StopProcessCommand{}, + }, +} diff --git a/cli/src/alluxio.org/cli/env/command.go b/cli/src/alluxio.org/cli/env/command.go new file mode 100644 index 000000000000..f69be504eb14 --- /dev/null +++ b/cli/src/alluxio.org/cli/env/command.go @@ -0,0 +1,103 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package env + +import ( + "fmt" + "os" + "os/exec" + "strings" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + + "alluxio.org/log" +) + +type Command interface { + ToCommand() *cobra.Command +} + +type BaseJavaCommand struct { + CommandName string + JavaClassName string + Parameters []string + + DebugMode bool + + UseServerClasspath bool // defaults to ALLUXIO_CLIENT_CLASSPATH, use ALLUXIO_SERVER_CLASSPATH if true + InlineJavaOpts []string // java opts provided by the user as part of the inline command + ShellJavaOpts string // default java opts encoded as part of the specific command +} + +func (c *BaseJavaCommand) InitRunJavaClassCmd(cmd *cobra.Command) *cobra.Command { + cmd.Flags().BoolVar(&c.DebugMode, "attach-debug", false, fmt.Sprintf("True to attach debug opts specified by $%v", ConfAlluxioUserAttachOpts.EnvVar)) + cmd.Flags().StringSliceVarP(&c.InlineJavaOpts, "java-opts", "D", nil, `Alluxio properties to apply, ex. -Dkey=value`) + return cmd +} + +// RunJavaClassCmd constructs a java command with a predetermined order of variable opts +// ${JAVA} ${ALLUXIO_USER_ATTACH_OPTS} -cp ${ALLUXIO_CLIENT_CLASSPATH} ${ALLUXIO_USER_JAVA_OPTS} \ +// {command default opts} {user inline opts} {command java class} {command parameter} {user inline args} +// where: +// - ${ALLUXIO_USER_*} are environment variables set by the user in alluxio-env.sh +// - {command *} are encoded as part of the command's definition +// - {user inline *} are specified by the user when entering the command +func (c *BaseJavaCommand) RunJavaClassCmd(args []string) *exec.Cmd { + var cmdArgs []string + if c.DebugMode { + if opts := Env.EnvVar.GetString(ConfAlluxioUserAttachOpts.EnvVar); opts != "" { + cmdArgs = append(cmdArgs, strings.Split(opts, " ")...) + } + } + classpath := EnvAlluxioClientClasspath + if c.UseServerClasspath { + classpath = EnvAlluxioServerClasspath + } + cmdArgs = append(cmdArgs, "-cp", Env.EnvVar.GetString(classpath)) + + if opts := Env.EnvVar.GetString(ConfAlluxioUserJavaOpts.EnvVar); opts != "" { + cmdArgs = append(cmdArgs, strings.Split(opts, " ")...) + } + if opts := strings.TrimSpace(c.ShellJavaOpts); opts != "" { + cmdArgs = append(cmdArgs, strings.Split(opts, " ")...) + } + for _, o := range c.InlineJavaOpts { + if opts := strings.TrimSpace(o); opts != "" { + cmdArgs = append(cmdArgs, fmt.Sprintf("-D%v", opts)) + } + } + cmdArgs = append(cmdArgs, c.JavaClassName) + if len(c.Parameters) > 0 { + cmdArgs = append(cmdArgs, c.Parameters...) + } + cmdArgs = append(cmdArgs, args...) + + ret := exec.Command(Env.EnvVar.GetString(ConfJava.EnvVar), cmdArgs...) + for _, k := range Env.EnvVar.AllKeys() { + ret.Env = append(ret.Env, fmt.Sprintf("%s=%v", k, Env.EnvVar.Get(k))) + } + return ret +} + +func (c *BaseJavaCommand) Run(args []string) error { + cmd := c.RunJavaClassCmd(args) + + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + log.Logger.Debugln(cmd.String()) + if err := cmd.Run(); err != nil { + return stacktrace.Propagate(err, "error running %v", c.CommandName) + } + return nil +} diff --git a/cli/src/alluxio.org/cli/env/env.go b/cli/src/alluxio.org/cli/env/env.go new file mode 100644 index 000000000000..df423daebe0f --- /dev/null +++ b/cli/src/alluxio.org/cli/env/env.go @@ -0,0 +1,255 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package env + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "sort" + "strings" + + "github.com/palantir/stacktrace" + "github.com/sirupsen/logrus" + "github.com/spf13/viper" + + "alluxio.org/log" +) + +const ( + userLoggerType = "USER_LOGGER" +) + +// Env is a global instance of the Alluxio environment +// It is assumed that InitAlluxioEnv() is run before access +// The package level variable exists to allow access by cobra.Command.RunE functions +var Env *AlluxioEnv + +type AlluxioEnv struct { + RootPath string + Version string + EnvVar *viper.Viper +} + +func InitAlluxioEnv(rootPath string, jarEnvVars map[string]string, appendClasspathJars map[string]func(*viper.Viper, string) string) error { + /* + Note the precedence order of viper, where earlier is retrieved first: + - Set, env, config, default (https://pkg.go.dev/github.com/dvln/viper#section-readme) + Since env > config, explicitly set any values read from config to take precedence over env + - Use single instance of viper to set values and retrieve values within go logic + - Use local viper instances to read configuration files to avoid confusion in precedence order + This effectively negates the possibility of retrieving config values to simplify the precedence order to be + - Set, env, default + */ + envVar := viper.New() + envVar.AutomaticEnv() // override defaults with existing environment variable values + + // read VERSION from version.sh + var ver string + { + v := viper.New() + v.AddConfigPath(filepath.Join(rootPath, "libexec")) + v.SetConfigName("version.sh") + v.SetConfigType("env") + if err := v.ReadInConfig(); err != nil { + return stacktrace.Propagate(err, "error reading version config") + } + ver = v.GetString(envVersion) + envVar.Set(envVersion, ver) + } + + // set default values + for k, v := range map[string]string{ + ConfAlluxioHome.EnvVar: rootPath, + ConfAlluxioConfDir.EnvVar: filepath.Join(rootPath, "conf"), + ConfAlluxioLogsDir.EnvVar: filepath.Join(rootPath, "logs"), + confAlluxioUserLogsDir.EnvVar: filepath.Join(rootPath, "logs", "user"), + } { + envVar.SetDefault(k, v) + } + + // set jar env vars + for envVarName, jarPathFormat := range jarEnvVars { + envVar.SetDefault(envVarName, filepath.Join(rootPath, fmt.Sprintf(jarPathFormat, ver))) + } + + // set user-specified environment variable values from alluxio-env.sh + { + // use ALLUXIO_CONF_DIR if set externally, otherwise default to above value + confPath := envVar.GetString(ConfAlluxioConfDir.EnvVar) + const alluxioEnv = "alluxio-env.sh" + log.Logger.Debugf("Loading %v configuration from directory %v", alluxioEnv, confPath) + v := viper.New() + if _, err := os.Stat(filepath.Join(confPath, alluxioEnv)); err == nil { + v.AddConfigPath(confPath) + v.SetConfigName(alluxioEnv) + v.SetConfigType("env") + v.AllowEmptyEnv(true) + if err := v.ReadInConfig(); err != nil { + return stacktrace.Propagate(err, "error reading alluxio-env config") + } + // for each entry found in alluxio-env.sh, explicitly set to global viper + for _, k := range v.AllKeys() { + key, val := strings.ToUpper(k), v.Get(k) + log.Logger.Debugf("Setting user provided %v=%v", key, val) + envVar.Set(key, val) + } + } + } + + // set classpath variables which are dependent on user configurable values + envVar.Set(EnvAlluxioClientClasspath, strings.Join([]string{ + envVar.GetString(ConfAlluxioConfDir.EnvVar) + "/", + envVar.GetString(confAlluxioClasspath.EnvVar), + envVar.GetString(EnvAlluxioAssemblyClientJar), + }, ":")) + envVar.Set(EnvAlluxioServerClasspath, strings.Join([]string{ + envVar.GetString(ConfAlluxioConfDir.EnvVar) + "/", + envVar.GetString(confAlluxioClasspath.EnvVar), + envVar.GetString(EnvAlluxioAssemblyServerJar), + }, ":")) + + for envVarName, envF := range appendClasspathJars { + envVar.Set(envVarName, envF(envVar, ver)) + } + + // check java executable and version + if err := checkAndSetJava(envVar); err != nil { + return stacktrace.Propagate(err, "error finding installed java") + } + if err := checkJavaVersion(envVar.GetString(ConfJava.EnvVar)); err != nil { + return stacktrace.Propagate(err, "error checking java version compatibility") + } + + // append default opts to ALLUXIO_JAVA_OPTS + alluxioJavaOpts := envVar.GetString(ConfAlluxioJavaOpts.EnvVar) + if alluxioJavaOpts != "" { + // warn about setting configuration through java opts that should be set through environment variables instead + for _, c := range []*AlluxioConfigEnvVar{ + ConfAlluxioConfDir, + ConfAlluxioLogsDir, + confAlluxioUserLogsDir, + } { + if strings.Contains(alluxioJavaOpts, c.configKey) { + log.Logger.Warnf("Setting %v through %v will be ignored. Use environment variable %v instead.", c.configKey, ConfAlluxioJavaOpts.EnvVar, c.EnvVar) + } + } + } + + for _, c := range []*AlluxioConfigEnvVar{ + ConfAlluxioHome, + ConfAlluxioConfDir, + ConfAlluxioLogsDir, + confAlluxioUserLogsDir, + } { + alluxioJavaOpts += c.ToJavaOpt(envVar, true) // mandatory java opts + } + + for _, c := range []*AlluxioConfigEnvVar{ + confAlluxioRamFolder, + confAlluxioMasterHostname, + confAlluxioMasterMountTableRootUfs, + confAlluxioWorkerRamdiskSize, + } { + alluxioJavaOpts += c.ToJavaOpt(envVar, false) // optional user provided java opts + } + + alluxioJavaOpts += fmt.Sprintf(JavaOptFormat, "log4j.configuration", "file:"+filepath.Join(envVar.GetString(ConfAlluxioConfDir.EnvVar), "log4j.properties")) + alluxioJavaOpts += fmt.Sprintf(JavaOptFormat, "org.apache.jasper.compiler.disablejsr199", true) + alluxioJavaOpts += fmt.Sprintf(JavaOptFormat, "java.net.preferIPv4Stack", true) + alluxioJavaOpts += fmt.Sprintf(JavaOptFormat, "org.apache.ratis.thirdparty.io.netty.allocator.useCacheForAllThreads", false) + + envVar.Set(ConfAlluxioJavaOpts.EnvVar, alluxioJavaOpts) + + for _, p := range ProcessRegistry { + p.SetEnvVars(envVar) + } + + // also set user environment variables, as they are not associated with a particular process + // ALLUXIO_USER_JAVA_OPTS = {default logger opts} ${ALLUXIO_JAVA_OPTS} {user provided opts} + userJavaOpts := fmt.Sprintf(JavaOptFormat, ConfAlluxioLoggerType, userLoggerType) + userJavaOpts += envVar.GetString(ConfAlluxioJavaOpts.EnvVar) + userJavaOpts += envVar.GetString(ConfAlluxioUserJavaOpts.EnvVar) + envVar.Set(ConfAlluxioUserJavaOpts.EnvVar, strings.TrimSpace(userJavaOpts)) // leading spaces need to be trimmed as a exec.Command argument + + if log.Logger.IsLevelEnabled(logrus.DebugLevel) { + keys := envVar.AllKeys() + sort.Strings(keys) + log.Logger.Debugln("Environment variables:") + for _, k := range keys { + log.Logger.Debugf("%v %v", strings.ToUpper(k), envVar.Get(k)) + } + log.Logger.Debugln() + } + + Env = &AlluxioEnv{ + RootPath: rootPath, + Version: ver, + EnvVar: envVar, + } + return nil +} + +func checkAndSetJava(envVar *viper.Viper) error { + if envVar.Get(ConfJava.EnvVar) != nil { + return nil + } + // check JAVA_HOME + if javaHome := envVar.Get(confJavaHome.EnvVar); javaHome != nil { + javaHomeStr, ok := javaHome.(string) + if !ok { + return stacktrace.NewError("error casting %v to string", javaHome) + } + javaHomeBinJava := filepath.Join(javaHomeStr, "bin", "java") + // check if JAVA_HOME exists with valid $JAVA_HOME/bin/java binary + if _, err := os.Stat(javaHomeBinJava); err == nil { + envVar.Set(ConfJava.EnvVar, javaHomeBinJava) + return nil + } + } + // check if java is available via `PATH` using `which` + whichJavaPath, err := exec.Command("which", "java").Output() + if err == nil { + envVar.Set(ConfJava.EnvVar, strings.TrimSpace(string(whichJavaPath))) + return nil + } + // cannot find java + // - ${JAVA} is not set + // - ${JAVA_HOME}/bin/java is not a valid path + // - java is not found as part of ${PATH} + return stacktrace.NewError(`Error: Cannot find 'java' on path or under $JAVA_HOME/bin/. Please set %v in alluxio-env.sh or user bash profile.`, confJavaHome.EnvVar) +} + +var ( + // matches the first 2 numbers in the version string encapsulated by double quotes, ex. "11.0.19" -> 11.0 + javaVersionRe = regexp.MustCompile(`.*"(?P\d+\.\d+)[\w.-]*".*`) + // must be either java 8 or 11 + requiredVersionRegex = regexp.MustCompile(`1\.8|11\.0`) +) + +func checkJavaVersion(javaPath string) error { + cmd := exec.Command("bash", "-c", fmt.Sprintf("%v -version", javaPath)) + javaVer, err := cmd.CombinedOutput() + if err != nil { + return stacktrace.Propagate(err, "error finding java version from `%v -version`", javaPath) + } + matches := javaVersionRe.FindStringSubmatch(string(javaVer)) + if len(matches) != 2 { + return stacktrace.NewError("java version output does not match expected regex pattern %v\n%v", javaVersionRe.String(), string(javaVer)) + } + if !requiredVersionRegex.MatchString(matches[1]) { + return stacktrace.NewError("Error: Alluxio requires Java 1.8 or 11.0, currently Java %v found.", matches[1]) + } + return nil +} diff --git a/cli/src/alluxio.org/cli/env/env_vars.go b/cli/src/alluxio.org/cli/env/env_vars.go new file mode 100644 index 000000000000..00f4c4cbb82c --- /dev/null +++ b/cli/src/alluxio.org/cli/env/env_vars.go @@ -0,0 +1,130 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package env + +import ( + "fmt" + + "github.com/spf13/viper" +) + +const ( + JavaOptFormat = " -D%v=%v" +) + +// alluxioEnvVarsTemplate lists the environment variables to include in conf/alluxio-env.sh.template +var alluxioEnvVarsTemplate = map[string]*AlluxioConfigEnvVar{} + +type AlluxioConfigEnvVar struct { + EnvVar string // environment variable read by startup script + description string // describes the environment variable in conf/alluxio-env.sh.template file + + // optional + configKey string // corresponding property key as defined in java + additionalAlluxioJavaOpts map[string]string // additional java opts to append if non-empty java opt is added +} + +func RegisterTemplateEnvVar(v *AlluxioConfigEnvVar) *AlluxioConfigEnvVar { + if _, ok := alluxioEnvVarsTemplate[v.EnvVar]; ok { + panic("Environment variable already registered: " + v.EnvVar) + } + alluxioEnvVarsTemplate[v.EnvVar] = v + return v +} + +const ( + envVersion = "VERSION" // VERSION is specified by libexec/version.sh and should not be overridden by the user + EnvAlluxioAssemblyClientJar = "ALLUXIO_ASSEMBLY_CLIENT_JAR" + EnvAlluxioAssemblyServerJar = "ALLUXIO_ASSEMBLY_SERVER_JAR" + EnvAlluxioClientClasspath = "ALLUXIO_CLIENT_CLASSPATH" + EnvAlluxioServerClasspath = "ALLUXIO_SERVER_CLASSPATH" +) + +var ( + ConfAlluxioConfDir = &AlluxioConfigEnvVar{ + configKey: "alluxio.conf.dir", + EnvVar: "ALLUXIO_CONF_DIR", + } + ConfAlluxioHome = &AlluxioConfigEnvVar{ + configKey: "alluxio.home", + EnvVar: "ALLUXIO_HOME", + } +) + +// environment variables that belong to the templatized alluxio-env.sh +// those with configKeys set will be appended to ALLUXIO_JAVA_OPTS +var ( + confJavaHome = RegisterTemplateEnvVar(&AlluxioConfigEnvVar{ + EnvVar: "JAVA_HOME", + }) + ConfJava = RegisterTemplateEnvVar(&AlluxioConfigEnvVar{ + EnvVar: "JAVA", + }) + ConfAlluxioLogsDir = RegisterTemplateEnvVar(&AlluxioConfigEnvVar{ + configKey: "alluxio.logs.dir", + EnvVar: "ALLUXIO_LOGS_DIR", + }) + confAlluxioUserLogsDir = RegisterTemplateEnvVar(&AlluxioConfigEnvVar{ + configKey: "alluxio.user.logs.dir", + EnvVar: "ALLUXIO_USER_LOGS_DIR", + }) + ConfAlluxioJavaOpts = RegisterTemplateEnvVar(&AlluxioConfigEnvVar{ + EnvVar: "ALLUXIO_JAVA_OPTS", + }) + confAlluxioClasspath = RegisterTemplateEnvVar(&AlluxioConfigEnvVar{ + EnvVar: "ALLUXIO_CLASSPATH", + }) + ConfAlluxioUserJavaOpts = RegisterTemplateEnvVar(&AlluxioConfigEnvVar{ + EnvVar: "ALLUXIO_USER_JAVA_OPTS", + }) + ConfAlluxioUserAttachOpts = RegisterTemplateEnvVar(&AlluxioConfigEnvVar{ + EnvVar: "ALLUXIO_USER_ATTACH_OPTS", + }) +) + +// environment variables with corresponding configKeys to append to ALLUXIO_JAVA_OPTS +var ( + confAlluxioRamFolder = &AlluxioConfigEnvVar{ + configKey: "alluxio.worker.tieredstore.level0.dirs.path", + EnvVar: "ALLUXIO_RAM_FOLDER", + additionalAlluxioJavaOpts: map[string]string{ + "alluxio.worker.tieredstore.level0.alias": "MEM", + }, + } + confAlluxioMasterHostname = &AlluxioConfigEnvVar{ + configKey: "alluxio.master.hostname", + EnvVar: "ALLUXIO_MASTER_HOSTNAME", + } + confAlluxioMasterMountTableRootUfs = &AlluxioConfigEnvVar{ + configKey: "alluxio.master.mount.table.root.ufs", + EnvVar: "ALLUXIO_MASTER_MOUNT_TABLE_ROOT_UFS", + } + confAlluxioWorkerRamdiskSize = &AlluxioConfigEnvVar{ + configKey: "alluxio.worker.ramdisk.size", + EnvVar: "ALLUXIO_WORKER_RAMDISK_SIZE", + } +) + +func (a *AlluxioConfigEnvVar) ToJavaOpt(env *viper.Viper, required bool) string { + v := env.Get(a.EnvVar) + if v == nil { + if required { + panic("No value set for required environment variable: " + a.EnvVar) + } + return "" + } + ret := fmt.Sprintf(JavaOptFormat, a.configKey, v) + for k2, v2 := range a.additionalAlluxioJavaOpts { + ret += fmt.Sprintf(JavaOptFormat, k2, v2) + } + return ret +} diff --git a/cli/src/alluxio.org/cli/env/process.go b/cli/src/alluxio.org/cli/env/process.go new file mode 100644 index 000000000000..848c27c997e5 --- /dev/null +++ b/cli/src/alluxio.org/cli/env/process.go @@ -0,0 +1,227 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package env + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "sync" + "syscall" + "time" + + "github.com/palantir/stacktrace" + "github.com/shirou/gopsutil/process" + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "alluxio.org/log" +) + +var ProcessRegistry = map[string]Process{} + +func RegisterProcess(p Process) Process { + name := p.Base().Name + if _, ok := ProcessRegistry[name]; ok { + panic(fmt.Sprintf("Process %v is already registered", name)) + } + ProcessRegistry[name] = p + return p +} + +type Process interface { + Base() *BaseProcess + SetEnvVars(*viper.Viper) + + Start(*StartProcessCommand) error + StartCmd(*cobra.Command) *cobra.Command + + Stop(*StopProcessCommand) error + StopCmd(*cobra.Command) *cobra.Command +} + +type BaseProcess struct { + Name string + JavaClassName string + JavaOptsEnvVarKey string + + // start + ProcessOutFile string + + // monitor + MonitorJavaClassName string +} + +func (p *BaseProcess) Launch(start *StartProcessCommand, args []string) error { + logsDir := Env.EnvVar.GetString(ConfAlluxioLogsDir.EnvVar) + if err := os.MkdirAll(logsDir, 0755); err != nil { + return stacktrace.Propagate(err, "error creating log directory at %v", logsDir) + } + + startCmd := exec.Command("nohup", args...) + for _, k := range Env.EnvVar.AllKeys() { + startCmd.Env = append(startCmd.Env, fmt.Sprintf("%s=%v", k, Env.EnvVar.Get(k))) + } + + outFile := filepath.Join(Env.EnvVar.GetString(ConfAlluxioLogsDir.EnvVar), p.ProcessOutFile) + f, err := os.OpenFile(outFile, os.O_WRONLY|os.O_CREATE, 0644) + if err != nil { + return stacktrace.Propagate(err, "error opening file at %v", outFile) + } + startCmd.Stdout = f + startCmd.Stderr = f + + log.Logger.Infof("Starting %v", p.Name) + log.Logger.Debugf("%v > %v 2>&1 &", startCmd.String(), outFile) + if err := startCmd.Start(); err != nil { + return stacktrace.Propagate(err, "error starting %v", p.Name) + } + + if start.AsyncStart { + log.Logger.Warnf("Skipping monitor checks for %v", p.Name) + } else { + // wait a little for main process to start before starting monitor process + time.Sleep(500 * time.Millisecond) + if err := p.Monitor(); err != nil { + return stacktrace.Propagate(err, "error monitoring process after launching") + } + } + return nil +} + +var debugOptsToRemove = []*regexp.Regexp{ + regexp.MustCompile(`-agentlib:jdwp=transport=dt_socket.*address=[0-9]*`), + regexp.MustCompile(`-Dcom.sun.management.jmxremote.port=[0-9]*`), +} + +// Monitor runs the corresponding monitoring java class for the process, running the command +// ${JAVA} -cp ${ALLUXIO_CLIENT_CLASSPATH} ${ALLUXIO_MONITOR_JAVA_OPTS} {monitor java class} +// where ${ALLUXIO_MONITOR_JAVA_OPTS} is equivalent to the process specific JAVA_OPTS +// but with several debugging related options removed +func (p *BaseProcess) Monitor() error { + cmdArgs := []string{"-cp", Env.EnvVar.GetString(EnvAlluxioClientClasspath)} + + javaOpts := strings.TrimSpace(Env.EnvVar.GetString(p.JavaOptsEnvVarKey)) + // remove debugging options from java opts for monitor process + for _, re := range debugOptsToRemove { + javaOpts = re.ReplaceAllString(javaOpts, "") + } + cmdArgs = append(cmdArgs, strings.Split(javaOpts, " ")...) + + cmdArgs = append(cmdArgs, p.MonitorJavaClassName) + + cmd := exec.Command(Env.EnvVar.GetString(ConfJava.EnvVar), cmdArgs...) + for _, k := range Env.EnvVar.AllKeys() { + cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%v", k, Env.EnvVar.Get(k))) + } + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + log.Logger.Debugln(cmd.String()) + if err := cmd.Run(); err != nil { + log.Logger.Errorf("[ FAILED ] The %v process is not serving requests", p.Name) + return stacktrace.Propagate(err, "error running monitor command for %v", p.Name) + } + log.Logger.Infof("[ OK ] The %v process is in a healthy state", p.Name) + return nil +} + +func (p *BaseProcess) Stop(cmd *StopProcessCommand) error { + processes, err := process.Processes() + if err != nil { + return stacktrace.Propagate(err, "error listing processes") + } + var matchingProcesses []*process.Process + for _, ps := range processes { + cmdline, err := ps.Cmdline() + if err != nil { + // process may have been terminated since listing + continue + } + if strings.Contains(cmdline, p.JavaClassName) { + matchingProcesses = append(matchingProcesses, ps) + } + } + if len(matchingProcesses) == 0 { + log.Logger.Warnf("No process to stop because could not find running process matching %v", p.JavaClassName) + return nil + } + log.Logger.Infof("Found %v running process(es) matching %v", len(matchingProcesses), p.JavaClassName) + wg, errs := &sync.WaitGroup{}, make([]error, len(matchingProcesses)) + for i, ps := range matchingProcesses { + i, ps := i, ps + wg.Add(1) + go func() { + defer wg.Done() + pid := int(ps.Pid) + proc, err := os.FindProcess(pid) + if err != nil { + errs[i] = stacktrace.Propagate(err, "error finding process with pid %v", pid) + return + } + if err := proc.Signal(syscall.SIGTERM); err != nil { // syscall.SIGTERM = kill -15 + errs[i] = stacktrace.Propagate(err, "error sending TERM signal to process for %v with pid %v", p.JavaClassName, pid) + return + } + // wait for process to exit + const killTimeoutSec = 120 + if ok := waitForProcessExit(proc, killTimeoutSec); !ok { + if cmd.SoftKill { + errs[i] = fmt.Errorf("Process for %v with pid %v did not terminate after %v seconds", p.JavaClassName, pid, killTimeoutSec) + return + } else { + log.Logger.Warnf("Force killing process for %v with pid %v", p.JavaClassName, pid) + if err := proc.Signal(syscall.SIGKILL); err != nil { // syscall.SIGKILL = kill -9 + errs[i] = stacktrace.Propagate(err, "error sending KILL signal to process for %v with pid %v", p.JavaClassName, pid) + return + } + } + } + }() + } + wg.Wait() + + var errMsg []string + for _, err := range errs { + if err != nil { + errMsg = append(errMsg, fmt.Sprintf("Failed to kill process: %v", err.Error())) + } + } + log.Logger.Infof("Successfully killed %v process(es)", len(matchingProcesses)-len(errMsg)) + if len(errMsg) != 0 { + log.Logger.Errorf("Failed to kill %v process(es):", len(errMsg)) + for _, msg := range errMsg { + log.Logger.Errorln(msg) + } + } + + return nil +} + +func waitForProcessExit(proc *os.Process, killTimeoutSec int) bool { + timeout := time.After(time.Duration(killTimeoutSec) * time.Second) + tick := time.Tick(time.Second * 10) + for { + select { + case <-timeout: + return false + case <-tick: + if err := proc.Signal(syscall.Signal(0)); err != nil { + // process is not found, done + return true + } + } + } +} diff --git a/cli/src/alluxio.org/cli/env/process_start.go b/cli/src/alluxio.org/cli/env/process_start.go new file mode 100644 index 000000000000..bf329c21d02e --- /dev/null +++ b/cli/src/alluxio.org/cli/env/process_start.go @@ -0,0 +1,46 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package env + +import ( + "github.com/spf13/cobra" +) + +const StartProcessName = "start" + +type StartProcessCommand struct { + AsyncStart bool + SkipKillOnStart bool +} + +func (c *StartProcessCommand) ToCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: StartProcessName, + Short: "Starts a process", + } + cmd.PersistentFlags().BoolVarP(&c.SkipKillOnStart, "skip-kill-prev", "N", false, "Avoid killing previous running processes when starting") + cmd.PersistentFlags().BoolVarP(&c.AsyncStart, "async", "a", false, "Asynchronously start processes without monitoring for start completion") + + for _, p := range ProcessRegistry { + p := p + cmd.AddCommand(p.StartCmd(&cobra.Command{ + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + if !c.SkipKillOnStart { + _ = p.Stop(&StopProcessCommand{}) + } + return p.Start(c) + }, + })) + } + return cmd +} diff --git a/cli/src/alluxio.org/cli/env/process_stop.go b/cli/src/alluxio.org/cli/env/process_stop.go new file mode 100644 index 000000000000..88446d7aeb73 --- /dev/null +++ b/cli/src/alluxio.org/cli/env/process_stop.go @@ -0,0 +1,39 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package env + +import "github.com/spf13/cobra" + +const StopProcessName = "stop" + +type StopProcessCommand struct { + SoftKill bool +} + +func (c *StopProcessCommand) ToCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: StopProcessName, + Short: "Stops a process", + } + cmd.PersistentFlags().BoolVarP(&c.SoftKill, "soft", "s", false, "Soft kill only, don't forcibly kill the process") + + for _, p := range ProcessRegistry { + p := p + cmd.AddCommand(p.StopCmd(&cobra.Command{ + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + return p.Stop(c) + }, + })) + } + return cmd +} diff --git a/cli/src/alluxio.org/cli/env/props.go b/cli/src/alluxio.org/cli/env/props.go new file mode 100644 index 000000000000..875310de5074 --- /dev/null +++ b/cli/src/alluxio.org/cli/env/props.go @@ -0,0 +1,17 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package env + +const ( + ConfAlluxioConfValidationEnabled = "alluxio.conf.validation.enabled" + ConfAlluxioLoggerType = "alluxio.logger.type" +) diff --git a/cli/src/alluxio.org/cli/env/service.go b/cli/src/alluxio.org/cli/env/service.go new file mode 100644 index 000000000000..274e2222856f --- /dev/null +++ b/cli/src/alluxio.org/cli/env/service.go @@ -0,0 +1,57 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package env + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +var serviceRegistry = map[string]*Service{} + +func RegisterService(p *Service) *Service { + if _, ok := serviceRegistry[p.Name]; ok { + panic(fmt.Sprintf("Service %v is already registered", p.Name)) + } + serviceRegistry[p.Name] = p + return p +} + +func InitServiceCommandTree(rootCmd *cobra.Command) { + for _, p := range serviceRegistry { + p.InitCommandTree(rootCmd) + } +} + +type Service struct { + Name string + Description string + Commands []Command +} + +func (s *Service) InitCommandTree(rootCmd *cobra.Command) { + cmd := &cobra.Command{ + Use: s.Name, + Short: s.Description, + } + rootCmd.AddCommand(cmd) + + for _, c := range s.Commands { + cmd.AddCommand(c.ToCommand()) + } +} + +func (s *Service) AddCommands(c ...Command) *Service { + s.Commands = append(s.Commands, c...) + return s +} diff --git a/cli/src/alluxio.org/cli/launch/launch.go b/cli/src/alluxio.org/cli/launch/launch.go new file mode 100644 index 000000000000..a78db7acf303 --- /dev/null +++ b/cli/src/alluxio.org/cli/launch/launch.go @@ -0,0 +1,57 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package launch + +import ( + "path/filepath" + + "github.com/palantir/stacktrace" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "alluxio.org/cli/env" + "alluxio.org/log" +) + +func Run(jarEnvVars map[bool]map[string]string, appendClasspathJars map[string]func(*viper.Viper, string) string) error { + rootCmd := &cobra.Command{ + Use: "bin/alluxio", + } + const rootPathName = "rootPath" + var flagRootPath string + rootCmd.PersistentFlags().StringVar(&flagRootPath, rootPathName, "", "Path to root of Alluxio installation") + if err := rootCmd.MarkPersistentFlagRequired(rootPathName); err != nil { + return stacktrace.Propagate(err, "error marking %v flag required", rootPathName) + } + if err := rootCmd.PersistentFlags().MarkHidden(rootPathName); err != nil { + return stacktrace.Propagate(err, "error marking %v flag hidden", rootPathName) + } + var flagDebugLog bool + rootCmd.PersistentFlags().BoolVar(&flagDebugLog, "debug-log", false, "True to enable debug logging") + var flagIsDeployed bool + rootCmd.PersistentFlags().BoolVar(&flagIsDeployed, "deployed-env", false, "True to set paths to be compatible with a deployed environment") + + rootCmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error { + if flagDebugLog { + log.Logger.SetLevel(logrus.DebugLevel) + } + if err := env.InitAlluxioEnv(filepath.Clean(flagRootPath), jarEnvVars[flagIsDeployed], appendClasspathJars); err != nil { + return stacktrace.Propagate(err, "error defining alluxio environment") + } + return nil + } + + env.InitServiceCommandTree(rootCmd) + + return rootCmd.Execute() +} diff --git a/cli/src/alluxio.org/cli/main.go b/cli/src/alluxio.org/cli/main.go new file mode 100644 index 000000000000..9d96a96bf846 --- /dev/null +++ b/cli/src/alluxio.org/cli/main.go @@ -0,0 +1,92 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package main + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/spf13/viper" + + "alluxio.org/cli/cmd/cache" + "alluxio.org/cli/cmd/conf" + "alluxio.org/cli/cmd/exec" + "alluxio.org/cli/cmd/fs" + "alluxio.org/cli/cmd/generate" + "alluxio.org/cli/cmd/info" + "alluxio.org/cli/cmd/job" + "alluxio.org/cli/cmd/journal" + "alluxio.org/cli/cmd/process" + "alluxio.org/cli/env" + "alluxio.org/cli/launch" + "alluxio.org/cli/processes" +) + +func main() { + for _, p := range []env.Process{ + processes.All, + processes.JobMaster, + processes.JobMasters, + processes.JobWorker, + processes.JobWorkers, + processes.Local, + processes.Master, + processes.Masters, + processes.Proxy, + processes.Proxies, + processes.Worker, + processes.Workers, + } { + env.RegisterProcess(p) + } + + for _, c := range []*env.Service{ + cache.Service, + conf.Service, + exec.Service, + fs.Service, + generate.Service, + info.Service, + job.Service, + journal.Service, + process.Service, + } { + env.RegisterService(c) + } + + // isDeployed bool -> env var key -> relative path from root with version placeholder + jarEnvVars := map[bool]map[string]string{ + false: { + env.EnvAlluxioAssemblyClientJar: filepath.Join("assembly", "client", "target", "alluxio-assembly-client-%v-jar-with-dependencies.jar"), + env.EnvAlluxioAssemblyServerJar: filepath.Join("assembly", "server", "target", "alluxio-assembly-server-%v-jar-with-dependencies.jar"), + }, + true: { + env.EnvAlluxioAssemblyClientJar: filepath.Join("assembly", "alluxio-client-%v.jar"), + env.EnvAlluxioAssemblyServerJar: filepath.Join("assembly", "alluxio-server-%v.jar"), + }, + } + appendClasspathJars := map[string]func(*viper.Viper, string) string{ + env.EnvAlluxioClientClasspath: func(envVar *viper.Viper, ver string) string { + return strings.Join([]string{ + envVar.GetString(env.EnvAlluxioClientClasspath), + filepath.Join(envVar.GetString(env.ConfAlluxioHome.EnvVar), "lib", fmt.Sprintf("alluxio-integration-tools-validation-%v.jar", ver)), + }, ":") + + }, + } + + if err := launch.Run(jarEnvVars, appendClasspathJars); err != nil { + os.Exit(1) + } +} diff --git a/cli/src/alluxio.org/cli/processes/all.go b/cli/src/alluxio.org/cli/processes/all.go new file mode 100644 index 000000000000..73ecd6aadb83 --- /dev/null +++ b/cli/src/alluxio.org/cli/processes/all.go @@ -0,0 +1,76 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package processes + +import ( + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "alluxio.org/cli/env" +) + +var All = &AllProcess{ + BaseProcess: &env.BaseProcess{ + Name: "all", + }, + Processes: []env.Process{ + Masters, + JobMasters, + Workers, + JobWorkers, + Proxies, + }, +} + +type AllProcess struct { + *env.BaseProcess + Processes []env.Process +} + +func (p *AllProcess) SetEnvVars(_ *viper.Viper) { + return +} + +func (p *AllProcess) Base() *env.BaseProcess { + return p.BaseProcess +} + +func (p *AllProcess) StartCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} + +func (p *AllProcess) StopCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} + +func (p *AllProcess) Start(cmd *env.StartProcessCommand) error { + for i := 0; i < len(p.Processes); i++ { + subprocess := p.Processes[i] + if err := subprocess.Start(cmd); err != nil { + return stacktrace.Propagate(err, "error starting subprocesses for %s", subprocess.Base().Name) + } + } + return nil +} + +func (p *AllProcess) Stop(cmd *env.StopProcessCommand) error { + for i := len(p.Processes) - 1; i >= 0; i-- { + subprocess := p.Processes[i] + if err := subprocess.Stop(cmd); err != nil { + return stacktrace.Propagate(err, "error stopping subprocesses for %s", subprocess.Base().Name) + } + } + return nil +} diff --git a/cli/src/alluxio.org/cli/processes/common.go b/cli/src/alluxio.org/cli/processes/common.go new file mode 100644 index 000000000000..1f88dcf48770 --- /dev/null +++ b/cli/src/alluxio.org/cli/processes/common.go @@ -0,0 +1,251 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package processes + +import ( + "fmt" + "io" + "io/ioutil" + "net" + "os" + "os/user" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/palantir/stacktrace" + "golang.org/x/crypto/ssh" + + "alluxio.org/cli/cmd/names" + "alluxio.org/cli/env" + "alluxio.org/log" +) + +type HostnamesFile struct { + Name string + Mutex *sync.Mutex + Once *sync.Once + Hostnames []string +} + +func NewHostnamesFile(name string) *HostnamesFile { + return &HostnamesFile{ + Name: name, + Mutex: &sync.Mutex{}, + Once: &sync.Once{}, + } +} + +// runSshCommand parses hostnames from the hosts files such as `conf/masters` and `conf/workers` +// For each hostname, create a SSH client and run the given command +func runSshCommand(command string, hostGroups ...string) error { + // prepare client config, ssh port info + config, port, err := prepareCommand() + if err != nil { + return stacktrace.Propagate(err, "prepare command failed") + } + + // get list of masters or workers, or both + hosts, err := getHostnames(hostGroups) + if err != nil { + return stacktrace.Propagate(err, "cannot read host names") + } + + // create wait group and channels + var wg sync.WaitGroup + errs := make(chan error) + for _, h := range hosts { + h := h + wg.Add(1) + + go func() { + defer wg.Done() + // dial on target remote address with given host, config and ssh port + dialAddr := fmt.Sprintf("%s:%d", h, port) + conn, err := ssh.Dial("tcp", dialAddr, config) + if err != nil { + errs <- stacktrace.Propagate(err, "dial failed to %v", h) + return + } + defer func(conn *ssh.Client) { + if err := conn.Close(); err != nil { + log.Logger.Warnf("connection to %s closed, error: %s", h, err) + } else { + log.Logger.Debugf("connection to %s closed", h) + } + }(conn) + + // create and set up a session + session, err := conn.NewSession() + if err != nil { + errs <- stacktrace.Propagate(err, "cannot create session at %v.", h) + return + } + defer func(session *ssh.Session) { + if err := session.Close(); err != nil && err != io.EOF { + log.Logger.Warnf("session at %s closed, error: %s", h, err) + } else { + log.Logger.Debugf("session at %s closed", h) + } + }(session) + + session.Stdout = os.Stdout + session.Stderr = os.Stderr + + // run command on the session, output errors + if err = session.Run(command); err != nil { + errs <- stacktrace.Propagate(err, "run command %v failed at %v", command, h) + return + } + }() + } + + go func() { + wg.Wait() + close(errs) + }() + + var errMsgs []string + for err := range errs { + errMsgs = append(errMsgs, err.Error()) + } + if len(errMsgs) > 0 { + return stacktrace.NewError("At least one error encountered:\n%v", strings.Join(errMsgs, "\n")) + } + log.Logger.Infof("Command %s successful on nodes: %s", command, hosts) + return nil +} + +func prepareCommand() (*ssh.ClientConfig, int, error) { + // get the current user + cu, err := user.Current() + if err != nil { + return nil, 0, stacktrace.Propagate(err, "cannot find current user") + } + cuName := cu.Username + log.Logger.Debugf("current user: %v", cuName) + + // get public key + signer, err := getSigner() + if err != nil { + return nil, 0, stacktrace.Propagate(err, "cannot get private key") + } + + // find default ssh port + port, e := net.LookupPort("tcp", "ssh") + if e != nil { + return nil, 0, stacktrace.Propagate(err, "cannot find default ssh port") + } + + // set client config with current user and signer + config := &ssh.ClientConfig{ + User: cuName, + Auth: []ssh.AuthMethod{ + ssh.PublicKeys(signer), + }, + Timeout: 5 * time.Second, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + } + return config, port, nil +} + +func addStartFlags(cmd *env.StartProcessCommand, arguments ...string) string { + cliPath := filepath.Join(env.Env.EnvVar.GetString(env.ConfAlluxioHome.EnvVar), names.BinAlluxio) + var command []string + command = append(command, cliPath) + command = append(command, arguments...) + if cmd.AsyncStart { + command = append(command, "-a") + } + if cmd.SkipKillOnStart { + command = append(command, "-N") + } + return strings.Join(command, " ") +} + +func addStopFlags(cmd *env.StopProcessCommand, arguments ...string) string { + cliPath := filepath.Join(env.Env.EnvVar.GetString(env.ConfAlluxioHome.EnvVar), names.BinAlluxio) + var command []string + command = append(command, cliPath) + command = append(command, arguments...) + if cmd.SoftKill { + command = append(command, "-s") + } + return strings.Join(command, " ") +} + +const ( + HostGroupMasters = "masters" + HostGroupWorkers = "workers" +) + +func getHostnames(hostGroups []string) ([]string, error) { + var hosts []string + for _, hostGroup := range hostGroups { + switch hostGroup { + case HostGroupMasters, HostGroupWorkers: + hostnames, err := NewHostnamesFile(hostGroup).getHostnames() + if err != nil { + return nil, stacktrace.Propagate(err, "error listing hostnames from %v", hostGroup) + } + hosts = append(hosts, hostnames...) + default: + return nil, stacktrace.NewError("unknown hosts file: %v", hostGroup) + } + + } + return hosts, nil +} + +func (f *HostnamesFile) getHostnames() ([]string, error) { + f.Mutex.Lock() + defer f.Mutex.Unlock() + var parseErr error + f.Once.Do(func() { + p := filepath.Join(env.Env.EnvVar.GetString(env.ConfAlluxioConfDir.EnvVar), f.Name) + file, err := ioutil.ReadFile(p) + if err != nil { + parseErr = err + } + for _, line := range strings.Split(string(file), "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "#") { + continue + } + if strings.TrimSpace(line) != "" { + f.Hostnames = append(f.Hostnames, line) + } + } + }) + if parseErr != nil { + return nil, stacktrace.Propagate(parseErr, "error parsing hostnames file") + } + return f.Hostnames, nil +} + +func getSigner() (ssh.Signer, error) { + // get private key + homePath, err := os.UserHomeDir() + if err != nil { + return nil, stacktrace.Propagate(err, "user home directory not found at %v", homePath) + } + privateKeyFile := filepath.Join(homePath, ".ssh", "id_rsa") + privateKey, err := os.ReadFile(privateKeyFile) + if err != nil { + return nil, stacktrace.Propagate(err, "private key file not found at %v", privateKeyFile) + } + parsedPrivateKey, err := ssh.ParsePrivateKey(privateKey) + if err != nil { + return nil, stacktrace.Propagate(err, "cannot parse public key at %v", privateKeyFile) + } + return parsedPrivateKey, nil +} diff --git a/cli/src/alluxio.org/cli/processes/job_master.go b/cli/src/alluxio.org/cli/processes/job_master.go new file mode 100644 index 000000000000..efb4a2ad36b9 --- /dev/null +++ b/cli/src/alluxio.org/cli/processes/job_master.go @@ -0,0 +1,99 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package processes + +import ( + "fmt" + "strings" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "alluxio.org/cli/env" +) + +var JobMaster = &JobMasterProcess{ + BaseProcess: &env.BaseProcess{ + Name: "job_master", + JavaClassName: "alluxio.master.AlluxioJobMaster", + JavaOptsEnvVarKey: ConfAlluxioJobMasterJavaOpts.EnvVar, + ProcessOutFile: "job_master.out", + MonitorJavaClassName: "alluxio.master.job.AlluxioJobMasterMonitor", + }, +} + +const ( + confAlluxioJobMasterAuditLoggerType = "alluxio.job.master.audit.logger.type" + envAlluxioAuditJobMasterLogger = "ALLUXIO_AUDIT_JOB_MASTER_LOGGER" + envAlluxioJobMasterLogger = "ALLUXIO_JOB_MASTER_LOGGER" + jobMasterAuditLoggerType = "JOB_MASTER_AUDIT_LOGGER" + jobMasterLoggerType = "JOB_MASTER_LOGGER" +) + +var ( + ConfAlluxioJobMasterJavaOpts = env.RegisterTemplateEnvVar(&env.AlluxioConfigEnvVar{ + EnvVar: "ALLUXIO_JOB_MASTER_JAVA_OPTS", + }) + confAlluxioJobMasterAttachOpts = env.RegisterTemplateEnvVar(&env.AlluxioConfigEnvVar{ + EnvVar: "ALLUXIO_JOB_MASTER_ATTACH_OPTS", + }) +) + +type JobMasterProcess struct { + *env.BaseProcess +} + +func (p *JobMasterProcess) Base() *env.BaseProcess { + return p.BaseProcess +} + +func (p *JobMasterProcess) SetEnvVars(envVar *viper.Viper) { + // ALLUXIO_JOB_MASTER_JAVA_OPTS = {default logger opts} ${ALLUXIO_JAVA_OPTS} ${ALLUXIO_JOB_MASTER_JAVA_OPTS} + envVar.SetDefault(envAlluxioJobMasterLogger, jobMasterLoggerType) + jobMasterJavaOpts := fmt.Sprintf(env.JavaOptFormat, env.ConfAlluxioLoggerType, envVar.Get(envAlluxioJobMasterLogger)) + envVar.SetDefault(envAlluxioAuditJobMasterLogger, jobMasterAuditLoggerType) + jobMasterJavaOpts += fmt.Sprintf(env.JavaOptFormat, confAlluxioJobMasterAuditLoggerType, envVar.Get(envAlluxioAuditJobMasterLogger)) + + jobMasterJavaOpts += envVar.GetString(env.ConfAlluxioJavaOpts.EnvVar) + jobMasterJavaOpts += envVar.GetString(p.JavaOptsEnvVarKey) + + envVar.Set(p.JavaOptsEnvVarKey, strings.TrimSpace(jobMasterJavaOpts)) // leading spaces need to be trimmed as a exec.Command argument +} + +func (p *JobMasterProcess) StartCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} + +func (p *JobMasterProcess) Start(cmd *env.StartProcessCommand) error { + cmdArgs := []string{env.Env.EnvVar.GetString(env.ConfJava.EnvVar)} + if attachOpts := env.Env.EnvVar.GetString(confAlluxioJobMasterAttachOpts.EnvVar); attachOpts != "" { + cmdArgs = append(cmdArgs, strings.Split(attachOpts, " ")...) + } + cmdArgs = append(cmdArgs, "-cp", env.Env.EnvVar.GetString(env.EnvAlluxioServerClasspath)) + + jobMasterJavaOpts := env.Env.EnvVar.GetString(p.JavaOptsEnvVarKey) + cmdArgs = append(cmdArgs, strings.Split(jobMasterJavaOpts, " ")...) + + cmdArgs = append(cmdArgs, p.JavaClassName) + + if err := p.Launch(cmd, cmdArgs); err != nil { + return stacktrace.Propagate(err, "error launching process") + } + return nil +} + +func (p *JobMasterProcess) StopCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} diff --git a/cli/src/alluxio.org/cli/processes/job_worker.go b/cli/src/alluxio.org/cli/processes/job_worker.go new file mode 100644 index 000000000000..671c8a7ae899 --- /dev/null +++ b/cli/src/alluxio.org/cli/processes/job_worker.go @@ -0,0 +1,94 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package processes + +import ( + "fmt" + "strings" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "alluxio.org/cli/env" +) + +var JobWorker = &WorkerProcess{ + BaseProcess: &env.BaseProcess{ + Name: "job_worker", + JavaClassName: "alluxio.worker.AlluxioJobWorker", + JavaOptsEnvVarKey: ConfAlluxioJobWorkerJavaOpts.EnvVar, + ProcessOutFile: "job_worker.out", + MonitorJavaClassName: "alluxio.worker.job.AlluxioJobWorkerMonitor", + }, +} + +const ( + envAlluxioJobWorkerLogger = "ALLUXIO_JOB_WORKER_LOGGER" + jobWorkerLoggerType = "JOB_WORKER_LOGGER" +) + +var ( + ConfAlluxioJobWorkerJavaOpts = env.RegisterTemplateEnvVar(&env.AlluxioConfigEnvVar{ + EnvVar: "ALLUXIO_JOB_WORKER_JAVA_OPTS", + }) + confAlluxioJobWorkerAttachOpts = env.RegisterTemplateEnvVar(&env.AlluxioConfigEnvVar{ + EnvVar: "ALLUXIO_JOB_WORKER_ATTACH_OPTS", + }) +) + +type JobWorkerProcess struct { + *env.BaseProcess +} + +func (p *JobWorkerProcess) Base() *env.BaseProcess { + return p.BaseProcess +} + +func (p *JobWorkerProcess) SetEnvVars(envVar *viper.Viper) { + // ALLUXIO_JOB_WORKER_JAVA_OPTS = {default logger opts} ${ALLUXIO_JAVA_OPTS} ${ALLUXIO_WORKER_JAVA_OPTS} + envVar.SetDefault(envAlluxioJobWorkerLogger, jobWorkerLoggerType) + jobWorkerJavaOpts := fmt.Sprintf(env.JavaOptFormat, env.ConfAlluxioLoggerType, envVar.Get(envAlluxioJobWorkerLogger)) + + jobWorkerJavaOpts += envVar.GetString(env.ConfAlluxioJavaOpts.EnvVar) + jobWorkerJavaOpts += envVar.GetString(p.JavaOptsEnvVarKey) + + envVar.Set(p.JavaOptsEnvVarKey, strings.TrimSpace(jobWorkerJavaOpts)) // leading spaces need to be trimmed as a exec.Command argument +} + +func (p *JobWorkerProcess) StartCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} + +func (p *JobWorkerProcess) Start(cmd *env.StartProcessCommand) error { + cmdArgs := []string{env.Env.EnvVar.GetString(env.ConfJava.EnvVar)} + if attachOpts := env.Env.EnvVar.GetString(confAlluxioJobWorkerAttachOpts.EnvVar); attachOpts != "" { + cmdArgs = append(cmdArgs, strings.Split(attachOpts, " ")...) + } + cmdArgs = append(cmdArgs, "-cp", env.Env.EnvVar.GetString(env.EnvAlluxioServerClasspath)) + + jobWorkerJavaOpts := env.Env.EnvVar.GetString(p.JavaOptsEnvVarKey) + cmdArgs = append(cmdArgs, strings.Split(jobWorkerJavaOpts, " ")...) + + cmdArgs = append(cmdArgs, p.JavaClassName) + + if err := p.Launch(cmd, cmdArgs); err != nil { + return stacktrace.Propagate(err, "error launching process") + } + return nil +} + +func (p *JobWorkerProcess) StopCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} diff --git a/cli/src/alluxio.org/cli/processes/local.go b/cli/src/alluxio.org/cli/processes/local.go new file mode 100644 index 000000000000..2aa2b6e108ca --- /dev/null +++ b/cli/src/alluxio.org/cli/processes/local.go @@ -0,0 +1,76 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package processes + +import ( + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "alluxio.org/cli/env" +) + +var Local = &LocalProcess{ + BaseProcess: &env.BaseProcess{ + Name: "local", + }, + Processes: []env.Process{ + Master, + JobMaster, + Worker, + JobWorker, + Proxy, + }, +} + +type LocalProcess struct { + *env.BaseProcess + Processes []env.Process +} + +func (p *LocalProcess) SetEnvVars(envVar *viper.Viper) { + return +} + +func (p *LocalProcess) Base() *env.BaseProcess { + return p.BaseProcess +} + +func (p *LocalProcess) StartCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} + +func (p *LocalProcess) StopCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} + +func (p *LocalProcess) Start(cmd *env.StartProcessCommand) error { + for i := 0; i < len(p.Processes); i++ { + subProcess := p.Processes[i] + if err := subProcess.Start(cmd); err != nil { + return stacktrace.Propagate(err, "Error starting subprocesses for %s", p.Processes[i]) + } + } + return nil +} + +func (p *LocalProcess) Stop(cmd *env.StopProcessCommand) error { + for i := len(p.Processes) - 1; i >= 0; i-- { + subProcess := p.Processes[i] + if err := subProcess.Stop(cmd); err != nil { + return stacktrace.Propagate(err, "Error stopping subprocesses for %s", p.Processes[i]) + } + } + return nil +} diff --git a/cli/src/alluxio.org/cli/processes/master.go b/cli/src/alluxio.org/cli/processes/master.go new file mode 100644 index 000000000000..70f85e7b6805 --- /dev/null +++ b/cli/src/alluxio.org/cli/processes/master.go @@ -0,0 +1,150 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package processes + +import ( + "fmt" + "os" + "strings" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "alluxio.org/cli/cmd/conf" + "alluxio.org/cli/cmd/journal" + "alluxio.org/cli/env" + "alluxio.org/log" +) + +var Master = &MasterProcess{ + BaseProcess: &env.BaseProcess{ + Name: "master", + JavaClassName: "alluxio.master.AlluxioMaster", + JavaOptsEnvVarKey: ConfAlluxioMasterJavaOpts.EnvVar, + ProcessOutFile: "master.out", + MonitorJavaClassName: "alluxio.master.AlluxioMasterMonitor", + }, +} + +const ( + confAlluxioMasterAuditLoggerType = "alluxio.master.audit.logger.type" + confAlluxioMasterJournalFolder = "alluxio.master.journal.folder" + confAlluxioMasterJournalInitFromBackup = "alluxio.master.journal.init.from.backup" + + envAlluxioAuditMasterLogger = "ALLUXIO_AUDIT_MASTER_LOGGER" + envAlluxioMasterLogger = "ALLUXIO_MASTER_LOGGER" + masterAuditLoggerType = "MASTER_AUDIT_LOGGER" + masterLoggerType = "MASTER_LOGGER" +) + +var ( + ConfAlluxioMasterJavaOpts = env.RegisterTemplateEnvVar(&env.AlluxioConfigEnvVar{ + EnvVar: "ALLUXIO_MASTER_JAVA_OPTS", + }) + confAlluxioMasterAttachOpts = env.RegisterTemplateEnvVar(&env.AlluxioConfigEnvVar{ + EnvVar: "ALLUXIO_MASTER_ATTACH_OPTS", + }) +) + +type MasterProcess struct { + *env.BaseProcess + + JournalBackupFile string +} + +func (p *MasterProcess) Base() *env.BaseProcess { + return p.BaseProcess +} + +func (p *MasterProcess) SetEnvVars(envVar *viper.Viper) { + // ALLUXIO_MASTER_JAVA_OPTS = {default logger opts} ${ALLUXIO_JAVA_OPTS} ${ALLUXIO_MASTER_JAVA_OPTS} + envVar.SetDefault(envAlluxioMasterLogger, masterLoggerType) + masterJavaOpts := fmt.Sprintf(env.JavaOptFormat, env.ConfAlluxioLoggerType, envVar.Get(envAlluxioMasterLogger)) + envVar.SetDefault(envAlluxioAuditMasterLogger, masterAuditLoggerType) + masterJavaOpts += fmt.Sprintf(env.JavaOptFormat, confAlluxioMasterAuditLoggerType, envVar.Get(envAlluxioAuditMasterLogger)) + + masterJavaOpts += envVar.GetString(env.ConfAlluxioJavaOpts.EnvVar) + masterJavaOpts += envVar.GetString(p.JavaOptsEnvVarKey) + + envVar.Set(p.JavaOptsEnvVarKey, strings.TrimSpace(masterJavaOpts)) // leading spaces need to be trimmed as a exec.Command argument +} + +func (p *MasterProcess) StartCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + cmd.Flags().StringVar(&p.JournalBackupFile, "journal-backup-file", "", "Path to journal backup file to restore the master from") + return cmd +} + +func (p *MasterProcess) Start(cmd *env.StartProcessCommand) error { + if err := p.checkJournal(); err != nil { + return stacktrace.Propagate(err, "error validating journal") + } + + cmdArgs := []string{env.Env.EnvVar.GetString(env.ConfJava.EnvVar)} + if attachOpts := env.Env.EnvVar.GetString(confAlluxioMasterAttachOpts.EnvVar); attachOpts != "" { + cmdArgs = append(cmdArgs, strings.Split(attachOpts, " ")...) + } + cmdArgs = append(cmdArgs, "-cp", env.Env.EnvVar.GetString(env.EnvAlluxioServerClasspath)) + + masterJavaOpts := env.Env.EnvVar.GetString(p.JavaOptsEnvVarKey) + cmdArgs = append(cmdArgs, strings.Split(masterJavaOpts, " ")...) + + if p.JournalBackupFile != "" { + cmdArgs = append(cmdArgs, strings.TrimSpace(fmt.Sprintf(env.JavaOptFormat, confAlluxioMasterJournalInitFromBackup, p.JournalBackupFile))) + } + // specify a default of -Xmx8g if no memory setting is specified + const xmxOpt = "-Xmx" + if !strings.Contains(masterJavaOpts, xmxOpt) && !strings.Contains(masterJavaOpts, "MaxRAMPercentage") { + cmdArgs = append(cmdArgs, fmt.Sprintf("%v8g", xmxOpt)) + } + // specify a default of -XX:MetaspaceSize=256M if not set + const metaspaceSizeOpt = "-XX:MetaspaceSize" + if !strings.Contains(masterJavaOpts, metaspaceSizeOpt) { + cmdArgs = append(cmdArgs, fmt.Sprintf("%v=256M", metaspaceSizeOpt)) + } + + cmdArgs = append(cmdArgs, p.JavaClassName) + + if err := p.Launch(cmd, cmdArgs); err != nil { + return stacktrace.Propagate(err, "error launching process") + } + return nil +} + +func (p *MasterProcess) StopCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} + +func (p *MasterProcess) checkJournal() error { + journalDir, err := conf.Get.FetchValue(confAlluxioMasterJournalFolder) + if err != nil { + return stacktrace.Propagate(err, "error fetching value for %v", confAlluxioMasterJournalFolder) + } + stat, err := os.Stat(journalDir) + if os.IsNotExist(err) { + log.Logger.Info("Journal directory does not exist, formatting") + if err := journal.Format.Format(); err != nil { + return stacktrace.Propagate(err, "error formatting journal") + } + return nil + } + if err != nil { + return stacktrace.Propagate(err, "error listing path at %v", journalDir) + } + if !stat.IsDir() { + return stacktrace.NewError("Journal location %v is not a directory. Please remove the file and retry.", journalDir) + } + // journal folder path exists and is a directory + return nil +} diff --git a/cli/src/alluxio.org/cli/processes/multi.go b/cli/src/alluxio.org/cli/processes/multi.go new file mode 100644 index 000000000000..4d8dc03bf861 --- /dev/null +++ b/cli/src/alluxio.org/cli/processes/multi.go @@ -0,0 +1,100 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package processes + +import ( + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "alluxio.org/cli/cmd/process" + "alluxio.org/cli/env" +) + +var ( + JobMasters = &MultiProcess{ + BaseProcess: &env.BaseProcess{ + Name: "job_masters", + }, + HostnameFiles: []string{HostGroupMasters}, + SingleProcessName: JobMaster.Name, + } + JobWorkers = &MultiProcess{ + BaseProcess: &env.BaseProcess{ + Name: "job_workers", + }, + HostnameFiles: []string{HostGroupWorkers}, + SingleProcessName: JobWorker.Name, + } + Masters = &MultiProcess{ + BaseProcess: &env.BaseProcess{ + Name: "masters", + }, + HostnameFiles: []string{HostGroupMasters}, + SingleProcessName: Master.Name, + } + Proxies = &MultiProcess{ + BaseProcess: &env.BaseProcess{ + Name: "proxies", + }, + HostnameFiles: []string{HostGroupMasters, HostGroupWorkers}, + SingleProcessName: Proxy.Name, + } + Workers = &MultiProcess{ + BaseProcess: &env.BaseProcess{ + Name: "workers", + }, + HostnameFiles: []string{HostGroupWorkers}, + SingleProcessName: Worker.Name, + } +) + +type MultiProcess struct { + *env.BaseProcess + HostnameFiles []string + SingleProcessName string +} + +func (p *MultiProcess) SetEnvVars(_ *viper.Viper) { + return +} + +func (p *MultiProcess) Base() *env.BaseProcess { + return p.BaseProcess +} + +func (p *MultiProcess) StartCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} + +func (p *MultiProcess) StopCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} + +func (p *MultiProcess) Start(cmd *env.StartProcessCommand) error { + return runSshCommand( + addStartFlags(cmd, + process.Service.Name, + env.StartProcessName, + p.SingleProcessName, + ), p.HostnameFiles...) +} + +func (p *MultiProcess) Stop(cmd *env.StopProcessCommand) error { + return runSshCommand( + addStopFlags(cmd, + process.Service.Name, + env.StopProcessName, + p.SingleProcessName, + ), p.HostnameFiles...) +} diff --git a/cli/src/alluxio.org/cli/processes/proxy.go b/cli/src/alluxio.org/cli/processes/proxy.go new file mode 100644 index 000000000000..1fd66cb01669 --- /dev/null +++ b/cli/src/alluxio.org/cli/processes/proxy.go @@ -0,0 +1,99 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package processes + +import ( + "fmt" + "strings" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "alluxio.org/cli/env" +) + +var Proxy = &ProxyProcess{ + BaseProcess: &env.BaseProcess{ + Name: "proxy", + JavaClassName: "alluxio.proxy.AlluxioProxy", + JavaOptsEnvVarKey: ConfAlluxioProxyJavaOpts.EnvVar, + ProcessOutFile: "proxy.out", + MonitorJavaClassName: "alluxio.proxy.AlluxioProxyMonitor", + }, +} + +const ( + confAlluxioProxyAuditLoggerType = "alluxio.proxy.audit.logger.type" + envAlluxioAuditProxyLogger = "ALLUXIO_AUDIT_PROXY_LOGGER" + envAlluxioProxyLogger = "ALLUXIO_PROXY_LOGGER" + proxyAuditLoggerType = "PROXY_AUDIT_LOGGER" + proxyLoggerType = "PROXY_LOGGER" +) + +var ( + ConfAlluxioProxyJavaOpts = env.RegisterTemplateEnvVar(&env.AlluxioConfigEnvVar{ + EnvVar: "ALLUXIO_PROXY_JAVA_OPTS", + }) + confAlluxioProxyAttachOpts = env.RegisterTemplateEnvVar(&env.AlluxioConfigEnvVar{ + EnvVar: "ALLUXIO_PROXY_ATTACH_OPTS", + }) +) + +type ProxyProcess struct { + *env.BaseProcess +} + +func (p *ProxyProcess) Base() *env.BaseProcess { + return p.BaseProcess +} + +func (p *ProxyProcess) SetEnvVars(envVar *viper.Viper) { + // ALLUXIO_PROXY_JAVA_OPTS = {default logger opts} ${ALLUXIO_JAVA_OPTS} ${ALLUXIO_PROXY_JAVA_OPTS} + envVar.SetDefault(envAlluxioProxyLogger, proxyLoggerType) + proxyJavaOpts := fmt.Sprintf(env.JavaOptFormat, env.ConfAlluxioLoggerType, envVar.Get(envAlluxioProxyLogger)) + envVar.SetDefault(envAlluxioAuditProxyLogger, proxyAuditLoggerType) + proxyJavaOpts += fmt.Sprintf(env.JavaOptFormat, confAlluxioProxyAuditLoggerType, envVar.Get(envAlluxioAuditProxyLogger)) + + proxyJavaOpts += envVar.GetString(env.ConfAlluxioJavaOpts.EnvVar) + proxyJavaOpts += envVar.GetString(p.JavaOptsEnvVarKey) + + envVar.Set(p.JavaOptsEnvVarKey, strings.TrimSpace(proxyJavaOpts)) // leading spaces need to be trimmed as a exec.Command argument +} + +func (p *ProxyProcess) StartCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} + +func (p *ProxyProcess) Start(cmd *env.StartProcessCommand) error { + cmdArgs := []string{env.Env.EnvVar.GetString(env.ConfJava.EnvVar)} + if attachOpts := env.Env.EnvVar.GetString(confAlluxioProxyAttachOpts.EnvVar); attachOpts != "" { + cmdArgs = append(cmdArgs, strings.Split(attachOpts, " ")...) + } + cmdArgs = append(cmdArgs, "-cp", env.Env.EnvVar.GetString(env.EnvAlluxioServerClasspath)) + + proxyJavaOpts := env.Env.EnvVar.GetString(p.JavaOptsEnvVarKey) + cmdArgs = append(cmdArgs, strings.Split(proxyJavaOpts, " ")...) + + cmdArgs = append(cmdArgs, p.JavaClassName) + + if err := p.Launch(cmd, cmdArgs); err != nil { + return stacktrace.Propagate(err, "error launching process") + } + return nil +} + +func (p *ProxyProcess) StopCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} diff --git a/cli/src/alluxio.org/cli/processes/worker.go b/cli/src/alluxio.org/cli/processes/worker.go new file mode 100644 index 000000000000..def4aa0adda1 --- /dev/null +++ b/cli/src/alluxio.org/cli/processes/worker.go @@ -0,0 +1,105 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package processes + +import ( + "fmt" + "strings" + + "github.com/palantir/stacktrace" + "github.com/spf13/cobra" + "github.com/spf13/viper" + + "alluxio.org/cli/env" +) + +var Worker = &WorkerProcess{ + BaseProcess: &env.BaseProcess{ + Name: "worker", + JavaClassName: "alluxio.worker.AlluxioWorker", + JavaOptsEnvVarKey: ConfAlluxioWorkerJavaOpts.EnvVar, + ProcessOutFile: "worker.out", + MonitorJavaClassName: "alluxio.worker.AlluxioWorkerMonitor", + }, +} + +const ( + envAlluxioWorkerLogger = "ALLUXIO_WORKER_LOGGER" + workerLoggerType = "WORKER_LOGGER" +) + +var ( + ConfAlluxioWorkerJavaOpts = env.RegisterTemplateEnvVar(&env.AlluxioConfigEnvVar{ + EnvVar: "ALLUXIO_WORKER_JAVA_OPTS", + }) + confAlluxioWorkerAttachOpts = env.RegisterTemplateEnvVar(&env.AlluxioConfigEnvVar{ + EnvVar: "ALLUXIO_WORKER_ATTACH_OPTS", + }) +) + +type WorkerProcess struct { + *env.BaseProcess +} + +func (p *WorkerProcess) Base() *env.BaseProcess { + return p.BaseProcess +} + +func (p *WorkerProcess) SetEnvVars(envVar *viper.Viper) { + // ALLUXIO_WORKER_JAVA_OPTS = {default logger opts} ${ALLUXIO_JAVA_OPTS} ${ALLUXIO_WORKER_JAVA_OPTS} + envVar.SetDefault(envAlluxioWorkerLogger, workerLoggerType) + workerJavaOpts := fmt.Sprintf(env.JavaOptFormat, env.ConfAlluxioLoggerType, envVar.Get(envAlluxioWorkerLogger)) + + workerJavaOpts += envVar.GetString(env.ConfAlluxioJavaOpts.EnvVar) + workerJavaOpts += envVar.GetString(p.JavaOptsEnvVarKey) + + envVar.Set(p.JavaOptsEnvVarKey, strings.TrimSpace(workerJavaOpts)) // leading spaces need to be trimmed as a exec.Command argument +} + +func (p *WorkerProcess) StartCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} + +func (p *WorkerProcess) Start(cmd *env.StartProcessCommand) error { + cmdArgs := []string{env.Env.EnvVar.GetString(env.ConfJava.EnvVar)} + if attachOpts := env.Env.EnvVar.GetString(confAlluxioWorkerAttachOpts.EnvVar); attachOpts != "" { + cmdArgs = append(cmdArgs, strings.Split(attachOpts, " ")...) + } + cmdArgs = append(cmdArgs, "-cp", env.Env.EnvVar.GetString(env.EnvAlluxioServerClasspath)) + + workerJavaOpts := env.Env.EnvVar.GetString(p.JavaOptsEnvVarKey) + cmdArgs = append(cmdArgs, strings.Split(workerJavaOpts, " ")...) + + // specify a default of -Xmx4g if no memory setting is specified + const xmxOpt = "-Xmx" + if !strings.Contains(workerJavaOpts, xmxOpt) && !strings.Contains(workerJavaOpts, "MaxRAMPercentage") { + cmdArgs = append(cmdArgs, fmt.Sprintf("%v4g", xmxOpt)) + } + // specify a default of -XX:MaxDirectMemorySize=4g if not set + const maxDirectMemorySize = "-XX:MaxDirectMemorySize" + if !strings.Contains(workerJavaOpts, maxDirectMemorySize) { + cmdArgs = append(cmdArgs, fmt.Sprintf("%v=4g", maxDirectMemorySize)) + } + + cmdArgs = append(cmdArgs, p.JavaClassName) + + if err := p.Launch(cmd, cmdArgs); err != nil { + return stacktrace.Propagate(err, "error launching process") + } + return nil +} + +func (p *WorkerProcess) StopCmd(cmd *cobra.Command) *cobra.Command { + cmd.Use = p.Name + return cmd +} diff --git a/cli/src/alluxio.org/go.mod b/cli/src/alluxio.org/go.mod new file mode 100644 index 000000000000..958b996eb25a --- /dev/null +++ b/cli/src/alluxio.org/go.mod @@ -0,0 +1,35 @@ +module alluxio.org + +go 1.18 + +require ( + github.com/palantir/stacktrace v0.0.0-20161112013806-78658fd2d177 + github.com/shirou/gopsutil v3.21.11+incompatible + github.com/sirupsen/logrus v1.9.0 + github.com/spf13/cobra v1.7.0 + github.com/spf13/viper v1.15.0 + golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e +) + +require ( + github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/magiconair/properties v1.8.7 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/pelletier/go-toml/v2 v2.0.6 // indirect + github.com/spf13/afero v1.9.3 // indirect + github.com/spf13/cast v1.5.0 // indirect + github.com/spf13/jwalterweatherman v1.1.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/stretchr/testify v1.8.2 // indirect + github.com/subosito/gotenv v1.4.2 // indirect + github.com/tklauser/go-sysconf v0.3.11 // indirect + github.com/tklauser/numcpus v0.6.0 // indirect + github.com/yusufpapurcu/wmi v1.2.3 // indirect + golang.org/x/sys v0.3.0 // indirect + golang.org/x/text v0.5.0 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/cli/src/alluxio.org/go.sum b/cli/src/alluxio.org/go.sum new file mode 100644 index 000000000000..c364d69e05c6 --- /dev/null +++ b/cli/src/alluxio.org/go.sum @@ -0,0 +1,504 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.44.3/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/palantir/stacktrace v0.0.0-20161112013806-78658fd2d177 h1:nRlQD0u1871kaznCnn1EvYiMbum36v7hw1DLPEjds4o= +github.com/palantir/stacktrace v0.0.0-20161112013806-78658fd2d177/go.mod h1:ao5zGxj8Z4x60IOVYZUbDSmt3R8Ddo080vEgPosHpak= +github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= +github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= +github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spf13/afero v1.9.3 h1:41FoI0fD7OR7mGcKE/aOiLkGreyf8ifIOQmJANWogMk= +github.com/spf13/afero v1.9.3/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= +github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= +github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.15.0 h1:js3yy885G8xwJa6iOISGFwd+qlUo5AvyXb7CiihdtiU= +github.com/spf13/viper v1.15.0/go.mod h1:fFcTBJxvhhzSJiZy8n+PeW6t8l+KeT/uTARa0jHOQLA= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= +github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= +github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= +github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= +github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms= +github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= +github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e h1:T8NU3HyQ8ClP4SEE+KbFlg6n0NhuTsN4MyznaarGsZM= +golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/cli/src/alluxio.org/log/log.go b/cli/src/alluxio.org/log/log.go new file mode 100644 index 000000000000..86e50b31187b --- /dev/null +++ b/cli/src/alluxio.org/log/log.go @@ -0,0 +1,30 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + +package log + +import ( + "os" + + "github.com/sirupsen/logrus" +) + +var Logger *logrus.Logger + +func init() { + l := logrus.New() + l.SetFormatter(&logrus.TextFormatter{ + FullTimestamp: true, + }) + l.SetOutput(os.Stdout) + + Logger = l +} diff --git a/dev/scripts/src/alluxio.org/build/profiles.yml b/dev/scripts/src/alluxio.org/build/profiles.yml index a63f3fe07836..9e91795ab612 100644 --- a/dev/scripts/src/alluxio.org/build/profiles.yml +++ b/dev/scripts/src/alluxio.org/build/profiles.yml @@ -26,14 +26,22 @@ profiles: - underFSStorage fileList: - bin/alluxio - - bin/alluxio-common.sh - - bin/alluxio-masters.sh - - bin/alluxio-monitor.sh + - bin/alluxio-bash + - bin/alluxio-common-bash.sh + - bin/alluxio-masters-bash.sh + - bin/alluxio-monitor-bash.sh + - bin/alluxio-mount-bash.sh - bin/alluxio-mount.sh + - bin/alluxio-start-bash.sh - bin/alluxio-start.sh + - bin/alluxio-stop-bash.sh - bin/alluxio-stop.sh - - bin/alluxio-workers.sh - - bin/launch-process + - bin/alluxio-workers-bash.sh + - bin/launch-process-bash + - cli/src/alluxio.org/cli/bin/alluxioCli-Darwin-aarch64 + - cli/src/alluxio.org/cli/bin/alluxioCli-Darwin-x86_64 + - cli/src/alluxio.org/cli/bin/alluxioCli-Linux-aarch64 + - cli/src/alluxio.org/cli/bin/alluxioCli-Linux-x86_64 - conf/alluxio-env.sh.template - conf/alluxio-site.properties.template - conf/core-site.xml.template @@ -74,14 +82,22 @@ profiles: - underFSStorage fileList: - bin/alluxio - - bin/alluxio-common.sh - - bin/alluxio-masters.sh - - bin/alluxio-monitor.sh + - bin/alluxio-bash + - bin/alluxio-common-bash.sh + - bin/alluxio-masters-bash.sh + - bin/alluxio-monitor-bash.sh + - bin/alluxio-mount-bash.sh - bin/alluxio-mount.sh + - bin/alluxio-start-bash.sh - bin/alluxio-start.sh + - bin/alluxio-stop-bash.sh - bin/alluxio-stop.sh - - bin/alluxio-workers.sh - - bin/launch-process + - bin/alluxio-workers-bash.sh + - bin/launch-process-bash + - cli/src/alluxio.org/cli/bin/alluxioCli-Darwin-aarch64 + - cli/src/alluxio.org/cli/bin/alluxioCli-Darwin-x86_64 + - cli/src/alluxio.org/cli/bin/alluxioCli-Linux-aarch64 + - cli/src/alluxio.org/cli/bin/alluxioCli-Linux-x86_64 - conf/alluxio-env.sh.template - conf/alluxio-site.properties.template - conf/core-site.xml.template @@ -122,14 +138,22 @@ profiles: - underFSStorage fileList: - bin/alluxio - - bin/alluxio-common.sh - - bin/alluxio-masters.sh - - bin/alluxio-monitor.sh + - bin/alluxio-bash + - bin/alluxio-common-bash.sh + - bin/alluxio-masters-bash.sh + - bin/alluxio-monitor-bash.sh + - bin/alluxio-mount-bash.sh - bin/alluxio-mount.sh + - bin/alluxio-start-bash.sh - bin/alluxio-start.sh + - bin/alluxio-stop-bash.sh - bin/alluxio-stop.sh - - bin/alluxio-workers.sh - - bin/launch-process + - bin/alluxio-workers-bash.sh + - bin/launch-process-bash + - cli/src/alluxio.org/cli/bin/alluxioCli-Darwin-aarch64 + - cli/src/alluxio.org/cli/bin/alluxioCli-Darwin-x86_64 + - cli/src/alluxio.org/cli/bin/alluxioCli-Linux-aarch64 + - cli/src/alluxio.org/cli/bin/alluxioCli-Linux-x86_64 - conf/alluxio-env.sh.template - conf/alluxio-site.properties.template - conf/core-site.xml.template diff --git a/pom.xml b/pom.xml index ece979f97f6a..732b7656cb99 100644 --- a/pom.xml +++ b/pom.xml @@ -1683,6 +1683,30 @@ --> true + + + + + org.codehaus.mojo + exec-maven-plugin + + + Build Golang CLI + + exec + + compile + + ${build.path}/cli/build-cli.sh + + -a + + + + + + +