From 14c81a00a7242c55fe2f003313bc0ddd9f82106a Mon Sep 17 00:00:00 2001 From: Andrew Woodlee Date: Wed, 23 Jul 2025 22:04:48 -0500 Subject: [PATCH] CLI: added `exec hosts` subcommand `list` --- .../unreleased/Added-20250723-220340.yaml | 3 + cmd/hosts.go | 40 +++++- cmd/root.go | 4 +- docs/content/cli/_index.md | 46 ++++--- pkg/backy/backy.go | 117 ++++++++++++++++++ tests/docker/Dockerfile | 4 +- tests/packageCommands.yml | 21 +++- 7 files changed, 209 insertions(+), 26 deletions(-) create mode 100644 .changes/unreleased/Added-20250723-220340.yaml diff --git a/.changes/unreleased/Added-20250723-220340.yaml b/.changes/unreleased/Added-20250723-220340.yaml new file mode 100644 index 0000000..7828a89 --- /dev/null +++ b/.changes/unreleased/Added-20250723-220340.yaml @@ -0,0 +1,3 @@ +kind: Added +body: 'CLI: added `exec hosts` subcommand `list`' +time: 2025-07-23T22:03:40.24191927-05:00 diff --git a/cmd/hosts.go b/cmd/hosts.go index 9f94765..b5f6ee5 100644 --- a/cmd/hosts.go +++ b/cmd/hosts.go @@ -1,6 +1,9 @@ package cmd import ( + "maps" + "slices" + "git.andrewnw.xyz/CyberShell/backy/pkg/backy" "git.andrewnw.xyz/CyberShell/backy/pkg/logging" "github.com/spf13/cobra" @@ -13,11 +16,17 @@ var ( Long: "Hosts executes specified commands on all the hosts defined in config file.\nUse the --commands or -c flag to choose the commands.", Run: Hosts, } + + hostsListExecCommand = &cobra.Command{ + Use: "list list1 list2 ...", + Short: "Runs lists in order specified defined in config file on all hosts.", + Long: "Lists executes specified lists on all the hosts defined in hosts config.\nPass the names of lists as arguments after command.", + Run: HostsList, + } ) func init() { - - hostsExecCommand.Flags().StringArrayVarP(&cmdList, "command", "c", nil, "Accepts space-separated names of commands. Specify multiple times for multiple commands.") + hostsExecCommand.AddCommand(hostsListExecCommand) parseS3Config() } @@ -53,3 +62,30 @@ func Hosts(cmd *cobra.Command, args []string) { backyConfOpts.ExecCmdsOnHosts(cmdList, hostsList) } + +func HostsList(cmd *cobra.Command, args []string) { + backyConfOpts := backy.NewConfigOptions(configFile, + backy.SetLogFile(logFile), + backy.EnableCommandStdOut(cmdStdOut), + backy.SetHostsConfigFile(hostsConfigFile)) + backyConfOpts.InitConfig() + + backyConfOpts.ParseConfigurationFile() + + if len(args) == 0 { + logging.ExitWithMSG("error: no lists specified", 1, &backyConfOpts.Logger) + } + + for _, l := range args { + _, listFound := backyConfOpts.CmdConfigLists[l] + if !listFound { + logging.ExitWithMSG("list "+l+" not found", 1, &backyConfOpts.Logger) + } + } + + maps.DeleteFunc(backyConfOpts.CmdConfigLists, func(k string, v *backy.CmdList) bool { + return !slices.Contains(args, k) + }) + + backyConfOpts.ExecuteListOnHosts(args) +} diff --git a/cmd/root.go b/cmd/root.go index 44e9075..2bf708d 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -36,13 +36,13 @@ func Execute() { } func init() { - rootCmd.PersistentFlags().StringVar(&logFile, "log-file", "", "log file to write to") + rootCmd.PersistentFlags().StringVar(&logFile, "logFile", "", "log file to write to") rootCmd.PersistentFlags().BoolVar(&cmdStdOut, "cmdStdOut", false, "Pass to print command output to stdout") rootCmd.PersistentFlags().StringVarP(&configFile, "config", "f", "", "config file to read from") rootCmd.PersistentFlags().StringVar(&hostsConfigFile, "hostsConfig", "", "yaml hosts file to read from") rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "Sets verbose level") - rootCmd.PersistentFlags().StringVar(&s3Endpoint, "s3-endpoint", "", "Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable.") + rootCmd.PersistentFlags().StringVar(&s3Endpoint, "s3Endpoint", "", "Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable.") rootCmd.AddCommand(backupCmd, execCmd, cronCmd, versionCmd, listCmd) } diff --git a/docs/content/cli/_index.md b/docs/content/cli/_index.md index 48f3d57..9bee796 100644 --- a/docs/content/cli/_index.md +++ b/docs/content/cli/_index.md @@ -26,8 +26,9 @@ Flags: --cmdStdOut Pass to print command output to stdout -f, --config string config file to read from -h, --help help for backy - --log-file string log file to write to - --s3-endpoint string Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable. + --hostsConfig string yaml hosts file to read from + --logFile string log file to write to + --s3Endpoint string Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable. -v, --verbose Sets verbose level Use "backy [command] --help" for more information about a command. @@ -51,8 +52,9 @@ Flags: Global Flags: --cmdStdOut Pass to print command output to stdout -f, --config string config file to read from - --log-file string log file to write to - --s3-endpoint string Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable. + --hostsConfig string yaml hosts file to read from + --logFile string log file to write to + --s3Endpoint string Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable. -v, --verbose Sets verbose level ``` @@ -70,8 +72,9 @@ Flags: Global Flags: --cmdStdOut Pass to print command output to stdout -f, --config string config file to read from - --log-file string log file to write to - --s3-endpoint string Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable. + --hostsConfig string yaml hosts file to read from + --logFile string log file to write to + --s3Endpoint string Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable. -v, --verbose Sets verbose level ``` @@ -86,6 +89,7 @@ Usage: Available Commands: host Runs command defined in config file on the hosts in order specified. + hosts Runs command defined in config file on the hosts in order specified. Flags: -h, --help help for exec @@ -93,8 +97,9 @@ Flags: Global Flags: --cmdStdOut Pass to print command output to stdout -f, --config string config file to read from - --log-file string log file to write to - --s3-endpoint string Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable. + --hostsConfig string yaml hosts file to read from + --logFile string log file to write to + --s3Endpoint string Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable. -v, --verbose Sets verbose level Use "backy exec [command] --help" for more information about a command. @@ -117,8 +122,9 @@ Flags: Global Flags: --cmdStdOut Pass to print command output to stdout -f, --config string config file to read from - --log-file string log file to write to - --s3-endpoint string Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable. + --hostsConfig string yaml hosts file to read from + --logFile string log file to write to + --s3Endpoint string Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable. -v, --verbose Sets verbose level ``` @@ -138,8 +144,9 @@ Flags: Global Flags: --cmdStdOut Pass to print command output to stdout -f, --config string config file to read from - --log-file string log file to write to - --s3-endpoint string Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable. + --hostsConfig string yaml hosts file to read from + --logFile string log file to write to + --s3Endpoint string Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable. -v, --verbose Sets verbose level ``` @@ -161,8 +168,9 @@ Flags: Global Flags: --cmdStdOut Pass to print command output to stdout -f, --config string config file to read from - --log-file string log file to write to - --s3-endpoint string Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable. + --hostsConfig string yaml hosts file to read from + --logFile string log file to write to + --s3Endpoint string Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable. -v, --verbose Sets verbose level Use "backy list [command] --help" for more information about a command. @@ -181,8 +189,9 @@ Flags: Global Flags: --cmdStdOut Pass to print command output to stdout -f, --config string config file to read from - --log-file string log file to write to - --s3-endpoint string Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable. + --hostsConfig string yaml hosts file to read from + --logFile string log file to write to + --s3Endpoint string Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable. -v, --verbose Sets verbose level ``` ## list lists @@ -199,7 +208,8 @@ Flags: Global Flags: --cmdStdOut Pass to print command output to stdout -f, --config string config file to read from - --log-file string log file to write to - --s3-endpoint string Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable. + --hostsConfig string yaml hosts file to read from + --logFile string log file to write to + --s3Endpoint string Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable. -v, --verbose Sets verbose level ``` diff --git a/pkg/backy/backy.go b/pkg/backy/backy.go index 1ef6d3f..d46aee7 100644 --- a/pkg/backy/backy.go +++ b/pkg/backy/backy.go @@ -11,6 +11,7 @@ import ( "io" "os" "os/exec" + "slices" "strings" "text/template" @@ -326,6 +327,71 @@ func cmdListWorker(msgTemps *msgTemplates, jobs <-chan *CmdList, results chan<- results <- "done" } } +func cmdListWorkerWithHosts(msgTemps *msgTemplates, jobs <-chan *CmdList, hosts <-chan *Host, results chan<- string, opts *ConfigOpts) { + for list := range jobs { + fieldsMap := map[string]interface{}{"list": list.Name} + var cmdLogger zerolog.Logger + var commandExecuted *Command + var cmdsRan []string + var outStructArr []outStruct + var hasError bool // Tracks if any command in the list failed + + for host := range hosts { + + for _, cmd := range list.Order { + cmdToRun := opts.Cmds[cmd] + if cmdToRun.Host != host.Host { + cmdToRun.Host = host.Host + cmdToRun.RemoteHost = host + } + commandExecuted = cmdToRun + currentCmd := cmdToRun.Name + fieldsMap["cmd"] = currentCmd + cmdLogger = cmdToRun.GenerateLogger(opts) + cmdLogger.Info().Fields(fieldsMap).Send() + + outputArr, runErr := cmdToRun.RunCmd(cmdLogger, opts) + cmdsRan = append(cmdsRan, cmd) + + if runErr != nil { + + cmdLogger.Err(runErr).Send() + + cmdToRun.ExecuteHooks("error", opts) + + // Notify failure + if list.NotifyConfig != nil { + notifyError(cmdLogger, msgTemps, list, cmdsRan, outStructArr, runErr, cmdToRun) + } + + // Execute error hooks for the failed command + hasError = true + break + } + + if list.GetCommandOutputInNotificationsOnSuccess || cmdToRun.Output.InList { + outStructArr = append(outStructArr, outStruct{ + CmdName: currentCmd, + CmdExecuted: currentCmd, + Output: outputArr, + }) + } + } + + if !hasError && list.NotifyConfig != nil && list.Notify.OnFailure { + notifySuccess(cmdLogger, msgTemps, list, cmdsRan, outStructArr) + } + + if !hasError { + commandExecuted.ExecuteHooks("success", opts) + } + + commandExecuted.ExecuteHooks("final", opts) + + } + results <- "done" + } +} func notifyError(logger zerolog.Logger, templates *msgTemplates, list *CmdList, cmdsRan []string, outStructArr []outStruct, err error, cmd *Command) { errStruct := map[string]interface{}{ @@ -397,6 +463,57 @@ func (opts *ConfigOpts) RunListConfig(cron string) { opts.closeHostConnections() } +func (opts *ConfigOpts) ExecuteListOnHosts(lists []string) { + + mTemps := &msgTemplates{ + err: template.Must(template.New("error.txt").ParseFS(templates, "templates/error.txt")), + success: template.Must(template.New("success.txt").ParseFS(templates, "templates/success.txt")), + } + for _, l := range opts.CmdConfigLists { + if !slices.Contains(lists, l.Name) { + delete(opts.CmdConfigLists, l.Name) + } + } + configListsLen := len(opts.CmdConfigLists) + listChan := make(chan *CmdList, configListsLen) + hostChan := make(chan *Host, len(opts.Hosts)) + results := make(chan string, configListsLen) + + // Start workers + for w := 1; w <= configListsLen; w++ { + go cmdListWorkerWithHosts(mTemps, listChan, hostChan, results, opts) + } + + // Enqueue jobs + for listName, cmdConfig := range opts.CmdConfigLists { + if cmdConfig.Name == "" { + cmdConfig.Name = listName + } + listChan <- cmdConfig + } + for _, h := range opts.Hosts { + if h.isProxyHost { + continue + } + hostChan <- h + // for _, proxyHost := range h.ProxyHost { + // if proxyHost.isProxyHost { + // continue + // } + // hostChan <- proxyHost + // } + } + close(listChan) + close(hostChan) + + // Process results + for a := 1; a <= configListsLen; a++ { + <-results + } + opts.closeHostConnections() + +} + func (opts *ConfigOpts) ExecuteCmds() { for _, cmd := range opts.executeCmds { cmdToRun := opts.Cmds[cmd] diff --git a/tests/docker/Dockerfile b/tests/docker/Dockerfile index 7502fdd..7dcfdea 100644 --- a/tests/docker/Dockerfile +++ b/tests/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:buster +FROM ubuntu:latest # Install SSH server RUN apt-get update && \ @@ -19,7 +19,7 @@ EXPOSE 22 RUN mkdir /var/run/sshd RUN chmod 0755 /var/run/sshd -# RUN apt-get update && apt-get install -y +RUN apt-get update && apt-get install -y curl # Start SSH service CMD ["/usr/sbin/sshd", "-D"] diff --git a/tests/packageCommands.yml b/tests/packageCommands.yml index b4518bf..dab9899 100644 --- a/tests/packageCommands.yml +++ b/tests/packageCommands.yml @@ -45,6 +45,16 @@ commands: - name: "jq" packageManager: apt packageOperation: install + output: + toLog: true + + testJq: + cmd: jq + shell: bash + Args: + - '--version' + output: + toLog: true checkDockerVersionWithInvalidRegex: type: package @@ -56,7 +66,7 @@ commands: version: '5:28\.0\K.4-1~([A-Za-z0-9]+(\.[A-Za-z0-9]+)+)*' packageManager: apt packageOperation: checkVersion - # host: mail.andrewnw.com + cmdLists: packageCommands: @@ -67,6 +77,13 @@ cmdLists: - checkDockerPartialVersionWithRegex - checkDockerVersionWithInvalidRegex - checkDockerNoVersion + + aptCommands: + output: + onFailure: true + order: + - installJq logging: - verbose: true \ No newline at end of file + verbose: true + cmd-std-out: true \ No newline at end of file