From 03f54c8714876363a90ce4c1845687ea9cce05ca Mon Sep 17 00:00:00 2001 From: Andrew Date: Fri, 20 Jan 2023 02:42:52 -0600 Subject: [PATCH] Added command and list execution (#1), small touchups - Added exec command to execute individual commands - Added --lists, -l flag to backup command - Run command lists (#1) - Small touchups and documentation --- README.md | 10 + cmd/backup.go | 20 +- cmd/exec.go | 31 +++ cmd/root.go | 1 + pkg/backy/backy.go | 239 ++++++++++++++---- pkg/backy/types.go | 30 +-- .../notification.go | 2 +- pkg/notifications/email.go | 5 - 8 files changed, 253 insertions(+), 85 deletions(-) create mode 100644 cmd/exec.go rename pkg/{notifications => notification}/notification.go (99%) delete mode 100644 pkg/notifications/email.go diff --git a/README.md b/README.md index 3401026..2863edc 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,16 @@ To install: This assumes you already have a working Go environment, if not please see [this page](https://golang.org/doc/install) first. +You can also download binaries [here](https://git.andrewnw.xyz/CyberShell/backy/releases) and [here](https://github.com/CybersShell/backy/releases). + +## Features + +- Define lists of commands and run them + +- Execute commands over SSH + +- More to come. + To run a config: `backy backup` diff --git a/cmd/backup.go b/cmd/backup.go index de352d1..f95e527 100644 --- a/cmd/backup.go +++ b/cmd/backup.go @@ -2,31 +2,33 @@ package cmd import ( "git.andrewnw.xyz/CyberShell/backy/pkg/backy" - "git.andrewnw.xyz/CyberShell/backy/pkg/notifications" + "git.andrewnw.xyz/CyberShell/backy/pkg/notification" "github.com/spf13/cobra" ) var ( backupCmd = &cobra.Command{ - Use: "backup [--commands==list1,list2]", + Use: "backup [--lists==list1,list2]", Short: "Runs commands defined in config file.", - Long: `Backup executes commands defined in config file, - use the -cmds flag to execute the specified commands.`, + Long: `Backup executes commands defined in config file. + Use the --lists flag to execute the specified commands.`, Run: Backup, } ) -var CmdList []string + +// Holds command list to run +var cmdList []string func init() { - // cobra.OnInitialize(initConfig) - backupCmd.Flags().StringSliceVar(&CmdList, "cmds", nil, "Accepts a comma-separated list of command lists to execute.") + backupCmd.Flags().StringSliceVarP(&cmdList, "lists", "l", nil, "Accepts a comma-separated names of command lists to execute.") } func Backup(cmd *cobra.Command, args []string) { - config := backy.ReadAndParseConfigFile(cfgFile) - notifications.SetupNotify(*config) + + config := backy.ReadAndParseConfigFile(cfgFile, cmdList) + notification.SetupNotify(*config) config.RunBackyConfig() } diff --git a/cmd/exec.go b/cmd/exec.go new file mode 100644 index 0000000..aa244ef --- /dev/null +++ b/cmd/exec.go @@ -0,0 +1,31 @@ +package cmd + +import ( + "git.andrewnw.xyz/CyberShell/backy/pkg/backy" + "git.andrewnw.xyz/CyberShell/backy/pkg/logging" + + "github.com/spf13/cobra" +) + +var ( + execCmd = &cobra.Command{ + Use: "exec command1 command2", + Short: "Runs commands defined in config file.", + Long: `Exec executes commands defined in config file.`, + Run: execute, + } +) + +func execute(cmd *cobra.Command, args []string) { + + if len(args) < 1 { + logging.ExitWithMSG("Please provide a command to run. Pass --help to see options.", 0, nil) + } + + opts := backy.NewOpts(cfgFile, backy.AddCommands(args)) + + commands := opts.GetCmdsInConfigFile() + + commands.ExecuteCmds() + +} diff --git a/cmd/root.go b/cmd/root.go index a97c6fb..2dcbdcd 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -41,6 +41,7 @@ func init() { rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "Sets verbose level") rootCmd.AddCommand(backupCmd) + rootCmd.AddCommand(execCmd) } func initConfig() { diff --git a/pkg/backy/backy.go b/pkg/backy/backy.go index d67a604..161bc13 100644 --- a/pkg/backy/backy.go +++ b/pkg/backy/backy.go @@ -27,23 +27,26 @@ var requiredKeys = []string{"commands", "cmd-configs"} var Sprintf = fmt.Sprintf -type BackyOptionFunc func(*BackyConfigOpts) - func (c *BackyConfigOpts) LogLvl(level string) BackyOptionFunc { + return func(bco *BackyConfigOpts) { c.BackyLogLvl = &level } } -func (c *BackyConfigOpts) GetConfig() { - c.ConfigFile = ReadAndParseConfigFile(c.ConfigFilePath) +func AddCommands(commands []string) BackyOptionFunc { + return func(bco *BackyConfigOpts) { + bco.executeCmds = append(bco.executeCmds, commands...) + } } func NewOpts(configFilePath string, opts ...BackyOptionFunc) *BackyConfigOpts { b := &BackyConfigOpts{} b.ConfigFilePath = configFilePath for _, opt := range opts { - opt(b) + if opt != nil { + opt(b) + } } return b } @@ -53,7 +56,7 @@ NewConfig initializes new config that holds information from the config file */ func NewConfig() *BackyConfigFile { return &BackyConfigFile{ - Cmds: make(map[string]Command), + Cmds: make(map[string]*Command), CmdConfigLists: make(map[string]*CmdConfig), Hosts: make(map[string]Host), Notifications: make(map[string]*NotificationsConfig), @@ -65,10 +68,12 @@ type environmentVars struct { env []string } -/* -* Runs a backup configuration - */ - +// RunCmd runs a Command. +// The environment of local commands will be the machine's environment plus any extra +// variables specified in the Env file or Environment. +// +// If host is specifed, the command will call ConnectToSSHHost, +// returning a client that is used to run the command. func (command *Command) RunCmd(log *zerolog.Logger) { var envVars = environmentVars{ @@ -118,6 +123,10 @@ func (command *Command) RunCmd(log *zerolog.Logger) { log.Error().Err(fmt.Errorf("error when running cmd: %s: %w", command.Cmd, err)).Send() } } else { + cmdExists := command.checkCmdExists() + if !cmdExists { + log.Error().Str(command.Cmd, "not found").Send() + } // shell := "/bin/bash" var err error if command.Shell != "" { @@ -159,12 +168,10 @@ func (command *Command) RunCmd(log *zerolog.Logger) { func cmdListWorker(id int, jobs <-chan *CmdConfig, config *BackyConfigFile, results chan<- string) { for j := range jobs { - // fmt.Println("worker", id, "started job", j) for _, cmd := range j.Order { cmdToRun := config.Cmds[cmd] cmdToRun.RunCmd(&config.Logger) } - // fmt.Println("worker", id, "finished job", j) results <- "done" } } @@ -198,8 +205,14 @@ func (config *BackyConfigFile) RunBackyConfig() { } +func (config *BackyConfigFile) ExecuteCmds() { + for _, cmd := range config.Cmds { + cmd.RunCmd(&config.Logger) + } +} + // ReadAndParseConfigFile validates and reads the config file. -func ReadAndParseConfigFile(configFile string) *BackyConfigFile { +func ReadAndParseConfigFile(configFile string, lists []string) *BackyConfigFile { backyConfigFile := NewConfig() @@ -218,7 +231,13 @@ func ReadAndParseConfigFile(configFile string) *BackyConfigFile { panic(fmt.Errorf("fatal error reading config file %s: %w", backyViper.ConfigFileUsed(), err)) } - CheckForConfigValues(backyViper) + CheckConfigValues(backyViper) + + for _, l := range lists { + if !backyViper.IsSet(getCmdListFromConfig(l)) { + logging.ExitWithMSG(Sprintf("list %s not found", l), 1, nil) + } + } var backyLoggingOpts *viper.Viper backyLoggingOptsSet := backyViper.IsSet("logging") @@ -229,7 +248,7 @@ func ReadAndParseConfigFile(configFile string) *BackyConfigFile { logFile := backyLoggingOpts.GetString("file") if verbose { - zerolog.SetGlobalLevel(zerolog.ErrorLevel) + zerolog.SetGlobalLevel(zerolog.InfoLevel) globalLvl := zerolog.GlobalLevel().String() os.Setenv("BACKY_LOGLEVEL", globalLvl) } @@ -281,7 +300,6 @@ func ReadAndParseConfigFile(configFile string) *BackyConfigFile { hostConfigsMap := make(map[string]*viper.Viper) for _, cmdName := range cmdNames { - var backupCmdStruct Command subCmd := backyViper.Sub(getNestedConfig("commands", cmdName)) hostSet := subCmd.IsSet("host") @@ -289,7 +307,6 @@ func ReadAndParseConfigFile(configFile string) *BackyConfigFile { if hostSet { log.Debug().Timestamp().Str(cmdName, "host is set").Str("host", host).Send() - backupCmdStruct.Host = &host if backyViper.IsSet(getNestedConfig("hosts", host)) { hostconfig := backyViper.Sub(getNestedConfig("hosts", host)) hostConfigsMap[host] = hostconfig @@ -298,8 +315,6 @@ func ReadAndParseConfigFile(configFile string) *BackyConfigFile { log.Debug().Timestamp().Str(cmdName, "host is not set").Send() } - // backyConfigFile.Cmds[cmdName] = backupCmdStruct - } cmdListCfg := backyViper.Sub("cmd-configs") @@ -310,7 +325,6 @@ func ReadAndParseConfigFile(configFile string) *BackyConfigFile { var cmdNotFoundSliceErr []error for cmdListName, cmdList := range backyConfigFile.CmdConfigLists { for _, cmdInList := range cmdList.Order { - // log.Info().Msgf("CmdList %s Cmd %s", cmdListName, cmdInList) _, cmdNameFound := backyConfigFile.Cmds[cmdInList] if !cmdNameFound { cmdNotFoundStr := fmt.Sprintf("command %s is not defined in config file", cmdInList) @@ -318,45 +332,48 @@ func ReadAndParseConfigFile(configFile string) *BackyConfigFile { cmdNotFoundSliceErr = append(cmdNotFoundSliceErr, cmdNotFoundErr) } else { log.Info().Str(cmdInList, "found in "+cmdListName).Send() - // backyConfigFile.CmdLists[cmdListName] = append(backyConfigFile.CmdLists[cmdListName], cmdInList) } } - } - if len(cmdNotFoundSliceErr) > 0 { - var cmdNotFoundErrorLog = log.Fatal() - for _, err := range cmdNotFoundSliceErr { - if err != nil { - cmdNotFoundErrorLog.Err(err) - } - } - cmdNotFoundErrorLog.Send() - } - - // var notificationSlice []string - for name, cmdCfg := range backyConfigFile.CmdConfigLists { - for _, notificationID := range cmdCfg.Notifications { - // if !contains(notificationSlice, notificationID) { + for _, notificationID := range cmdList.Notifications { - cmdCfg.NotificationsConfig = make(map[string]*NotificationsConfig) + cmdList.NotificationsConfig = make(map[string]*NotificationsConfig) notifConfig := backyViper.Sub(getNestedConfig("notifications", notificationID)) config := &NotificationsConfig{ Config: notifConfig, Enabled: true, } - cmdCfg.NotificationsConfig[notificationID] = config + cmdList.NotificationsConfig[notificationID] = config // First we get a "copy" of the entry - if entry, ok := cmdCfg.NotificationsConfig[notificationID]; ok { + if entry, ok := cmdList.NotificationsConfig[notificationID]; ok { // Then we modify the copy entry.Config = notifConfig entry.Enabled = true // Then we reassign the copy - cmdCfg.NotificationsConfig[notificationID] = entry + cmdList.NotificationsConfig[notificationID] = entry + } + backyConfigFile.CmdConfigLists[cmdListName].NotificationsConfig[notificationID] = config + + } + } + + if len(lists) > 0 { + for l := range backyConfigFile.CmdConfigLists { + if !contains(lists, l) { + delete(backyConfigFile.CmdConfigLists, l) } - backyConfigFile.CmdConfigLists[name].NotificationsConfig[notificationID] = config } - // } + } + + if len(cmdNotFoundSliceErr) > 0 { + var cmdNotFoundErrorLog = log.Fatal() + for _, err := range cmdNotFoundSliceErr { + if err != nil { + cmdNotFoundErrorLog.Err(err) + } + } + cmdNotFoundErrorLog.Send() } var notificationsMap = make(map[string]interface{}) @@ -370,16 +387,119 @@ func ReadAndParseConfigFile(configFile string) *BackyConfigFile { } backyConfigFile.Notifications[id] = config } + } + + return backyConfigFile +} + +// GetCmdsInConfigFile validates and reads the config file for commands. +func (opts *BackyConfigOpts) GetCmdsInConfigFile() *BackyConfigFile { + + backyConfigFile := NewConfig() + + backyViper := viper.New() + + if opts.ConfigFilePath != strings.TrimSpace("") { + backyViper.SetConfigFile(opts.ConfigFilePath) + } else { + backyViper.SetConfigName("backy.yaml") // name of config file (with extension) + backyViper.SetConfigType("yaml") // REQUIRED if the config file does not have the extension in the name + backyViper.AddConfigPath(".") // optionally look for config in the working directory + backyViper.AddConfigPath("$HOME/.config/backy") // call multiple times to add many search paths + } + err := backyViper.ReadInConfig() // Find and read the config file + if err != nil { // Handle errors reading the config file + panic(fmt.Errorf("fatal error reading config file %s: %w", backyViper.ConfigFileUsed(), err)) + } + + CheckConfigValues(backyViper) + for _, c := range opts.executeCmds { + if !backyViper.IsSet(getCmdFromConfig(c)) { + logging.ExitWithMSG(Sprintf("command %s is not in config file %s", c, backyViper.ConfigFileUsed()), 1, nil) + } + } + var backyLoggingOpts *viper.Viper + backyLoggingOptsSet := backyViper.IsSet("logging") + if backyLoggingOptsSet { + backyLoggingOpts = backyViper.Sub("logging") + } + verbose := backyLoggingOpts.GetBool("verbose") + + logFile := backyLoggingOpts.GetString("file") + if verbose { + zerolog.SetGlobalLevel(zerolog.InfoLevel) + globalLvl := zerolog.GlobalLevel().String() + os.Setenv("BACKY_LOGLEVEL", globalLvl) + } + output := zerolog.ConsoleWriter{Out: os.Stdout, TimeFormat: time.RFC1123} + output.FormatLevel = func(i interface{}) string { + return strings.ToUpper(fmt.Sprintf("| %-6s|", i)) + } + output.FormatMessage = func(i interface{}) string { + return fmt.Sprintf("%s", i) + } + output.FormatFieldName = func(i interface{}) string { + return fmt.Sprintf("%s: ", i) + } + output.FormatFieldValue = func(i interface{}) string { + return strings.ToUpper(fmt.Sprintf("%s", i)) + } + + fileLogger := &lumberjack.Logger{ + MaxSize: 500, // megabytes + MaxBackups: 3, + MaxAge: 28, //days + Compress: true, // disabled by default + } + if strings.TrimSpace(logFile) != "" { + fileLogger.Filename = logFile + } else { + fileLogger.Filename = "./backy.log" + } + + // UNIX Time is faster and smaller than most timestamps + zerolog.TimeFieldFormat = zerolog.TimeFormatUnix + // zerolog.TimeFieldFormat = time.RFC1123 + writers := zerolog.MultiLevelWriter(os.Stdout, fileLogger) + log := zerolog.New(writers).With().Timestamp().Logger() + + backyConfigFile.Logger = log + + commandsMap := backyViper.GetStringMapString("commands") + commandsMapViper := backyViper.Sub("commands") + unmarshalErr := commandsMapViper.Unmarshal(&backyConfigFile.Cmds) + if unmarshalErr != nil { + panic(fmt.Errorf("error unmarshalling cmds struct: %w", unmarshalErr)) + } + + var cmdNames []string + for c := range commandsMap { + if contains(opts.executeCmds, c) { + cmdNames = append(cmdNames, c) + } + if !contains(opts.executeCmds, c) { + delete(backyConfigFile.Cmds, c) + } + } + + hostConfigsMap := make(map[string]*viper.Viper) + + for _, cmdName := range cmdNames { + subCmd := backyViper.Sub(getNestedConfig("commands", cmdName)) + + hostSet := subCmd.IsSet("host") + host := subCmd.GetString("host") + + if hostSet { + log.Debug().Timestamp().Str(cmdName, "host is set").Str("host", host).Send() + if backyViper.IsSet(getNestedConfig("hosts", host)) { + hostconfig := backyViper.Sub(getNestedConfig("hosts", host)) + hostConfigsMap[host] = hostconfig + } + } else { + log.Debug().Timestamp().Str(cmdName, "host is not set").Send() + } - // for _, notif := range backyConfigFile.Notifications { - // fmt.Printf("Type: %s\n", notif.Config.GetString("type")) - // notificationID := notif.Config.GetString("id") - // if !contains(notificationSlice, notificationID) { - // config := backyConfigFile.Notifications[notificationID] - // config.Enabled = false - // backyConfigFile.Notifications[notificationID] = config - // } - // } } return backyConfigFile @@ -389,6 +509,13 @@ func getNestedConfig(nestedConfig, key string) string { return fmt.Sprintf("%s.%s", nestedConfig, key) } +func getCmdFromConfig(key string) string { + return fmt.Sprintf("commands.%s", key) +} +func getCmdListFromConfig(list string) string { + return fmt.Sprintf("cmd-configs.%s", list) +} + func resolveDir(path string) (string, error) { usr, err := user.Current() if err != nil { @@ -410,7 +537,7 @@ func injectEnvIntoSSH(envVarsToInject environmentVars, process *ssh.Session, log if envVarsToInject.file != "" { envPath, envPathErr := resolveDir(envVarsToInject.file) if envPathErr != nil { - log.Error().Err(envPathErr).Send() + log.Err(envPathErr).Send() } file, err := os.Open(envPath) if err != nil { @@ -466,6 +593,12 @@ func injectEnvIntoLocalCMD(envVarsToInject environmentVars, process *exec.Cmd, l } } } + envVarsToInject.env = append(envVarsToInject.env, os.Environ()...) +} + +func (cmd *Command) checkCmdExists() bool { + _, err := exec.LookPath(cmd.Cmd) + return err == nil } func contains(s []string, e string) bool { @@ -477,7 +610,7 @@ func contains(s []string, e string) bool { return false } -func CheckForConfigValues(config *viper.Viper) { +func CheckConfigValues(config *viper.Viper) { for _, key := range requiredKeys { isKeySet := config.IsSet(key) diff --git a/pkg/backy/types.go b/pkg/backy/types.go index 7f787bd..3eba0b7 100644 --- a/pkg/backy/types.go +++ b/pkg/backy/types.go @@ -57,6 +57,8 @@ type Command struct { Environment []string `yaml:"environment,omitempty"` } +type BackyOptionFunc func(*BackyConfigOpts) + type CmdConfig struct { Order []string `yaml:"order,omitempty"` Notifications []string `yaml:"notifications,omitempty"` @@ -64,27 +66,20 @@ type CmdConfig struct { } type BackyConfigFile struct { - /* - Cmds holds the commands for a list. - Key is the name of the command, - */ - Cmds map[string]Command `yaml:"commands"` - /* - CmdLConfigists holds the lists of commands to be run in order. - Key is the command list name. - */ + // Cmds holds the commands for a list. + // Key is the name of the command, + Cmds map[string]*Command `yaml:"commands"` + + // CmdConfigLists holds the lists of commands to be run in order. + // Key is the command list name. CmdConfigLists map[string]*CmdConfig `yaml:"cmd-configs"` - /* - Hosts holds the Host config. - key is the host. - */ + // Hosts holds the Host config. + // key is the host. Hosts map[string]Host `yaml:"hosts"` - /* - Notifications holds the config for different notifications. - */ + // Notifications holds the config for different notifications. Notifications map[string]*NotificationsConfig Logger zerolog.Logger @@ -95,7 +90,8 @@ type BackyConfigOpts struct { ConfigFile *BackyConfigFile // Holds config file ConfigFilePath string - + // Holds commands to execute for the exec command + executeCmds []string // Global log level BackyLogLvl *string } diff --git a/pkg/notifications/notification.go b/pkg/notification/notification.go similarity index 99% rename from pkg/notifications/notification.go rename to pkg/notification/notification.go index 816f1e4..3a2437c 100644 --- a/pkg/notifications/notification.go +++ b/pkg/notification/notification.go @@ -1,7 +1,7 @@ // notification.go // Copyright (C) Andrew Woodlee 2023 // License: Apache-2.0 -package notifications +package notification import ( "fmt" diff --git a/pkg/notifications/email.go b/pkg/notifications/email.go deleted file mode 100644 index aa6a308..0000000 --- a/pkg/notifications/email.go +++ /dev/null @@ -1,5 +0,0 @@ -package notifications - -func GetConfig() { - -}