From 37c20aaafa69aa4c335686834cd601dfcf46bf4e Mon Sep 17 00:00:00 2001 From: Andrew Date: Sat, 11 Feb 2023 23:50:19 -0600 Subject: [PATCH] added new features - expanding `environment` vars in cmd section - support for connecting to one proxy/bastion host - better notification text layout - better error message on private key failing to open --- .goreleaser.yaml | 7 +- README.md | 93 ++------------ cmd/backup.go | 2 +- cmd/config.go | 31 +++++ examples/backy.yaml | 14 +- pkg/backy/backy.go | 64 +++++++--- pkg/backy/config.go | 128 ++++++++++--------- pkg/backy/cron.go | 14 +- pkg/backy/notification.go | 1 + pkg/backy/ssh.go | 220 +++++++++++++++++++++++--------- pkg/backy/templates/error.txt | 8 ++ pkg/backy/templates/success.txt | 7 + pkg/backy/types.go | 12 +- pkg/backy/utils.go | 32 +++++ pkg/logging/logging.go | 14 +- 15 files changed, 413 insertions(+), 234 deletions(-) create mode 100644 cmd/config.go create mode 100644 pkg/backy/templates/error.txt create mode 100644 pkg/backy/templates/success.txt diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 2a0be51..bf8dccb 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -8,10 +8,15 @@ builds: - env: - CGO_ENABLED=0 goos: + - freebsd - linux + goarch: + - "386" + - amd64 + - arm64 archives: - - format: tar.gz + - format: binary # this name template makes the OS and Arch compatible with the results of uname. name_template: >- {{ .ProjectName }}_{{ .Version }}_ diff --git a/README.md b/README.md index aa704c0..2675356 100644 --- a/README.md +++ b/README.md @@ -14,11 +14,17 @@ You can also download binaries [here](https://git.andrewnw.xyz/CyberShell/backy/ ## Features -- Define lists of commands and run them +- Allows easy configuration of executable commands -- Execute commands over SSH +- Allows for commands to be run on many hosts over SSH -- More to come. +- Commands can be grouped in list to run in specific order + +- Notifications on completion and failure + +- Run in cron mode + +- For any command, especially backup commands To run a config: @@ -29,84 +35,11 @@ Or to use a specific file: If you leave the config path blank, the following paths will be searched in order: +- `./backy.yml` - `./backy.yaml` +- `~/.config/backy.yml` - `~/.config/backy.yaml` -Create a file at `~/.config/backy.yaml`: +Create a file at `~/.config/backy.yml`. -```yaml -commands: - stop-docker-container: - cmd: docker - Args: - - compose - - -f /some/path/to/docker-compose.yaml - - down - # if host is not defined, cmd will be run locally - host: some-host - backup-docker-container-script: - cmd: /path/to/script - # The host has to be defined in the config file - host: some-host - shell-cmd: - cmd: rsync - shell: bash - Args: - - -av some-host:/path/to/data ~/Docker/Backups/docker-data - hostname: - cmd: hostname - -cmd-configs: - cmds-to-run: # this can be any name you want - # all commands have to be defined - order: - - stop-docker-container - - backup-docker-container-script - - shell-cmd - - hostname - notifications: - - matrix - name: backup-some-server - hostname: - name: hostname - order: - - hostname - notifications: - - prod-email - -hosts: - some-host: - hostname: some-hostname - config: ~/.ssh/config - user: user - privatekeypath: /path/to/private/key - port: 22 - password: - - -logging: - verbose: true - file: /path/to/logs/commands.log - console: false - cmd-std-out: false - - -notifications: - prod-email: - id: prod-email - type: mail - host: yourhost.tld:port - senderAddress: email@domain.tld - to: - - admin@domain.tld - username: smtp-username@domain.tld - password: your-password-here - matrix: - id: matrix - type: matrix - home-server: your-home-server.tld - room-id: room-id - access-token: your-access-token - user-id: your-user-id - -``` +See the config file in the examples directory to configure it. diff --git a/cmd/backup.go b/cmd/backup.go index e715580..dfeda08 100644 --- a/cmd/backup.go +++ b/cmd/backup.go @@ -25,7 +25,7 @@ var cmdLists []string func init() { - backupCmd.Flags().StringSliceVarP(&cmdLists, "lists", "l", nil, "Accepts a comma-separated names of command lists to execute.") + backupCmd.Flags().StringSliceVarP(&cmdLists, "lists", "l", nil, "Accepts comma-separated names of command lists to execute.") } diff --git a/cmd/config.go b/cmd/config.go new file mode 100644 index 0000000..80c3da1 --- /dev/null +++ b/cmd/config.go @@ -0,0 +1,31 @@ +package cmd + +// import ( +// "git.andrewnw.xyz/CyberShell/backy/pkg/backy" + +// "github.com/spf13/cobra" +// ) + +// var ( +// configCmd = &cobra.Command{ +// Use: "config list ...", +// Short: "Runs commands defined in config file.", +// Long: `Cron executes commands at the time defined in config file.`, +// Run: config, +// } + +// cmds []string +// lists []string +// ) + +// func config(cmd *cobra.Command, args []string) { + +// opts := backy.NewOpts(cfgFile, backy.UseCron()) +// opts.InitConfig() + +// } + +// func init() { + +// configCmd.PersistentFlags().StringArrayVarP(&cmds, "cmds", "c", nil, "Accepts comma-seperated list of commands to list") +// } diff --git a/examples/backy.yaml b/examples/backy.yaml index e1bb24e..6ea27cd 100644 --- a/examples/backy.yaml +++ b/examples/backy.yaml @@ -11,6 +11,9 @@ commands: cmd: /path/to/script # The host has to be defined in the config file host: some-host + environment: + - FOO=BAR + - APP=$VAR shell-cmd: cmd: rsync shell: bash @@ -38,15 +41,19 @@ cmd-configs: - prod-email hosts: + # any ssh_config(5) keys/values not listed here will be looked up in the config file or the default config file some-host: hostname: some-hostname config: ~/.ssh/config user: user privatekeypath: /path/to/private/key port: 22 - password: - + # can also be env:VAR + password: file:/path/to/file + # only one is supported for now + proxyjump: some-proxy-host +# optional logging: verbose: true file: /path/to/logs/commands.log @@ -58,7 +65,8 @@ notifications: prod-email: id: prod-email type: mail - host: yourhost.tld:port + host: yourhost.tld + port: 587 senderAddress: email@domain.tld to: - admin@domain.tld diff --git a/pkg/backy/backy.go b/pkg/backy/backy.go index 0d4bb4c..540c373 100644 --- a/pkg/backy/backy.go +++ b/pkg/backy/backy.go @@ -11,11 +11,17 @@ import ( "io" "os" "os/exec" + "text/template" + + "embed" "github.com/rs/zerolog" ) -var requiredKeys = []string{"commands", "cmd-configs", "logging"} +//go:embed templates/*.txt +var templates embed.FS + +var requiredKeys = []string{"commands", "cmd-configs"} var Sprintf = fmt.Sprintf @@ -23,7 +29,7 @@ var Sprintf = fmt.Sprintf // The environment of local commands will be the machine's environment plus any extra // variables specified in the Env file or Environment. // Dir can also be specified for local commands. -func (command *Command) RunCmd(log *zerolog.Logger) error { +func (command *Command) RunCmd(log *zerolog.Logger, hosts map[string]*Host) error { var ( ArgsStr string @@ -44,12 +50,12 @@ func (command *Command) RunCmd(log *zerolog.Logger) error { if command.Host != nil { log.Info().Str("Command", fmt.Sprintf("Running command: %s %s on host %s", command.Cmd, ArgsStr, *command.Host)).Send() - sshc, err := command.RemoteHost.ConnectToSSHHost(log) + err := command.RemoteHost.ConnectToSSHHost(log, hosts) if err != nil { return err } - defer sshc.Close() - commandSession, err := sshc.NewSession() + defer command.RemoteHost.SshClient.Close() + commandSession, err := command.RemoteHost.SshClient.NewSession() if err != nil { log.Err(fmt.Errorf("new ssh session: %w", err)).Send() return err @@ -158,7 +164,7 @@ func cmdListWorker(id int, jobs <-chan *CmdList, config *BackyConfigFile, result fieldsMap["list"] = list.Name cmdLog := config.Logger.Info() var count int - var Msg string + var cmdsRan []string for _, cmd := range list.Order { currentCmd = config.Cmds[cmd].Cmd fieldsMap["cmd"] = config.Cmds[cmd].Cmd @@ -167,12 +173,22 @@ func cmdListWorker(id int, jobs <-chan *CmdList, config *BackyConfigFile, result cmdLogger := config.Logger.With(). Str("backy-cmd", cmd). Logger() - runOutErr := cmdToRun.RunCmd(&cmdLogger) + runOutErr := cmdToRun.RunCmd(&cmdLogger, config.Hosts) count++ if runOutErr != nil { + var errMsg bytes.Buffer if list.NotifyConfig != nil { - notifySendErr := list.NotifyConfig.Send(context.Background(), fmt.Sprintf("List %s failed on command %s ", list.Name, cmd), - fmt.Sprintf("List %s failed on command %s running command %s. \n Error: %v", list.Name, cmd, currentCmd, runOutErr)) + errStruct := make(map[string]interface{}) + errStruct["listName"] = list.Name + errStruct["Command"] = currentCmd + errStruct["Err"] = runOutErr + errStruct["CmdsRan"] = cmdsRan + t := template.Must(template.New("error.txt").ParseFS(templates, "templates/error.txt")) + tmpErr := t.Execute(&errMsg, errStruct) + if tmpErr != nil { + config.Logger.Err(tmpErr).Send() + } + notifySendErr := list.NotifyConfig.Send(context.Background(), fmt.Sprintf("List %s failed on command %s ", list.Name, cmd), errMsg.String()) if notifySendErr != nil { config.Logger.Err(notifySendErr).Send() } @@ -182,22 +198,32 @@ func cmdListWorker(id int, jobs <-chan *CmdList, config *BackyConfigFile, result } else { if count == len(list.Order) { - Msg += fmt.Sprintf("%s ", cmd) + cmdsRan = append(cmdsRan, cmd) + var successMsg bytes.Buffer if list.NotifyConfig != nil { - err := list.NotifyConfig.Send(context.Background(), fmt.Sprintf("List %s succeded", list.Name), - fmt.Sprintf("Command list %s was completed successfully. The following commands ran:\n %s", list.Name, Msg)) + successStruct := make(map[string]interface{}) + successStruct["listName"] = list.Name + successStruct["CmdsRan"] = cmdsRan + t := template.Must(template.New("success.txt").ParseFS(templates, "templates/success.txt")) + tmpErr := t.Execute(&successMsg, successStruct) + if tmpErr != nil { + config.Logger.Err(tmpErr).Send() + break + } + err := list.NotifyConfig.Send(context.Background(), fmt.Sprintf("List %s succeded", list.Name), successMsg.String()) if err != nil { config.Logger.Err(err).Send() } } } else { - Msg += fmt.Sprintf("%s, ", cmd) + cmdsRan = append(cmdsRan, cmd) } } } results <- "done" } + } // RunBackyConfig runs a command list from the BackyConfigFile. @@ -208,7 +234,7 @@ func (config *BackyConfigFile) RunBackyConfig(cron string) { // This starts up 3 workers, initially blocked // because there are no jobs yet. - for w := 1; w <= 3; w++ { + for w := 1; w <= configListsLen; w++ { go cmdListWorker(w, listChan, config, results) } @@ -216,7 +242,10 @@ func (config *BackyConfigFile) RunBackyConfig(cron string) { // Here we send 5 `jobs` and then `close` that // channel to indicate that's all the work we have. // configChan <- config.Cmds - for _, cmdConfig := range config.CmdConfigLists { + for listName, cmdConfig := range config.CmdConfigLists { + if cmdConfig.Name == "" { + cmdConfig.Name = listName + } if cron != "" { if cron == cmdConfig.Cron { listChan <- cmdConfig @@ -235,6 +264,9 @@ func (config *BackyConfigFile) RunBackyConfig(cron string) { func (config *BackyConfigFile) ExecuteCmds() { for _, cmd := range config.Cmds { - cmd.RunCmd(&config.Logger) + runErr := cmd.RunCmd(&config.Logger, config.Hosts) + if runErr != nil { + config.Logger.Err(runErr).Send() + } } } diff --git a/pkg/backy/config.go b/pkg/backy/config.go index daa56ac..07b5c93 100644 --- a/pkg/backy/config.go +++ b/pkg/backy/config.go @@ -12,7 +12,6 @@ import ( "github.com/mattn/go-isatty" "github.com/rs/zerolog" "github.com/spf13/viper" - "mvdan.cc/sh/v3/shell" ) // ReadConfig validates and reads the config file. @@ -28,13 +27,12 @@ func ReadConfig(opts *BackyConfigOpts) *BackyConfigFile { backyConfigFile := NewConfig() backyViper := opts.viper - // loadEnv(backyViper) + opts.loadEnv() envFileInConfigDir := fmt.Sprintf("%s/.env", path.Dir(backyViper.ConfigFileUsed())) - envFileErr := godotenv.Load() - if envFileErr != nil { - _ = godotenv.Load(envFileInConfigDir) - } + // load the .env file in config file directory + _ = godotenv.Load(envFileInConfigDir) + if backyViper.GetBool(getNestedConfig("logging", "cmd-std-out")) { os.Setenv("BACKY_STDOUT", "enabled") } @@ -52,24 +50,28 @@ func ReadConfig(opts *BackyConfigOpts) *BackyConfigFile { } } - var backyLoggingOpts *viper.Viper - isBackyLoggingOptsSet := backyViper.IsSet("logging") - if isBackyLoggingOptsSet { - backyLoggingOpts = backyViper.Sub("logging") - } - verbose := backyLoggingOpts.GetBool("verbose") + var ( + // backyLoggingOpts *viper.Viper + verbose bool + logFile string + ) - logFile := backyLoggingOpts.GetString("file") + verbose = backyViper.GetBool(getLoggingKeyFromConfig("verbose")) + + logFile = fmt.Sprintf("%s/backy.log", path.Dir(backyViper.ConfigFileUsed())) + if backyViper.IsSet(getLoggingKeyFromConfig("file")) { + logFile = backyViper.GetString(getLoggingKeyFromConfig("file")) + } zerolog.SetGlobalLevel(zerolog.InfoLevel) if verbose { zerolog.SetGlobalLevel(zerolog.DebugLevel) globalLvl := zerolog.GlobalLevel() - os.Setenv("BACKY_LOGLEVEL", Sprintf("%x", globalLvl)) + os.Setenv("BACKY_LOGLEVEL", Sprintf("%v", globalLvl)) } - consoleLoggingEnabled := backyLoggingOpts.GetBool("console") + consoleLoggingEnabled := backyViper.GetBool(getLoggingKeyFromConfig("console")) // Other qualifiers can go here as well if consoleLoggingEnabled { @@ -78,12 +80,13 @@ func ReadConfig(opts *BackyConfigOpts) *BackyConfigFile { os.Setenv("BACKY_CONSOLE_LOGGING", "") } - writers := logging.SetLoggingWriters(backyLoggingOpts, logFile) + writers := logging.SetLoggingWriters(logFile) log := zerolog.New(writers).With().Timestamp().Logger() backyConfigFile.Logger = log + log.Info().Str("config file", backyViper.ConfigFileUsed()).Send() commandsMap := backyViper.GetStringMapString("commands") commandsMapViper := backyViper.Sub("commands") unmarshalErr := commandsMapViper.Unmarshal(&backyConfigFile.Cmds) @@ -100,6 +103,8 @@ func ReadConfig(opts *BackyConfigOpts) *BackyConfigFile { os.Exit(1) } + expandEnvVars(opts.backyEnv, cmdConf.Environment) + host := cmdConf.Host if host != nil { if backyViper.IsSet(getNestedConfig("hosts", *host)) { @@ -114,12 +119,37 @@ func ReadConfig(opts *BackyConfigOpts) *BackyConfigFile { if unmarshalErr != nil { panic(fmt.Errorf("error unmarshalling hosts struct: %w", unmarshalErr)) } - for _, v := range backyConfigFile.Hosts { - - if v.JumpHost != "" { - proxyHost, defined := backyConfigFile.Hosts[v.JumpHost] - if defined { - v.ProxyHost = proxyHost + for _, host := range backyConfigFile.Hosts { + if host.ProxyJump != "" { + proxyHosts := strings.Split(host.ProxyJump, ",") + if len(proxyHosts) > 1 { + for hostNum, h := range proxyHosts { + if hostNum > 1 { + proxyHost, defined := backyConfigFile.Hosts[h] + if defined { + host.ProxyHost = append(host.ProxyHost, proxyHost) + } else { + newProxy := &Host{Host: h} + host.ProxyHost = append(host.ProxyHost, newProxy) + } + } else { + proxyHost, defined := backyConfigFile.Hosts[h] + if defined { + host.ProxyHost = append(host.ProxyHost, proxyHost) + } else { + newHost := &Host{Host: h} + host.ProxyHost = append(host.ProxyHost, newHost) + } + } + } + } else { + proxyHost, defined := backyConfigFile.Hosts[proxyHosts[0]] + if defined { + host.ProxyHost = append(host.ProxyHost, proxyHost) + } else { + newProxy := &Host{Host: proxyHosts[0]} + host.ProxyHost = append(host.ProxyHost, newProxy) + } } } } @@ -157,10 +187,7 @@ func ReadConfig(opts *BackyConfigOpts) *BackyConfigFile { cmdNotFoundErrorLog.Errs("commands not found", cmdNotFoundSliceErr).Send() } - if opts.useCron && len(backyConfigFile.CmdConfigLists) > 0 { - log.Info().Msg("Starting cron mode...") - - } else if opts.useCron && (len(backyConfigFile.CmdConfigLists) == 0) { + if opts.useCron && (len(backyConfigFile.CmdConfigLists) == 0) { logging.ExitWithMSG("No cron fields detected in any command lists", 1, nil) } @@ -217,6 +244,14 @@ func getNestedConfig(nestedConfig, key string) string { func getCmdFromConfig(key string) string { return fmt.Sprintf("commands.%s", key) } + +func getLoggingKeyFromConfig(key string) string { + if key == "" { + return "logging" + } + return fmt.Sprintf("logging.%s", key) +} + func getCmdListFromConfig(list string) string { return fmt.Sprintf("cmd-configs.%s", list) } @@ -228,8 +263,13 @@ func (opts *BackyConfigOpts) InitConfig() { backyViper := viper.New() if strings.TrimSpace(opts.ConfigFilePath) != "" { + err := testFile(opts.ConfigFilePath) + if err != nil { + logging.ExitWithMSG(fmt.Sprintf("Could not open config file %s: %v", opts.ConfigFilePath, err), 1, nil) + } backyViper.SetConfigFile(opts.ConfigFilePath) } else { + backyViper.SetConfigName("backy.yml") // name of config file (with extension) backyViper.SetConfigName("backy.yaml") // name of config file (with extension) backyViper.SetConfigType("yaml") // REQUIRED if the config file does not have the extension in the name backyViper.AddConfigPath(".") // optionally look for config in the working directory @@ -237,40 +277,8 @@ func (opts *BackyConfigOpts) InitConfig() { } err := backyViper.ReadInConfig() // Find and read the config file if err != nil { // Handle errors reading the config file - panic(fmt.Errorf("fatal error reading config file %s: %w", backyViper.ConfigFileUsed(), err)) + msg := fmt.Sprintf("fatal error reading config file %s: %v", backyViper.ConfigFileUsed(), err) + logging.ExitWithMSG(msg, 1, nil) } opts.viper = backyViper } - -func loadEnv(backyViper *viper.Viper) { - envFileInConfigDir := fmt.Sprintf("%s/.env", path.Dir(backyViper.ConfigFileUsed())) - var backyEnv map[string]string - backyEnv, envFileErr := godotenv.Read() - - // envFile, envFileErr := os.Open(".env") - if envFileErr != nil { - backyEnv, _ = godotenv.Read(envFileInConfigDir) - } - envFileErr = godotenv.Load() - if envFileErr != nil { - _ = godotenv.Load(envFileInConfigDir) - } - - env := func(name string) string { - name = strings.ToUpper(name) - envVar, found := backyEnv[name] - if found { - return envVar - } - return "" - } - envVars := []string{"APP=${BACKY_APP}"} - - for indx, v := range envVars { - if strings.Contains(v, "$") || (strings.Contains(v, "${") && strings.Contains(v, "}")) { - out, _ := shell.Expand(v, env) - envVars[indx] = out - // println(out) - } - } -} diff --git a/pkg/backy/cron.go b/pkg/backy/cron.go index b601b4b..48b88ef 100644 --- a/pkg/backy/cron.go +++ b/pkg/backy/cron.go @@ -14,15 +14,21 @@ import ( func (conf *BackyConfigFile) Cron() { s := gocron.NewScheduler(time.Local) s.TagsUnique() - for _, config := range conf.CmdConfigLists { - if strings.TrimSpace(config.Cron) != "" { - _, err := s.CronWithSeconds(config.Cron).Tag(config.Name).Do(func(cron string) { + for listName, config := range conf.CmdConfigLists { + if config.Name == "" { + config.Name = listName + } + cron := strings.TrimSpace(config.Cron) + if cron != "" { + conf.Logger.Info().Str("Scheduling cron list", config.Name).Str("Time", cron).Send() + _, err := s.CronWithSeconds(cron).Tag(config.Name).Do(func(cron string) { conf.RunBackyConfig(cron) - }, config.Cron) + }, cron) if err != nil { panic(err) } } } + conf.Logger.Info().Msg("Starting cron mode...") s.StartBlocking() } diff --git a/pkg/backy/notification.go b/pkg/backy/notification.go index d255335..6367038 100644 --- a/pkg/backy/notification.go +++ b/pkg/backy/notification.go @@ -87,5 +87,6 @@ func setupMail(config mailConfig) *mail.Mail { mailClient := mail.New(config.senderaddress, config.host+":"+config.port) mailClient.AuthenticateSMTP("", config.username, config.password, config.host) mailClient.AddReceivers(config.to...) + mailClient.BodyFormat(mail.PlainText) return mailClient } diff --git a/pkg/backy/ssh.go b/pkg/backy/ssh.go index 4caed05..07a2321 100644 --- a/pkg/backy/ssh.go +++ b/pkg/backy/ssh.go @@ -7,9 +7,9 @@ package backy import ( "bufio" "fmt" - "log" "os" "os/user" + "strconv" "strings" "time" @@ -20,83 +20,98 @@ import ( "golang.org/x/crypto/ssh/knownhosts" ) -var ErrPrivateKeyFileFailedToOpen = errors.New("Private key file failed to open.") +var ErrPrivateKeyFileFailedToOpen = errors.New("Failed to open private key file. If encrypted, make sure the password is specified.") var TS = strings.TrimSpace // ConnectToSSHHost connects to a host by looking up the config values in the directory ~/.ssh/config // It uses any set values and looks up an unset values in the config files // It returns an ssh.Client used to run commands against. -func (remoteConfig *Host) ConnectToSSHHost(log *zerolog.Logger) (*ssh.Client, error) { +// If configFile is empty, any required configuration is looked up in the default config files +// If any value is not found, defaults are used +func (remoteConfig *Host) ConnectToSSHHost(log *zerolog.Logger, hosts map[string]*Host) error { - var sshClient *ssh.Client + // var sshClient *ssh.Client var connectErr error // TODO: add JumpHost config check - // if !remoteConfig.UseConfigFiles { - // log.Info().Msg("Not using config files") - // } if TS(remoteConfig.ConfigFilePath) == "" { remoteConfig.useDefaultConfig = true } + if remoteConfig.ProxyHost != nil { + for _, proxyHost := range remoteConfig.ProxyHost { + log.Info().Msgf("Proxy Host %s", proxyHost.Host) + err := proxyHost.GetProxyJumpConfig(hosts) + if err != nil { + return err + } + } + } khPath, khPathErr := GetKnownHosts(remoteConfig.KnownHostsFile) if khPathErr != nil { - return nil, khPathErr + return khPathErr } if remoteConfig.ClientConfig == nil { remoteConfig.ClientConfig = &ssh.ClientConfig{} } - var sshConfigFile *os.File + var configFile *os.File var sshConfigFileOpenErr error if !remoteConfig.useDefaultConfig { - - sshConfigFile, sshConfigFileOpenErr = os.Open(remoteConfig.ConfigFilePath) + configFile, sshConfigFileOpenErr = os.Open(remoteConfig.ConfigFilePath) if sshConfigFileOpenErr != nil { - return nil, sshConfigFileOpenErr + return sshConfigFileOpenErr } } else { defaultConfig, _ := resolveDir("~/.ssh/config") - sshConfigFile, sshConfigFileOpenErr = os.Open(defaultConfig) + configFile, sshConfigFileOpenErr = os.Open(defaultConfig) if sshConfigFileOpenErr != nil { - return nil, sshConfigFileOpenErr + return sshConfigFileOpenErr } } + remoteConfig.SSHConfigFile = &sshConfigFile{} remoteConfig.SSHConfigFile.DefaultUserSettings = ssh_config.DefaultUserSettings - - cfg, decodeErr := ssh_config.Decode(sshConfigFile) + var decodeErr error + remoteConfig.SSHConfigFile.SshConfigFile, decodeErr = ssh_config.Decode(configFile) if decodeErr != nil { - return nil, decodeErr + return decodeErr } - remoteConfig.SSHConfigFile.SshConfigFile = cfg - remoteConfig.GetPrivateKeyFromConfig() - remoteConfig.GetHostNameWithPort() + remoteConfig.ClientConfig.Timeout = time.Second * 30 + remoteConfig.GetPrivateKeyFileFromConfig() + remoteConfig.GetPort() + remoteConfig.GetHostName() + remoteConfig.CombineHostNameWithPort() remoteConfig.GetSshUserFromConfig() - log.Info().Msgf("Port: %v", remoteConfig.Port) if remoteConfig.HostName == "" { - return nil, errors.New("No hostname found or specified") + return errors.New("No hostname found or specified") } err := remoteConfig.GetAuthMethods() if err != nil { - return nil, err + return err } - // TODO: Add value/option to config for host key and add bool to check for host key hostKeyCallback, err := knownhosts.New(khPath) if err != nil { - return nil, errors.Wrap(err, "could not create hostkeycallback function") + return errors.Wrap(err, "could not create hostkeycallback function") } remoteConfig.ClientConfig.HostKeyCallback = hostKeyCallback log.Info().Str("user", remoteConfig.ClientConfig.User).Send() - log.Info().Msgf("Connecting to host %s", remoteConfig.HostName) - remoteConfig.ClientConfig.Timeout = time.Second * 30 - sshClient, connectErr = ssh.Dial("tcp", remoteConfig.HostName, remoteConfig.ClientConfig) + remoteConfig.SshClient, connectErr = remoteConfig.ConnectThroughBastion(log) if connectErr != nil { - return nil, connectErr + return connectErr } - return sshClient, nil + if remoteConfig.SshClient != nil { + return nil + } + + log.Info().Msgf("Connecting to host %s", remoteConfig.HostName) + remoteConfig.SshClient, connectErr = ssh.Dial("tcp", remoteConfig.HostName, remoteConfig.ClientConfig) + if connectErr != nil { + return connectErr + } + return nil } func (remoteHost *Host) GetSshUserFromConfig() { @@ -155,9 +170,9 @@ func (remoteHost *Host) GetAuthMethods() error { // GetPrivateKeyFromConfig checks to see if the privateKeyPath is empty. // If not, it keeps the value. // If empty, the key is looked for in the specified config file. -// If that path is empty, the default config file is searched +// If that path is empty, the default config file is searched. // If not found in the default file, the privateKeyPath is set to ~/.ssh/id_rsa -func (remoteHost *Host) GetPrivateKeyFromConfig() { +func (remoteHost *Host) GetPrivateKeyFileFromConfig() { var identityFile string if remoteHost.PrivateKeyPath == "" { identityFile, _ = remoteHost.SSHConfigFile.SshConfigFile.Get(remoteHost.Host, "IdentityFile") @@ -175,18 +190,11 @@ func (remoteHost *Host) GetPrivateKeyFromConfig() { remoteHost.PrivateKeyPath, _ = resolveDir(identityFile) } -// GetHostNameWithPort checks if the port from the config file is 0 +// GetPort checks if the port from the config file is 0 // If it is the port is searched in the SSH config file(s) -func (remoteHost *Host) GetHostNameWithPort() { +func (remoteHost *Host) GetPort() { port := fmt.Sprintf("%v", remoteHost.Port) - - if remoteHost.HostName == "" { - remoteHost.HostName, _ = remoteHost.SSHConfigFile.SshConfigFile.Get(remoteHost.Host, "HostName") - if remoteHost.HostName == "" { - remoteHost.HostName = remoteHost.SSHConfigFile.DefaultUserSettings.Get(remoteHost.Host, "HostName") - } - } - // no port specifed + // port specifed? if port == "0" { port, _ = remoteHost.SSHConfigFile.SshConfigFile.Get(remoteHost.Host, "Port") if port == "" { @@ -195,16 +203,34 @@ func (remoteHost *Host) GetHostNameWithPort() { port = "22" } } - println(port) } - if !strings.HasSuffix(remoteHost.HostName, ":"+port) { - remoteHost.HostName = remoteHost.HostName + ":" + port + portNum, _ := strconv.ParseUint(port, 10, 32) + remoteHost.Port = uint16(portNum) +} + +func (remoteHost *Host) CombineHostNameWithPort() { + remoteHost.HostName = fmt.Sprintf("%s:%v", remoteHost.HostName, remoteHost.Port) +} + +func (remoteHost *Host) GetHostName() { + + if remoteHost.HostName == "" { + remoteHost.HostName, _ = remoteHost.SSHConfigFile.SshConfigFile.Get(remoteHost.Host, "HostName") + if remoteHost.HostName == "" { + remoteHost.HostName = remoteHost.SSHConfigFile.DefaultUserSettings.Get(remoteHost.Host, "HostName") + } } } -func (remoteHost *Host) ConnectThroughBastion() (*ssh.Client, error) { +func (remoteHost *Host) ConnectThroughBastion(log *zerolog.Logger) (*ssh.Client, error) { + if remoteHost.ProxyHost == nil { + return nil, nil + } + + log.Info().Msgf("Connecting to proxy host %s", remoteHost.ProxyHost[0].HostName) + // connect to the bastion host - bClient, err := ssh.Dial("tcp", remoteHost.ProxyHost.HostName, remoteHost.ProxyHost.ClientConfig) + bClient, err := ssh.Dial("tcp", remoteHost.ProxyHost[0].HostName, remoteHost.ProxyHost[0].ClientConfig) if err != nil { return nil, err } @@ -214,10 +240,10 @@ func (remoteHost *Host) ConnectThroughBastion() (*ssh.Client, error) { if err != nil { return nil, err } - + log.Info().Msgf("Connecting to host %s", remoteHost.HostName) ncc, chans, reqs, err := ssh.NewClientConn(conn, remoteHost.HostName, remoteHost.ClientConfig) if err != nil { - log.Fatal(err) + return nil, err } sClient := ssh.NewClient(ncc, chans, reqs) @@ -258,14 +284,14 @@ func GetPrivateKeyPassword(key string) (string, error) { return prKeyPassword, nil } -func GetPassword(key string) (string, error) { - key = strings.TrimSpace(key) - if key == "" { +func GetPassword(pass string) (string, error) { + pass = strings.TrimSpace(pass) + if pass == "" { return "", nil } var password string - if strings.HasPrefix(key, "file:") { - passFilePath := strings.TrimPrefix(key, "file:") + if strings.HasPrefix(pass, "file:") { + passFilePath := strings.TrimPrefix(pass, "file:") passFilePath, _ = resolveDir(passFilePath) keyFile, keyFileErr := os.Open(passFilePath) if keyFileErr != nil { @@ -275,14 +301,94 @@ func GetPassword(key string) (string, error) { for passwordScanner.Scan() { password = passwordScanner.Text() } - } else if strings.HasPrefix(key, "env:") { - passEnv := strings.TrimPrefix(key, "env:") + } else if strings.HasPrefix(pass, "env:") { + passEnv := strings.TrimPrefix(pass, "env:") passEnv = strings.TrimPrefix(passEnv, "${") passEnv = strings.TrimSuffix(passEnv, "}") passEnv = strings.TrimPrefix(passEnv, "$") password = os.Getenv(passEnv) } else { - password = key + password = pass } return password, nil } + +func (remoteConfig *Host) GetProxyJumpFromConfig(hosts map[string]*Host) error { + proxyJump, _ := remoteConfig.SSHConfigFile.SshConfigFile.Get(remoteConfig.Host, "ProxyJump") + if proxyJump == "" { + proxyJump = remoteConfig.SSHConfigFile.DefaultUserSettings.Get(remoteConfig.Host, "ProxyJump") + } + if remoteConfig.ProxyJump == "" && proxyJump != "" { + remoteConfig.ProxyJump = proxyJump + } + proxyJumpHosts := strings.Split(remoteConfig.ProxyJump, ",") + if remoteConfig.ProxyHost == nil && len(proxyJumpHosts) == 1 { + remoteConfig.ProxyJump = proxyJump + proxyHost, proxyHostFound := hosts[proxyJump] + if proxyHostFound { + remoteConfig.ProxyHost = append(remoteConfig.ProxyHost, proxyHost) + } else { + newProxy := &Host{Host: proxyJump} + remoteConfig.ProxyHost = append(remoteConfig.ProxyHost, newProxy) + } + } + + return nil +} +func (remoteConfig *Host) GetProxyJumpConfig(hosts map[string]*Host) error { + if TS(remoteConfig.ConfigFilePath) == "" { + remoteConfig.useDefaultConfig = true + } + + // log.Info().Msgf("Proxy Host %s", remoteConfig.ProxyHost[0].Host) + khPath, khPathErr := GetKnownHosts(remoteConfig.KnownHostsFile) + + if khPathErr != nil { + return khPathErr + } + if remoteConfig.ClientConfig == nil { + remoteConfig.ClientConfig = &ssh.ClientConfig{} + } + var configFile *os.File + var sshConfigFileOpenErr error + if !remoteConfig.useDefaultConfig { + + configFile, sshConfigFileOpenErr = os.Open(remoteConfig.ConfigFilePath) + if sshConfigFileOpenErr != nil { + return sshConfigFileOpenErr + } + } else { + defaultConfig, _ := resolveDir("~/.ssh/config") + configFile, sshConfigFileOpenErr = os.Open(defaultConfig) + if sshConfigFileOpenErr != nil { + return sshConfigFileOpenErr + } + } + remoteConfig.SSHConfigFile = &sshConfigFile{} + remoteConfig.SSHConfigFile.DefaultUserSettings = ssh_config.DefaultUserSettings + var decodeErr error + remoteConfig.SSHConfigFile.SshConfigFile, decodeErr = ssh_config.Decode(configFile) + if decodeErr != nil { + return decodeErr + } + remoteConfig.GetPrivateKeyFileFromConfig() + remoteConfig.GetPort() + remoteConfig.GetHostName() + remoteConfig.CombineHostNameWithPort() + remoteConfig.GetSshUserFromConfig() + if remoteConfig.HostName == "" { + return errors.New("No hostname found or specified") + } + err := remoteConfig.GetAuthMethods() + if err != nil { + return err + } + + // TODO: Add value/option to config for host key and add bool to check for host key + hostKeyCallback, err := knownhosts.New(khPath) + if err != nil { + return errors.Wrap(err, "could not create hostkeycallback function") + } + remoteConfig.ClientConfig.HostKeyCallback = hostKeyCallback + return nil +} diff --git a/pkg/backy/templates/error.txt b/pkg/backy/templates/error.txt new file mode 100644 index 0000000..b40f345 --- /dev/null +++ b/pkg/backy/templates/error.txt @@ -0,0 +1,8 @@ +Command list {{.listName }} failed on running {{.Command}}. + +The error was {{ .Err }} + +The following commands ran: +{{- range .CmdsRan}} + - {{. -}} +{{end}} \ No newline at end of file diff --git a/pkg/backy/templates/success.txt b/pkg/backy/templates/success.txt new file mode 100644 index 0000000..0212c86 --- /dev/null +++ b/pkg/backy/templates/success.txt @@ -0,0 +1,7 @@ +Command list {{ .listName }} was completed successfully. + + +The following commands ran: +{{- range .CmdsRan}} + - {{. -}} +{{end}} diff --git a/pkg/backy/types.go b/pkg/backy/types.go index d206fa2..4c6abe6 100644 --- a/pkg/backy/types.go +++ b/pkg/backy/types.go @@ -41,17 +41,18 @@ type Host struct { HostName string `yaml:"hostname,omitempty"` KnownHostsFile string `yaml:"knownhostsfile,omitempty"` ClientConfig *ssh.ClientConfig - SSHConfigFile sshConfigFile + SSHConfigFile *sshConfigFile + SshClient *ssh.Client Port uint16 `yaml:"port,omitempty"` - JumpHost string `yaml:"jumphost,omitempty"` + ProxyJump string `yaml:"proxyjump,omitempty"` Password string `yaml:"password,omitempty"` PrivateKeyPath string `yaml:"privatekeypath,omitempty"` PrivateKeyPassword string `yaml:"privatekeypassword,omitempty"` UseConfigFiles bool `yaml:"use_config_files,omitempty"` useDefaultConfig bool User string `yaml:"user,omitempty"` - // ProxyHost holds the configuration for a JumpHost host - ProxyHost *Host + // ProxyHost holds the configuration for a ProxyJump host + ProxyHost []*Host } type sshConfigFile struct { @@ -144,6 +145,9 @@ type BackyConfigOpts struct { // Holds commands to execute for the exec command executeLists []string + // Holds env vars from .env file + backyEnv map[string]string + viper *viper.Viper } diff --git a/pkg/backy/utils.go b/pkg/backy/utils.go index c094050..b09a4c2 100644 --- a/pkg/backy/utils.go +++ b/pkg/backy/utils.go @@ -9,6 +9,7 @@ import ( "fmt" "os" "os/exec" + "path" "path/filepath" "strings" @@ -17,6 +18,7 @@ import ( "github.com/rs/zerolog" "github.com/spf13/viper" "golang.org/x/crypto/ssh" + "mvdan.cc/sh/v3/shell" ) func injectEnvIntoSSH(envVarsToInject environmentVars, process *ssh.Session, log *zerolog.Logger) { @@ -207,3 +209,33 @@ func resolveDir(path string) (string, error) { } return path, nil } + +func (opts *BackyConfigOpts) loadEnv() { + envFileInConfigDir := fmt.Sprintf("%s/.env", path.Dir(opts.viper.ConfigFileUsed())) + var backyEnv map[string]string + backyEnv, envFileErr := godotenv.Read(envFileInConfigDir) + if envFileErr != nil { + return + } + + opts.backyEnv = backyEnv +} + +func expandEnvVars(backyEnv map[string]string, envVars []string) { + + env := func(name string) string { + name = strings.ToUpper(name) + envVar, found := backyEnv[name] + if found { + return envVar + } + return "" + } + + for indx, v := range envVars { + if strings.Contains(v, "$") || (strings.Contains(v, "${") && strings.Contains(v, "}")) { + out, _ := shell.Expand(v, env) + envVars[indx] = out + } + } +} diff --git a/pkg/logging/logging.go b/pkg/logging/logging.go index 7e20881..03a2ba2 100644 --- a/pkg/logging/logging.go +++ b/pkg/logging/logging.go @@ -7,7 +7,6 @@ import ( "time" "github.com/rs/zerolog" - "github.com/spf13/viper" "gopkg.in/natefinch/lumberjack.v2" ) @@ -25,7 +24,7 @@ func ExitWithMSG(msg string, code int, log *zerolog.Logger) { os.Exit(code) } -func SetLoggingWriters(v *viper.Viper, logFile string) (writers zerolog.LevelWriter) { +func SetLoggingWriters(logFile string) (writers zerolog.LevelWriter) { console := zerolog.ConsoleWriter{} if IsConsoleLoggingEnabled() { @@ -55,12 +54,7 @@ func SetLoggingWriters(v *viper.Viper, logFile string) (writers zerolog.LevelWri MaxAge: 28, //days Compress: true, // disabled by default } - if strings.TrimSpace(logFile) != "" { - fileLogger.Filename = logFile - } else { - fileLogger.Filename = "./backy.log" - } - + fileLogger.Filename = logFile // UNIX Time is faster and smaller than most timestamps zerolog.TimeFieldFormat = zerolog.TimeFormatUnix // zerolog.TimeFieldFormat = time.RFC1123 @@ -75,3 +69,7 @@ func SetLoggingWriters(v *viper.Viper, logFile string) (writers zerolog.LevelWri func IsConsoleLoggingEnabled() bool { return os.Getenv("BACKY_CONSOLE_LOGGING") == "enabled" } + +// func IsTerminal() bool { +// return os.Getenv("BACKY_TERM") == "enabled" +// }