make remote commands run and not fail if an SSH session failed to be created

This reverts commit affdd0abfd.

.changes/v0.3.1.md

@@ -0,0 +1,3 @@
+## v0.3.1 - 2023-07-20
+### Changed
+* If an SSH session failed to be created, the command would fail. This would be caused when restarting the SSH host. The SSH connection is attempted to be created again. If successful, the command is executed normally.

CHANGELOG.md

@@ -6,6 +6,19 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html),
 and is generated by [Changie](https://github.com/miniscruff/changie).
 
 
+## v0.3.1 - 2023-07-20
+### Changed
+* If an SSH session failed to be created, the command would fail. This would be caused when restarting the SSH host. The SSH connection is attempted to be created again. If successful, the command is executed normally.
+
+## v0.3.0 - 2023-01-07
+### Added
+* Getting environment variables and passwords from Vault (not tested yet)
+* Vault configuration to config (not tested yet)
+* Ability to run scripts from file on local machine on the remote host
+* Ability to get ouput in the notification of a list for individual commands or all commands
+### Changed
+* Make SSH connections close after all commands have been run; reuse previous connections if needed
+
 ## 0.2.4 - 2023-02-18
 ### Added
 * Notifications now display errors and the output of the failed command.

cmd/version.go

@@ -7,7 +7,7 @@ import (
 	"github.com/spf13/cobra"
 )
 
-const versionStr = "0.3.0"
+const versionStr = "0.3.1"
 
 var (
 	versionCmd = &cobra.Command{

docs/content/config/commands.md

@@ -16,11 +16,12 @@ commands:
       - -f /some/path/to/docker-compose.yaml
       - down
     # if host is not defined, command will be run locally
+    # The host has to be defined in either the config file or the SSH Config files
     host: some-host 
   backup-docker-container-script:
-    cmd: /path/to/script/on/some-host
-    # The host has to be defined in either the config file or the SSH Config files
-    host: some-host
+    cmd: /path/to/local/script
+    # script file is input as stdin to SSH
+    type: scriptFile # also can be script
     environment:
       - FOO=BAR
       - APP=$VAR
@@ -33,6 +34,7 @@ Values available for this section:
 | `cmd` | Defines the command to execute | `string` | yes |
 | `args` | Defines the arguments to the command | `[]string` | no |
 | `environment` | Defines evironment variables for the command | `[]string` | no |
+| `type` | May be `scriptFile` or `script`. Runs script from local machine on remote | `string` | no |
 | `getOutput` | Command(s) output is in the notification(s) | `bool` | no |
 | `host` | If not specified, the command will execute locally. | `string` | no |
 | `shell` | Only applicable when host is not specified | `string` | no |
@@ -84,6 +86,14 @@ If I assign a value to host as `host: web-prod` and don't specify this value in
 If shell is defined and host is NOT defined, the command will run in the specified shell.
 Make sure to escape any shell input.
 
+### type
+
+May be `scriptFile` or `script`. Runs script from local machine on remote host.
+
+If `script` is specified, `cmd` is used as the script.
+
+Script file is input as stdin to SSH. 
+
 ### environment
 
 The environment variables support expansion:

pkg/backy/backy.go

@@ -64,9 +64,20 @@ func (command *Command) RunCmd(log zerolog.Logger, backyConf *ConfigFile, opts *
 			}
 		}
 		commandSession, err := command.RemoteHost.SshClient.NewSession()
+
+		// Retry connecting to host; if that fails, error. If it does not fail, try to create new session
 		if err != nil {
-			return nil, err
+			connErr := command.RemoteHost.ConnectToSSHHost(opts, backyConf)
+			if connErr != nil {
+				return nil, fmt.Errorf("error creating session: %v, and error creating new connection to host: %v", err, connErr)
+			}
+			commandSession, err = command.RemoteHost.SshClient.NewSession()
+
+			if err != nil {
+				return nil, fmt.Errorf("error creating session: %v", err)
+			}
 		}
+
 		defer commandSession.Close()
 
 		injectEnvIntoSSH(envVars, commandSession, opts, log)
@@ -390,15 +401,12 @@ func (config *ConfigFile) RunListConfig(cron string, opts *ConfigOpts) {
 	listChan := make(chan *CmdList, configListsLen)
 	results := make(chan string)
 
-	// This starts up 3 workers, initially blocked
+	// This starts up list workers, initially blocked
 	// because there are no jobs yet.
 	for w := 1; w <= configListsLen; w++ {
 		go cmdListWorker(mTemps, listChan, config, results, opts)
 	}
 
-	// Here we send 5 `jobs` and then `close` that
-	// channel to indicate that's all the work we have.
-	// configChan <- config.Cmds
 	for listName, cmdConfig := range config.CmdConfigLists {
 		if cmdConfig.Name == "" {
 			cmdConfig.Name = listName
@@ -438,6 +446,7 @@ func (config *ConfigFile) ExecuteCmds(opts *ConfigOpts) {
 
 func (c *ConfigFile) closeHostConnections() {
 	for _, host := range c.Hosts {
+		c.Logger.Info().Str("server", host.HostName)
 		if host.isProxyHost {
 			continue
 		}
@@ -445,6 +454,7 @@ func (c *ConfigFile) closeHostConnections() {
 			if _, err := host.SshClient.NewSession(); err == nil {
 				c.Logger.Info().Msgf("Closing host connection %s", host.HostName)
 				host.SshClient.Close()
+				host.SshClient = nil
 			}
 		}
 		for _, proxyHost := range host.ProxyHost {
@@ -455,6 +465,7 @@ func (c *ConfigFile) closeHostConnections() {
 				if _, err := host.SshClient.NewSession(); err == nil {
 					c.Logger.Info().Msgf("Closing connection to proxy host %s", host.HostName)
 					host.SshClient.Close()
+					host.SshClient = nil
 				}
 			}
 		}
@@ -464,6 +475,7 @@ func (c *ConfigFile) closeHostConnections() {
 			if _, err := host.SshClient.NewSession(); err == nil {
 				c.Logger.Info().Msgf("Closing proxy host connection %s", host.HostName)
 				host.SshClient.Close()
+				host.SshClient = nil
 			}
 		}
 	}

pkg/backy/ssh.go

@@ -117,6 +117,7 @@ func (remoteConfig *Host) ConnectToSSHHost(opts *ConfigOpts, config *ConfigFile)
 	if connectErr != nil {
 		return connectErr
 	}
+
 	config.Hosts[remoteConfig.Host] = remoteConfig
 	return nil
 }
@@ -134,6 +135,7 @@ func (remoteHost *Host) GetSshUserFromConfig() {
 	}
 	remoteHost.ClientConfig.User = remoteHost.User
 }
+
 func (remoteHost *Host) GetAuthMethods(opts *ConfigOpts) error {
 	var signer ssh.Signer
 	var err error
@@ -265,6 +267,7 @@ func (remoteHost *Host) ConnectThroughBastion(log zerolog.Logger) (*ssh.Client,
 }
 
 func GetKnownHosts(khPath string) (string, error) {
+
 	if TS(khPath) != "" {
 		return resolveDir(khPath)
 	}
@@ -272,6 +275,7 @@ func GetKnownHosts(khPath string) (string, error) {
 }
 
 func GetPrivateKeyPassword(key string, opts *ConfigOpts, log zerolog.Logger) (string, error) {
+
 	var prKeyPassword string
 	if strings.HasPrefix(key, "file:") {
 		privKeyPassFilePath := strings.TrimPrefix(key, "file:")
@@ -298,6 +302,7 @@ func GetPrivateKeyPassword(key string, opts *ConfigOpts, log zerolog.Logger) (st
 }
 
 func GetPassword(pass string, opts *ConfigOpts, log zerolog.Logger) (string, error) {
+
 	pass = strings.TrimSpace(pass)
 	if pass == "" {
 		return "", nil
@@ -329,6 +334,7 @@ func GetPassword(pass string, opts *ConfigOpts, log zerolog.Logger) (string, err
 }
 
 func (remoteConfig *Host) GetProxyJumpFromConfig(hosts map[string]*Host) error {
+
 	proxyJump, _ := remoteConfig.SSHConfigFile.SshConfigFile.Get(remoteConfig.Host, "ProxyJump")
 	if proxyJump == "" {
 		proxyJump = remoteConfig.SSHConfigFile.DefaultUserSettings.Get(remoteConfig.Host, "ProxyJump")
@@ -354,6 +360,7 @@ func (remoteConfig *Host) GetProxyJumpFromConfig(hosts map[string]*Host) error {
 }
 
 func (remoteConfig *Host) GetProxyJumpConfig(hosts map[string]*Host, opts *ConfigOpts) error {
+
 	if TS(remoteConfig.ConfigFilePath) == "" {
 		remoteConfig.useDefaultConfig = true
 	}