make remote commands run and not fail if an SSH session failed to be created

CHANGELOG.md

@@ -6,6 +6,19 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html),
 and is generated by [Changie](https://github.com/miniscruff/changie).
 
 
+## v0.3.1 - 2023-07-20
+### Changed
+* If an SSH session failed to be created, the command would fail. This would be caused when restarting the SSH host. The SSH connection is attempted to be created again. If successful, the command is executed normally.
+
+## v0.3.0 - 2023-01-07
+### Added
+* Getting environment variables and passwords from Vault (not tested yet)
+* Vault configuration to config (not tested yet)
+* Ability to run scripts from file on local machine on the remote host
+* Ability to get ouput in the notification of a list for individual commands or all commands
+### Changed
+* Make SSH connections close after all commands have been run; reuse previous connections if needed
+
 ## 0.2.4 - 2023-02-18
 ### Added
 * Notifications now display errors and the output of the failed command.

cmd/version.go

@@ -7,7 +7,7 @@ import (
 	"github.com/spf13/cobra"
 )
 
-const versionStr = "0.3.0"
+const versionStr = "0.3.1"
 
 var (
 	versionCmd = &cobra.Command{

pkg/backy/backy.go

@@ -64,9 +64,20 @@ func (command *Command) RunCmd(log zerolog.Logger, backyConf *ConfigFile, opts *
 			}
 		}
 		commandSession, err := command.RemoteHost.SshClient.NewSession()
+
+		// Retry connecting to host; if that fails, error. If it does not fail, try to create new session
 		if err != nil {
-			return nil, err
+			connErr := command.RemoteHost.ConnectToSSHHost(opts, backyConf)
+			if connErr != nil {
+				return nil, fmt.Errorf("error creating session: %v, and error creating new connection to host: %v", err, connErr)
 			}
+			commandSession, err = command.RemoteHost.SshClient.NewSession()
+
+			if err != nil {
+				return nil, fmt.Errorf("error creating session: %v", err)
+			}
+		}
+
 		defer commandSession.Close()
 
 		injectEnvIntoSSH(envVars, commandSession, opts, log)
@@ -390,15 +401,12 @@ func (config *ConfigFile) RunListConfig(cron string, opts *ConfigOpts) {
 	listChan := make(chan *CmdList, configListsLen)
 	results := make(chan string)
 
-	// This starts up 3 workers, initially blocked
+	// This starts up list workers, initially blocked
 	// because there are no jobs yet.
 	for w := 1; w <= configListsLen; w++ {
 		go cmdListWorker(mTemps, listChan, config, results, opts)
 	}
 
-	// Here we send 5 `jobs` and then `close` that
-	// channel to indicate that's all the work we have.
-	// configChan <- config.Cmds
 	for listName, cmdConfig := range config.CmdConfigLists {
 		if cmdConfig.Name == "" {
 			cmdConfig.Name = listName
@@ -438,6 +446,7 @@ func (config *ConfigFile) ExecuteCmds(opts *ConfigOpts) {
 
 func (c *ConfigFile) closeHostConnections() {
 	for _, host := range c.Hosts {
+		c.Logger.Info().Str("server", host.HostName)
 		if host.isProxyHost {
 			continue
 		}
@@ -445,6 +454,7 @@ func (c *ConfigFile) closeHostConnections() {
 			if _, err := host.SshClient.NewSession(); err == nil {
 				c.Logger.Info().Msgf("Closing host connection %s", host.HostName)
 				host.SshClient.Close()
+				host.SshClient = nil
 			}
 		}
 		for _, proxyHost := range host.ProxyHost {
@@ -455,6 +465,7 @@ func (c *ConfigFile) closeHostConnections() {
 				if _, err := host.SshClient.NewSession(); err == nil {
 					c.Logger.Info().Msgf("Closing connection to proxy host %s", host.HostName)
 					host.SshClient.Close()
+					host.SshClient = nil
 				}
 			}
 		}
@@ -464,6 +475,7 @@ func (c *ConfigFile) closeHostConnections() {
 			if _, err := host.SshClient.NewSession(); err == nil {
 				c.Logger.Info().Msgf("Closing proxy host connection %s", host.HostName)
 				host.SshClient.Close()
+				host.SshClient = nil
 			}
 		}
 	}

pkg/backy/ssh.go

@@ -117,6 +117,7 @@ func (remoteConfig *Host) ConnectToSSHHost(opts *ConfigOpts, config *ConfigFile)
 	if connectErr != nil {
 		return connectErr
 	}
+
 	config.Hosts[remoteConfig.Host] = remoteConfig
 	return nil
 }
@@ -134,6 +135,7 @@ func (remoteHost *Host) GetSshUserFromConfig() {
 	}
 	remoteHost.ClientConfig.User = remoteHost.User
 }
+
 func (remoteHost *Host) GetAuthMethods(opts *ConfigOpts) error {
 	var signer ssh.Signer
 	var err error
@@ -265,6 +267,7 @@ func (remoteHost *Host) ConnectThroughBastion(log zerolog.Logger) (*ssh.Client,
 }
 
 func GetKnownHosts(khPath string) (string, error) {
+
 	if TS(khPath) != "" {
 		return resolveDir(khPath)
 	}
@@ -272,6 +275,7 @@ func GetKnownHosts(khPath string) (string, error) {
 }
 
 func GetPrivateKeyPassword(key string, opts *ConfigOpts, log zerolog.Logger) (string, error) {
+
 	var prKeyPassword string
 	if strings.HasPrefix(key, "file:") {
 		privKeyPassFilePath := strings.TrimPrefix(key, "file:")
@@ -298,6 +302,7 @@ func GetPrivateKeyPassword(key string, opts *ConfigOpts, log zerolog.Logger) (st
 }
 
 func GetPassword(pass string, opts *ConfigOpts, log zerolog.Logger) (string, error) {
+
 	pass = strings.TrimSpace(pass)
 	if pass == "" {
 		return "", nil
@@ -329,6 +334,7 @@ func GetPassword(pass string, opts *ConfigOpts, log zerolog.Logger) (string, err
 }
 
 func (remoteConfig *Host) GetProxyJumpFromConfig(hosts map[string]*Host) error {
+
 	proxyJump, _ := remoteConfig.SSHConfigFile.SshConfigFile.Get(remoteConfig.Host, "ProxyJump")
 	if proxyJump == "" {
 		proxyJump = remoteConfig.SSHConfigFile.DefaultUserSettings.Get(remoteConfig.Host, "ProxyJump")
@@ -354,6 +360,7 @@ func (remoteConfig *Host) GetProxyJumpFromConfig(hosts map[string]*Host) error {
 }
 
 func (remoteConfig *Host) GetProxyJumpConfig(hosts map[string]*Host, opts *ConfigOpts) error {
+
 	if TS(remoteConfig.ConfigFilePath) == "" {
 		remoteConfig.useDefaultConfig = true
 	}