change: Commands: host can now be localhost or 127.0.0.1 to run commands locally

.changes/unreleased/Changed-20250321-090849.yaml

@@ -0,0 +1,3 @@
+kind: Changed
+body: 'Commands: `host` can now be `localhost` or `127.0.0.1` to run commands locally'
+time: 2025-03-21T09:08:49.871021144-05:00

.changes/v0.10.2.md

@@ -0,0 +1,6 @@
+## v0.10.2 - 2025-03-19
+### Added
+* Notifications: http service added
+* Variable support. Can be referenced with `%{var:nameOfVar}%` in select string fields.
+### Changed
+* vault: initialize vault before validating config

.woodpecker/go-lint.yml

@@ -5,7 +5,7 @@ steps:
       - go build
       - go test
   release:
-    image: golangci/golangci-lint:v1.53.3
+    image: golangci/golangci-lint:v1.64.7
     commands:
       - golangci-lint run -v --timeout 5m
 

CHANGELOG.md

@@ -6,6 +6,13 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html),
 and is generated by [Changie](https://github.com/miniscruff/changie).
 
 
+## v0.10.2 - 2025-03-19
+### Added
+* Notifications: http service added
+* Variable support. Can be referenced with `%{var:nameOfVar}%` in select string fields.
+### Changed
+* vault: initialize vault before validating config
+
 ## v0.10.1 - 2025-03-11
 ### Added
 * UserCommands: add ssh public keys when running locally

cmd/version.go

@@ -7,7 +7,7 @@ import (
 	"github.com/spf13/cobra"
 )
 
-const versionStr = "0.10.1"
+const versionStr = "0.10.2"
 
 var (
 	versionCmd = &cobra.Command{

go.mod

@@ -2,9 +2,7 @@ module git.andrewnw.xyz/CyberShell/backy
 
 go 1.23
 
-toolchain go1.23.6
+toolchain go1.23.7
-
-replace git.andrewnw.xyz/CyberShell/backy => /home/andrew/Projects/backy
 
 require (
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.76.0

pkg/backy/allowedexternaldirectives_enumer.go

@@ -0,0 +1,145 @@
+// Code generated by "enumer -linecomment -yaml -text -json -type=AllowedExternalDirectives"; DO NOT EDIT.
+
+package backy
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+)
+
+const _AllowedExternalDirectivesName = "DefaultExternalDirvaultvault-filevault-file-envfile-envfileenv"
+
+var _AllowedExternalDirectivesIndex = [...]uint8{0, 18, 23, 33, 47, 55, 59, 62}
+
+const _AllowedExternalDirectivesLowerName = "defaultexternaldirvaultvault-filevault-file-envfile-envfileenv"
+
+func (i AllowedExternalDirectives) String() string {
+	if i < 0 || i >= AllowedExternalDirectives(len(_AllowedExternalDirectivesIndex)-1) {
+		return fmt.Sprintf("AllowedExternalDirectives(%d)", i)
+	}
+	return _AllowedExternalDirectivesName[_AllowedExternalDirectivesIndex[i]:_AllowedExternalDirectivesIndex[i+1]]
+}
+
+// An "invalid array index" compiler error signifies that the constant values have changed.
+// Re-run the stringer command to generate them again.
+func _AllowedExternalDirectivesNoOp() {
+	var x [1]struct{}
+	_ = x[DefaultExternalDir-(0)]
+	_ = x[AllowedExternalDirectiveVault-(1)]
+	_ = x[AllowedExternalDirectiveVaultFile-(2)]
+	_ = x[AllowedExternalDirectiveAll-(3)]
+	_ = x[AllowedExternalDirectiveFileEnv-(4)]
+	_ = x[AllowedExternalDirectiveFile-(5)]
+	_ = x[AllowedExternalDirectiveEnv-(6)]
+}
+
+var _AllowedExternalDirectivesValues = []AllowedExternalDirectives{DefaultExternalDir, AllowedExternalDirectiveVault, AllowedExternalDirectiveVaultFile, AllowedExternalDirectiveAll, AllowedExternalDirectiveFileEnv, AllowedExternalDirectiveFile, AllowedExternalDirectiveEnv}
+
+var _AllowedExternalDirectivesNameToValueMap = map[string]AllowedExternalDirectives{
+	_AllowedExternalDirectivesName[0:18]:       DefaultExternalDir,
+	_AllowedExternalDirectivesLowerName[0:18]:  DefaultExternalDir,
+	_AllowedExternalDirectivesName[18:23]:      AllowedExternalDirectiveVault,
+	_AllowedExternalDirectivesLowerName[18:23]: AllowedExternalDirectiveVault,
+	_AllowedExternalDirectivesName[23:33]:      AllowedExternalDirectiveVaultFile,
+	_AllowedExternalDirectivesLowerName[23:33]: AllowedExternalDirectiveVaultFile,
+	_AllowedExternalDirectivesName[33:47]:      AllowedExternalDirectiveAll,
+	_AllowedExternalDirectivesLowerName[33:47]: AllowedExternalDirectiveAll,
+	_AllowedExternalDirectivesName[47:55]:      AllowedExternalDirectiveFileEnv,
+	_AllowedExternalDirectivesLowerName[47:55]: AllowedExternalDirectiveFileEnv,
+	_AllowedExternalDirectivesName[55:59]:      AllowedExternalDirectiveFile,
+	_AllowedExternalDirectivesLowerName[55:59]: AllowedExternalDirectiveFile,
+	_AllowedExternalDirectivesName[59:62]:      AllowedExternalDirectiveEnv,
+	_AllowedExternalDirectivesLowerName[59:62]: AllowedExternalDirectiveEnv,
+}
+
+var _AllowedExternalDirectivesNames = []string{
+	_AllowedExternalDirectivesName[0:18],
+	_AllowedExternalDirectivesName[18:23],
+	_AllowedExternalDirectivesName[23:33],
+	_AllowedExternalDirectivesName[33:47],
+	_AllowedExternalDirectivesName[47:55],
+	_AllowedExternalDirectivesName[55:59],
+	_AllowedExternalDirectivesName[59:62],
+}
+
+// AllowedExternalDirectivesString retrieves an enum value from the enum constants string name.
+// Throws an error if the param is not part of the enum.
+func AllowedExternalDirectivesString(s string) (AllowedExternalDirectives, error) {
+	if val, ok := _AllowedExternalDirectivesNameToValueMap[s]; ok {
+		return val, nil
+	}
+
+	if val, ok := _AllowedExternalDirectivesNameToValueMap[strings.ToLower(s)]; ok {
+		return val, nil
+	}
+	return 0, fmt.Errorf("%s does not belong to AllowedExternalDirectives values", s)
+}
+
+// AllowedExternalDirectivesValues returns all values of the enum
+func AllowedExternalDirectivesValues() []AllowedExternalDirectives {
+	return _AllowedExternalDirectivesValues
+}
+
+// AllowedExternalDirectivesStrings returns a slice of all String values of the enum
+func AllowedExternalDirectivesStrings() []string {
+	strs := make([]string, len(_AllowedExternalDirectivesNames))
+	copy(strs, _AllowedExternalDirectivesNames)
+	return strs
+}
+
+// IsAAllowedExternalDirectives returns "true" if the value is listed in the enum definition. "false" otherwise
+func (i AllowedExternalDirectives) IsAAllowedExternalDirectives() bool {
+	for _, v := range _AllowedExternalDirectivesValues {
+		if i == v {
+			return true
+		}
+	}
+	return false
+}
+
+// MarshalJSON implements the json.Marshaler interface for AllowedExternalDirectives
+func (i AllowedExternalDirectives) MarshalJSON() ([]byte, error) {
+	return json.Marshal(i.String())
+}
+
+// UnmarshalJSON implements the json.Unmarshaler interface for AllowedExternalDirectives
+func (i *AllowedExternalDirectives) UnmarshalJSON(data []byte) error {
+	var s string
+	if err := json.Unmarshal(data, &s); err != nil {
+		return fmt.Errorf("AllowedExternalDirectives should be a string, got %s", data)
+	}
+
+	var err error
+	*i, err = AllowedExternalDirectivesString(s)
+	return err
+}
+
+// MarshalText implements the encoding.TextMarshaler interface for AllowedExternalDirectives
+func (i AllowedExternalDirectives) MarshalText() ([]byte, error) {
+	return []byte(i.String()), nil
+}
+
+// UnmarshalText implements the encoding.TextUnmarshaler interface for AllowedExternalDirectives
+func (i *AllowedExternalDirectives) UnmarshalText(text []byte) error {
+	var err error
+	*i, err = AllowedExternalDirectivesString(string(text))
+	return err
+}
+
+// MarshalYAML implements a YAML Marshaler for AllowedExternalDirectives
+func (i AllowedExternalDirectives) MarshalYAML() (interface{}, error) {
+	return i.String(), nil
+}
+
+// UnmarshalYAML implements a YAML Unmarshaler for AllowedExternalDirectives
+func (i *AllowedExternalDirectives) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var s string
+	if err := unmarshal(&s); err != nil {
+		return err
+	}
+
+	var err error
+	*i, err = AllowedExternalDirectivesString(s)
+	return err
+}

pkg/backy/backy.go

@@ -61,7 +61,8 @@ func (command *Command) RunCmd(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([
 		}
 	}
 
-	if command.Host != nil {
+	if !IsHostLocal(command.Host) {
+
 		outputArr, errSSH = command.RunCmdSSH(cmdCtxLogger, opts)
 		if errSSH != nil {
 			return outputArr, errSSH
@@ -217,7 +218,13 @@ func (command *Command) RunCmd(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([
 					command.UserHome = strings.TrimSpace(string(userHome))
 					userSshDir := fmt.Sprintf("%s/.ssh", command.UserHome)
 
-					os.MkdirAll(userSshDir, 0700)
+					if _, err := os.Stat(userSshDir); os.IsNotExist(err) {
+						err := os.MkdirAll(userSshDir, 0700)
+						if err != nil {
+							return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error creating directory %s %v", userSshDir, err)
+						}
+					}
+
 					if _, err := os.Stat(fmt.Sprintf("%s/authorized_keys", userSshDir)); os.IsNotExist(err) {
 						_, err := os.Create(fmt.Sprintf("%s/authorized_keys", userSshDir))
 						if err != nil {
@@ -457,7 +464,7 @@ func (cmd *Command) ExecuteHooks(hookType string, opts *ConfigOpts) {
 			cmdLogger := opts.Logger.With().
 				Str("backy-cmd", v).Str("hookType", "error").
 				Logger()
-			errCmd.RunCmd(cmdLogger, opts)
+			_, _ = errCmd.RunCmd(cmdLogger, opts)
 		}
 
 	case "success":
@@ -467,7 +474,7 @@ func (cmd *Command) ExecuteHooks(hookType string, opts *ConfigOpts) {
 			cmdLogger := opts.Logger.With().
 				Str("backy-cmd", v).Str("hookType", "success").
 				Logger()
-			successCmd.RunCmd(cmdLogger, opts)
+			_, _ = successCmd.RunCmd(cmdLogger, opts)
 		}
 	case "final":
 		for _, v := range cmd.Hooks.Final {
@@ -476,7 +483,7 @@ func (cmd *Command) ExecuteHooks(hookType string, opts *ConfigOpts) {
 			cmdLogger := opts.Logger.With().
 				Str("backy-cmd", v).Str("hookType", "final").
 				Logger()
-			finalCmd.RunCmd(cmdLogger, opts)
+			_, _ = finalCmd.RunCmd(cmdLogger, opts)
 		}
 	}
 }
@@ -486,11 +493,10 @@ func (cmd *Command) GenerateLogger(opts *ConfigOpts) zerolog.Logger {
 		Str("Backy-cmd", cmd.Name).Str("Host", "local machine").
 		Logger()
 
-	if cmd.Host != nil {
+	if !IsHostLocal(cmd.Host) {
 		cmdLogger = opts.Logger.With().
-			Str("Backy-cmd", cmd.Name).Str("Host", *cmd.Host).
+			Str("Backy-cmd", cmd.Name).Str("Host", cmd.Host).
 			Logger()
-
 	}
 	return cmdLogger
 }
@@ -502,7 +508,7 @@ func (opts *ConfigOpts) ExecCmdsSSH(cmdList []string, hostsList []string) {
 		for _, c := range cmdList {
 			cmd := opts.Cmds[c]
 			cmd.RemoteHost = host
-			cmd.Host = &host.Host
+			cmd.Host = host.Host
 			opts.Logger.Info().Str("host", h).Str("cmd", c).Send()
 			_, err := cmd.RunCmdSSH(cmd.GenerateLogger(opts), opts)
 			if err != nil {
@@ -531,6 +537,13 @@ func logCommandOutput(command *Command, cmdOutBuf bytes.Buffer, cmdCtxLogger zer
 	return outputArr
 }
 
+func (c *Command) GetVariablesFromConf(opts *ConfigOpts) {
+	c.ScriptEnvFile = replaceVarInString(opts.Vars, c.ScriptEnvFile, opts.Logger)
+	c.Name = replaceVarInString(opts.Vars, c.Name, opts.Logger)
+	c.OutputFile = replaceVarInString(opts.Vars, c.OutputFile, opts.Logger)
+	c.Host = replaceVarInString(opts.Vars, c.Host, opts.Logger)
+}
+
 // func executeUserCommands() []string {
 
 // }

pkg/backy/config.go

@@ -103,6 +103,12 @@ func (opts *ConfigOpts) ReadConfig() *ConfigOpts {
 
 	backyKoanf := opts.koanf
 
+	if backyKoanf.Exists("variables") {
+		unmarshalConfigIntoStruct(backyKoanf, "variables", &opts.Vars, opts.Logger)
+	}
+
+	getConfigDir(opts)
+
 	opts.loadEnv()
 
 	if backyKoanf.Bool(getNestedConfig("logging", "cmd-std-out")) {
@@ -125,14 +131,23 @@ func (opts *ConfigOpts) ReadConfig() *ConfigOpts {
 
 	log.Info().Str("config file", opts.ConfigFilePath).Send()
 
-	unmarshalConfig(backyKoanf, "commands", &opts.Cmds, opts.Logger)
+	if err := opts.initVault(); err != nil {
+		log.Err(err).Send()
+	}
+
+	unmarshalConfigIntoStruct(backyKoanf, "commands", &opts.Cmds, opts.Logger)
 
 	getCommandEnvironments(opts)
 
-	unmarshalConfig(backyKoanf, "hosts", &opts.Hosts, opts.Logger)
+	unmarshalConfigIntoStruct(backyKoanf, "hosts", &opts.Hosts, opts.Logger)
 
 	resolveHostConfigs(opts)
 
+	for k, v := range opts.Vars {
+		v = getExternalConfigDirectiveValue(v, opts)
+		opts.Vars[k] = v
+	}
+
 	loadCommandLists(opts, backyKoanf)
 
 	validateCommandLists(opts)
@@ -148,15 +163,11 @@ func (opts *ConfigOpts) ReadConfig() *ConfigOpts {
 	filterExecuteLists(opts)
 
 	if backyKoanf.Exists("notifications") {
-		unmarshalConfig(backyKoanf, "notifications", &opts.NotificationConf, opts.Logger)
+		unmarshalConfigIntoStruct(backyKoanf, "notifications", &opts.NotificationConf, opts.Logger)
 	}
 
 	opts.SetupNotify()
 
-	if err := opts.setupVault(); err != nil {
-		log.Err(err).Send()
-	}
-
 	return opts
 }
 
@@ -220,6 +231,7 @@ func setLoggingOptions(k *koanf.Koanf, opts *ConfigOpts) {
 		logFile = k.String(getLoggingKeyFromConfig("file"))
 		opts.LogFilePath = logFile
 	}
+	opts.LogFilePath = logFile
 
 	zerolog.SetGlobalLevel(zerolog.InfoLevel)
 	if isLoggingVerbose {
@@ -239,7 +251,7 @@ func setupLogger(opts *ConfigOpts) zerolog.Logger {
 	return zerolog.New(writers).With().Timestamp().Logger()
 }
 
-func unmarshalConfig(k *koanf.Koanf, key string, target interface{}, log zerolog.Logger) {
+func unmarshalConfigIntoStruct(k *koanf.Koanf, key string, target interface{}, log zerolog.Logger) {
 	if err := k.UnmarshalWithConf(key, target, koanf.UnmarshalConf{Tag: "yaml"}); err != nil {
 		logging.ExitWithMSG(fmt.Sprintf("error unmarshaling key %s into struct: %v", key, err), 1, &log)
 	}
@@ -281,16 +293,22 @@ func resolveProxyHosts(host *Host, opts *ConfigOpts) {
 	}
 }
 
+func getConfigDir(opts *ConfigOpts) {
+	if isRemoteURL(opts.ConfigFilePath) {
+		p, _ := getRemoteDir(opts.ConfigFilePath)
+		opts.ConfigDir = p
+	} else {
+		opts.ConfigDir = path.Dir(opts.ConfigFilePath)
+	}
+}
+
 func loadCommandLists(opts *ConfigOpts, backyKoanf *koanf.Koanf) {
 	var listConfigFiles []string
 	var u *url.URL
 	var p string
-	// if config file is remote, use the directory of the remote file
 	if isRemoteURL(opts.ConfigFilePath) {
 		p, u = getRemoteDir(opts.ConfigFilePath)
 		opts.ConfigDir = p
-		println(p)
-		// // Still use local list files if a remote config file is used, but use them last
 		listConfigFiles = []string{u.JoinPath("lists.yml").String(), u.JoinPath("lists.yaml").String()}
 	} else {
 		opts.ConfigDir = path.Dir(opts.ConfigFilePath)
@@ -313,7 +331,7 @@ func loadCommandLists(opts *ConfigOpts, backyKoanf *koanf.Koanf) {
 		if backyKoanf.Exists("cmdLists.file") {
 			loadCmdListsFile(backyKoanf, listsConfig, opts)
 		} else {
-			unmarshalConfig(backyKoanf, "cmdLists", &opts.CmdConfigLists, opts.Logger)
+			unmarshalConfigIntoStruct(backyKoanf, "cmdLists", &opts.CmdConfigLists, opts.Logger)
 		}
 	}
 }
@@ -353,7 +371,7 @@ func loadListConfigFile(filePath string, k *koanf.Koanf, opts *ConfigOpts) bool
 		return false
 	}
 
-	unmarshalConfig(k, "cmdLists", &opts.CmdConfigLists, opts.Logger)
+	unmarshalConfigIntoStruct(k, "cmdLists", &opts.CmdConfigLists, opts.Logger)
 	keyNotSupported("cmd-lists", "cmdLists", k, opts, true)
 	opts.CmdListFile = filePath
 	return true
@@ -381,7 +399,7 @@ func loadCmdListsFile(backyKoanf *koanf.Koanf, listsConfig *koanf.Koanf, opts *C
 	}
 
 	keyNotSupported("cmd-lists", "cmdLists", listsConfig, opts, true)
-	unmarshalConfig(listsConfig, "cmdLists", &opts.CmdConfigLists, opts.Logger)
+	unmarshalConfigIntoStruct(listsConfig, "cmdLists", &opts.CmdConfigLists, opts.Logger)
 	opts.Logger.Info().Str("using lists config file", opts.CmdListFile).Send()
 }
 
@@ -438,7 +456,7 @@ func getLoggingKeyFromConfig(key string) string {
 // 	return fmt.Sprintf("cmdLists.%s", list)
 // }
 
-func (opts *ConfigOpts) setupVault() error {
+func (opts *ConfigOpts) initVault() error {
 	if !opts.koanf.Bool("vault.enabled") {
 		return nil
 	}
@@ -459,7 +477,7 @@ func (opts *ConfigOpts) setupVault() error {
 		token = os.Getenv("VAULT_TOKEN")
 	}
 	if strings.TrimSpace(token) == "" {
-		return fmt.Errorf("no token found, but one was required. \n\nSet the config key vault.token or the environment variable VAULT_TOKEN")
+		return fmt.Errorf("no token found. One is required. \n\nSet the config key vault.token or the environment variable VAULT_TOKEN")
 	}
 
 	client.SetToken(token)
@@ -471,6 +489,11 @@ func (opts *ConfigOpts) setupVault() error {
 
 	opts.vaultClient = client
 
+	for _, v := range opts.VaultKeys {
+		v.Name = replaceVarInString(opts.Vars, v.Key, opts.Logger)
+		v.MountPath = replaceVarInString(opts.Vars, v.MountPath, opts.Logger)
+	}
+
 	return nil
 }
 
@@ -478,7 +501,10 @@ func processCmds(opts *ConfigOpts) error {
 
 	// process commands
 	for cmdName, cmd := range opts.Cmds {
-
+		for i, v := range cmd.Args {
+			v = replaceVarInString(opts.Vars, v, opts.Logger)
+			cmd.Args[i] = v
+		}
 		if cmd.Name == "" {
 			cmd.Name = cmdName
 		}
@@ -501,9 +527,13 @@ func processCmds(opts *ConfigOpts) error {
 			}
 		}
 
-		// resolve hosts
+		if !IsHostLocal(cmd.Host) {
-		if cmd.Host != nil {
+
-			host, hostFound := opts.Hosts[*cmd.Host]
+			cmdHost := replaceVarInString(opts.Vars, cmd.Host, opts.Logger)
+			if cmdHost != cmd.Host {
+				cmd.Host = cmdHost
+			}
+			host, hostFound := opts.Hosts[cmd.Host]
 			if hostFound {
 				cmd.RemoteHost = host
 				cmd.RemoteHost.Host = host.Host
@@ -511,12 +541,12 @@ func processCmds(opts *ConfigOpts) error {
 					cmd.RemoteHost.HostName = host.HostName
 				}
 			} else {
-				opts.Logger.Info().Msgf("adding host %s to host list", *cmd.Host)
+				opts.Logger.Info().Msgf("adding host %s to host list", cmd.Host)
 				if opts.Hosts == nil {
 					opts.Hosts = make(map[string]*Host)
 				}
-				opts.Hosts[*cmd.Host] = &Host{Host: *cmd.Host}
+				opts.Hosts[cmd.Host] = &Host{Host: cmd.Host}
-				cmd.RemoteHost = &Host{Host: *cmd.Host}
+				cmd.RemoteHost = &Host{Host: cmd.Host}
 			}
 		} else {
 
@@ -562,7 +592,7 @@ func processCmds(opts *ConfigOpts) error {
 			if cmd.Username == "" {
 				return fmt.Errorf("username is required for user command %s", cmd.Name)
 			}
-
+			cmd.Username = replaceVarInString(opts.Vars, cmd.Username, opts.Logger)
 			err := detectOSType(cmd, opts)
 			if err != nil {
 				opts.Logger.Info().Err(err).Str("command", cmdName).Send()
@@ -577,8 +607,10 @@ func processCmds(opts *ConfigOpts) error {
 					opts.Logger.Debug().Msg("changing password for user: " + cmd.Username)
 					cmd.UserPassword = getExternalConfigDirectiveValue(cmd.UserPassword, opts)
 				}
-				if cmd.Host != nil {
+
-					host, ok := opts.Hosts[*cmd.Host]
+				if !IsHostLocal(cmd.Host) {
+
+					host, ok := opts.Hosts[cmd.Host]
 					if ok {
 						cmd.userMan, err = usermanager.NewUserManager(host.OS)
 					}
@@ -647,7 +679,9 @@ func processHooks(cmd *Command, hooks []string, opts *ConfigOpts, hookType strin
 }
 
 func detectOSType(cmd *Command, opts *ConfigOpts) error {
-	if cmd.Host == nil {
+
+	if IsHostLocal(cmd.Host) {
+
 		if runtime.GOOS == "linux" {
 			cmd.OS = "linux"
 			opts.Logger.Info().Msg("Unix/Linux type OS detected")
@@ -656,7 +690,7 @@ func detectOSType(cmd *Command, opts *ConfigOpts) error {
 		return fmt.Errorf("using an os that is not yet supported for user commands")
 	}
 
-	host, ok := opts.Hosts[*cmd.Host]
+	host, ok := opts.Hosts[cmd.Host]
 	if ok {
 		if host.OS != "" {
 			return nil
@@ -688,3 +722,24 @@ func keyNotSupported(oldKey, newKey string, koanf *koanf.Koanf, opts *ConfigOpts
 		}
 	}
 }
+
+func replaceVarInString(vars map[string]string, str string, logger zerolog.Logger) string {
+	if strings.Contains(str, "%{var:") && strings.Contains(str, "}%") {
+		logger.Debug().Msgf("replacing vars in string %s", str)
+		for k, v := range vars {
+			if strings.Contains(str, "%{var:"+k+"}%") {
+				str = strings.ReplaceAll(str, "%{var:"+k+"}%", v)
+			}
+		}
+		if strings.Contains(str, "%{var:") && strings.Contains(str, "}%") {
+			logger.Warn().Msg("could not replace all vars in string")
+		}
+	}
+	return str
+}
+
+func VariadicFunctionParameterTest(allowedKeys ...string) {
+	if contains(allowedKeys, "file") {
+		println("file param included")
+	}
+}

pkg/backy/list.go

@@ -46,9 +46,9 @@ func (opts *ConfigOpts) ListCommand(cmd string) {
 		}
 
 		// is it remote or local
-		if cmdInfo.Host != nil {
+		if !IsHostLocal(cmdInfo.Host) {
 			println()
-			print("Host: ", *cmdInfo.Host)
+			print("Host: ", cmdInfo.Host)
 			println()
 
 		} else {

pkg/backy/notification.go

@@ -9,6 +9,7 @@ import (
 
 	"git.andrewnw.xyz/CyberShell/backy/pkg/logging"
 	"github.com/nikoksr/notify"
+	"github.com/nikoksr/notify/service/http"
 	"github.com/nikoksr/notify/service/mail"
 	"github.com/nikoksr/notify/service/matrix"
 	"maunium.net/go/mautrix/id"
@@ -30,6 +31,12 @@ type MailConfig struct {
 	Password      string   `yaml:"password"`
 }
 
+type HttpConfig struct {
+	URL     string              `yaml:"url"`
+	Method  string              `yaml:"method"`
+	Headers map[string][]string `yaml:"headers"`
+}
+
 // SetupNotify sets up notify instances for each command list.
 func (opts *ConfigOpts) SetupNotify() {
 
@@ -59,6 +66,7 @@ func (opts *ConfigOpts) SetupNotify() {
 					continue
 				}
 				conf.Password = getExternalConfigDirectiveValue(conf.Password, opts)
+				opts.Logger.Debug().Str("list", confName).Str("id", confId).Msg("adding mail notification service")
 				mailConf := setupMail(conf)
 				services = append(services, mailConf)
 			case "matrix":
@@ -68,13 +76,22 @@ func (opts *ConfigOpts) SetupNotify() {
 					continue
 				}
 				conf.AccessToken = getExternalConfigDirectiveValue(conf.AccessToken, opts)
+				opts.Logger.Debug().Str("list", confName).Str("id", confId).Msg("adding matrix notification service")
 				mtrxConf, mtrxErr := setupMatrix(conf)
 				if mtrxErr != nil {
 					opts.Logger.Info().Str("list", confName).Err(fmt.Errorf("error: configuring matrix id %s failed during setup: %w", id, mtrxErr))
 					continue
 				}
 				services = append(services, mtrxConf)
-
+			case "http":
+				conf, ok := opts.NotificationConf.HttpConfig[confId]
+				if !ok {
+					opts.Logger.Info().Err(fmt.Errorf("error: ID %s not found in http object", confId)).Str("list", confName).Send()
+					continue
+				}
+				opts.Logger.Debug().Str("list", confName).Str("id", confId).Msg("adding http notification service")
+				httpConf := setupHttp(conf)
+				services = append(services, httpConf)
 			default:
 				opts.Logger.Info().Err(fmt.Errorf("id %s not found", id)).Str("list", confName).Send()
 			}
@@ -100,3 +117,19 @@ func setupMail(config MailConfig) *mail.Mail {
 	mailClient.BodyFormat(mail.PlainText)
 	return mailClient
 }
+
+func setupHttp(httpConf HttpConfig) *http.Service {
+
+	httpService := http.New()
+	httpService.AddReceivers(&http.Webhook{
+		URL:         httpConf.URL,
+		Header:      httpConf.Headers,
+		ContentType: "text/plain",
+		Method:      httpConf.Method,
+		BuildPayload: func(subject, message string) (payload any) {
+			return subject + "\n\n" + message
+		},
+	})
+
+	return httpService
+}

pkg/backy/ssh.go

@@ -441,8 +441,8 @@ func (command *Command) RunCmdSSH(cmdCtxLogger zerolog.Logger, opts *ConfigOpts)
 
 	cmdCtxLogger.Info().
 		Str("Command", command.Name).
-		Str("Host", *command.Host).
+		Str("Host", command.Host).
-		Msgf("Running %s on host %s", getCommandTypeAndSetCommandInfoLabel(command.Type), *command.Host)
+		Msgf("Running %s on host %s", getCommandTypeAndSetCommandInfoLabel(command.Type), command.Host)
 
 	// cmdCtxLogger.Debug().Str("cmd", command.Cmd).Strs("args", command.Args).Send()
 
@@ -523,12 +523,19 @@ func (command *Command) RunCmdSSH(cmdCtxLogger zerolog.Logger, opts *ConfigOpts)
 				return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error creating file /tmp/%s: %v", uuidFile.String(), passFileErr)
 			}
 
-			passFile.Write([]byte(userNamePass))
+			_, err = passFile.Write([]byte(userNamePass))
+			if err != nil {
+				return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error writing to file /tmp/%s: %v", uuidFile.String(), err)
+			}
 
 			ArgsStr = fmt.Sprintf("cat %s | chpasswd", passFilePath)
 			defer passFile.Close()
 
-			defer client.Remove(passFilePath)
+			rmFileFunc := func() {
+				_ = client.Remove(passFilePath)
+			}
+
+			defer rmFileFunc()
 			// commandSession.Stdin = command.stdin
 		}
 		if err := commandSession.Run(ArgsStr); err != nil {
@@ -561,7 +568,10 @@ func (command *Command) RunCmdSSH(cmdCtxLogger zerolog.Logger, opts *ConfigOpts)
 						return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error creating sftp client: %v", err)
 					}
 
-					client.MkdirAll(userSshDir)
+					err = client.MkdirAll(userSshDir)
+					if err != nil {
+						return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error creating directory %s: %v", userSshDir, err)
+					}
 					_, err = client.Create(fmt.Sprintf("%s/authorized_keys", userSshDir))
 					if err != nil {
 						return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error opening file %s/authorized_keys: %v", userSshDir, err)
@@ -801,3 +811,8 @@ func CheckIfHostHasHostName(host string) (bool, string) {
 	println(HostName)
 	return HostName != "", HostName
 }
+
+func IsHostLocal(host string) bool {
+	host = strings.ToLower(host)
+	return host == "127.0.0.1" || host == "localhost" || host == ""
+}

pkg/backy/types.go

@@ -56,7 +56,7 @@ type (
 		// See CommandType enum further down the page for acceptable values
 		Type CommandType `yaml:"type,omitempty"`
 
-		Host *string `yaml:"host,omitempty"`
+		Host string `yaml:"host,omitempty"`
 
 		Hooks *Hooks `yaml:"hooks,omitempty"`
 
@@ -205,6 +205,8 @@ type (
 
 		List ListConfig
 
+		Vars map[string]string `yaml:"variables"`
+
 		VaultKeys []*VaultKey `yaml:"keys"`
 
 		koanf *koanf.Koanf
@@ -239,6 +241,7 @@ type (
 	Notifications struct {
 		MailConfig   map[string]MailConfig   `yaml:"mail,omitempty"`
 		MatrixConfig map[string]MatrixStruct `yaml:"matrix,omitempty"`
+		HttpConfig   map[string]HttpConfig   `yaml:"http,omitempty"`
 	}
 
 	CmdOutput struct {
@@ -275,8 +278,9 @@ type (
 	}
 
 	// use ints so we can use enums
-	CommandType      int
+	CommandType               int
-	PackageOperation int
+	PackageOperation          int
+	AllowedExternalDirectives int
 )
 
 //go:generate go run github.com/dmarkham/enumer -linecomment -yaml -text -json -type=CommandType
@@ -299,3 +303,14 @@ const (
 	PackOpCheckVersion                         // checkVersion
 	PackOpIsInstalled                          // isInstalled
 )
+
+//go:generate go run github.com/dmarkham/enumer -linecomment -yaml -text -json -type=AllowedExternalDirectives
+const (
+	DefaultExternalDir                AllowedExternalDirectives = iota
+	AllowedExternalDirectiveVault                               // vault
+	AllowedExternalDirectiveVaultFile                           // vault-file
+	AllowedExternalDirectiveAll                                 // vault-file-env
+	AllowedExternalDirectiveFileEnv                             // file-env
+	AllowedExternalDirectiveFile                                // file
+	AllowedExternalDirectiveEnv                                 // env
+)

pkg/backy/utils.go

@@ -110,7 +110,11 @@ func injectEnvIntoSSH(envVarsToInject environmentVars, process *ssh.Session, opt
 			goto errEnvFile
 		}
 		for key, val := range envMap {
-			process.Setenv(key, GetVaultKey(val, opts, log))
+			err = process.Setenv(key, GetVaultKey(val, opts, log))
+			if err != nil {
+				log.Error().Err(err).Send()
+
+			}
 		}
 	}
 
@@ -121,7 +125,11 @@ errEnvFile:
 		if strings.Contains(envVal, "=") {
 			envVarArr := strings.Split(envVal, "=")
 
-			process.Setenv(envVarArr[0], getExternalConfigDirectiveValue(envVarArr[1], opts))
+			err := process.Setenv(envVarArr[0], getExternalConfigDirectiveValue(envVarArr[1], opts))
+			if err != nil {
+				log.Error().Err(err).Send()
+
+			}
 		}
 	}
 }
@@ -354,6 +362,7 @@ func getExternalConfigDirectiveValue(key string, opts *ConfigOpts) string {
 	if !(strings.HasPrefix(key, externDirectiveStart) && strings.HasSuffix(key, externDirectiveEnd)) {
 		return key
 	}
+	key = replaceVarInString(opts.Vars, key, opts.Logger)
 	opts.Logger.Debug().Str("expanding external key", key).Send()
 	if strings.HasPrefix(key, envExternDirectiveStart) {
 		key = strings.TrimPrefix(key, envExternDirectiveStart)
@@ -385,7 +394,6 @@ func getExternalConfigDirectiveValue(key string, opts *ConfigOpts) string {
 		key = strings.TrimSuffix(key, externDirectiveEnd)
 		key = GetVaultKey(key, opts, opts.Logger)
 	}
-	println(key)
 
 	return key
 }
@@ -410,7 +418,6 @@ func getVaultSecret(vaultClient *vault.Client, key *VaultKey) (string, error) {
 	}
 
 	value, ok := secret.Data[key.Key].(string)
-	println(value)
 	if !ok {
 		return "", fmt.Errorf("value type assertion failed for vault key %s: %T %#v", key.Name, secret.Data[key.Name], secret.Data[key.Name])
 	}

pkg/remotefetcher/cache.go

@@ -116,7 +116,7 @@ func (c *Cache) Set(source, hash string, data []byte, dataType string) (CacheDat
 	path := filepath.Join(c.dir, fmt.Sprintf("%s-%s", fileName, sourceHash))
 
 	if _, err := os.Stat(path); os.IsNotExist(err) {
-		os.MkdirAll(c.dir, 0700)
+		_ = os.MkdirAll(c.dir, 0700)
 	}
 
 	err := os.WriteFile(path, data, 0644)
@@ -171,7 +171,7 @@ func (cf *CachedFetcher) Hash(data []byte) string {
 func LoadMetadataFromFile(filePath string) ([]*CacheData, error) {
 	if _, err := os.Stat(filePath); os.IsNotExist(err) {
 		// Create the file if it does not exist
-		os.MkdirAll(path.Dir(filePath), 0700)
+		_ = os.MkdirAll(path.Dir(filePath), 0700)
 		emptyData := []byte("[]")
 		err := os.WriteFile(filePath, emptyData, 0644)
 		if err != nil {