change: Commands: host can now be localhost or 127.0.0.1 to run commands locally

.changes/unreleased/Changed-20250321-090849.yaml

@@ -0,0 +1,3 @@
+kind: Changed
+body: 'Commands: `host` can now be `localhost` or `127.0.0.1` to run commands locally'
+time: 2025-03-21T09:08:49.871021144-05:00

.changes/v0.10.1.md

@@ -0,0 +1,8 @@
+## v0.10.1 - 2025-03-11
+### Added
+* UserCommands: add ssh public keys when running locally
+* UserCommands: add field CreateUserHome
+### Changed
+* UserCommands: create temp file when modifing password over SSH
+* UserCommands: change field name
+* Vault: keys are now referenced by `name`, and the actual data by `data`

.changes/v0.10.2.md

@@ -0,0 +1,6 @@
+## v0.10.2 - 2025-03-19
+### Added
+* Notifications: http service added
+* Variable support. Can be referenced with `%{var:nameOfVar}%` in select string fields.
+### Changed
+* vault: initialize vault before validating config

.woodpecker/gitea.yml

@@ -1,9 +1,7 @@
-name: goreleaser release
 steps:
   golang:
     image: golang:1.23
     commands:
-      - go mod tidy
       - go install github.com/goreleaser/goreleaser/v2@v2.7.0
       - goreleaser release -f .goreleaser/gitea.yml --release-notes=".changes/$(go run backy.go version -V).md"
     environment:

.woodpecker/go-lint.yml

@@ -5,7 +5,7 @@ steps:
       - go build
       - go test
   release:
-    image: golangci/golangci-lint:v1.53.3
+    image: golangci/golangci-lint:v1.64.7
     commands:
       - golangci-lint run -v --timeout 5m
 

CHANGELOG.md

@@ -6,6 +6,22 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html),
 and is generated by [Changie](https://github.com/miniscruff/changie).
 
 
+## v0.10.2 - 2025-03-19
+### Added
+* Notifications: http service added
+* Variable support. Can be referenced with `%{var:nameOfVar}%` in select string fields.
+### Changed
+* vault: initialize vault before validating config
+
+## v0.10.1 - 2025-03-11
+### Added
+* UserCommands: add ssh public keys when running locally
+* UserCommands: add field CreateUserHome
+### Changed
+* UserCommands: create temp file when modifing password over SSH
+* UserCommands: change field name
+* Vault: keys are now referenced by `name`, and the actual data by `data`
+
 ## v0.10.0 - 2025-03-08
 ### Added
 * Hooks: improved logging when executing

cmd/version.go

@@ -7,7 +7,7 @@ import (
 	"github.com/spf13/cobra"
 )
 
-const versionStr = "0.10.0"
+const versionStr = "0.10.2"
 
 var (
 	versionCmd = &cobra.Command{

docs/content/config/commands/_index.md

@@ -16,7 +16,7 @@ Values available for this section **(case-sensitive)**:
 | ----------------| ------------------------------------------------------------------------------------------------------- | --------------------- | -------- |----------------------------|
 | `cmd`           | Defines the command to execute                                                                          | `string`              | yes      | No                         |
 | `Args`          | Defines the arguments to the command                                                                    | `[]string`            | no       | No                         |
-| `environment`   | Defines environment variables for the command                                                           | `[]string`            | no       | No                         |
+| `environment`   | Defines environment variables for the command                                                           | `[]string`            | no       | Partial                    |
 | `type`          | See documentation further down the page. Additional fields may be required.                             | `string`              | no       | No                         |
 | `getOutput`     | Command(s) output is in the notification(s)                                                             | `bool`                | no       | No                         |
 | `host`          | If not specified, the command will execute locally.                                                     | `string`              | no       | No                         |
@@ -95,8 +95,9 @@ The following options are available:
 The environment variables support expansion:
 
 - using escaped values `$VAR` or `${VAR}`
+- using any external directive, and if using the env directive, the variable will be read from a `.env` file
 
-For now, the variables have to be defined in an `.env` file in the same directory that the program is run from.
+<!-- For now, the variables expanded have to be defined in an `.env` file in the same directory that the program is run from. -->
 
 If using it with host specified, the SSH server has to be configured to accept those env variables.
 

docs/content/config/commands/user-commands.md

@@ -6,16 +6,18 @@ description: This is dedicated to user commands.
 
 This is dedicated to `user` commands. The command `type` field must be `user`. User is a type that allows one to perform user operations. There are several additional options available when `type` is `user`:
 
-| name | notes | type | required |
+| name            | notes                                                        | type       | required | External directive support
-| --- | --- | --- | --- |
+| ----------------| -------------------------------------------------------------| ---------- | ---------| --------------------------|
-| `userName` | The name of a user to be configured. | `string` | yes |
+| `userName`      | The name of a user to be configured.                         | `string`   | yes      | no 						 |
-| `userOperation` | The type of operation to perform. | `string` | yes |
+| `userOperation` | The type of operation to perform.                            | `string`   | yes      | no 						 |
-| `userID` | The user ID to use. | `string` | no |
+| `userID`        | The user ID to use.                                          | `string`   | no       | no 						 |
-| `userGroups` | The groups the user should be added to. | `[]string` | no |
+| `userGroups`    | The groups the user should be added to.                      | `[]string` | no       | no 						 |
-| `userSshPubKeys` | The keys to add to the user's authorized keys. | `[]string` | no |
+| `systemUser`    | Create a system user.                                        | `bool`     | no       | no 						 |
-| `userShell` | The shell for the user. | `string` | no |
+| `userCreateHome`| Create the home directory.                                   | `bool`     | no       | no 						 |
-| `userHome` | The user's home directory. | `string` | no |
+| `userSshPubKeys`| The keys to add to the user's authorized keys.               | `[]string` | no       | yes 						 |
-| `userPassword` | The new password value when using the `password` operation. Can be specified by using external directive. | `string` | no |
+| `userShell`     | The shell for the user.                                      | `string`   | no       | no 						 |
+| `userHome`      | The user's home directory.                                   | `string`   | no       | no 						 |
+| `userPassword`  | The new password value when using the `password` operation.  | `string`   | no       | yes						 |
 
 
 #### example

docs/content/config/vault.md

@@ -6,7 +6,7 @@ description: Set up and configure vault.
 
 [Vault](https://www.vaultproject.io/) is a tool for storing secrets and other data securely.
 
-Vault config can be used by prefixing `vault:` in front of a password or ENV var.
+A Vault key can be used by prefixing `%{vault:vault.keys.name}%` in a field that supports external directives.
 
 This is the object in the config file:
 
@@ -18,10 +18,12 @@ vault:
   keys:
     - name: mongourl
       mountpath: secret
+      key: data
       path: mongo/url
-      type:  # KVv1 or KVv2
+      type: KVv2  # KVv1 or KVv2
-    - name:
+    - name: someKeyName
-      path:
+      mountpath: secret
-      type:
+      key: keyData
-      mountpath:
+      type: KVv2
+      path: some/path
 ```

go.mod

@@ -2,14 +2,13 @@ module git.andrewnw.xyz/CyberShell/backy
 
 go 1.23
 
-toolchain go1.23.6
+toolchain go1.23.7
-
-replace git.andrewnw.xyz/CyberShell/backy => /home/andrew/Projects/backy
 
 require (
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.76.0
 	github.com/dmarkham/enumer v1.5.11
 	github.com/go-co-op/gocron v1.37.0
+	github.com/google/uuid v1.6.0
 	github.com/hashicorp/vault/api v1.15.0
 	github.com/joho/godotenv v1.5.1
 	github.com/kevinburke/ssh_config v1.2.0
@@ -51,7 +50,6 @@ require (
 	github.com/go-jose/go-jose/v4 v4.0.1 // indirect
 	github.com/go-viper/mapstructure/v2 v2.2.1 // indirect
 	github.com/goccy/go-json v0.10.5 // indirect
-	github.com/google/uuid v1.6.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect

pkg/backy/allowedexternaldirectives_enumer.go

@@ -0,0 +1,145 @@
+// Code generated by "enumer -linecomment -yaml -text -json -type=AllowedExternalDirectives"; DO NOT EDIT.
+
+package backy
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+)
+
+const _AllowedExternalDirectivesName = "DefaultExternalDirvaultvault-filevault-file-envfile-envfileenv"
+
+var _AllowedExternalDirectivesIndex = [...]uint8{0, 18, 23, 33, 47, 55, 59, 62}
+
+const _AllowedExternalDirectivesLowerName = "defaultexternaldirvaultvault-filevault-file-envfile-envfileenv"
+
+func (i AllowedExternalDirectives) String() string {
+	if i < 0 || i >= AllowedExternalDirectives(len(_AllowedExternalDirectivesIndex)-1) {
+		return fmt.Sprintf("AllowedExternalDirectives(%d)", i)
+	}
+	return _AllowedExternalDirectivesName[_AllowedExternalDirectivesIndex[i]:_AllowedExternalDirectivesIndex[i+1]]
+}
+
+// An "invalid array index" compiler error signifies that the constant values have changed.
+// Re-run the stringer command to generate them again.
+func _AllowedExternalDirectivesNoOp() {
+	var x [1]struct{}
+	_ = x[DefaultExternalDir-(0)]
+	_ = x[AllowedExternalDirectiveVault-(1)]
+	_ = x[AllowedExternalDirectiveVaultFile-(2)]
+	_ = x[AllowedExternalDirectiveAll-(3)]
+	_ = x[AllowedExternalDirectiveFileEnv-(4)]
+	_ = x[AllowedExternalDirectiveFile-(5)]
+	_ = x[AllowedExternalDirectiveEnv-(6)]
+}
+
+var _AllowedExternalDirectivesValues = []AllowedExternalDirectives{DefaultExternalDir, AllowedExternalDirectiveVault, AllowedExternalDirectiveVaultFile, AllowedExternalDirectiveAll, AllowedExternalDirectiveFileEnv, AllowedExternalDirectiveFile, AllowedExternalDirectiveEnv}
+
+var _AllowedExternalDirectivesNameToValueMap = map[string]AllowedExternalDirectives{
+	_AllowedExternalDirectivesName[0:18]:       DefaultExternalDir,
+	_AllowedExternalDirectivesLowerName[0:18]:  DefaultExternalDir,
+	_AllowedExternalDirectivesName[18:23]:      AllowedExternalDirectiveVault,
+	_AllowedExternalDirectivesLowerName[18:23]: AllowedExternalDirectiveVault,
+	_AllowedExternalDirectivesName[23:33]:      AllowedExternalDirectiveVaultFile,
+	_AllowedExternalDirectivesLowerName[23:33]: AllowedExternalDirectiveVaultFile,
+	_AllowedExternalDirectivesName[33:47]:      AllowedExternalDirectiveAll,
+	_AllowedExternalDirectivesLowerName[33:47]: AllowedExternalDirectiveAll,
+	_AllowedExternalDirectivesName[47:55]:      AllowedExternalDirectiveFileEnv,
+	_AllowedExternalDirectivesLowerName[47:55]: AllowedExternalDirectiveFileEnv,
+	_AllowedExternalDirectivesName[55:59]:      AllowedExternalDirectiveFile,
+	_AllowedExternalDirectivesLowerName[55:59]: AllowedExternalDirectiveFile,
+	_AllowedExternalDirectivesName[59:62]:      AllowedExternalDirectiveEnv,
+	_AllowedExternalDirectivesLowerName[59:62]: AllowedExternalDirectiveEnv,
+}
+
+var _AllowedExternalDirectivesNames = []string{
+	_AllowedExternalDirectivesName[0:18],
+	_AllowedExternalDirectivesName[18:23],
+	_AllowedExternalDirectivesName[23:33],
+	_AllowedExternalDirectivesName[33:47],
+	_AllowedExternalDirectivesName[47:55],
+	_AllowedExternalDirectivesName[55:59],
+	_AllowedExternalDirectivesName[59:62],
+}
+
+// AllowedExternalDirectivesString retrieves an enum value from the enum constants string name.
+// Throws an error if the param is not part of the enum.
+func AllowedExternalDirectivesString(s string) (AllowedExternalDirectives, error) {
+	if val, ok := _AllowedExternalDirectivesNameToValueMap[s]; ok {
+		return val, nil
+	}
+
+	if val, ok := _AllowedExternalDirectivesNameToValueMap[strings.ToLower(s)]; ok {
+		return val, nil
+	}
+	return 0, fmt.Errorf("%s does not belong to AllowedExternalDirectives values", s)
+}
+
+// AllowedExternalDirectivesValues returns all values of the enum
+func AllowedExternalDirectivesValues() []AllowedExternalDirectives {
+	return _AllowedExternalDirectivesValues
+}
+
+// AllowedExternalDirectivesStrings returns a slice of all String values of the enum
+func AllowedExternalDirectivesStrings() []string {
+	strs := make([]string, len(_AllowedExternalDirectivesNames))
+	copy(strs, _AllowedExternalDirectivesNames)
+	return strs
+}
+
+// IsAAllowedExternalDirectives returns "true" if the value is listed in the enum definition. "false" otherwise
+func (i AllowedExternalDirectives) IsAAllowedExternalDirectives() bool {
+	for _, v := range _AllowedExternalDirectivesValues {
+		if i == v {
+			return true
+		}
+	}
+	return false
+}
+
+// MarshalJSON implements the json.Marshaler interface for AllowedExternalDirectives
+func (i AllowedExternalDirectives) MarshalJSON() ([]byte, error) {
+	return json.Marshal(i.String())
+}
+
+// UnmarshalJSON implements the json.Unmarshaler interface for AllowedExternalDirectives
+func (i *AllowedExternalDirectives) UnmarshalJSON(data []byte) error {
+	var s string
+	if err := json.Unmarshal(data, &s); err != nil {
+		return fmt.Errorf("AllowedExternalDirectives should be a string, got %s", data)
+	}
+
+	var err error
+	*i, err = AllowedExternalDirectivesString(s)
+	return err
+}
+
+// MarshalText implements the encoding.TextMarshaler interface for AllowedExternalDirectives
+func (i AllowedExternalDirectives) MarshalText() ([]byte, error) {
+	return []byte(i.String()), nil
+}
+
+// UnmarshalText implements the encoding.TextUnmarshaler interface for AllowedExternalDirectives
+func (i *AllowedExternalDirectives) UnmarshalText(text []byte) error {
+	var err error
+	*i, err = AllowedExternalDirectivesString(string(text))
+	return err
+}
+
+// MarshalYAML implements a YAML Marshaler for AllowedExternalDirectives
+func (i AllowedExternalDirectives) MarshalYAML() (interface{}, error) {
+	return i.String(), nil
+}
+
+// UnmarshalYAML implements a YAML Unmarshaler for AllowedExternalDirectives
+func (i *AllowedExternalDirectives) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var s string
+	if err := unmarshal(&s); err != nil {
+		return err
+	}
+
+	var err error
+	*i, err = AllowedExternalDirectivesString(s)
+	return err
+}

pkg/backy/backy.go

@@ -11,6 +11,7 @@ import (
 	"io"
 	"os"
 	"os/exec"
+	"strings"
 	"text/template"
 
 	"embed"
@@ -60,7 +61,8 @@ func (command *Command) RunCmd(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([
 		}
 	}
 
-	if command.Host != nil {
+	if !IsHostLocal(command.Host) {
+
 		outputArr, errSSH = command.RunCmdSSH(cmdCtxLogger, opts)
 		if errSSH != nil {
 			return outputArr, errSSH
@@ -95,7 +97,7 @@ func (command *Command) RunCmd(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([
 				command.Shell = "sh"
 			}
 			localCMD = exec.Command(command.Shell, command.Args...)
-			injectEnvIntoLocalCMD(envVars, localCMD, cmdCtxLogger)
+			injectEnvIntoLocalCMD(envVars, localCMD, cmdCtxLogger, opts)
 
 			cmdOutWriters = io.MultiWriter(&cmdOutBuf)
 
@@ -176,7 +178,7 @@ func (command *Command) RunCmd(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([
 			localCMD.Dir = *command.Dir
 		}
 
-		injectEnvIntoLocalCMD(envVars, localCMD, cmdCtxLogger)
+		injectEnvIntoLocalCMD(envVars, localCMD, cmdCtxLogger, opts)
 
 		cmdOutWriters = io.MultiWriter(&cmdOutBuf)
 
@@ -194,6 +196,63 @@ func (command *Command) RunCmd(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([
 			cmdCtxLogger.Error().Err(fmt.Errorf("error when running cmd %s: %w", command.Name, err)).Send()
 			return outputArr, err
 		}
+
+		if command.Type == UserCT {
+
+			if command.UserOperation == "add" {
+				if command.UserSshPubKeys != nil {
+					var (
+						f        *os.File
+						err      error
+						userHome []byte
+					)
+
+					cmdCtxLogger.Info().Msg("adding SSH Keys")
+
+					localCMD := exec.Command(fmt.Sprintf("grep \"%s\" /etc/passwd | cut -d: -f6", command.Username))
+					userHome, err = localCMD.CombinedOutput()
+					if err != nil {
+						return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error finding user home from /etc/passwd: %v", err)
+					}
+
+					command.UserHome = strings.TrimSpace(string(userHome))
+					userSshDir := fmt.Sprintf("%s/.ssh", command.UserHome)
+
+					if _, err := os.Stat(userSshDir); os.IsNotExist(err) {
+						err := os.MkdirAll(userSshDir, 0700)
+						if err != nil {
+							return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error creating directory %s %v", userSshDir, err)
+						}
+					}
+
+					if _, err := os.Stat(fmt.Sprintf("%s/authorized_keys", userSshDir)); os.IsNotExist(err) {
+						_, err := os.Create(fmt.Sprintf("%s/authorized_keys", userSshDir))
+						if err != nil {
+							return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error creating file %s/authorized_keys: %v", userSshDir, err)
+						}
+					}
+
+					f, err = os.OpenFile(fmt.Sprintf("%s/authorized_keys", userSshDir), 0700, os.ModeAppend)
+					if err != nil {
+						return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error opening file %s/authorized_keys: %v", userSshDir, err)
+					}
+					defer f.Close()
+					for _, k := range command.UserSshPubKeys {
+						buf := bytes.NewBufferString(k)
+						cmdCtxLogger.Info().Str("key", k).Msg("adding SSH key")
+						if _, err := f.ReadFrom(buf); err != nil {
+							return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error adding to authorized keys: %v", err)
+						}
+					}
+					localCMD = exec.Command(fmt.Sprintf("chown -R %s:%s %s", command.Username, command.Username, userHome))
+					_, err = localCMD.CombinedOutput()
+					if err != nil {
+						return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), err
+					}
+
+				}
+			}
+		}
 	}
 	return outputArr, nil
 }
@@ -405,7 +464,7 @@ func (cmd *Command) ExecuteHooks(hookType string, opts *ConfigOpts) {
 			cmdLogger := opts.Logger.With().
 				Str("backy-cmd", v).Str("hookType", "error").
 				Logger()
-			errCmd.RunCmd(cmdLogger, opts)
+			_, _ = errCmd.RunCmd(cmdLogger, opts)
 		}
 
 	case "success":
@@ -415,7 +474,7 @@ func (cmd *Command) ExecuteHooks(hookType string, opts *ConfigOpts) {
 			cmdLogger := opts.Logger.With().
 				Str("backy-cmd", v).Str("hookType", "success").
 				Logger()
-			successCmd.RunCmd(cmdLogger, opts)
+			_, _ = successCmd.RunCmd(cmdLogger, opts)
 		}
 	case "final":
 		for _, v := range cmd.Hooks.Final {
@@ -424,7 +483,7 @@ func (cmd *Command) ExecuteHooks(hookType string, opts *ConfigOpts) {
 			cmdLogger := opts.Logger.With().
 				Str("backy-cmd", v).Str("hookType", "final").
 				Logger()
-			finalCmd.RunCmd(cmdLogger, opts)
+			_, _ = finalCmd.RunCmd(cmdLogger, opts)
 		}
 	}
 }
@@ -434,11 +493,10 @@ func (cmd *Command) GenerateLogger(opts *ConfigOpts) zerolog.Logger {
 		Str("Backy-cmd", cmd.Name).Str("Host", "local machine").
 		Logger()
 
-	if cmd.Host != nil {
+	if !IsHostLocal(cmd.Host) {
 		cmdLogger = opts.Logger.With().
-			Str("Backy-cmd", cmd.Name).Str("Host", *cmd.Host).
+			Str("Backy-cmd", cmd.Name).Str("Host", cmd.Host).
 			Logger()
-
 	}
 	return cmdLogger
 }
@@ -450,7 +508,7 @@ func (opts *ConfigOpts) ExecCmdsSSH(cmdList []string, hostsList []string) {
 		for _, c := range cmdList {
 			cmd := opts.Cmds[c]
 			cmd.RemoteHost = host
-			cmd.Host = &host.Host
+			cmd.Host = host.Host
 			opts.Logger.Info().Str("host", h).Str("cmd", c).Send()
 			_, err := cmd.RunCmdSSH(cmd.GenerateLogger(opts), opts)
 			if err != nil {
@@ -479,6 +537,13 @@ func logCommandOutput(command *Command, cmdOutBuf bytes.Buffer, cmdCtxLogger zer
 	return outputArr
 }
 
+func (c *Command) GetVariablesFromConf(opts *ConfigOpts) {
+	c.ScriptEnvFile = replaceVarInString(opts.Vars, c.ScriptEnvFile, opts.Logger)
+	c.Name = replaceVarInString(opts.Vars, c.Name, opts.Logger)
+	c.OutputFile = replaceVarInString(opts.Vars, c.OutputFile, opts.Logger)
+	c.Host = replaceVarInString(opts.Vars, c.Host, opts.Logger)
+}
+
 // func executeUserCommands() []string {
 
 // }

pkg/backy/config.go

@@ -1,7 +1,6 @@
 package backy
 
 import (
-	"context"
 	"errors"
 	"fmt"
 	"net/url"
@@ -104,6 +103,12 @@ func (opts *ConfigOpts) ReadConfig() *ConfigOpts {
 
 	backyKoanf := opts.koanf
 
+	if backyKoanf.Exists("variables") {
+		unmarshalConfigIntoStruct(backyKoanf, "variables", &opts.Vars, opts.Logger)
+	}
+
+	getConfigDir(opts)
+
 	opts.loadEnv()
 
 	if backyKoanf.Bool(getNestedConfig("logging", "cmd-std-out")) {
@@ -126,14 +131,23 @@ func (opts *ConfigOpts) ReadConfig() *ConfigOpts {
 
 	log.Info().Str("config file", opts.ConfigFilePath).Send()
 
-	unmarshalConfig(backyKoanf, "commands", &opts.Cmds, opts.Logger)
+	if err := opts.initVault(); err != nil {
+		log.Err(err).Send()
+	}
+
+	unmarshalConfigIntoStruct(backyKoanf, "commands", &opts.Cmds, opts.Logger)
 
 	getCommandEnvironments(opts)
 
-	unmarshalConfig(backyKoanf, "hosts", &opts.Hosts, opts.Logger)
+	unmarshalConfigIntoStruct(backyKoanf, "hosts", &opts.Hosts, opts.Logger)
 
 	resolveHostConfigs(opts)
 
+	for k, v := range opts.Vars {
+		v = getExternalConfigDirectiveValue(v, opts)
+		opts.Vars[k] = v
+	}
+
 	loadCommandLists(opts, backyKoanf)
 
 	validateCommandLists(opts)
@@ -149,15 +163,11 @@ func (opts *ConfigOpts) ReadConfig() *ConfigOpts {
 	filterExecuteLists(opts)
 
 	if backyKoanf.Exists("notifications") {
-		unmarshalConfig(backyKoanf, "notifications", &opts.NotificationConf, opts.Logger)
+		unmarshalConfigIntoStruct(backyKoanf, "notifications", &opts.NotificationConf, opts.Logger)
 	}
 
 	opts.SetupNotify()
 
-	if err := opts.setupVault(); err != nil {
-		log.Err(err).Send()
-	}
-
 	return opts
 }
 
@@ -221,6 +231,7 @@ func setLoggingOptions(k *koanf.Koanf, opts *ConfigOpts) {
 		logFile = k.String(getLoggingKeyFromConfig("file"))
 		opts.LogFilePath = logFile
 	}
+	opts.LogFilePath = logFile
 
 	zerolog.SetGlobalLevel(zerolog.InfoLevel)
 	if isLoggingVerbose {
@@ -240,7 +251,7 @@ func setupLogger(opts *ConfigOpts) zerolog.Logger {
 	return zerolog.New(writers).With().Timestamp().Logger()
 }
 
-func unmarshalConfig(k *koanf.Koanf, key string, target interface{}, log zerolog.Logger) {
+func unmarshalConfigIntoStruct(k *koanf.Koanf, key string, target interface{}, log zerolog.Logger) {
 	if err := k.UnmarshalWithConf(key, target, koanf.UnmarshalConf{Tag: "yaml"}); err != nil {
 		logging.ExitWithMSG(fmt.Sprintf("error unmarshaling key %s into struct: %v", key, err), 1, &log)
 	}
@@ -248,6 +259,9 @@ func unmarshalConfig(k *koanf.Koanf, key string, target interface{}, log zerolog
 
 func getCommandEnvironments(opts *ConfigOpts) {
 	for cmdName, cmdConf := range opts.Cmds {
+		if cmdConf.Env == "" {
+			continue
+		}
 		opts.Logger.Debug().Str("env file", cmdConf.Env).Str("cmd", cmdName).Send()
 		if err := testFile(cmdConf.Env); err != nil {
 			logging.ExitWithMSG("Could not open file"+cmdConf.Env+": "+err.Error(), 1, &opts.Logger)
@@ -279,16 +293,22 @@ func resolveProxyHosts(host *Host, opts *ConfigOpts) {
 	}
 }
 
+func getConfigDir(opts *ConfigOpts) {
+	if isRemoteURL(opts.ConfigFilePath) {
+		p, _ := getRemoteDir(opts.ConfigFilePath)
+		opts.ConfigDir = p
+	} else {
+		opts.ConfigDir = path.Dir(opts.ConfigFilePath)
+	}
+}
+
 func loadCommandLists(opts *ConfigOpts, backyKoanf *koanf.Koanf) {
 	var listConfigFiles []string
 	var u *url.URL
 	var p string
-	// if config file is remote, use the directory of the remote file
 	if isRemoteURL(opts.ConfigFilePath) {
 		p, u = getRemoteDir(opts.ConfigFilePath)
 		opts.ConfigDir = p
-		println(p)
-		// // Still use local list files if a remote config file is used, but use them last
 		listConfigFiles = []string{u.JoinPath("lists.yml").String(), u.JoinPath("lists.yaml").String()}
 	} else {
 		opts.ConfigDir = path.Dir(opts.ConfigFilePath)
@@ -311,7 +331,7 @@ func loadCommandLists(opts *ConfigOpts, backyKoanf *koanf.Koanf) {
 		if backyKoanf.Exists("cmdLists.file") {
 			loadCmdListsFile(backyKoanf, listsConfig, opts)
 		} else {
-			unmarshalConfig(backyKoanf, "cmdLists", &opts.CmdConfigLists, opts.Logger)
+			unmarshalConfigIntoStruct(backyKoanf, "cmdLists", &opts.CmdConfigLists, opts.Logger)
 		}
 	}
 }
@@ -351,7 +371,7 @@ func loadListConfigFile(filePath string, k *koanf.Koanf, opts *ConfigOpts) bool
 		return false
 	}
 
-	unmarshalConfig(k, "cmdLists", &opts.CmdConfigLists, opts.Logger)
+	unmarshalConfigIntoStruct(k, "cmdLists", &opts.CmdConfigLists, opts.Logger)
 	keyNotSupported("cmd-lists", "cmdLists", k, opts, true)
 	opts.CmdListFile = filePath
 	return true
@@ -379,7 +399,7 @@ func loadCmdListsFile(backyKoanf *koanf.Koanf, listsConfig *koanf.Koanf, opts *C
 	}
 
 	keyNotSupported("cmd-lists", "cmdLists", listsConfig, opts, true)
-	unmarshalConfig(listsConfig, "cmdLists", &opts.CmdConfigLists, opts.Logger)
+	unmarshalConfigIntoStruct(listsConfig, "cmdLists", &opts.CmdConfigLists, opts.Logger)
 	opts.Logger.Info().Str("using lists config file", opts.CmdListFile).Send()
 }
 
@@ -436,7 +456,7 @@ func getLoggingKeyFromConfig(key string) string {
 // 	return fmt.Sprintf("cmdLists.%s", list)
 // }
 
-func (opts *ConfigOpts) setupVault() error {
+func (opts *ConfigOpts) initVault() error {
 	if !opts.koanf.Bool("vault.enabled") {
 		return nil
 	}
@@ -457,7 +477,7 @@ func (opts *ConfigOpts) setupVault() error {
 		token = os.Getenv("VAULT_TOKEN")
 	}
 	if strings.TrimSpace(token) == "" {
-		return fmt.Errorf("no token found, but one was required. \n\nSet the config key vault.token or the environment variable VAULT_TOKEN")
+		return fmt.Errorf("no token found. One is required. \n\nSet the config key vault.token or the environment variable VAULT_TOKEN")
 	}
 
 	client.SetToken(token)
@@ -469,70 +489,22 @@ func (opts *ConfigOpts) setupVault() error {
 
 	opts.vaultClient = client
 
+	for _, v := range opts.VaultKeys {
+		v.Name = replaceVarInString(opts.Vars, v.Key, opts.Logger)
+		v.MountPath = replaceVarInString(opts.Vars, v.MountPath, opts.Logger)
+	}
+
 	return nil
 }
 
-func getVaultSecret(vaultClient *vault.Client, key *VaultKey) (string, error) {
-	var (
-		secret *vault.KVSecret
-		err    error
-	)
-
-	if key.ValueType == "KVv2" {
-		secret, err = vaultClient.KVv2(key.MountPath).Get(context.Background(), key.Path)
-	} else if key.ValueType == "KVv1" {
-		secret, err = vaultClient.KVv1(key.MountPath).Get(context.Background(), key.Path)
-	} else if key.ValueType != "" {
-		return "", fmt.Errorf("type %s for key %s not known. Valid types are KVv1 or KVv2", key.ValueType, key.Name)
-	} else {
-		return "", fmt.Errorf("type for key %s must be specified. Valid types are KVv1 or KVv2", key.Name)
-
-	}
-	if err != nil {
-		return "", fmt.Errorf("unable to read secret: %v", err)
-	}
-
-	value, ok := secret.Data[key.Name].(string)
-	if !ok {
-		return "", fmt.Errorf("value type assertion failed: %T %#v", secret.Data[key.Name], secret.Data[key.Name])
-	}
-
-	return value, nil
-}
-
-func parseVaultKey(keyName string, keys []*VaultKey) (*VaultKey, error) {
-
-	for _, k := range keys {
-		if k.Name == keyName {
-			return k, nil
-		}
-	}
-	return nil, fmt.Errorf("key %s not found in vault keys", keyName)
-}
-
-func GetVaultKey(str string, opts *ConfigOpts, log zerolog.Logger) string {
-	key, err := parseVaultKey(str, opts.VaultKeys)
-	if key == nil && err == nil {
-		return str
-	}
-	if err != nil && key == nil {
-		log.Err(err).Send()
-		return ""
-	}
-
-	value, secretErr := getVaultSecret(opts.vaultClient, key)
-	if secretErr != nil {
-		log.Err(secretErr).Send()
-		return value
-	}
-	return value
-}
-
 func processCmds(opts *ConfigOpts) error {
 
 	// process commands
 	for cmdName, cmd := range opts.Cmds {
-
+		for i, v := range cmd.Args {
+			v = replaceVarInString(opts.Vars, v, opts.Logger)
+			cmd.Args[i] = v
+		}
 		if cmd.Name == "" {
 			cmd.Name = cmdName
 		}
@@ -555,9 +527,13 @@ func processCmds(opts *ConfigOpts) error {
 			}
 		}
 
-		// resolve hosts
+		if !IsHostLocal(cmd.Host) {
-		if cmd.Host != nil {
+
-			host, hostFound := opts.Hosts[*cmd.Host]
+			cmdHost := replaceVarInString(opts.Vars, cmd.Host, opts.Logger)
+			if cmdHost != cmd.Host {
+				cmd.Host = cmdHost
+			}
+			host, hostFound := opts.Hosts[cmd.Host]
 			if hostFound {
 				cmd.RemoteHost = host
 				cmd.RemoteHost.Host = host.Host
@@ -565,12 +541,12 @@ func processCmds(opts *ConfigOpts) error {
 					cmd.RemoteHost.HostName = host.HostName
 				}
 			} else {
-				opts.Logger.Info().Msgf("adding host %s to host list", *cmd.Host)
+				opts.Logger.Info().Msgf("adding host %s to host list", cmd.Host)
 				if opts.Hosts == nil {
 					opts.Hosts = make(map[string]*Host)
 				}
-				opts.Hosts[*cmd.Host] = &Host{Host: *cmd.Host}
+				opts.Hosts[cmd.Host] = &Host{Host: cmd.Host}
-				cmd.RemoteHost = &Host{Host: *cmd.Host}
+				cmd.RemoteHost = &Host{Host: cmd.Host}
 			}
 		} else {
 
@@ -616,7 +592,7 @@ func processCmds(opts *ConfigOpts) error {
 			if cmd.Username == "" {
 				return fmt.Errorf("username is required for user command %s", cmd.Name)
 			}
-
+			cmd.Username = replaceVarInString(opts.Vars, cmd.Username, opts.Logger)
 			err := detectOSType(cmd, opts)
 			if err != nil {
 				opts.Logger.Info().Err(err).Str("command", cmdName).Send()
@@ -631,8 +607,10 @@ func processCmds(opts *ConfigOpts) error {
 					opts.Logger.Debug().Msg("changing password for user: " + cmd.Username)
 					cmd.UserPassword = getExternalConfigDirectiveValue(cmd.UserPassword, opts)
 				}
-				if cmd.Host != nil {
+
-					host, ok := opts.Hosts[*cmd.Host]
+				if !IsHostLocal(cmd.Host) {
+
+					host, ok := opts.Hosts[cmd.Host]
 					if ok {
 						cmd.userMan, err = usermanager.NewUserManager(host.OS)
 					}
@@ -701,7 +679,9 @@ func processHooks(cmd *Command, hooks []string, opts *ConfigOpts, hookType strin
 }
 
 func detectOSType(cmd *Command, opts *ConfigOpts) error {
-	if cmd.Host == nil {
+
+	if IsHostLocal(cmd.Host) {
+
 		if runtime.GOOS == "linux" {
 			cmd.OS = "linux"
 			opts.Logger.Info().Msg("Unix/Linux type OS detected")
@@ -710,7 +690,7 @@ func detectOSType(cmd *Command, opts *ConfigOpts) error {
 		return fmt.Errorf("using an os that is not yet supported for user commands")
 	}
 
-	host, ok := opts.Hosts[*cmd.Host]
+	host, ok := opts.Hosts[cmd.Host]
 	if ok {
 		if host.OS != "" {
 			return nil
@@ -742,3 +722,24 @@ func keyNotSupported(oldKey, newKey string, koanf *koanf.Koanf, opts *ConfigOpts
 		}
 	}
 }
+
+func replaceVarInString(vars map[string]string, str string, logger zerolog.Logger) string {
+	if strings.Contains(str, "%{var:") && strings.Contains(str, "}%") {
+		logger.Debug().Msgf("replacing vars in string %s", str)
+		for k, v := range vars {
+			if strings.Contains(str, "%{var:"+k+"}%") {
+				str = strings.ReplaceAll(str, "%{var:"+k+"}%", v)
+			}
+		}
+		if strings.Contains(str, "%{var:") && strings.Contains(str, "}%") {
+			logger.Warn().Msg("could not replace all vars in string")
+		}
+	}
+	return str
+}
+
+func VariadicFunctionParameterTest(allowedKeys ...string) {
+	if contains(allowedKeys, "file") {
+		println("file param included")
+	}
+}

pkg/backy/list.go

@@ -46,9 +46,9 @@ func (opts *ConfigOpts) ListCommand(cmd string) {
 		}
 
 		// is it remote or local
-		if cmdInfo.Host != nil {
+		if !IsHostLocal(cmdInfo.Host) {
 			println()
-			print("Host: ", *cmdInfo.Host)
+			print("Host: ", cmdInfo.Host)
 			println()
 
 		} else {

pkg/backy/notification.go

@@ -9,6 +9,7 @@ import (
 
 	"git.andrewnw.xyz/CyberShell/backy/pkg/logging"
 	"github.com/nikoksr/notify"
+	"github.com/nikoksr/notify/service/http"
 	"github.com/nikoksr/notify/service/mail"
 	"github.com/nikoksr/notify/service/matrix"
 	"maunium.net/go/mautrix/id"
@@ -30,6 +31,12 @@ type MailConfig struct {
 	Password      string   `yaml:"password"`
 }
 
+type HttpConfig struct {
+	URL     string              `yaml:"url"`
+	Method  string              `yaml:"method"`
+	Headers map[string][]string `yaml:"headers"`
+}
+
 // SetupNotify sets up notify instances for each command list.
 func (opts *ConfigOpts) SetupNotify() {
 
@@ -59,6 +66,7 @@ func (opts *ConfigOpts) SetupNotify() {
 					continue
 				}
 				conf.Password = getExternalConfigDirectiveValue(conf.Password, opts)
+				opts.Logger.Debug().Str("list", confName).Str("id", confId).Msg("adding mail notification service")
 				mailConf := setupMail(conf)
 				services = append(services, mailConf)
 			case "matrix":
@@ -68,13 +76,22 @@ func (opts *ConfigOpts) SetupNotify() {
 					continue
 				}
 				conf.AccessToken = getExternalConfigDirectiveValue(conf.AccessToken, opts)
+				opts.Logger.Debug().Str("list", confName).Str("id", confId).Msg("adding matrix notification service")
 				mtrxConf, mtrxErr := setupMatrix(conf)
 				if mtrxErr != nil {
 					opts.Logger.Info().Str("list", confName).Err(fmt.Errorf("error: configuring matrix id %s failed during setup: %w", id, mtrxErr))
 					continue
 				}
 				services = append(services, mtrxConf)
-
+			case "http":
+				conf, ok := opts.NotificationConf.HttpConfig[confId]
+				if !ok {
+					opts.Logger.Info().Err(fmt.Errorf("error: ID %s not found in http object", confId)).Str("list", confName).Send()
+					continue
+				}
+				opts.Logger.Debug().Str("list", confName).Str("id", confId).Msg("adding http notification service")
+				httpConf := setupHttp(conf)
+				services = append(services, httpConf)
 			default:
 				opts.Logger.Info().Err(fmt.Errorf("id %s not found", id)).Str("list", confName).Send()
 			}
@@ -100,3 +117,19 @@ func setupMail(config MailConfig) *mail.Mail {
 	mailClient.BodyFormat(mail.PlainText)
 	return mailClient
 }
+
+func setupHttp(httpConf HttpConfig) *http.Service {
+
+	httpService := http.New()
+	httpService.AddReceivers(&http.Webhook{
+		URL:         httpConf.URL,
+		Header:      httpConf.Headers,
+		ContentType: "text/plain",
+		Method:      httpConf.Method,
+		BuildPayload: func(subject, message string) (payload any) {
+			return subject + "\n\n" + message
+		},
+	})
+
+	return httpService
+}

pkg/backy/ssh.go

@@ -15,6 +15,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/google/uuid"
 	"github.com/kevinburke/ssh_config"
 	"github.com/pkg/errors"
 	"github.com/pkg/sftp"
@@ -440,8 +441,8 @@ func (command *Command) RunCmdSSH(cmdCtxLogger zerolog.Logger, opts *ConfigOpts)
 
 	cmdCtxLogger.Info().
 		Str("Command", command.Name).
-		Str("Host", *command.Host).
+		Str("Host", command.Host).
-		Msgf("Running %s on host %s", getCommandTypeAndSetCommandInfoLabel(command.Type), *command.Host)
+		Msgf("Running %s on host %s", getCommandTypeAndSetCommandInfoLabel(command.Type), command.Host)
 
 	// cmdCtxLogger.Debug().Str("cmd", command.Cmd).Strs("args", command.Args).Send()
 
@@ -509,9 +510,32 @@ func (command *Command) RunCmdSSH(cmdCtxLogger zerolog.Logger, opts *ConfigOpts)
 
 		if command.Type == UserCT && command.UserOperation == "password" {
 			// cmdCtxLogger.Debug().Msgf("adding stdin")
-			userNamePass := fmt.Sprintf("%s:%s", command.Username, command.UserPassword)
-			ArgsStr = fmt.Sprintf("echo %s | chpasswd", userNamePass)
 
+			userNamePass := fmt.Sprintf("%s:%s", command.Username, command.UserPassword)
+			client, err := sftp.NewClient(command.RemoteHost.SshClient)
+			if err != nil {
+				return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error creating sftp client: %v", err)
+			}
+			uuidFile := uuid.New()
+			passFilePath := fmt.Sprintf("/tmp/%s", uuidFile.String())
+			passFile, passFileErr := client.Create(passFilePath)
+			if passFileErr != nil {
+				return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error creating file /tmp/%s: %v", uuidFile.String(), passFileErr)
+			}
+
+			_, err = passFile.Write([]byte(userNamePass))
+			if err != nil {
+				return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error writing to file /tmp/%s: %v", uuidFile.String(), err)
+			}
+
+			ArgsStr = fmt.Sprintf("cat %s | chpasswd", passFilePath)
+			defer passFile.Close()
+
+			rmFileFunc := func() {
+				_ = client.Remove(passFilePath)
+			}
+
+			defer rmFileFunc()
 			// commandSession.Stdin = command.stdin
 		}
 		if err := commandSession.Run(ArgsStr); err != nil {
@@ -544,7 +568,10 @@ func (command *Command) RunCmdSSH(cmdCtxLogger zerolog.Logger, opts *ConfigOpts)
 						return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error creating sftp client: %v", err)
 					}
 
-					client.MkdirAll(userSshDir)
+					err = client.MkdirAll(userSshDir)
+					if err != nil {
+						return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error creating directory %s: %v", userSshDir, err)
+					}
 					_, err = client.Create(fmt.Sprintf("%s/authorized_keys", userSshDir))
 					if err != nil {
 						return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error opening file %s/authorized_keys: %v", userSshDir, err)
@@ -784,3 +811,8 @@ func CheckIfHostHasHostName(host string) (bool, string) {
 	println(HostName)
 	return HostName != "", HostName
 }
+
+func IsHostLocal(host string) bool {
+	host = strings.ToLower(host)
+	return host == "127.0.0.1" || host == "localhost" || host == ""
+}

pkg/backy/types.go

@@ -56,7 +56,7 @@ type (
 		// See CommandType enum further down the page for acceptable values
 		Type CommandType `yaml:"type,omitempty"`
 
-		Host *string `yaml:"host,omitempty"`
+		Host string `yaml:"host,omitempty"`
 
 		Hooks *Hooks `yaml:"hooks,omitempty"`
 
@@ -115,7 +115,9 @@ type (
 
 		UserShell string `yaml:"userShell,omitempty"`
 
-		SystemUser bool `yaml:"systemUser,omitempty"`
+		UserCreateHome bool `yaml:"userCreateHome,omitempty"`
+
+		UserIsSystem bool `yaml:"userIsSystem,omitempty"`
 
 		UserPassword string `yaml:"userPassword,omitempty"`
 
@@ -203,6 +205,8 @@ type (
 
 		List ListConfig
 
+		Vars map[string]string `yaml:"variables"`
+
 		VaultKeys []*VaultKey `yaml:"keys"`
 
 		koanf *koanf.Koanf
@@ -221,6 +225,7 @@ type (
 
 	VaultKey struct {
 		Name      string `yaml:"name"`
+		Key       string `yaml:"key"`
 		Path      string `yaml:"path"`
 		ValueType string `yaml:"type"`
 		MountPath string `yaml:"mountpath"`
@@ -236,6 +241,7 @@ type (
 	Notifications struct {
 		MailConfig   map[string]MailConfig   `yaml:"mail,omitempty"`
 		MatrixConfig map[string]MatrixStruct `yaml:"matrix,omitempty"`
+		HttpConfig   map[string]HttpConfig   `yaml:"http,omitempty"`
 	}
 
 	CmdOutput struct {
@@ -272,8 +278,9 @@ type (
 	}
 
 	// use ints so we can use enums
-	CommandType      int
+	CommandType               int
-	PackageOperation int
+	PackageOperation          int
+	AllowedExternalDirectives int
 )
 
 //go:generate go run github.com/dmarkham/enumer -linecomment -yaml -text -json -type=CommandType
@@ -296,3 +303,14 @@ const (
 	PackOpCheckVersion                         // checkVersion
 	PackOpIsInstalled                          // isInstalled
 )
+
+//go:generate go run github.com/dmarkham/enumer -linecomment -yaml -text -json -type=AllowedExternalDirectives
+const (
+	DefaultExternalDir                AllowedExternalDirectives = iota
+	AllowedExternalDirectiveVault                               // vault
+	AllowedExternalDirectiveVaultFile                           // vault-file
+	AllowedExternalDirectiveAll                                 // vault-file-env
+	AllowedExternalDirectiveFileEnv                             // file-env
+	AllowedExternalDirectiveFile                                // file
+	AllowedExternalDirectiveEnv                                 // env
+)

pkg/backy/utils.go

@@ -6,6 +6,7 @@ package backy
 
 import (
 	"bytes"
+	"context"
 	"errors"
 	"fmt"
 	"os"
@@ -16,6 +17,7 @@ import (
 
 	"git.andrewnw.xyz/CyberShell/backy/pkg/logging"
 	"git.andrewnw.xyz/CyberShell/backy/pkg/remotefetcher"
+	vault "github.com/hashicorp/vault/api"
 	"github.com/joho/godotenv"
 	"github.com/knadh/koanf/v2"
 	"github.com/rs/zerolog"
@@ -108,7 +110,11 @@ func injectEnvIntoSSH(envVarsToInject environmentVars, process *ssh.Session, opt
 			goto errEnvFile
 		}
 		for key, val := range envMap {
-			process.Setenv(key, GetVaultKey(val, opts, log))
+			err = process.Setenv(key, GetVaultKey(val, opts, log))
+			if err != nil {
+				log.Error().Err(err).Send()
+
+			}
 		}
 	}
 
@@ -119,12 +125,16 @@ errEnvFile:
 		if strings.Contains(envVal, "=") {
 			envVarArr := strings.Split(envVal, "=")
 
-			process.Setenv(envVarArr[0], GetVaultKey(envVarArr[1], opts, log))
+			err := process.Setenv(envVarArr[0], getExternalConfigDirectiveValue(envVarArr[1], opts))
+			if err != nil {
+				log.Error().Err(err).Send()
+
+			}
 		}
 	}
 }
 
-func injectEnvIntoLocalCMD(envVarsToInject environmentVars, process *exec.Cmd, log zerolog.Logger) {
+func injectEnvIntoLocalCMD(envVarsToInject environmentVars, process *exec.Cmd, log zerolog.Logger, opts *ConfigOpts) {
 	if envVarsToInject.file != "" {
 		envPath, _ := getFullPathWithHomeDir(envVarsToInject.file)
 
@@ -148,7 +158,8 @@ errEnvFile:
 
 	for _, envVal := range envVarsToInject.env {
 		if strings.Contains(envVal, "=") {
-			process.Env = append(process.Env, envVal)
+			envVarArr := strings.Split(envVal, "=")
+			process.Env = append(process.Env, fmt.Sprintf("%s=%s", envVarArr[0], getExternalConfigDirectiveValue(envVarArr[1], opts)))
 		}
 	}
 	process.Env = append(process.Env, os.Environ()...)
@@ -249,7 +260,6 @@ func (opts *ConfigOpts) loadEnv() {
 func expandEnvVars(backyEnv map[string]string, envVars []string) {
 
 	env := func(name string) string {
-		name = strings.ToUpper(name)
 		envVar, found := backyEnv[name]
 		if found {
 			return envVar
@@ -258,14 +268,14 @@ func expandEnvVars(backyEnv map[string]string, envVars []string) {
 	}
 
 	for indx, v := range envVars {
-		if strings.HasPrefix(v, externDirectiveStart) && strings.HasSuffix(v, externDirectiveEnd) {
+
-			if strings.HasPrefix(v, envExternDirectiveStart) {
+		if strings.HasPrefix(v, envExternDirectiveStart) && strings.HasSuffix(v, externDirectiveEnd) {
-				v = strings.TrimPrefix(v, envExternDirectiveStart)
+			v = strings.TrimPrefix(v, envExternDirectiveStart)
-				v = strings.TrimRight(v, externDirectiveEnd)
+			v = strings.TrimRight(v, externDirectiveEnd)
-				out, _ := shell.Expand(v, env)
+			out, _ := shell.Expand(v, env)
-				envVars[indx] = out
+			envVars[indx] = out
-			}
 		}
+
 	}
 }
 
@@ -293,7 +303,8 @@ func getCommandTypeAndSetCommandInfo(command *Command) *Command {
 				command.Username,
 				command.UserHome,
 				command.UserShell,
-				command.SystemUser,
+				command.UserIsSystem,
+				command.UserCreateHome,
 				command.UserGroups,
 				command.Args)
 		case "modify":
@@ -351,6 +362,7 @@ func getExternalConfigDirectiveValue(key string, opts *ConfigOpts) string {
 	if !(strings.HasPrefix(key, externDirectiveStart) && strings.HasSuffix(key, externDirectiveEnd)) {
 		return key
 	}
+	key = replaceVarInString(opts.Vars, key, opts.Logger)
 	opts.Logger.Debug().Str("expanding external key", key).Send()
 	if strings.HasPrefix(key, envExternDirectiveStart) {
 		key = strings.TrimPrefix(key, envExternDirectiveStart)
@@ -385,3 +397,57 @@ func getExternalConfigDirectiveValue(key string, opts *ConfigOpts) string {
 
 	return key
 }
+
+func getVaultSecret(vaultClient *vault.Client, key *VaultKey) (string, error) {
+	var (
+		secret *vault.KVSecret
+		err    error
+	)
+
+	if key.ValueType == "KVv2" {
+		secret, err = vaultClient.KVv2(key.MountPath).Get(context.Background(), key.Path)
+	} else if key.ValueType == "KVv1" {
+		secret, err = vaultClient.KVv1(key.MountPath).Get(context.Background(), key.Path)
+	} else if key.ValueType != "" {
+		return "", fmt.Errorf("type %s for key %s not known. Valid types are KVv1 or KVv2", key.ValueType, key.Name)
+	} else {
+		return "", fmt.Errorf("type for key %s must be specified. Valid types are KVv1 or KVv2", key.Name)
+	}
+	if err != nil {
+		return "", fmt.Errorf("unable to read secret: %v", err)
+	}
+
+	value, ok := secret.Data[key.Key].(string)
+	if !ok {
+		return "", fmt.Errorf("value type assertion failed for vault key %s: %T %#v", key.Name, secret.Data[key.Name], secret.Data[key.Name])
+	}
+
+	return value, nil
+}
+
+func getVaultKeyData(keyName string, keys []*VaultKey) (*VaultKey, error) {
+	for _, k := range keys {
+		if k.Name == keyName {
+			return k, nil
+		}
+	}
+	return nil, fmt.Errorf("key %s not found in vault keys", keyName)
+}
+
+func GetVaultKey(str string, opts *ConfigOpts, log zerolog.Logger) string {
+	key, err := getVaultKeyData(str, opts.VaultKeys)
+	if key == nil && err == nil {
+		return str
+	}
+	if err != nil && key == nil {
+		log.Err(err).Send()
+		return ""
+	}
+
+	value, secretErr := getVaultSecret(opts.vaultClient, key)
+	if secretErr != nil {
+		log.Err(secretErr).Send()
+		return value
+	}
+	return value
+}

pkg/remotefetcher/cache.go

@@ -116,7 +116,7 @@ func (c *Cache) Set(source, hash string, data []byte, dataType string) (CacheDat
 	path := filepath.Join(c.dir, fmt.Sprintf("%s-%s", fileName, sourceHash))
 
 	if _, err := os.Stat(path); os.IsNotExist(err) {
-		os.MkdirAll(c.dir, 0700)
+		_ = os.MkdirAll(c.dir, 0700)
 	}
 
 	err := os.WriteFile(path, data, 0644)
@@ -171,7 +171,7 @@ func (cf *CachedFetcher) Hash(data []byte) string {
 func LoadMetadataFromFile(filePath string) ([]*CacheData, error) {
 	if _, err := os.Stat(filePath); os.IsNotExist(err) {
 		// Create the file if it does not exist
-		os.MkdirAll(path.Dir(filePath), 0700)
+		_ = os.MkdirAll(path.Dir(filePath), 0700)
 		emptyData := []byte("[]")
 		err := os.WriteFile(filePath, emptyData, 0644)
 		if err != nil {

pkg/usermanager/linux/linux.go

@@ -15,7 +15,7 @@ func (l LinuxUserManager) NewLinuxManager() *LinuxUserManager {
 }
 
 // AddUser adds a new user to the system.
-func (l LinuxUserManager) AddUser(username, homeDir, shell string, isSystem bool, groups, args []string) (string, []string) {
+func (l LinuxUserManager) AddUser(username, homeDir, shell string, isSystem, createHome bool, groups, args []string) (string, []string) {
 	baseArgs := []string{}
 
 	if isSystem {
@@ -38,6 +38,10 @@ func (l LinuxUserManager) AddUser(username, homeDir, shell string, isSystem bool
 		baseArgs = append(baseArgs, args...)
 	}
 
+	if createHome {
+		baseArgs = append(baseArgs, "-m")
+	}
+
 	args = append(baseArgs, username)
 
 	cmd := "useradd"

pkg/usermanager/userman.go

@@ -10,7 +10,7 @@ import (
 // UserManager defines the interface for user management operations.
 // All functions but one return a string for the command and any args.
 type UserManager interface {
-	AddUser(username, homeDir, shell string, isSystem bool, groups, args []string) (string, []string)
+	AddUser(username, homeDir, shell string, createHome, isSystem bool, groups, args []string) (string, []string)
 	RemoveUser(username string) (string, []string)
 	ModifyUser(username, homeDir, shell string, groups []string) (string, []string)
 	// Modify password uses chpasswd for Linux systems to build the command to change the password

release

@@ -1,5 +1,6 @@
 #!/bin/bash
 set -eou pipefail
+go mod tidy
 go generate ./...
 CURRENT_TAG="$(go run backy.go version -V)"
 goreleaser -f .goreleaser/github.yml check

tests/VaultTest.yml

@@ -0,0 +1,27 @@
+commands:
+  vaultEnvVar:
+    cmd: echo
+    shell: /bin/zsh
+    Args:
+      - ${VAULT_VAR}
+    environment:
+      "VAULT_VAR=%{vault:vaultTestSecret}%"
+
+logging:
+  verbose: true
+
+vault:
+  token: root
+  address: http://127.0.0.1:8200
+  enabled: true
+  keys:
+    - name: vaultTestSecret
+      key: data
+      mountpath: secret
+      path: test/var
+      type: KVv2 # KVv1 or KVv2
+
+cmdLists:
+  addUsers:
+    order:
+      - vaultEnvVar