change file

.changes/unreleased/Added-20250409-174528.yaml

@@ -0,0 +1,3 @@
+kind: Added
+body: 'feat: Package operation `versionCheck` supports regular expressions (see [regexp](https://pkg.go.dev/regexp) package for docs)'
+time: 2025-04-09T17:45:28.836497149-05:00

.changes/unreleased/Added-20250501-110745.yaml

@@ -0,0 +1,3 @@
+kind: Added
+body: 'Command lists: added `cmdLists.[name].notify` object'
+time: 2025-05-01T11:07:45.96164753-05:00

.changes/unreleased/Added-20250704-085917.yaml

@@ -0,0 +1,3 @@
+kind: Added
+body: Testing setup with Docker
+time: 2025-07-04T08:59:17.430373451-05:00

.changes/unreleased/Added-20250704-102126.yaml

@@ -0,0 +1,3 @@
+kind: Added
+body: 'CLI: add global flag --hostsConfig that allows hosts to be dynamic in relation to the main config'
+time: 2025-07-04T10:21:26.864635558-05:00

.changes/unreleased/Changed-20250321-090849.yaml

@@ -0,0 +1,3 @@
+kind: Changed
+body: 'Commands: `host` can now be `localhost` or `127.0.0.1` to run commands locally'
+time: 2025-03-21T09:08:49.871021144-05:00

.changes/unreleased/Changed-20250325-003357.yaml

@@ -0,0 +1,3 @@
+kind: Changed
+body: lists loaded from external files only if no list config present in current file
+time: 2025-03-25T00:33:57.039431409-05:00

.changes/unreleased/Changed-20250407-223020.yaml

@@ -0,0 +1,3 @@
+kind: Changed
+body: "`PackageManager.Parse` renamed to `ParseRemotePackageManagerVersionOutput`. This now returns arrays of PackageManagerCommon.Package and errors."
+time: 2025-04-07T22:30:20.342177323-05:00

.changes/unreleased/Changed-20250418-133440.yaml

@@ -0,0 +1,3 @@
+kind: Changed
+body: 'Internal: refactoring and renaming functions'
+time: 2025-04-18T13:34:40.842541658-05:00

.changes/unreleased/Changed-20250501-110534.yaml

@@ -0,0 +1,3 @@
+kind: Changed
+body: 'Commands: moved output-prefixed keys to the `commands.[name].output` object'
+time: 2025-05-01T11:05:34.90130087-05:00

.changes/unreleased/Changed-20250609-072601.yaml

@@ -0,0 +1,3 @@
+kind: Changed
+body: Change internal method name for better understanding
+time: 2025-06-09T07:26:01.819927627-05:00

.changes/unreleased/Fixed-20250418-095747.yaml

@@ -0,0 +1,3 @@
+kind: Fixed
+body: 'Command Lists: hooks now run correctly when commands finish'
+time: 2025-04-18T09:57:47.39035092-05:00

.changes/unreleased/Fixed-20250424-225711.yaml

@@ -0,0 +1,3 @@
+kind: Fixed
+body: Log file passed using `--log-file` correctly used
+time: 2025-04-24T22:57:11.592829277-05:00

.changes/v0.10.0.md

@@ -0,0 +1,16 @@
+## v0.10.0 - 2025-03-08
+### Added
+* Hooks: improved logging when executing
+* User commands: adding SSH keys using config key `userSshPubKeys`
+* directives: added support for fetching values using directive `%{externalSource:key}%`
+### Changed
+* Commands: if dir is not specified, run in config dir
+* FileDirective: use the config directory if path is not absolute
+* Host: changes to case of some keys
+* Notifications: added external directive to sensitive keys
+### Fixed
+* LocalFetcher: return fetch error
+* Lists: load file key before attempting to load from current file
+* fix: host not in config file, but in ssh config, properly added to hosts struct
+* SSH: password authentication bugs
+* User commands: change user password works

.changes/v0.10.1.md

@@ -0,0 +1,8 @@
+## v0.10.1 - 2025-03-11
+### Added
+* UserCommands: add ssh public keys when running locally
+* UserCommands: add field CreateUserHome
+### Changed
+* UserCommands: create temp file when modifing password over SSH
+* UserCommands: change field name
+* Vault: keys are now referenced by `name`, and the actual data by `data`

.changes/v0.10.2.md

@@ -0,0 +1,6 @@
+## v0.10.2 - 2025-03-19
+### Added
+* Notifications: http service added
+* Variable support. Can be referenced with `%{var:nameOfVar}%` in select string fields.
+### Changed
+* vault: initialize vault before validating config

.gitignore

@@ -5,6 +5,6 @@ dist/
 .codegpt
 
 *.log
-*.sh
+/*.sh
 /*.yaml
 /*.yml

.vscode/settings.json

@@ -9,5 +9,6 @@
         "mautrix",
         "nikoksr",
         "Strs"
-    ]
+    ],
+    "CodeGPT.apiKey": "CodeGPT Plus Beta"
 }

.woodpecker/gitea.yml

@@ -1,9 +1,7 @@
-name: goreleaser release
 steps:
   golang:
     image: golang:1.23
     commands:
-      - go mod tidy
       - go install github.com/goreleaser/goreleaser/v2@v2.7.0
       - goreleaser release -f .goreleaser/gitea.yml --release-notes=".changes/$(go run backy.go version -V).md"
     environment:

.woodpecker/go-lint.yml

@@ -5,7 +5,7 @@ steps:
       - go build
       - go test
   release:
-    image: golangci/golangci-lint:v1.53.3
+    image: golangci/golangci-lint:v1.64.7
     commands:
       - golangci-lint run -v --timeout 5m
 

CHANGELOG.md

@@ -6,6 +6,39 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html),
 and is generated by [Changie](https://github.com/miniscruff/changie).
 
 
+## v0.10.2 - 2025-03-19
+### Added
+* Notifications: http service added
+* Variable support. Can be referenced with `%{var:nameOfVar}%` in select string fields.
+### Changed
+* vault: initialize vault before validating config
+
+## v0.10.1 - 2025-03-11
+### Added
+* UserCommands: add ssh public keys when running locally
+* UserCommands: add field CreateUserHome
+### Changed
+* UserCommands: create temp file when modifing password over SSH
+* UserCommands: change field name
+* Vault: keys are now referenced by `name`, and the actual data by `data`
+
+## v0.10.0 - 2025-03-08
+### Added
+* Hooks: improved logging when executing
+* User commands: adding SSH keys using config key `userSshPubKeys`
+* directives: added support for fetching values using directive `%{externalSource:key}%`
+### Changed
+* Commands: if dir is not specified, run in config dir
+* FileDirective: use the config directory if path is not absolute
+* Host: changes to case of some keys
+* Notifications: added external directive to sensitive keys
+### Fixed
+* LocalFetcher: return fetch error
+* Lists: load file key before attempting to load from current file
+* fix: host not in config file, but in ssh config, properly added to hosts struct
+* SSH: password authentication bugs
+* User commands: change user password works
+
 ## v0.9.1 - 2025-03-01
 ### Changed
 * Use EnvVar AWS_PROFILE to get S3 profile

backy.code-workspace

@@ -18,6 +18,8 @@
 			"maunium",
 			"mautrix",
 			"nikoksr",
+			"packagemanagercommon",
+			"rawbytes",
 			"remotefetcher",
 			"Strs"
 		]

cmd/backup.go

@@ -30,9 +30,14 @@ func init() {
 }
 
 func Backup(cmd *cobra.Command, args []string) {
-	backyConfOpts := backy.NewOpts(cfgFile, backy.AddCommandLists(cmdLists), backy.SetLogFile(logFile), backy.SetCmdStdOut(cmdStdOut))
+	backyConfOpts := backy.NewConfigOptions(configFile,
+		backy.AddCommandLists(cmdLists),
+		backy.SetLogFile(logFile),
+		backy.EnableCommandStdOut(cmdStdOut),
+		backy.SetHostsConfigFile(hostsConfigFile))
+
 	backyConfOpts.InitConfig()
-	backyConfOpts.ReadConfig()
+	backyConfOpts.ParseConfigurationFile()
 
 	backyConfOpts.RunListConfig("")
 	for _, host := range backyConfOpts.Hosts {

cmd/backup_test.go

@@ -0,0 +1,54 @@
+package cmd
+
+// import (
+// 	"bufio"
+// 	"encoding/json"
+// 	"os"
+// 	"os/exec"
+// 	"strings"
+// 	"testing"
+
+// 	"github.com/stretchr/testify/assert"
+// )
+
+// // TestConfigOptions tests the configuration options for the backy package.
+// func Test_ErrorHook(t *testing.T) {
+// 	configFile := "-f ../../tests/ErrorHook.yml"
+// 	logFile := "--log-file=ErrorHook.log"
+// 	backyCommand := exec.Command("go", "run", "../../backy.go", configFile, logFile, "backup")
+// 	backyCommand.Stderr = os.Stdout
+// 	backyCommand.Stdout = os.Stdout
+// 	err := backyCommand.Run()
+// 	assert.Nil(t, err)
+// 	os.Remove("ErrorHook.log")
+// 	logFileData, logFileErr := os.ReadFile("ErrorHook.log")
+// 	if logFileErr != nil {
+// 		assert.FailNow(t, logFileErr.Error())
+
+// 	}
+// 	var JsonData []map[string]interface{}
+// 	jsonScanner := bufio.NewScanner(strings.NewReader(string(logFileData)))
+
+// 	for jsonScanner.Scan() {
+// 		var jsonDataLine map[string]interface{}
+// 		err = json.Unmarshal(jsonScanner.Bytes(), &jsonDataLine)
+// 		assert.Nil(t, err)
+// 		JsonData = append(JsonData, jsonDataLine)
+// 	}
+// 	for _, v := range JsonData {
+// 		_, ok := v["error"]
+// 		if !ok {
+// 			assert.FailNow(t, "error does not exist\n")
+// 			// return
+// 		}
+// 	}
+// 	// t.Logf("%s", logFileData)
+// 	// t.Logf("%v", JsonData)
+// }
+
+// func TestBackupErrorHook(t *testing.T) {
+// 	logFile = "ErrorHook.log"
+
+// 	configFile = "../tests/ErrorHook.yml"
+
+// }

cmd/config.go

@@ -20,7 +20,7 @@ package cmd
 
 // func config(cmd *cobra.Command, args []string) {
 
-// 	opts := backy.NewOpts(cfgFile, backy.cronEnabled())
+// 	opts := backy.NewConfigOptions(configFile, backy.cronEnabled())
 // 	opts.InitConfig()
 
 // }

cmd/cron.go

@@ -18,13 +18,14 @@ var (
 func cron(cmd *cobra.Command, args []string) {
 	parseS3Config()
 
-	opts := backy.NewOpts(cfgFile,
+	opts := backy.NewConfigOptions(configFile,
 		backy.EnableCron(),
 		backy.SetLogFile(logFile),
-		backy.SetCmdStdOut(cmdStdOut))
+		backy.EnableCommandStdOut(cmdStdOut),
+		backy.SetHostsConfigFile(hostsConfigFile))
 
 	opts.InitConfig()
-	opts.ReadConfig()
+	opts.ParseConfigurationFile()
 
 	opts.Cron()
 }

cmd/exec.go

@@ -32,8 +32,12 @@ func execute(cmd *cobra.Command, args []string) {
 		logging.ExitWithMSG("Please provide a command to run. Pass --help to see options.", 1, nil)
 	}
 
-	opts := backy.NewOpts(cfgFile, backy.AddCommands(args), backy.SetLogFile(logFile), backy.SetCmdStdOut(cmdStdOut))
+	opts := backy.NewConfigOptions(configFile,
+		backy.AddCommands(args),
+		backy.SetLogFile(logFile),
+		backy.EnableCommandStdOut(cmdStdOut),
+		backy.SetHostsConfigFile(hostsConfigFile))
 	opts.InitConfig()
-	opts.ReadConfig()
+	opts.ParseConfigurationFile()
 	opts.ExecuteCmds()
 }

cmd/host.go

@@ -35,10 +35,13 @@ func init() {
 //    2. stdin (on command line) (TODO)
 
 func Host(cmd *cobra.Command, args []string) {
-	backyConfOpts := backy.NewOpts(cfgFile, backy.SetLogFile(logFile), backy.SetCmdStdOut(cmdStdOut))
+	backyConfOpts := backy.NewConfigOptions(configFile,
+		backy.SetLogFile(logFile),
+		backy.EnableCommandStdOut(cmdStdOut),
+		backy.SetHostsConfigFile(hostsConfigFile))
 	backyConfOpts.InitConfig()
 
-	backyConfOpts.ReadConfig()
+	backyConfOpts.ParseConfigurationFile()
 
 	// check CLI input
 	if hostsList == nil {
@@ -46,14 +49,20 @@ func Host(cmd *cobra.Command, args []string) {
 	}
 
 	for _, h := range hostsList {
+		if backy.IsHostLocal(h) {
+			continue
+		}
 		// check if h exists in the config file
 		_, hostFound := backyConfOpts.Hosts[h]
 		if !hostFound {
 			// check if h exists in the SSH config file
-			hostFoundInConfig, s := backy.CheckIfHostHasHostName(h)
+			hostFoundInConfig, s := backy.DoesHostHaveHostName(h)
 			if !hostFoundInConfig {
 				logging.ExitWithMSG("host "+h+" not found", 1, &backyConfOpts.Logger)
 			}
+			if backyConfOpts.Hosts == nil {
+				backyConfOpts.Hosts = make(map[string]*backy.Host)
+			}
 			// create host with hostname and host
 			backyConfOpts.Hosts[h] = &backy.Host{Host: h, HostName: s}
 		}
@@ -68,5 +77,5 @@ func Host(cmd *cobra.Command, args []string) {
 		}
 	}
 
-	backyConfOpts.ExecCmdsSSH(cmdList, hostsList)
+	backyConfOpts.ExecCmdsOnHosts(cmdList, hostsList)
 }

cmd/list.go

@@ -22,13 +22,13 @@ var (
 		Use:   "cmds [cmd1 cmd2 cmd3...]",
 		Short: "List commands defined in config file.",
 		Long:  "List commands defined in config file",
-		Run:   ListCmds,
+		Run:   ListCommands,
 	}
 	listCmdLists = &cobra.Command{
 		Use:   "lists [list1 list2 ...]",
 		Short: "List lists defined in config file.",
 		Long:  "List lists defined in config file",
-		Run:   ListCmdLists,
+		Run:   ListCommandLists,
 	}
 )
 
@@ -40,7 +40,7 @@ func init() {
 
 }
 
-func ListCmds(cmd *cobra.Command, args []string) {
+func ListCommands(cmd *cobra.Command, args []string) {
 
 	// setup based on whats passed in:
 	//   - cmds
@@ -54,17 +54,19 @@ func ListCmds(cmd *cobra.Command, args []string) {
 
 	parseS3Config()
 
-	opts := backy.NewOpts(cfgFile, backy.SetLogFile(logFile))
+	opts := backy.NewConfigOptions(configFile,
+		backy.SetLogFile(logFile),
+		backy.SetHostsConfigFile(hostsConfigFile))
 
 	opts.InitConfig()
-	opts.ReadConfig()
+	opts.ParseConfigurationFile()
 
 	for _, v := range cmdsToList {
 		opts.ListCommand(v)
 	}
 }
 
-func ListCmdLists(cmd *cobra.Command, args []string) {
+func ListCommandLists(cmd *cobra.Command, args []string) {
 
 	parseS3Config()
 
@@ -74,10 +76,12 @@ func ListCmdLists(cmd *cobra.Command, args []string) {
 		logging.ExitWithMSG("Error: lists subcommand needs lists", 1, nil)
 	}
 
-	opts := backy.NewOpts(cfgFile, backy.SetLogFile(logFile))
+	opts := backy.NewConfigOptions(configFile,
+		backy.SetLogFile(logFile),
+		backy.SetHostsConfigFile(hostsConfigFile))
 
 	opts.InitConfig()
-	opts.ReadConfig()
+	opts.ParseConfigurationFile()
 
 	for _, v := range listsToList {
 		opts.ListCommandList(v)

cmd/root.go

@@ -13,11 +13,12 @@ import (
 
 var (
 	// Used for flags.
-	cfgFile    string
+	configFile      string
-	verbose    bool
+	hostsConfigFile string
-	cmdStdOut  bool
+	verbose         bool
-	logFile    string
+	cmdStdOut       bool
-	s3Endpoint string
+	logFile         string
+	s3Endpoint      string
 
 	rootCmd = &cobra.Command{
 		Use:   "backy",
@@ -38,7 +39,8 @@ func init() {
 	rootCmd.PersistentFlags().StringVar(&logFile, "log-file", "", "log file to write to")
 	rootCmd.PersistentFlags().BoolVar(&cmdStdOut, "cmdStdOut", false, "Pass to print command output to stdout")
 
-	rootCmd.PersistentFlags().StringVarP(&cfgFile, "config", "f", "", "config file to read from")
+	rootCmd.PersistentFlags().StringVarP(&configFile, "config", "f", "", "config file to read from")
+	rootCmd.PersistentFlags().StringVar(&hostsConfigFile, "hostsConfig", "", "yaml hosts file to read from")
 	rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "Sets verbose level")
 	rootCmd.PersistentFlags().StringVar(&s3Endpoint, "s3-endpoint", "", "Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable.")
 	rootCmd.AddCommand(backupCmd, execCmd, cronCmd, versionCmd, listCmd)

cmd/version.go

@@ -7,7 +7,7 @@ import (
 	"github.com/spf13/cobra"
 )
 
-const versionStr = "0.9.1"
+const versionStr = "0.10.2"
 
 var (
 	versionCmd = &cobra.Command{

docs/content/cli/exec.md

@@ -15,5 +15,5 @@ The `exec` subcommand can do some things that the configuration file can't do ye
 The commands have to be defined in the config file. The hosts need to at least be in the ssh_config(5) file.
 
 ```sh
-backy exec host [--commands=command1 -commands=command2 ... | -c command1 -c command2 ...] [--hosts=host1 --hosts=hosts2 ... | -m host1 -c host2 ...]  [flags]
+backy exec host [--commands=command1 -commands=command2 ... | -c command1 -c command2 ...] [--hosts=host1 --hosts=hosts2 ... | -m host1 -m host2 ...]  [flags]
 ```

docs/content/cli/list.md

@@ -0,0 +1,29 @@
+---
+title: List
+---
+
+
+List commands, lists, or hosts defined in config file
+
+Usage:
+```
+  backy list [command]
+```
+
+Available Commands:
+  cmds        List commands defined in config file.
+  lists       List lists defined in config file.
+
+Flags:
+```
+  -h, --help   help for list
+```
+
+Global Flags:
+```
+      --cmdStdOut            Pass to print command output to stdout
+  -f, --config string        config file to read from
+      --log-file string      log file to write to
+      --s3-endpoint string   Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable.
+  -v, --verbose              Sets verbose level
+```

docs/content/config/command-lists.md

@@ -35,12 +35,12 @@ If a remote config file is specified (on the command-line using `-f`) and the li
 ```
 
 | key | description | type | required
-| --- | --- | --- | --- |
+| --- | --- | --- | ---
-| `order` | Defines the sequence of commands to execute | `[]string` | yes |
+| `order` | Defines the sequence of commands to execute | `[]string` | yes
-| `getOutput` | Command(s) output is in the notification(s) | `bool` | no |
+| `sendNotificationOnSuccess` | Whether to send notification on list success with the commands' output | `bool` | no
-| `notifications` | The notification service(s) and ID(s) to use on success and failure. Must be *`service.id`*. See the [notifications documentation page](/config/notifications/) for more | `[]string` | no |
+| `notifications` | The notification service(s) and ID(s) to use on success and failure. Must be *`service.id`*. See the [notifications documentation page](/config/notifications/) for more | `[]string` | no
-| `name` | Optional name of the list | `string` | no |
+| `name` | Optional name of the list | `string` | no
-| `cron` | Time at which to schedule the list. Only has affect when cron subcommand is run. | `string` | no |
+| `cron` | Time at which to schedule the list. Only has affect when cron subcommand is run. | `string` | no
 
 ### Order
 

docs/content/config/commands/_index.md

@@ -8,46 +8,21 @@ weight: 1
 
 ### Example Config
 
-```yaml
+{{% code file="/examples/example.yml" language="yaml" %}}
-commands:
-  stop-docker-container:
-    cmd: docker
-    Args:
-      - compose
-      - -f /some/path/to/docker-compose.yaml
-      - down
-    # if host is not defined, command will be run locally
-    # The host has to be defined in either the config file or the SSH Config files
-    host: some-host
-    hooks
-      error:
-        - some-other-command-when-failing
-      success:
-        - success-command
-      final:
-        - final-command
-  backup-docker-container-script:
-    cmd: /path/to/local/script
-    # script file is input as stdin to SSH
-    type: scriptFile # also can be script
-    environment:
-      - FOO=BAR
-      - APP=$VAR
-```
 
 Values available for this section **(case-sensitive)**:
 
-| name | notes | type | required
+| name            | notes                                                                                                   | type                  | required | External directive support |
-| --- | --- | --- | --- |
+| ----------------| ------------------------------------------------------------------------------------------------------- | --------------------- | -------- |----------------------------|
-| `cmd` | Defines the command to execute | `string` | yes |
+| `cmd`           | Defines the command to execute                                                                          | `string`              | yes      | No                         |
-| `Args` | Defines the arguments to the command | `[]string` | no |
+| `Args`          | Defines the arguments to the command                                                                    | `[]string`            | no       | No                         |
-| `environment` | Defines environment variables for the command | `[]string` | no |
+| `environment`   | Defines environment variables for the command                                                           | `[]string`            | no       | Partial                    |
-| `type` | See documentation further down the page. Additional fields may be required. | `string` | no |
+| `type`          | See documentation further down the page. Additional fields may be required.                             | `string`              | no       | No                         |
-| `getOutput` | Command(s) output is in the notification(s) | `bool` | no |
+| `getOutput`     | Command(s) output is in the notification(s)                                                             | `bool`                | no       | No                         |
-| `host` | If not specified, the command will execute locally. | `string` | no |
+| `host`          | If not specified, the command will execute locally.                                                     | `string`              | no       | No                         |
-| `scriptEnvFile` | When type is `scriptFile` or `script`, this file is prepended to the input. | `string` | no |
+| `scriptEnvFile` | When type is `scriptFile` or `script`, this file is prepended to the input.                             | `string`              | no       | No                         |
-| `shell` | Run the command in the shell | `string` | no |
+| `shell`         | Run the command in the shell                                                                            | `string`              | no       | No                         |
-| `hooks` | Hooks are used at the end of the individual command. Must have at least `error`, `success`, or `final`. | `map[string][]string` | no |
+| `hooks`         | Hooks are used at the end of the individual command. Must have at least `error`, `success`, or `final`. | `map[string][]string` | no       | No                         |
 
 #### cmd
 
@@ -120,8 +95,9 @@ The following options are available:
 The environment variables support expansion:
 
 - using escaped values `$VAR` or `${VAR}`
+- using any external directive, and if using the env directive, the variable will be read from a `.env` file
 
-For now, the variables have to be defined in an `.env` file in the same directory that the program is run from.
+<!-- For now, the variables expanded have to be defined in an `.env` file in the same directory that the program is run from. -->
 
 If using it with host specified, the SSH server has to be configured to accept those env variables.
 

docs/content/config/commands/user-commands.md

@@ -6,14 +6,18 @@ description: This is dedicated to user commands.
 
 This is dedicated to `user` commands. The command `type` field must be `user`. User is a type that allows one to perform user operations. There are several additional options available when `type` is `user`:
 
-| name | notes | type | required |
+| name            | notes                                                        | type       | required | External directive support
-| --- | --- | --- | --- |
+| ----------------| -------------------------------------------------------------| ---------- | ---------| --------------------------|
-| `userName` | The name of a user to be configured. | `string` | yes |
+| `userName`      | The name of a user to be configured.                         | `string`   | yes      | no 						 |
-| `userOperation` | The type of operation to perform. | `string` | yes |
+| `userOperation` | The type of operation to perform.                            | `string`   | yes      | no 						 |
-| `userID` | The user ID to use. | `string` | yes |
+| `userID`        | The user ID to use.                                          | `string`   | no       | no 						 |
-| `userGroups` | The groups the user should be added to. | `[]string` | yes |
+| `userGroups`    | The groups the user should be added to.                      | `[]string` | no       | no 						 |
-| `userShell` | The shell for the user. | `string` | yes |
+| `systemUser`    | Create a system user.                                        | `bool`     | no       | no 						 |
-| `userHome` | The user's home directory. | `string` | no |
+| `userCreateHome`| Create the home directory.                                   | `bool`     | no       | no 						 |
+| `userSshPubKeys`| The keys to add to the user's authorized keys.               | `[]string` | no       | yes 						 |
+| `userShell`     | The shell for the user.                                      | `string`   | no       | no 						 |
+| `userHome`      | The user's home directory.                                   | `string`   | no       | no 						 |
+| `userPassword`  | The new password value when using the `password` operation.  | `string`   | no       | yes						 |
 
 
 #### example

docs/content/config/directives.md

@@ -0,0 +1,15 @@
+---
+title: "External Directives"
+weight: 2
+description: How to set up external directives.
+---
+
+External directives are for including data that should not be in the config file. The following directives are supported:
+
+- `%{file:path/to/file}%`
+- `%{env:ENV_VAR}%`
+- `%{vault:vault-key}%`
+
+See the docs of each command if the field is supported.
+
+If the file path does not begin with a `/`, the config file's directory will be used as the starting point.

docs/content/config/hosts.md

@@ -5,20 +5,20 @@ description: >
   This page tells you how to use hosts.
 ---
 
-| Key                  | Description                                                   | Type     | Required |
+| Key                  | Description                                                   | Type     | Required | External directive support |
-|----------------------|---------------------------------------------------------------|----------|----------|
+|----------------------|---------------------------------------------------------------|----------|----------|----------------------------|
-| `OS`                 | Operating system of the host (used for package commands)      | `string` | no       |
+| `OS`                 | Operating system of the host (used for package commands)      | `string` | no       | No                         |
-| `config`             | Path to the SSH config file                                   | `string` | no       |
+| `config`             | Path to the SSH config file                                   | `string` | no       | No                         |
-| `host`               | Specifies the `Host` ssh_config(5) directive                  | `string` | yes      |
+| `host`               | Specifies the `Host` ssh_config(5) directive                  | `string` | yes      | No                         |
-| `hostname`           | Hostname of the host                                          | `string` | no       |
+| `hostname`           | Hostname of the host                                          | `string` | no       | No                         |
-| `knownhostsfile`     | Path to the known hosts file                                  | `string` | no       |
+| `knownHostsFile`     | Path to the known hosts file                                  | `string` | no       | No                         |
-| `port`               | Port number to connect to                                     | `uint16` | no       |
+| `port`               | Port number to connect to                                     | `uint16` | no       | No                         |
-| `proxyjump`          | Proxy jump hosts, comma-separated                             | `string` | no       |
+| `proxyjump`          | Proxy jump hosts, comma-separated                             | `string` | no       | No                         |
-| `password`           | Password for SSH authentication                               | `string` | no       |
+| `password`           | Password for SSH authentication                               | `string` | no       | No                         |
-| `privatekeypath`     | Path to the private key file                                  | `string` | no       |
+| `privateKeyPath`     | Path to the private key file                                  | `string` | no       | No                         |
-| `privatekeypassword` | Password for the private key file                             | `string` | no       |
+| `privateKeyPassword` | Password for the private key file                             | `string` | no       | Yes                        |
-| `user`               | Username for SSH authentication                               | `string` | no       |
+| `user`               | Username for SSH authentication                               | `string` | no       | No                         |
 
 ## exec host subcommand
 
-Backy has a subcommand `exec host`. This subcommand takes the flags of `-m host1 -m host2`. For now these hosts need to be defined in the config file.
+Backy has a subcommand `exec host`. This subcommand takes the flags of `-m host1 -m host2`. For now these hosts need to be defined in the config file.

docs/content/config/notifications.md

@@ -39,23 +39,23 @@ There must be a section with an id (eg. `mail.test-svr`) following one of these
 
 ### mail
 
-| key | description | type
+| key | description | type | External directive support |
-| --- | --- | ---
+| --- | --- | --- | --- |
-| `host` | Specifies the SMTP host to connect to | `string`
+| `host` | Specifies the SMTP host to connect to | `string` | no
-| `port` | Specifies the SMTP port | `uint16`
+| `port` | Specifies the SMTP port | `uint16` | no
-| `senderaddress` | Address from which to send mail | `string`
+| `senderaddress` | Address from which to send mail | `string` | no
-| `to` | Recipients to send emails to | `[]string`
+| `to` | Recipients to send emails to | `[]string` | no
-| `username` | SMTP username | `string`
+| `username` | SMTP username | `string` | no
-| `password` | SMTP password | `string`
+| `password` | SMTP password | `string` | yes
 
 ### matrix
 
-| key | description | type
+| key | description | type | External directive support |
-| --- | --- | ---
+| --- | --- | ---| ---- |
-| `home-server` | Specifies the Matrix server connect to | `string`
+| `home-server` | Specifies the Matrix server connect to | `string` | no
-| `room-id` | Specifies the room ID of the room to send messages to | `string`
+| `room-id` | Specifies the room ID of the room to send messages to | `string` | no
-| `access-token` | Matrix access token | `string`
+| `access-token` | Matrix access token  | `string` | yes
-| `user-id` | Matrix user ID | `string`
+| `user-id` | Matrix user ID | `string` | no
 
 To get your access token (assumes you are using [Element](https://element.io/)) :
 

docs/content/config/vault.md

@@ -6,7 +6,7 @@ description: Set up and configure vault.
 
 [Vault](https://www.vaultproject.io/) is a tool for storing secrets and other data securely.
 
-Vault config can be used by prefixing `vault:` in front of a password or ENV var.
+A Vault key can be used by prefixing `%{vault:vault.keys.name}%` in a field that supports external directives.
 
 This is the object in the config file:
 
@@ -18,10 +18,12 @@ vault:
   keys:
     - name: mongourl
       mountpath: secret
+      key: data
       path: mongo/url
-      type:  # KVv1 or KVv2
+      type: KVv2  # KVv1 or KVv2
-    - name:
+    - name: someKeyName
-      path:
+      mountpath: secret
-      type:
+      key: keyData
-      mountpath:
+      type: KVv2
+      path: some/path
 ```

docs/content/examples/backy.yaml

@@ -0,0 +1,108 @@
+commands:
+  stop-docker-container:
+    cmd: docker
+    Args:
+      - compose
+      - -f /some/path/to/docker-compose.yaml
+      - down
+    # if host is not defined, cmd will be run locally
+    host: some-host
+    hooks:
+      final:
+        - hostname
+      error:
+        - hostname
+  backup-docker-container-script:
+    cmd: /path/to/script
+    # The host has to be defined in the config file
+    host: some-host
+    environment:
+      - FOO=BAR
+      - APP=$VAR
+  shell-cmd:
+    cmd: rsync
+    shell: bash
+    Args:
+      - -av some-host:/path/to/data ~/Docker/Backups/docker-data
+  hostname:
+    cmd: hostname
+  update-docker:
+    type: package
+    shell: zsh # best to run package commands in a shell
+    packageName: docker-ce
+    Args:
+      - docker-ce-cli
+    packageManager: apt
+    packageOperation: install
+  update-dockerApt:
+    # type: package
+    shell: zsh
+    cmd: apt
+    Args:
+      - update
+      - "&&"
+      - apt install -y docker-ce
+      - docker-ce-cli
+    packageManager: apt
+    packageOperation: install
+
+cmd-lists:
+  cmds-to-run: # this can be any name you want
+    # all commands have to be defined
+    order:
+      - stop-docker-container
+      - backup-docker-container-script
+      - shell-cmd
+      - hostname
+    notifications:
+      - matrix.matrix
+    name: backup-some-server
+    cron: "0 0 1 * * *"
+  hostname:
+    name: hostname
+    order:
+      - hostname
+    notifications:
+      - mail.prod-email
+
+hosts:
+  # any ssh_config(5) keys/values not listed here will be looked up in the config file or the default config file
+  some-host:
+    hostname: some-hostname
+    config: ~/.ssh/config
+    user: user
+    privateKeyPath: /path/to/private/key
+    port: 22
+    # can also be env:VAR
+    password: file:/path/to/file
+    # only one is supported for now
+    proxyjump: some-proxy-host
+
+# optional
+logging:
+  verbose: true
+  file: ./backy.log
+  console: false
+  cmd-std-out: false
+
+
+notifications:
+  mail:
+    prod-email:
+      id: prod-email
+      type: mail
+      host: yourhost.tld
+      port: 587
+      senderAddress: email@domain.tld
+      to:
+        - admin@domain.tld
+      username: smtp-username@domain.tld
+      password: your-password-here
+  matrix:
+    matrix:
+      id: matrix
+      type: matrix
+      home-server: your-home-server.tld
+      room-id: room-id
+      access-token: your-access-token
+      user-id: your-user-id

docs/content/examples/example.yml

@@ -0,0 +1,24 @@
+commands:
+  stop-docker-container:
+    cmd: docker
+    Args:
+      - compose
+      - -f /some/path/to/docker-compose.yaml
+      - down
+    # if host is not defined, command will be run locally
+    # The host has to be defined in either the config file or the SSH Config files
+    host: some-host
+    hooks:
+      error:
+        - some-other-command-when-failing
+      success:
+        - success-command
+      final:
+        - final-command
+  backup-docker-container-script:
+    cmd: /path/to/local/script
+    # script file is input as stdin to SSH
+    type: scriptFile # also can be script
+    environment:
+      - FOO=BAR
+      - APP=$VAR

docs/content/getting-started/config.md

@@ -130,7 +130,7 @@ If logfile is not defined, the log file will be written to the config directory
 
 `console-disabled` controls whether the logging messages are echoed to StdOut. Default is false.
 
-`verbose` basically does nothing as all necessary info is already output.
+`verbose` prints out debugging messages.
 
 ```yaml
 logging:
@@ -144,7 +144,7 @@ logging:
 
 [Vault](https://www.vaultproject.io/) can be used to get some configuration values and ENV variables securely.
 
-```
+```yaml
 vault:
   token: hvs.tXqcASvTP8wg92f7riyvGyuf
   address: http://127.0.0.1:8200

docs/layouts/shortcodes/code.html

@@ -0,0 +1,3 @@
+{{ $file := .Get "file" | readFile }}
+{{ $lang := .Get "language" }}
+{{ (print "```" $lang "\n" $file "\n```") }}

examples/example.yml

@@ -0,0 +1,24 @@
+commands:
+  stop-docker-container:
+    cmd: docker
+    Args:
+      - compose
+      - -f /some/path/to/docker-compose.yaml
+      - down
+    # if host is not defined, command will be run locally
+    # The host has to be defined in either the config file or the SSH Config files
+    host: some-host
+    hooks:
+      error:
+        - some-other-command-when-failing
+      success:
+        - success-command
+      final:
+        - final-command
+  backup-docker-container-script:
+    cmd: /path/to/local/script
+    # script file is input as stdin to SSH
+    type: scriptFile # also can be script
+    environment:
+      - FOO=BAR
+      - APP=$VAR

go.mod

@@ -1,15 +1,14 @@
 module git.andrewnw.xyz/CyberShell/backy
 
-go 1.23
+go 1.23.0
 
-toolchain go1.23.6
+toolchain go1.23.7
-
-replace git.andrewnw.xyz/CyberShell/backy => /home/andrew/Projects/backy
 
 require (
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.76.0
 	github.com/dmarkham/enumer v1.5.11
 	github.com/go-co-op/gocron v1.37.0
+	github.com/google/uuid v1.6.0
 	github.com/hashicorp/vault/api v1.15.0
 	github.com/joho/godotenv v1.5.1
 	github.com/kevinburke/ssh_config v1.2.0
@@ -21,10 +20,13 @@ require (
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/nikoksr/notify v1.3.0
 	github.com/pkg/errors v0.9.1
+	github.com/pkg/sftp v1.13.7
 	github.com/rs/zerolog v1.33.0
 	github.com/sethvargo/go-password v0.3.1
 	github.com/spf13/cobra v1.8.1
-	golang.org/x/crypto v0.33.0
+	github.com/stretchr/testify v1.10.0
+	github.com/testcontainers/testcontainers-go v0.37.0
+	golang.org/x/crypto v0.38.0
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1
 	gopkg.in/yaml.v3 v3.0.1
 	maunium.net/go/mautrix v0.23.0
@@ -32,7 +34,10 @@ require (
 )
 
 require (
+	dario.cat/mergo v1.0.1 // indirect
 	filippo.io/edwards25519 v1.1.0 // indirect
+	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
+	github.com/Microsoft/go-winio v0.6.2 // indirect
 	github.com/aws/aws-sdk-go-v2 v1.36.1 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.32 // indirect
@@ -44,13 +49,25 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.13 // indirect
 	github.com/aws/smithy-go v1.22.2 // indirect
 	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
+	github.com/containerd/log v0.1.0 // indirect
+	github.com/containerd/platforms v0.2.1 // indirect
+	github.com/cpuguy83/dockercfg v0.3.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/distribution/reference v0.6.0 // indirect
+	github.com/docker/docker v28.0.1+incompatible // indirect
+	github.com/docker/go-connections v0.5.0 // indirect
+	github.com/docker/go-units v0.5.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/ebitengine/purego v0.8.2 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/go-ini/ini v1.67.0 // indirect
 	github.com/go-jose/go-jose/v4 v4.0.1 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/go-viper/mapstructure/v2 v2.2.1 // indirect
 	github.com/goccy/go-json v0.10.5 // indirect
-	github.com/google/uuid v1.6.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
@@ -65,31 +82,57 @@ require (
 	github.com/klauspost/compress v1.17.11 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.9 // indirect
 	github.com/knadh/koanf/maps v0.1.1 // indirect
+	github.com/kr/fs v0.1.0 // indirect
+	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
+	github.com/magiconair/properties v1.8.10 // indirect
 	github.com/mattn/go-colorable v0.1.14 // indirect
 	github.com/minio/md5-simd v1.1.2 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
+	github.com/moby/docker-image-spec v1.3.1 // indirect
+	github.com/moby/patternmatcher v0.6.0 // indirect
+	github.com/moby/sys/sequential v0.5.0 // indirect
+	github.com/moby/sys/user v0.1.0 // indirect
+	github.com/moby/sys/userns v0.1.0 // indirect
+	github.com/moby/term v0.5.0 // indirect
+	github.com/morikuni/aec v1.0.0 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.1.1 // indirect
 	github.com/pascaldekloe/name v1.0.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
 	github.com/robfig/cron/v3 v3.0.1 // indirect
 	github.com/rs/xid v1.6.0 // indirect
 	github.com/ryanuber/go-glob v1.0.0 // indirect
+	github.com/shirou/gopsutil/v4 v4.25.1 // indirect
+	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/spf13/pflag v1.0.6 // indirect
 	github.com/stretchr/objx v0.5.2 // indirect
-	github.com/stretchr/testify v1.10.0 // indirect
 	github.com/tidwall/gjson v1.18.0 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.1 // indirect
 	github.com/tidwall/sjson v1.2.5 // indirect
+	github.com/tklauser/go-sysconf v0.3.12 // indirect
+	github.com/tklauser/numcpus v0.6.1 // indirect
+	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	go.mau.fi/util v0.8.4 // indirect
+	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.57.0 // indirect
+	go.opentelemetry.io/otel v1.36.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0 // indirect
+	go.opentelemetry.io/otel/metric v1.36.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.36.0 // indirect
+	go.opentelemetry.io/otel/trace v1.36.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.7.0 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
 	golang.org/x/exp v0.0.0-20250207012021-f9890c6ad9f3 // indirect
 	golang.org/x/mod v0.23.0 // indirect
-	golang.org/x/net v0.35.0 // indirect
+	golang.org/x/net v0.40.0 // indirect
-	golang.org/x/sync v0.11.0 // indirect
+	golang.org/x/sync v0.14.0 // indirect
-	golang.org/x/sys v0.30.0 // indirect
+	golang.org/x/sys v0.33.0 // indirect
-	golang.org/x/text v0.22.0 // indirect
+	golang.org/x/text v0.25.0 // indirect
 	golang.org/x/time v0.10.0 // indirect
 	golang.org/x/tools v0.30.0 // indirect
+	google.golang.org/protobuf v1.36.6 // indirect
 )

go.sum

@@ -1,5 +1,13 @@
+dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s=
+dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
 filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
 filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
+github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 h1:bvDV9vkmnHYOMsOr4WLk+Vo07yKIzd94sVoIqshQ4bU=
+github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8=
+github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
+github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
+github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
+github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
 github.com/aws/aws-sdk-go-v2 v1.36.1 h1:iTDl5U6oAhkNPba0e1t1hrwAo02ZMqbrGq4k5JBWM5E=
 github.com/aws/aws-sdk-go-v2 v1.36.1/go.mod h1:5PMILGVKiW32oDzjj6RU52yrNrDPUHcbZQYr1sM7qmM=
 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8 h1:zAxi9p3wsZMIaVCdoiQp2uZ9k1LsZvmAnoTBeZPXom0=
@@ -24,24 +32,51 @@ github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ=
 github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
 github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
 github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
+github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
+github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
+github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A=
+github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw=
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
+github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA=
+github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc=
 github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/creack/pty v1.1.23 h1:4M6+isWdcStXEf15G/RbrMPOQj1dZ7HPZCGwE4kOeP0=
+github.com/creack/pty v1.1.23/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
+github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
 github.com/dmarkham/enumer v1.5.11 h1:quorLCaEfzjJ23Pf7PB9lyyaHseh91YfTM/sAD/4Mbo=
 github.com/dmarkham/enumer v1.5.11/go.mod h1:yixql+kDDQRYqcuBM2n9Vlt7NoT9ixgXhaXry8vmRg8=
+github.com/docker/docker v28.0.1+incompatible h1:FCHjSRdXhNRFjlHMTv4jUNlIBbTeRjrWfeFuJp7jpo0=
+github.com/docker/docker v28.0.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
+github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
+github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
+github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/ebitengine/purego v0.8.2 h1:jPPGWs2sZ1UgOSgD2bClL0MJIqu58nOmIcBuXr62z1I=
+github.com/ebitengine/purego v0.8.2/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
 github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
 github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
+github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
+github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/go-co-op/gocron v1.37.0 h1:ZYDJGtQ4OMhTLKOKMIch+/CY70Brbb1dGdooLEhh7b0=
 github.com/go-co-op/gocron v1.37.0/go.mod h1:3L/n6BkO7ABj+TrfSVXLRzsP26zmikL4ISkLQ0O8iNY=
 github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
 github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
 github.com/go-jose/go-jose/v4 v4.0.1 h1:QVEPDE3OluqXBQZDcnNvQrInro2h0e4eqNbnZSWqS6U=
 github.com/go-jose/go-jose/v4 v4.0.1/go.mod h1:WVf9LFMHh/QVrmqrOfqun0C45tMe3RoiKJMPvgWwLfY=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
+github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
+github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
 github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI=
 github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow=
 github.com/go-test/deep v1.0.2 h1:onZX1rnHT3Wv6cqNgYyFOOlgVKJrksuCMCRvJStbMYw=
@@ -51,11 +86,16 @@ github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlnd
 github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
 github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
-github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
-github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3 h1:5ZPtiqj0JL5oKWmcsq4VMaAW5ukBEgSGXEN89zeH1Jo=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.26.3/go.mod h1:ndYquD05frm2vACXE1nsccT4oJzjhw2arTS2cpUD1PI=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
 github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
@@ -87,6 +127,8 @@ github.com/jordan-wright/email v4.0.1-0.20210109023952-943e75fe5223+incompatible
 github.com/jordan-wright/email v4.0.1-0.20210109023952-943e75fe5223+incompatible/go.mod h1:1c7szIrayyPPB/987hsnvNzLushdWf4o/79s3P08L8A=
 github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=
 github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
 github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
 github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
@@ -100,6 +142,8 @@ github.com/knadh/koanf/providers/rawbytes v0.1.0 h1:dpzgu2KO6uf6oCb4aP05KDmKmAmI
 github.com/knadh/koanf/providers/rawbytes v0.1.0/go.mod h1:mMTB1/IcJ/yE++A2iEZbY1MLygX7vttU+C+S/YmPu9c=
 github.com/knadh/koanf/v2 v2.1.2 h1:I2rtLRqXRy1p01m/utEtpZSSA6dcJbgGVuE27kW2PzQ=
 github.com/knadh/koanf/v2 v2.1.2/go.mod h1:Gphfaen0q1Fc1HTgJgSTC4oRX9R2R5ErYMZJy8fLJBo=
+github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
+github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
@@ -109,6 +153,10 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
+github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
+github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE=
+github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
 github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
@@ -128,15 +176,37 @@ github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyua
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
 github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
+github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
+github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
+github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk=
+github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc=
+github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc=
+github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo=
+github.com/moby/sys/user v0.1.0 h1:WmZ93f5Ux6het5iituh9x2zAG7NFY9Aqi49jjE1PaQg=
+github.com/moby/sys/user v0.1.0/go.mod h1:fKJhFOnsCN6xZ5gSfbM6zaHGgDJMrqt9/reuj4T7MmU=
+github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g=
+github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28=
+github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
+github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
+github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
+github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
 github.com/nikoksr/notify v1.3.0 h1:UxzfxzAYGQD9a5JYLBTVx0lFMxeHCke3rPCkfWdPgLs=
 github.com/nikoksr/notify v1.3.0/go.mod h1:Xor2hMmkvrCfkCKvXGbcrESez4brac2zQjhd6U2BbeM=
+github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
+github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
+github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=
+github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M=
 github.com/pascaldekloe/name v1.0.0 h1:n7LKFgHixETzxpRv2R77YgPUFo85QHGZKrdaYm7eY5U=
 github.com/pascaldekloe/name v1.0.0/go.mod h1:Z//MfYJnH4jVpQ9wkclwu2I2MkHmXTlT9wR5UZScttM=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/sftp v1.13.7 h1:uv+I3nNJvlKZIQGSr8JVQLNHFU9YhhNpvC14Y6KgmSM=
+github.com/pkg/sftp v1.13.7/go.mod h1:KMKI0t3T6hfA+lTR/ssZdunHo+uwq7ghoN09/FSu3DY=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw=
+github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
 github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
 github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
 github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
@@ -153,6 +223,10 @@ github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkB
 github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
 github.com/sethvargo/go-password v0.3.1 h1:WqrLTjo7X6AcVYfC6R7GtSyuUQR9hGyAj/f1PYQZCJU=
 github.com/sethvargo/go-password v0.3.1/go.mod h1:rXofC1zT54N7R8K/h1WDUdkf9BOx5OptoxrMBcrXzvs=
+github.com/shirou/gopsutil/v4 v4.25.1 h1:QSWkTc+fu9LTAWfkZwZ6j8MSUk4A2LV7rbH0ZqmLjXs=
+github.com/shirou/gopsutil/v4 v4.25.1/go.mod h1:RoUCUpndaJFtT+2zsZzzmhvbfGoDCJ7nFXKJf8GqJbI=
+github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
+github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
 github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
@@ -164,11 +238,14 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE
 github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
 github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
 github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/testcontainers/testcontainers-go v0.37.0 h1:L2Qc0vkTw2EHWQ08djon0D2uw7Z/PtHS/QzZZ5Ra/hg=
+github.com/testcontainers/testcontainers-go v0.37.0/go.mod h1:QPzbxZhQ6Bclip9igjLFj6z0hs01bU8lrl2dHQmgFGM=
 github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY=
 github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
@@ -179,34 +256,128 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
 github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
 github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
 github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
+github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU=
+github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI=
+github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk=
+github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
+github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
 go.mau.fi/util v0.8.4 h1:mVKlJcXWfVo8ZW3f4vqtjGpqtZqJvX4ETekxawt2vnQ=
 go.mau.fi/util v0.8.4/go.mod h1:MOfGTs1CBuK6ERTcSL4lb5YU7/ujz09eOPVEDckuazY=
+go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
+go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.57.0 h1:DheMAlT6POBP+gh8RUH19EOTnQIor5QE0uSRPtzCpSw=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.57.0/go.mod h1:wZcGmeVO9nzP67aYSLDqXNWK87EZWhi7JWj1v7ZXf94=
+go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg=
+go.opentelemetry.io/otel v1.36.0/go.mod h1:/TcFMXYjyRNh8khOAO9ybYkqaDBb/70aVwkNML4pP8E=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0 h1:dNzwXjZKpMpE2JhmO+9HsPl42NIXFIFSUSSs0fiqra0=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.36.0/go.mod h1:90PoxvaEB5n6AOdZvi+yWJQoE95U8Dhhw2bSyRqnTD0=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU=
+go.opentelemetry.io/otel/metric v1.36.0 h1:MoWPKVhQvJ+eeXWHFBOPoBOi20jh6Iq2CcCREuTYufE=
+go.opentelemetry.io/otel/metric v1.36.0/go.mod h1:zC7Ks+yeyJt4xig9DEw9kuUFe5C3zLbVjV2PzT6qzbs=
+go.opentelemetry.io/otel/sdk v1.36.0 h1:b6SYIuLRs88ztox4EyrvRti80uXIFy+Sqzoh9kFULbs=
+go.opentelemetry.io/otel/sdk v1.36.0/go.mod h1:+lC+mTgD+MUWfjJubi2vvXWcVxyr9rmlshZni72pXeY=
+go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w=
+go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA=
+go.opentelemetry.io/proto/otlp v1.7.0 h1:jX1VolD6nHuFzOYso2E73H85i92Mv8JQYk0K9vz09os=
+go.opentelemetry.io/proto/otlp v1.7.0/go.mod h1:fSKjH6YJ7HDlwzltzyMj036AJ3ejJLCgCSHGj4efDDo=
 go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
 go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
-golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
+golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8=
+golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw=
 golang.org/x/exp v0.0.0-20250207012021-f9890c6ad9f3 h1:qNgPs5exUA+G0C96DrPwNrvLSj7GT/9D+3WMWUcUg34=
 golang.org/x/exp v0.0.0-20250207012021-f9890c6ad9f3/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.23.0 h1:Zb7khfcRGKk+kqfxFaP5tZqCnDZMjC5VtUBs87Hr6QM=
 golang.org/x/mod v0.23.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
-golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY=
+golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.14.0 h1:woo0S4Yywslg6hp4eUFjTVOyKt0RookbpAHG4c1HmhQ=
+golang.org/x/sync v0.14.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
+golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
-golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU=
+golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
-golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
+golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
+golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg=
+golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4=
+golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA=
 golang.org/x/time v0.10.0 h1:3usCWA8tQn0L8+hFJQNgzpWbd89begxN66o1Ojdn5L4=
 golang.org/x/time v0.10.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/tools v0.30.0 h1:BgcpHewrV5AUp2G9MebG4XPFI1E2W41zU1SaqVA9vJY=
 golang.org/x/tools v0.30.0/go.mod h1:c347cR/OJfw5TI+GfX7RUPNMdDRRbjvYTS0jPyvsVtY=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/genproto v0.0.0-20241118233622-e639e219e697 h1:ToEetK57OidYuqD4Q5w+vfEnPvPpuTwedCNVohYJfNk=
+google.golang.org/genproto/googleapis/api v0.0.0-20250528174236-200df99c418a h1:SGktgSolFCo75dnHJF2yMvnns6jCmHFJ0vE4Vn2JKvQ=
+google.golang.org/genproto/googleapis/api v0.0.0-20250528174236-200df99c418a/go.mod h1:a77HrdMjoeKbnd2jmgcWdaS++ZLZAEq3orIOAEIKiVw=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a h1:v2PbRU4K3llS09c7zodFpNePeamkAwG3mPrAery9VeE=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A=
+google.golang.org/grpc v1.72.2 h1:TdbGzwb82ty4OusHWepvFWGLgIbNo1/SUynEN0ssqv8=
+google.golang.org/grpc v1.72.2/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM=
+google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY=
+google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
@@ -217,6 +388,8 @@ gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYs
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU=
+gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU=
 maunium.net/go/mautrix v0.23.0 h1:HNlR19eew5lvrNSL2muhExaGhYdaGk5FfEiA82QqUP4=
 maunium.net/go/mautrix v0.23.0/go.mod h1:AGnnaz3ylGikUo1I1MJVn9QLsl2No1/ZNnGDyO0QD5s=
 mvdan.cc/sh/v3 v3.10.0 h1:v9z7N1DLZ7owyLM/SXZQkBSXcwr2IGMm2LY2pmhVXj4=

pkg/backy/allowedexternaldirectives_enumer.go

@@ -0,0 +1,149 @@
+// Code generated by "enumer -linecomment -yaml -text -json -type=AllowedExternalDirectives"; DO NOT EDIT.
+
+package backy
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+)
+
+const _AllowedExternalDirectivesName = "DefaultExternalDirvaultvault-envvault-filevault-file-envfile-envfileenv"
+
+var _AllowedExternalDirectivesIndex = [...]uint8{0, 18, 23, 32, 42, 56, 64, 68, 71}
+
+const _AllowedExternalDirectivesLowerName = "defaultexternaldirvaultvault-envvault-filevault-file-envfile-envfileenv"
+
+func (i AllowedExternalDirectives) String() string {
+	if i < 0 || i >= AllowedExternalDirectives(len(_AllowedExternalDirectivesIndex)-1) {
+		return fmt.Sprintf("AllowedExternalDirectives(%d)", i)
+	}
+	return _AllowedExternalDirectivesName[_AllowedExternalDirectivesIndex[i]:_AllowedExternalDirectivesIndex[i+1]]
+}
+
+// An "invalid array index" compiler error signifies that the constant values have changed.
+// Re-run the stringer command to generate them again.
+func _AllowedExternalDirectivesNoOp() {
+	var x [1]struct{}
+	_ = x[DefaultExternalDir-(0)]
+	_ = x[AllowedExternalDirectiveVault-(1)]
+	_ = x[AllowedExternalDirectiveVaultEnv-(2)]
+	_ = x[AllowedExternalDirectiveVaultFile-(3)]
+	_ = x[AllowedExternalDirectiveAll-(4)]
+	_ = x[AllowedExternalDirectiveFileEnv-(5)]
+	_ = x[AllowedExternalDirectiveFile-(6)]
+	_ = x[AllowedExternalDirectiveEnv-(7)]
+}
+
+var _AllowedExternalDirectivesValues = []AllowedExternalDirectives{DefaultExternalDir, AllowedExternalDirectiveVault, AllowedExternalDirectiveVaultEnv, AllowedExternalDirectiveVaultFile, AllowedExternalDirectiveAll, AllowedExternalDirectiveFileEnv, AllowedExternalDirectiveFile, AllowedExternalDirectiveEnv}
+
+var _AllowedExternalDirectivesNameToValueMap = map[string]AllowedExternalDirectives{
+	_AllowedExternalDirectivesName[0:18]:       DefaultExternalDir,
+	_AllowedExternalDirectivesLowerName[0:18]:  DefaultExternalDir,
+	_AllowedExternalDirectivesName[18:23]:      AllowedExternalDirectiveVault,
+	_AllowedExternalDirectivesLowerName[18:23]: AllowedExternalDirectiveVault,
+	_AllowedExternalDirectivesName[23:32]:      AllowedExternalDirectiveVaultEnv,
+	_AllowedExternalDirectivesLowerName[23:32]: AllowedExternalDirectiveVaultEnv,
+	_AllowedExternalDirectivesName[32:42]:      AllowedExternalDirectiveVaultFile,
+	_AllowedExternalDirectivesLowerName[32:42]: AllowedExternalDirectiveVaultFile,
+	_AllowedExternalDirectivesName[42:56]:      AllowedExternalDirectiveAll,
+	_AllowedExternalDirectivesLowerName[42:56]: AllowedExternalDirectiveAll,
+	_AllowedExternalDirectivesName[56:64]:      AllowedExternalDirectiveFileEnv,
+	_AllowedExternalDirectivesLowerName[56:64]: AllowedExternalDirectiveFileEnv,
+	_AllowedExternalDirectivesName[64:68]:      AllowedExternalDirectiveFile,
+	_AllowedExternalDirectivesLowerName[64:68]: AllowedExternalDirectiveFile,
+	_AllowedExternalDirectivesName[68:71]:      AllowedExternalDirectiveEnv,
+	_AllowedExternalDirectivesLowerName[68:71]: AllowedExternalDirectiveEnv,
+}
+
+var _AllowedExternalDirectivesNames = []string{
+	_AllowedExternalDirectivesName[0:18],
+	_AllowedExternalDirectivesName[18:23],
+	_AllowedExternalDirectivesName[23:32],
+	_AllowedExternalDirectivesName[32:42],
+	_AllowedExternalDirectivesName[42:56],
+	_AllowedExternalDirectivesName[56:64],
+	_AllowedExternalDirectivesName[64:68],
+	_AllowedExternalDirectivesName[68:71],
+}
+
+// AllowedExternalDirectivesString retrieves an enum value from the enum constants string name.
+// Throws an error if the param is not part of the enum.
+func AllowedExternalDirectivesString(s string) (AllowedExternalDirectives, error) {
+	if val, ok := _AllowedExternalDirectivesNameToValueMap[s]; ok {
+		return val, nil
+	}
+
+	if val, ok := _AllowedExternalDirectivesNameToValueMap[strings.ToLower(s)]; ok {
+		return val, nil
+	}
+	return 0, fmt.Errorf("%s does not belong to AllowedExternalDirectives values", s)
+}
+
+// AllowedExternalDirectivesValues returns all values of the enum
+func AllowedExternalDirectivesValues() []AllowedExternalDirectives {
+	return _AllowedExternalDirectivesValues
+}
+
+// AllowedExternalDirectivesStrings returns a slice of all String values of the enum
+func AllowedExternalDirectivesStrings() []string {
+	strs := make([]string, len(_AllowedExternalDirectivesNames))
+	copy(strs, _AllowedExternalDirectivesNames)
+	return strs
+}
+
+// IsAAllowedExternalDirectives returns "true" if the value is listed in the enum definition. "false" otherwise
+func (i AllowedExternalDirectives) IsAAllowedExternalDirectives() bool {
+	for _, v := range _AllowedExternalDirectivesValues {
+		if i == v {
+			return true
+		}
+	}
+	return false
+}
+
+// MarshalJSON implements the json.Marshaler interface for AllowedExternalDirectives
+func (i AllowedExternalDirectives) MarshalJSON() ([]byte, error) {
+	return json.Marshal(i.String())
+}
+
+// UnmarshalJSON implements the json.Unmarshaler interface for AllowedExternalDirectives
+func (i *AllowedExternalDirectives) UnmarshalJSON(data []byte) error {
+	var s string
+	if err := json.Unmarshal(data, &s); err != nil {
+		return fmt.Errorf("AllowedExternalDirectives should be a string, got %s", data)
+	}
+
+	var err error
+	*i, err = AllowedExternalDirectivesString(s)
+	return err
+}
+
+// MarshalText implements the encoding.TextMarshaler interface for AllowedExternalDirectives
+func (i AllowedExternalDirectives) MarshalText() ([]byte, error) {
+	return []byte(i.String()), nil
+}
+
+// UnmarshalText implements the encoding.TextUnmarshaler interface for AllowedExternalDirectives
+func (i *AllowedExternalDirectives) UnmarshalText(text []byte) error {
+	var err error
+	*i, err = AllowedExternalDirectivesString(string(text))
+	return err
+}
+
+// MarshalYAML implements a YAML Marshaler for AllowedExternalDirectives
+func (i AllowedExternalDirectives) MarshalYAML() (interface{}, error) {
+	return i.String(), nil
+}
+
+// UnmarshalYAML implements a YAML Unmarshaler for AllowedExternalDirectives
+func (i *AllowedExternalDirectives) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var s string
+	if err := unmarshal(&s); err != nil {
+		return err
+	}
+
+	var err error
+	*i, err = AllowedExternalDirectivesString(s)
+	return err
+}

pkg/backy/backy.go

@@ -11,6 +11,7 @@ import (
 	"io"
 	"os"
 	"os/exec"
+	"strings"
 	"text/template"
 
 	"embed"
@@ -34,7 +35,7 @@ var Sprintf = fmt.Sprintf
 func (command *Command) RunCmd(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([]string, error) {
 
 	var (
-		ArgsStr       string // concatenating the arguments
+		ArgsStr       string
 		cmdOutBuf     bytes.Buffer
 		cmdOutWriters io.Writer
 		errSSH        error
@@ -54,26 +55,35 @@ func (command *Command) RunCmd(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([
 		ArgsStr += fmt.Sprintf(" %s", v)
 	}
 
-	if command.Type == UserCT {
+	if command.Type == UserCommandType {
 		if command.UserOperation == "password" {
 			cmdCtxLogger.Info().Str("password", command.UserPassword).Msg("user password to be updated")
 		}
 	}
 
-	if command.Host != nil {
+	if !IsHostLocal(command.Host) {
-		outputArr, errSSH = command.RunCmdSSH(cmdCtxLogger, opts)
+
+		outputArr, errSSH = command.RunCmdOnHost(cmdCtxLogger, opts)
 		if errSSH != nil {
 			return outputArr, errSSH
 		}
 	} else {
 
 		// Handle package operations
-		if command.Type == PackageCT && command.PackageOperation == PackOpCheckVersion {
+		if command.Type == PackageCommandType && command.PackageOperation == PackageOperationCheckVersion {
-			cmdCtxLogger.Info().Str("package", command.PackageName).Msg("Checking package versions")
+			opts.Logger.Info().Msg("")
+			for _, p := range command.Packages {
+				cmdCtxLogger.Info().Str("package", p.Name).Msg("Checking installed and remote package versions")
+			}
+			opts.Logger.Info().Msg("")
 
 			// Execute the package version command
 			cmd := exec.Command(command.Cmd, command.Args...)
 			cmdOutWriters = io.MultiWriter(&cmdOutBuf)
+
+			if IsCmdStdOutEnabled() {
+				cmdOutWriters = io.MultiWriter(os.Stdout, &cmdOutBuf)
+			}
 			cmd.Stdout = cmdOutWriters
 			cmd.Stderr = cmdOutWriters
 
@@ -85,7 +95,8 @@ func (command *Command) RunCmd(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([
 		}
 
 		var localCMD *exec.Cmd
-		if command.Type == RemoteScriptCT {
+
+		if command.Type == RemoteScriptCommandType {
 			script, err := command.Fetcher.Fetch(command.Cmd)
 			if err != nil {
 				return nil, err
@@ -95,15 +106,15 @@ func (command *Command) RunCmd(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([
 				command.Shell = "sh"
 			}
 			localCMD = exec.Command(command.Shell, command.Args...)
-			injectEnvIntoLocalCMD(envVars, localCMD, cmdCtxLogger)
+			injectEnvIntoLocalCMD(envVars, localCMD, cmdCtxLogger, opts)
 
 			cmdOutWriters = io.MultiWriter(&cmdOutBuf)
 
 			if IsCmdStdOutEnabled() {
 				cmdOutWriters = io.MultiWriter(os.Stdout, &cmdOutBuf)
 			}
-			if command.OutputFile != "" {
+			if command.Output.File != "" {
-				file, err := os.Create(command.OutputFile)
+				file, err := os.Create(command.Output.File)
 				if err != nil {
 					return nil, fmt.Errorf("error creating output file: %w", err)
 				}
@@ -136,7 +147,7 @@ func (command *Command) RunCmd(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([
 				if str, ok := outMap["output"].(string); ok {
 					outputArr = append(outputArr, str)
 				}
-				if command.OutputToLog {
+				if command.Output.ToLog {
 					cmdCtxLogger.Info().Fields(outMap).Send()
 				}
 			}
@@ -144,6 +155,7 @@ func (command *Command) RunCmd(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([
 		}
 
 		var err error
+
 		if command.Shell != "" {
 			cmdCtxLogger.Info().Str("Command", fmt.Sprintf("Running command %s on local machine in %s", command.Name, command.Shell)).Send()
 
@@ -156,8 +168,10 @@ func (command *Command) RunCmd(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([
 			cmdCtxLogger.Info().Str("Command", fmt.Sprintf("Running command %s on local machine", command.Name)).Send()
 
 			// execute package commands in a shell
-			if command.Type == PackageCT {
+			if command.Type == PackageCommandType {
-				cmdCtxLogger.Info().Str("package", command.PackageName).Msg("Executing package command")
+				for _, p := range command.Packages {
+					cmdCtxLogger.Info().Str("packages", p.Name).Msg("Executing package command")
+				}
 				ArgsStr = fmt.Sprintf("%s %s", command.Cmd, ArgsStr)
 				localCMD = exec.Command("/bin/sh", "-c", ArgsStr)
 			} else {
@@ -165,11 +179,17 @@ func (command *Command) RunCmd(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([
 			}
 		}
 
+		if command.Type == UserCommandType {
+			if command.UserOperation == "password" {
+				localCMD.Stdin = command.stdin
+				cmdCtxLogger.Info().Str("password", command.UserPassword).Msg("user password to be updated")
+			}
+		}
 		if command.Dir != nil {
 			localCMD.Dir = *command.Dir
 		}
 
-		injectEnvIntoLocalCMD(envVars, localCMD, cmdCtxLogger)
+		injectEnvIntoLocalCMD(envVars, localCMD, cmdCtxLogger, opts)
 
 		cmdOutWriters = io.MultiWriter(&cmdOutBuf)
 
@@ -182,38 +202,84 @@ func (command *Command) RunCmd(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([
 
 		err = localCMD.Run()
 
-		outScanner := bufio.NewScanner(&cmdOutBuf)
+		outputArr = logCommandOutput(command, cmdOutBuf, cmdCtxLogger, outputArr)
-
-		for outScanner.Scan() {
-			outMap := make(map[string]interface{})
-			outMap["cmd"] = command.Cmd
-			outMap["output"] = outScanner.Text()
-
-			if str, ok := outMap["output"].(string); ok {
-				outputArr = append(outputArr, str)
-			}
-			// if command.GetOutput {
-			cmdCtxLogger.Info().Fields(outMap).Send()
-			// }
-		}
 		if err != nil {
 			cmdCtxLogger.Error().Err(fmt.Errorf("error when running cmd %s: %w", command.Name, err)).Send()
 			return outputArr, err
 		}
+
+		if command.Type == UserCommandType {
+
+			if command.UserOperation == "add" {
+				if command.UserSshPubKeys != nil {
+					var (
+						authorizedKeysFile *os.File
+						err                error
+						userHome           []byte
+					)
+
+					cmdCtxLogger.Info().Msg("adding SSH Keys")
+
+					localCMD := exec.Command(fmt.Sprintf("grep \"%s\" /etc/passwd | cut -d: -f6", command.Username))
+					userHome, err = localCMD.CombinedOutput()
+					if err != nil {
+						return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.Output.ToLog), fmt.Errorf("error finding user home from /etc/passwd: %v", err)
+					}
+
+					command.UserHome = strings.TrimSpace(string(userHome))
+					userSshDir := fmt.Sprintf("%s/.ssh", command.UserHome)
+
+					if _, err := os.Stat(userSshDir); os.IsNotExist(err) {
+						err := os.MkdirAll(userSshDir, 0700)
+						if err != nil {
+							return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.Output.ToLog), fmt.Errorf("error creating directory %s %v", userSshDir, err)
+						}
+					}
+
+					if _, err := os.Stat(fmt.Sprintf("%s/authorized_keys", userSshDir)); os.IsNotExist(err) {
+						_, err := os.Create(fmt.Sprintf("%s/authorized_keys", userSshDir))
+						if err != nil {
+							return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.Output.ToLog), fmt.Errorf("error creating file %s/authorized_keys: %v", userSshDir, err)
+						}
+					}
+
+					authorizedKeysFile, err = os.OpenFile(fmt.Sprintf("%s/authorized_keys", userSshDir), 0700, os.ModeAppend)
+					if err != nil {
+						return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.Output.ToLog), fmt.Errorf("error opening file %s/authorized_keys: %v", userSshDir, err)
+					}
+					defer authorizedKeysFile.Close()
+					for _, k := range command.UserSshPubKeys {
+						buf := bytes.NewBufferString(k)
+						cmdCtxLogger.Info().Str("key", k).Msg("adding SSH key")
+						if _, err := authorizedKeysFile.ReadFrom(buf); err != nil {
+							return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.Output.ToLog), fmt.Errorf("error adding to authorized keys: %v", err)
+						}
+					}
+					localCMD = exec.Command(fmt.Sprintf("chown -R %s:%s %s", command.Username, command.Username, userHome))
+					_, err = localCMD.CombinedOutput()
+					if err != nil {
+						return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.Output.ToLog), err
+					}
+
+				}
+			}
+		}
 	}
 	return outputArr, nil
 }
 
-func cmdListWorker(msgTemps *msgTemplates, jobs <-chan *CmdList, results chan<- CmdResult, opts *ConfigOpts) {
+func cmdListWorker(msgTemps *msgTemplates, jobs <-chan *CmdList, results chan<- string, opts *ConfigOpts) {
 	for list := range jobs {
 		fieldsMap := map[string]interface{}{"list": list.Name}
 		var cmdLogger zerolog.Logger
+		var commandExecuted *Command
 		var cmdsRan []string
 		var outStructArr []outStruct
 		var hasError bool // Tracks if any command in the list failed
 
 		for _, cmd := range list.Order {
 			cmdToRun := opts.Cmds[cmd]
+			commandExecuted = cmdToRun
 			currentCmd := cmdToRun.Name
 			fieldsMap["cmd"] = currentCmd
 			cmdLogger = cmdToRun.GenerateLogger(opts)
@@ -224,23 +290,21 @@ func cmdListWorker(msgTemps *msgTemplates, jobs <-chan *CmdList, results chan<-
 
 			if runErr != nil {
 
-				// Log the error and send a failed result
 				cmdLogger.Err(runErr).Send()
-				results <- CmdResult{CmdName: cmd, ListName: list.Name, Error: runErr}
 
-				// Execute error hooks for the failed command
 				cmdToRun.ExecuteHooks("error", opts)
 
 				// Notify failure
 				if list.NotifyConfig != nil {
 					notifyError(cmdLogger, msgTemps, list, cmdsRan, outStructArr, runErr, cmdToRun)
 				}
+
+				// Execute error hooks for the failed command
 				hasError = true
 				break
 			}
 
-			// Collect output if required
+			if list.GetCommandOutputInNotificationsOnSuccess || cmdToRun.Output.InList {
-			if list.GetOutput || cmdToRun.GetOutput {
 				outStructArr = append(outStructArr, outStruct{
 					CmdName:     currentCmd,
 					CmdExecuted: currentCmd,
@@ -249,44 +313,20 @@ func cmdListWorker(msgTemps *msgTemplates, jobs <-chan *CmdList, results chan<-
 			}
 		}
 
-		// Notify success if no errors occurred
+		if !hasError && list.NotifyConfig != nil && list.Notify.OnFailure {
-		if !hasError && list.NotifyConfig != nil && (list.NotifyOnSuccess || list.GetOutput) {
 			notifySuccess(cmdLogger, msgTemps, list, cmdsRan, outStructArr)
 		}
 
-		// Execute success and final hooks for all commands
+		if !hasError {
-		for _, cmd := range list.Order {
+			commandExecuted.ExecuteHooks("success", opts)
-			cmdToRun := opts.Cmds[cmd]
-
-			// Execute success hooks if the command succeeded
-			if !hasError || cmdsRanContains(cmd, cmdsRan) {
-				cmdToRun.ExecuteHooks("success", opts)
-			}
-
-			// Execute final hooks for every command
-			cmdToRun.ExecuteHooks("final", opts)
 		}
 
-		// Send the final result for the list
+		commandExecuted.ExecuteHooks("final", opts)
-		if hasError {
+
-			results <- CmdResult{CmdName: cmdsRan[len(cmdsRan)-1], ListName: list.Name, Error: fmt.Errorf("list execution failed")}
+		results <- "done"
-		} else {
-			results <- CmdResult{CmdName: cmdsRan[len(cmdsRan)-1], ListName: list.Name, Error: nil}
-		}
 	}
 }
 
-// Helper to check if a command is in the list of executed commands
-func cmdsRanContains(cmd string, cmdsRan []string) bool {
-	for _, c := range cmdsRan {
-		if c == cmd {
-			return true
-		}
-	}
-	return false
-}
-
-// Helper to notify errors
 func notifyError(logger zerolog.Logger, templates *msgTemplates, list *CmdList, cmdsRan []string, outStructArr []outStruct, err error, cmd *Command) {
 	errStruct := map[string]interface{}{
 		"listName":  list.Name,
@@ -332,7 +372,7 @@ func (opts *ConfigOpts) RunListConfig(cron string) {
 	}
 	configListsLen := len(opts.CmdConfigLists)
 	listChan := make(chan *CmdList, configListsLen)
-	results := make(chan CmdResult, configListsLen)
+	results := make(chan string, configListsLen)
 
 	// Start workers
 	for w := 1; w <= configListsLen; w++ {
@@ -352,10 +392,7 @@ func (opts *ConfigOpts) RunListConfig(cron string) {
 
 	// Process results
 	for a := 1; a <= configListsLen; a++ {
-		result := <-results
+		<-results
-		opts.Logger.Debug().Msgf("Processing result for list %s, command %s", result.ListName, result.CmdName)
-
-		// Process final hooks for the list (already handled in worker)
 	}
 	opts.closeHostConnections()
 }
@@ -367,10 +404,8 @@ func (opts *ConfigOpts) ExecuteCmds() {
 		_, runErr := cmdToRun.RunCmd(cmdLogger, opts)
 		if runErr != nil {
 			opts.Logger.Err(runErr).Send()
-
 			cmdToRun.ExecuteHooks("error", opts)
 		} else {
-
 			cmdToRun.ExecuteHooks("success", opts)
 		}
 
@@ -378,7 +413,6 @@ func (opts *ConfigOpts) ExecuteCmds() {
 	}
 
 	opts.closeHostConnections()
-
 }
 
 func (c *ConfigOpts) closeHostConnections() {
@@ -426,26 +460,31 @@ func (cmd *Command) ExecuteHooks(hookType string, opts *ConfigOpts) {
 		for _, v := range cmd.Hooks.Error {
 			errCmd := opts.Cmds[v]
 			cmdLogger := opts.Logger.With().
-				Str("backy-cmd", v).
+				Str("backy-cmd", v).Str("hookType", "error").
 				Logger()
-			errCmd.RunCmd(cmdLogger, opts)
+			cmdLogger.Info().Msgf("Running error hook command %s", v)
+			// URGENT: Never returns
+			_, _ = errCmd.RunCmd(cmdLogger, opts)
+			return
 		}
 
 	case "success":
 		for _, v := range cmd.Hooks.Success {
 			successCmd := opts.Cmds[v]
 			cmdLogger := opts.Logger.With().
-				Str("backy-cmd", v).
+				Str("backy-cmd", v).Str("hookType", "success").
 				Logger()
-			successCmd.RunCmd(cmdLogger, opts)
+			cmdLogger.Info().Msgf("Running success hook command %s", v)
+			_, _ = successCmd.RunCmd(cmdLogger, opts)
 		}
 	case "final":
 		for _, v := range cmd.Hooks.Final {
 			finalCmd := opts.Cmds[v]
 			cmdLogger := opts.Logger.With().
-				Str("backy-cmd", v).
+				Str("backy-cmd", v).Str("hookType", "final").
 				Logger()
-			finalCmd.RunCmd(cmdLogger, opts)
+			cmdLogger.Info().Msgf("Running final hook command %s", v)
+			_, _ = finalCmd.RunCmd(cmdLogger, opts)
 		}
 	}
 }
@@ -455,32 +494,59 @@ func (cmd *Command) GenerateLogger(opts *ConfigOpts) zerolog.Logger {
 		Str("Backy-cmd", cmd.Name).Str("Host", "local machine").
 		Logger()
 
-	if cmd.Host != nil {
+	if !IsHostLocal(cmd.Host) {
 		cmdLogger = opts.Logger.With().
-			Str("Backy-cmd", cmd.Name).Str("Host", *cmd.Host).
+			Str("Backy-cmd", cmd.Name).Str("Host", cmd.Host).
 			Logger()
-
 	}
 	return cmdLogger
 }
 
-func (opts *ConfigOpts) ExecCmdsSSH(cmdList []string, hostsList []string) {
+func (opts *ConfigOpts) ExecCmdsOnHosts(cmdList []string, hostsList []string) {
 	// Iterate over hosts and exec commands
 	for _, h := range hostsList {
 		host := opts.Hosts[h]
 		for _, c := range cmdList {
 			cmd := opts.Cmds[c]
 			cmd.RemoteHost = host
-			cmd.Host = &host.Host
+			cmd.Host = h
-			opts.Logger.Info().Str("host", h).Str("cmd", c).Send()
+			if IsHostLocal(h) {
-			_, err := cmd.RunCmdSSH(cmd.GenerateLogger(opts), opts)
+				_, err := cmd.RunCmd(cmd.GenerateLogger(opts), opts)
-			if err != nil {
+				if err != nil {
-				opts.Logger.Err(err).Str("host", h).Str("cmd", c).Send()
+					opts.Logger.Err(err).Str("host", h).Str("cmd", c).Send()
+				}
+			} else {
+
+				cmd.Host = host.Host
+				opts.Logger.Info().Str("host", h).Str("cmd", c).Send()
+				_, err := cmd.RunCmdOnHost(cmd.GenerateLogger(opts), opts)
+				if err != nil {
+					opts.Logger.Err(err).Str("host", h).Str("cmd", c).Send()
+				}
 			}
 		}
 	}
 }
 
+func logCommandOutput(command *Command, cmdOutBuf bytes.Buffer, cmdCtxLogger zerolog.Logger, outputArr []string) []string {
+
+	outScanner := bufio.NewScanner(&cmdOutBuf)
+
+	for outScanner.Scan() {
+		outMap := make(map[string]interface{})
+		outMap["cmd"] = command.Name
+		outMap["output"] = outScanner.Text()
+
+		if str, ok := outMap["output"].(string); ok {
+			outputArr = append(outputArr, str)
+		}
+		if command.Output.ToLog {
+			cmdCtxLogger.Info().Fields(outMap).Send()
+		}
+	}
+	return outputArr
+}
+
 // func executeUserCommands() []string {
 
 // }

pkg/backy/backy_test.go

@@ -0,0 +1,83 @@
+package backy
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"log"
+	"testing"
+
+	"git.andrewnw.xyz/CyberShell/backy/pkg/pkgman"
+	packagemanagercommon "git.andrewnw.xyz/CyberShell/backy/pkg/pkgman/common"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/testcontainers/testcontainers-go"
+)
+
+// TestConfigOptions tests the configuration options for the backy package.
+func Test_ErrorHook(t *testing.T) {
+
+	configFile := "../../tests/ErrorHook.yml"
+	logFile := "ErrorHook.log"
+	backyConfigOptions := NewConfigOptions(configFile, SetLogFile(logFile))
+	backyConfigOptions.InitConfig()
+	backyConfigOptions.ParseConfigurationFile()
+	backyConfigOptions.RunListConfig("")
+
+}
+
+func TestSettingCommandInfoPackageCommandDnf(t *testing.T) {
+
+	packagecommand := &Command{
+		Type:             PackageCommandType,
+		PackageManager:   "dnf",
+		Shell:            "zsh",
+		PackageOperation: PackageOperationCheckVersion,
+		Packages:         []packagemanagercommon.Package{{Name: "docker-ce"}},
+	}
+	dnfPackage, _ := pkgman.PackageManagerFactory("dnf", pkgman.WithoutAuth())
+
+	packagecommand.pkgMan = dnfPackage
+	PackageCommand := getCommandTypeAndSetCommandInfo(packagecommand)
+
+	assert.Equal(t, "dnf", PackageCommand.Cmd)
+
+}
+
+func TestWithDockerFile(t *testing.T) {
+	ctx := context.Background()
+
+	docker, err := testcontainers.Run(ctx, "",
+		testcontainers.WithDockerfile(testcontainers.FromDockerfile{
+			Context:    "../../tests/docker",
+			Dockerfile: "Dockerfile",
+			KeepImage:  false,
+			// BuildOptionsModifier: func(buildOptions *types.ImageBuildOptions) {
+			// 	buildOptions.Target = "target2"
+			// },
+		}),
+	)
+	// docker.
+
+	if err != nil {
+		log.Printf("failed to start container: %v", err)
+		return
+	}
+
+	r, err := docker.Logs(ctx)
+	if err != nil {
+		log.Printf("failed to get logs: %v", err)
+		return
+	}
+
+	logs, err := io.ReadAll(r)
+	if err != nil {
+		log.Printf("failed to read logs: %v", err)
+		return
+	}
+
+	fmt.Println(string(logs))
+
+	require.NoError(t, err)
+}

pkg/backy/commandtype_enumer.go

@@ -25,29 +25,29 @@ func (i CommandType) String() string {
 // Re-run the stringer command to generate them again.
 func _CommandTypeNoOp() {
 	var x [1]struct{}
-	_ = x[DefaultCT-(0)]
+	_ = x[DefaultCommandType-(0)]
-	_ = x[ScriptCT-(1)]
+	_ = x[ScriptCommandType-(1)]
-	_ = x[ScriptFileCT-(2)]
+	_ = x[ScriptFileCommandType-(2)]
-	_ = x[RemoteScriptCT-(3)]
+	_ = x[RemoteScriptCommandType-(3)]
-	_ = x[PackageCT-(4)]
+	_ = x[PackageCommandType-(4)]
-	_ = x[UserCT-(5)]
+	_ = x[UserCommandType-(5)]
 }
 
-var _CommandTypeValues = []CommandType{DefaultCT, ScriptCT, ScriptFileCT, RemoteScriptCT, PackageCT, UserCT}
+var _CommandTypeValues = []CommandType{DefaultCommandType, ScriptCommandType, ScriptFileCommandType, RemoteScriptCommandType, PackageCommandType, UserCommandType}
 
 var _CommandTypeNameToValueMap = map[string]CommandType{
-	_CommandTypeName[0:0]:        DefaultCT,
+	_CommandTypeName[0:0]:        DefaultCommandType,
-	_CommandTypeLowerName[0:0]:   DefaultCT,
+	_CommandTypeLowerName[0:0]:   DefaultCommandType,
-	_CommandTypeName[0:6]:        ScriptCT,
+	_CommandTypeName[0:6]:        ScriptCommandType,
-	_CommandTypeLowerName[0:6]:   ScriptCT,
+	_CommandTypeLowerName[0:6]:   ScriptCommandType,
-	_CommandTypeName[6:16]:       ScriptFileCT,
+	_CommandTypeName[6:16]:       ScriptFileCommandType,
-	_CommandTypeLowerName[6:16]:  ScriptFileCT,
+	_CommandTypeLowerName[6:16]:  ScriptFileCommandType,
-	_CommandTypeName[16:28]:      RemoteScriptCT,
+	_CommandTypeName[16:28]:      RemoteScriptCommandType,
-	_CommandTypeLowerName[16:28]: RemoteScriptCT,
+	_CommandTypeLowerName[16:28]: RemoteScriptCommandType,
-	_CommandTypeName[28:35]:      PackageCT,
+	_CommandTypeName[28:35]:      PackageCommandType,
-	_CommandTypeLowerName[28:35]: PackageCT,
+	_CommandTypeLowerName[28:35]: PackageCommandType,
-	_CommandTypeName[35:39]:      UserCT,
+	_CommandTypeName[35:39]:      UserCommandType,
-	_CommandTypeLowerName[35:39]: UserCT,
+	_CommandTypeLowerName[35:39]: UserCommandType,
 }
 
 var _CommandTypeNames = []string{

pkg/backy/config.go

@@ -1,7 +1,6 @@
 package backy
 
 import (
-	"context"
 	"errors"
 	"fmt"
 	"net/url"
@@ -22,10 +21,13 @@ import (
 	"github.com/rs/zerolog"
 )
 
-const macroStart string = "%{"
+const (
-const macroEnd string = "}%"
+	externDirectiveStart      string = "%{"
-const envMacroStart string = "%{env:"
+	externDirectiveEnd        string = "}%"
-const vaultMacroStart string = "%{vault:"
+	externFileDirectiveStart  string = "%{file:"
+	envExternDirectiveStart   string = "%{env:"
+	vaultExternDirectiveStart string = "%{vault:"
+)
 
 func (opts *ConfigOpts) InitConfig() {
 	var err error
@@ -78,30 +80,117 @@ func (opts *ConfigOpts) InitConfig() {
 		logging.ExitWithMSG(fmt.Sprintf("error initializing cache: %v", err), 1, nil)
 	}
 
-	fetcher, err := remotefetcher.NewRemoteFetcher(opts.ConfigFilePath, opts.Cache)
-
 	if isRemoteURL(opts.ConfigFilePath) {
 		p, _ := getRemoteDir(opts.ConfigFilePath)
 		opts.ConfigDir = p
 	}
+
+	fetcher, err := remotefetcher.NewRemoteFetcher(opts.ConfigFilePath, opts.Cache)
 	if err != nil {
 		logging.ExitWithMSG(fmt.Sprintf("error initializing config fetcher: %v", err), 1, nil)
 	}
+
 	if opts.ConfigFilePath != "" {
 		loadConfigFile(fetcher, opts.ConfigFilePath, backyKoanf, opts)
 	} else {
 		loadDefaultConfigFiles(fetcher, configFiles, backyKoanf, opts)
 	}
+
 	opts.koanf = backyKoanf
 }
 
-func loadConfigFile(fetcher remotefetcher.RemoteFetcher, filePath string, k *koanf.Koanf, opts *ConfigOpts) {
+func (opts *ConfigOpts) ParseConfigurationFile() *ConfigOpts {
-	data, err := fetcher.Fetch(filePath)
+	setTerminalEnv()
-	if err != nil {
+
-		logging.ExitWithMSG(fmt.Sprintf("Could not fetch config file %s: %v", filePath, err), 1, nil)
+	backyKoanf := opts.koanf
+
+	if backyKoanf.Exists("variables") {
+		unmarshalConfigIntoStruct(backyKoanf, "variables", &opts.Vars, opts.Logger)
 	}
 
-	if err := k.Load(rawbytes.Provider(data), yaml.Parser()); err != nil {
+	getConfigDir(opts)
+
+	opts.loadEnv()
+
+	if backyKoanf.Bool(getNestedConfig("logging", "cmd-std-out")) {
+		os.Setenv("BACKY_CMDSTDOUT", "enabled")
+	}
+
+	// override the default value of cmd-std-out if flag is set
+	if opts.CmdStdOut {
+		os.Setenv("BACKY_CMDSTDOUT", "enabled")
+	}
+
+	CheckConfigValues(backyKoanf, opts.ConfigFilePath)
+
+	validateExecCommandsFromCLI(backyKoanf, opts)
+
+	setLoggingOptions(backyKoanf, opts)
+
+	log := setupLogger(opts)
+	opts.Logger = log
+
+	hostsFetcher, err := remotefetcher.NewRemoteFetcher(opts.HostsFilePath, opts.Cache)
+	opts.Logger.Info().Str("hosts file", opts.HostsFilePath).Send()
+	if err != nil {
+		logging.ExitWithMSG(fmt.Sprintf("error initializing config fetcher: %v", err), 1, nil)
+	}
+
+	var hostKoanf = koanf.New(".")
+	if opts.HostsFilePath != "" {
+		loadConfigFile(hostsFetcher, opts.HostsFilePath, hostKoanf, opts)
+		unmarshalConfigIntoStruct(hostKoanf, "hosts", &opts.Hosts, opts.Logger)
+	} else {
+		unmarshalConfigIntoStruct(backyKoanf, "hosts", &opts.Hosts, opts.Logger)
+	}
+
+	log.Info().Str("config file", opts.ConfigFilePath).Send()
+
+	if err := opts.initializeVault(); err != nil {
+		log.Err(err).Send()
+	}
+
+	unmarshalConfigIntoStruct(backyKoanf, "commands", &opts.Cmds, opts.Logger)
+
+	getCommandEnvironments(opts)
+
+	getHostConfigs(opts)
+
+	for k, v := range opts.Vars {
+		v = getExternalConfigDirectiveValue(v, opts, AllowedExternalDirectiveAll)
+		opts.Vars[k] = v
+	}
+
+	loadCommandLists(opts, backyKoanf)
+
+	validateCommandLists(opts)
+
+	if opts.cronEnabled && len(opts.CmdConfigLists) == 0 {
+		logging.ExitWithMSG("No cron fields detected in any command lists", 1, nil)
+	}
+
+	if err := processCmds(opts); err != nil {
+		logging.ExitWithMSG(err.Error(), 1, &opts.Logger)
+	}
+
+	filterExecuteLists(opts)
+
+	if backyKoanf.Exists("notifications") {
+		unmarshalConfigIntoStruct(backyKoanf, "notifications", &opts.NotificationConf, opts.Logger)
+	}
+
+	opts.SetupNotify()
+
+	return opts
+}
+
+func loadConfigFile(fetcher remotefetcher.RemoteFetcher, filePath string, koanfConfigParser *koanf.Koanf, opts *ConfigOpts) {
+	data, err := fetcher.Fetch(filePath)
+	if err != nil {
+		logging.ExitWithMSG(generateFileFetchErrorString(filePath, "config", err), 1, nil)
+	}
+
+	if err := koanfConfigParser.Load(rawbytes.Provider(data), yaml.Parser()); err != nil {
 		logging.ExitWithMSG(fmt.Sprintf("error loading config: %v", err), 1, &opts.Logger)
 	}
 }
@@ -130,68 +219,6 @@ func loadDefaultConfigFiles(fetcher remotefetcher.RemoteFetcher, configFiles []s
 	}
 }
 
-func (opts *ConfigOpts) ReadConfig() *ConfigOpts {
-	setTerminalEnv()
-
-	backyKoanf := opts.koanf
-
-	opts.loadEnv()
-
-	if backyKoanf.Bool(getNestedConfig("logging", "cmd-std-out")) {
-		os.Setenv("BACKY_CMDSTDOUT", "enabled")
-	}
-
-	// override the default value of cmd-std-out if flag is set
-	if opts.CmdStdOut {
-		os.Setenv("BACKY_CMDSTDOUT", "enabled")
-	}
-
-	CheckConfigValues(backyKoanf, opts.ConfigFilePath)
-
-	validateCommands(backyKoanf, opts)
-
-	setLoggingOptions(backyKoanf, opts)
-
-	log := setupLogger(opts)
-	opts.Logger = log
-
-	log.Info().Str("config file", opts.ConfigFilePath).Send()
-
-	unmarshalConfig(backyKoanf, "commands", &opts.Cmds, opts.Logger)
-
-	validateCommandEnvironments(opts)
-
-	unmarshalConfig(backyKoanf, "hosts", &opts.Hosts, opts.Logger)
-
-	resolveHostConfigs(opts)
-
-	loadCommandLists(opts, backyKoanf)
-
-	validateCommandLists(opts)
-
-	if opts.cronEnabled && len(opts.CmdConfigLists) == 0 {
-		logging.ExitWithMSG("No cron fields detected in any command lists", 1, nil)
-	}
-
-	if err := processCmds(opts); err != nil {
-		logging.ExitWithMSG(err.Error(), 1, &opts.Logger)
-	}
-
-	filterExecuteLists(opts)
-
-	if backyKoanf.Exists("notifications") {
-		unmarshalConfig(backyKoanf, "notifications", &opts.NotificationConf, opts.Logger)
-	}
-
-	opts.SetupNotify()
-
-	if err := opts.setupVault(); err != nil {
-		log.Err(err).Send()
-	}
-
-	return opts
-}
-
 func setTerminalEnv() {
 	if isatty.IsTerminal(os.Stdout.Fd()) || isatty.IsCygwinTerminal(os.Stdout.Fd()) {
 		os.Setenv("BACKY_TERM", "enabled")
@@ -200,7 +227,7 @@ func setTerminalEnv() {
 	}
 }
 
-func validateCommands(k *koanf.Koanf, opts *ConfigOpts) {
+func validateExecCommandsFromCLI(k *koanf.Koanf, opts *ConfigOpts) {
 	for _, c := range opts.executeCmds {
 		if !k.Exists(getCmdFromConfig(c)) {
 			logging.ExitWithMSG(fmt.Sprintf("command %s is not in config file %s", c, opts.ConfigFilePath), 1, nil)
@@ -208,23 +235,23 @@ func validateCommands(k *koanf.Koanf, opts *ConfigOpts) {
 	}
 }
 
-func setLoggingOptions(k *koanf.Koanf, opts *ConfigOpts) {
+func setLoggingOptions(backyKoanf *koanf.Koanf, opts *ConfigOpts) {
-	isLoggingVerbose := k.Bool(getLoggingKeyFromConfig("verbose"))
+	isVerboseLoggingSetInConfig := backyKoanf.Bool(getLoggingKeyFromConfig("verbose"))
 
 	// if log file is set in config file and not set on command line, use "./backy.log"
 	logFile := "./backy.log"
-	if opts.LogFilePath == "" && k.Exists(getLoggingKeyFromConfig("file")) {
+	if opts.LogFilePath == "" && backyKoanf.Exists(getLoggingKeyFromConfig("file")) {
-		logFile = k.String(getLoggingKeyFromConfig("file"))
+		logFile = backyKoanf.String(getLoggingKeyFromConfig("file"))
 		opts.LogFilePath = logFile
 	}
 
 	zerolog.SetGlobalLevel(zerolog.InfoLevel)
-	if isLoggingVerbose {
+	if isVerboseLoggingSetInConfig {
 		zerolog.SetGlobalLevel(zerolog.DebugLevel)
 		os.Setenv("BACKY_LOGLEVEL", fmt.Sprintf("%v", zerolog.GlobalLevel()))
 	}
 
-	if k.Bool(getLoggingKeyFromConfig("console-disabled")) {
+	if backyKoanf.Bool(getLoggingKeyFromConfig("console-disabled")) {
 		os.Setenv("BACKY_CONSOLE_LOGGING", "")
 	} else {
 		os.Setenv("BACKY_CONSOLE_LOGGING", "enabled")
@@ -236,34 +263,37 @@ func setupLogger(opts *ConfigOpts) zerolog.Logger {
 	return zerolog.New(writers).With().Timestamp().Logger()
 }
 
-func unmarshalConfig(k *koanf.Koanf, key string, target interface{}, log zerolog.Logger) {
+func unmarshalConfigIntoStruct(k *koanf.Koanf, key string, target interface{}, log zerolog.Logger) {
 	if err := k.UnmarshalWithConf(key, target, koanf.UnmarshalConf{Tag: "yaml"}); err != nil {
-		logging.ExitWithMSG(fmt.Sprintf("error unmarshalling key %s into struct: %v", key, err), 1, &log)
+		logging.ExitWithMSG(fmt.Sprintf("error unmarshaling key %s into struct: %v", key, err), 1, &log)
 	}
 }
 
-func validateCommandEnvironments(opts *ConfigOpts) {
+func getCommandEnvironments(opts *ConfigOpts) {
 	for cmdName, cmdConf := range opts.Cmds {
+		if cmdConf.Env == "" {
+			continue
+		}
+		opts.Logger.Debug().Str("env file", cmdConf.Env).Str("cmd", cmdName).Send()
 		if err := testFile(cmdConf.Env); err != nil {
-			opts.Logger.Info().Str("cmd", cmdName).Err(err).Send()
+			logging.ExitWithMSG("Could not open file"+cmdConf.Env+": "+err.Error(), 1, &opts.Logger)
-			os.Exit(1)
 		}
 		expandEnvVars(opts.backyEnv, cmdConf.Environment)
 	}
 }
 
-func resolveHostConfigs(opts *ConfigOpts) {
+func getHostConfigs(opts *ConfigOpts) {
 	for hostConfigName, host := range opts.Hosts {
 		if host.Host == "" {
 			host.Host = hostConfigName
 		}
 		if host.ProxyJump != "" {
-			resolveProxyHosts(host, opts)
+			getProxyHosts(host, opts)
 		}
 	}
 }
 
-func resolveProxyHosts(host *Host, opts *ConfigOpts) {
+func getProxyHosts(host *Host, opts *ConfigOpts) {
 	proxyHosts := strings.Split(host.ProxyJump, ",")
 	for _, h := range proxyHosts {
 		proxyHost, defined := opts.Hosts[h]
@@ -275,16 +305,22 @@ func resolveProxyHosts(host *Host, opts *ConfigOpts) {
 	}
 }
 
+func getConfigDir(opts *ConfigOpts) {
+	if isRemoteURL(opts.ConfigFilePath) {
+		p, _ := getRemoteDir(opts.ConfigFilePath)
+		opts.ConfigDir = p
+	} else {
+		opts.ConfigDir = path.Dir(opts.ConfigFilePath)
+	}
+}
+
 func loadCommandLists(opts *ConfigOpts, backyKoanf *koanf.Koanf) {
 	var listConfigFiles []string
 	var u *url.URL
 	var p string
-	// if config file is remote, use the directory of the remote file
 	if isRemoteURL(opts.ConfigFilePath) {
 		p, u = getRemoteDir(opts.ConfigFilePath)
 		opts.ConfigDir = p
-		println(p)
-		// // Still use local list files if a remote config file is used, but use them last
 		listConfigFiles = []string{u.JoinPath("lists.yml").String(), u.JoinPath("lists.yaml").String()}
 	} else {
 		opts.ConfigDir = path.Dir(opts.ConfigFilePath)
@@ -297,16 +333,19 @@ func loadCommandLists(opts *ConfigOpts, backyKoanf *koanf.Koanf) {
 
 	listsConfig := koanf.New(".")
 
-	for _, l := range listConfigFiles {
+	if backyKoanf.Exists("cmdLists") {
-		if loadListConfigFile(l, listsConfig, opts) {
+		if backyKoanf.Exists("cmdLists.file") {
-			break
+			loadCmdListsFile(backyKoanf, listsConfig, opts)
+		} else {
+			unmarshalConfigIntoStruct(backyKoanf, "cmdLists", &opts.CmdConfigLists, opts.Logger)
 		}
 	}
 
-	if backyKoanf.Exists("cmdLists") {
+	if opts.CmdConfigLists == nil {
-		unmarshalConfig(backyKoanf, "cmdLists", &opts.CmdConfigLists, opts.Logger)
+		for _, l := range listConfigFiles {
-		if backyKoanf.Exists("cmdLists.file") {
+			if loadListConfigFile(l, listsConfig, opts) {
-			loadCmdListsFile(backyKoanf, listsConfig, opts)
+				break
+			}
 		}
 	}
 }
@@ -346,7 +385,7 @@ func loadListConfigFile(filePath string, k *koanf.Koanf, opts *ConfigOpts) bool
 		return false
 	}
 
-	unmarshalConfig(k, "cmdLists", &opts.CmdConfigLists, opts.Logger)
+	unmarshalConfigIntoStruct(k, "cmdLists", &opts.CmdConfigLists, opts.Logger)
 	keyNotSupported("cmd-lists", "cmdLists", k, opts, true)
 	opts.CmdListFile = filePath
 	return true
@@ -355,6 +394,7 @@ func loadListConfigFile(filePath string, k *koanf.Koanf, opts *ConfigOpts) bool
 func loadCmdListsFile(backyKoanf *koanf.Koanf, listsConfig *koanf.Koanf, opts *ConfigOpts) {
 	opts.CmdListFile = strings.TrimSpace(backyKoanf.String("cmdLists.file"))
 	if !path.IsAbs(opts.CmdListFile) {
+		// TODO: Needs testing - might cause undefined/unexpected behavior if remote config path is used
 		opts.CmdListFile = path.Join(path.Dir(opts.ConfigFilePath), opts.CmdListFile)
 	}
 
@@ -366,7 +406,7 @@ func loadCmdListsFile(backyKoanf *koanf.Koanf, listsConfig *koanf.Koanf, opts *C
 
 	data, err := fetcher.Fetch(opts.CmdListFile)
 	if err != nil {
-		logging.ExitWithMSG(fmt.Sprintf("Could not fetch config file %s: %v", opts.CmdListFile, err), 1, nil)
+		logging.ExitWithMSG(generateFileFetchErrorString(opts.CmdListFile, "list config", err), 1, nil)
 	}
 
 	if err := listsConfig.Load(rawbytes.Provider(data), yaml.Parser()); err != nil {
@@ -374,10 +414,14 @@ func loadCmdListsFile(backyKoanf *koanf.Koanf, listsConfig *koanf.Koanf, opts *C
 	}
 
 	keyNotSupported("cmd-lists", "cmdLists", listsConfig, opts, true)
-	unmarshalConfig(listsConfig, "cmdLists", &opts.CmdConfigLists, opts.Logger)
+	unmarshalConfigIntoStruct(listsConfig, "cmdLists", &opts.CmdConfigLists, opts.Logger)
 	opts.Logger.Info().Str("using lists config file", opts.CmdListFile).Send()
 }
 
+func generateFileFetchErrorString(file, fileType string, err error) string {
+	return fmt.Sprintf("Could not fetch %s file %s: %v", file, fileType, err)
+}
+
 func validateCommandLists(opts *ConfigOpts) {
 	var cmdNotFoundSliceErr []error
 	for cmdListName, cmdList := range opts.CmdConfigLists {
@@ -427,7 +471,7 @@ func getLoggingKeyFromConfig(key string) string {
 // 	return fmt.Sprintf("cmdLists.%s", list)
 // }
 
-func (opts *ConfigOpts) setupVault() error {
+func (opts *ConfigOpts) initializeVault() error {
 	if !opts.koanf.Bool("vault.enabled") {
 		return nil
 	}
@@ -448,97 +492,41 @@ func (opts *ConfigOpts) setupVault() error {
 		token = os.Getenv("VAULT_TOKEN")
 	}
 	if strings.TrimSpace(token) == "" {
-		return fmt.Errorf("no token found, but one was required. \n\nSet the config key vault.token or the environment variable VAULT_TOKEN")
+		return fmt.Errorf("no token found. One is required. \n\nSet the config key vault.token or the environment variable VAULT_TOKEN")
 	}
 
 	client.SetToken(token)
 
 	unmarshalErr := opts.koanf.UnmarshalWithConf("vault.keys", &opts.VaultKeys, koanf.UnmarshalConf{Tag: "yaml"})
 	if unmarshalErr != nil {
-		logging.ExitWithMSG(fmt.Sprintf("error unmarshalling vault.keys into struct: %v", unmarshalErr), 1, &opts.Logger)
+		logging.ExitWithMSG(fmt.Sprintf("error unmarshaling vault.keys into struct: %v", unmarshalErr), 1, &opts.Logger)
 	}
 
 	opts.vaultClient = client
 
+	for _, v := range opts.VaultKeys {
+		v.Name = replaceVarInString(opts.Vars, v.Key, opts.Logger)
+		v.MountPath = replaceVarInString(opts.Vars, v.MountPath, opts.Logger)
+	}
+
 	return nil
 }
 
-func getVaultSecret(vaultClient *vault.Client, key *VaultKey) (string, error) {
-	var (
-		secret *vault.KVSecret
-		err    error
-	)
-
-	if key.ValueType == "KVv2" {
-		secret, err = vaultClient.KVv2(key.MountPath).Get(context.Background(), key.Path)
-	} else if key.ValueType == "KVv1" {
-		secret, err = vaultClient.KVv1(key.MountPath).Get(context.Background(), key.Path)
-	} else if key.ValueType != "" {
-		return "", fmt.Errorf("type %s for key %s not known. Valid types are KVv1 or KVv2", key.ValueType, key.Name)
-	} else {
-		return "", fmt.Errorf("type for key %s must be specified. Valid types are KVv1 or KVv2", key.Name)
-
-	}
-	if err != nil {
-		return "", fmt.Errorf("unable to read secret: %v", err)
-	}
-
-	value, ok := secret.Data[key.Name].(string)
-	if !ok {
-		return "", fmt.Errorf("value type assertion failed: %T %#v", secret.Data[key.Name], secret.Data[key.Name])
-	}
-
-	return value, nil
-}
-
-func isVaultKey(str string) (string, bool) {
-	str = strings.TrimSpace(str)
-	return strings.TrimPrefix(str, "vault:"), strings.HasPrefix(str, "vault:")
-}
-
-func parseVaultKey(str string, keys []*VaultKey) (*VaultKey, error) {
-	keyName, isKey := isVaultKey(str)
-	if !isKey {
-		return nil, nil
-	}
-
-	for _, k := range keys {
-		if k.Name == keyName {
-			return k, nil
-		}
-	}
-	return nil, fmt.Errorf("key %s not found in vault keys", keyName)
-}
-
-func GetVaultKey(str string, opts *ConfigOpts, log zerolog.Logger) string {
-	key, err := parseVaultKey(str, opts.VaultKeys)
-	if key == nil && err == nil {
-		return str
-	}
-	if err != nil && key == nil {
-		log.Err(err).Send()
-		return ""
-	}
-
-	value, secretErr := getVaultSecret(opts.vaultClient, key)
-	if secretErr != nil {
-		log.Err(secretErr).Send()
-		return value
-	}
-	return value
-}
-
 func processCmds(opts *ConfigOpts) error {
 
 	// process commands
 	for cmdName, cmd := range opts.Cmds {
-
+		cmd.GetVariablesFromConf(opts)
+		cmd.Cmd = replaceVarInString(opts.Vars, cmd.Cmd, opts.Logger)
+		for i, v := range cmd.Args {
+			v = replaceVarInString(opts.Vars, v, opts.Logger)
+			cmd.Args[i] = v
+		}
 		if cmd.Name == "" {
 			cmd.Name = cmdName
 		}
-		// println("Cmd.Name = " + cmd.Name)
+
 		hooks := cmd.Hooks
-		// resolve hooks
 		if hooks != nil {
 
 			processHookSuccess := processHooks(cmd, hooks.Error, opts, "error")
@@ -555,9 +543,13 @@ func processCmds(opts *ConfigOpts) error {
 			}
 		}
 
-		// resolve hosts
+		if !IsHostLocal(cmd.Host) {
-		if cmd.Host != nil {
+
-			host, hostFound := opts.Hosts[*cmd.Host]
+			cmdHost := replaceVarInString(opts.Vars, cmd.Host, opts.Logger)
+			if cmdHost != cmd.Host {
+				cmd.Host = cmdHost
+			}
+			host, hostFound := opts.Hosts[cmd.Host]
 			if hostFound {
 				cmd.RemoteHost = host
 				cmd.RemoteHost.Host = host.Host
@@ -565,8 +557,12 @@ func processCmds(opts *ConfigOpts) error {
 					cmd.RemoteHost.HostName = host.HostName
 				}
 			} else {
-				opts.Hosts[*cmd.Host] = &Host{Host: *cmd.Host}
+				opts.Logger.Info().Msgf("adding host %s to host list", cmd.Host)
-				cmd.RemoteHost = &Host{Host: *cmd.Host}
+				if opts.Hosts == nil {
+					opts.Hosts = make(map[string]*Host)
+				}
+				opts.Hosts[cmd.Host] = &Host{Host: cmd.Host}
+				cmd.RemoteHost = &Host{Host: cmd.Host}
 			}
 		} else {
 
@@ -577,19 +573,20 @@ func processCmds(opts *ConfigOpts) error {
 					return err
 				}
 				cmd.Dir = &cmdDir
+			} else {
+				cmd.Dir = &opts.ConfigDir
 			}
 		}
 
-		// Parse package commands
+		if cmd.Type == PackageCommandType {
-		if cmd.Type == PackageCT {
 			if cmd.PackageManager == "" {
-				return fmt.Errorf("package manager is required for package command %s", cmd.PackageName)
+				return fmt.Errorf("package manager is required for package command %s", cmd.Name)
 			}
 			if cmd.PackageOperation.String() == "" {
-				return fmt.Errorf("package operation is required for package command %s", cmd.PackageName)
+				return fmt.Errorf("package operation is required for package command %s", cmd.Name)
 			}
-			if cmd.PackageName == "" {
+			if cmd.Packages == nil {
-				return fmt.Errorf("package name is required for package command %s", cmd.PackageName)
+				return fmt.Errorf("package name is required for package command %s", cmd.Name)
 			}
 			var err error
 
@@ -607,24 +604,38 @@ func processCmds(opts *ConfigOpts) error {
 		}
 
 		// Parse user commands
-		if cmd.Type == UserCT {
+		if cmd.Type == UserCommandType {
 			if cmd.Username == "" {
 				return fmt.Errorf("username is required for user command %s", cmd.Name)
 			}
-
+			cmd.Username = replaceVarInString(opts.Vars, cmd.Username, opts.Logger)
-			detectOSType(cmd, opts)
+			err := detectOSType(cmd, opts)
-			var err error
+			if err != nil {
+				opts.Logger.Info().Err(err).Str("command", cmdName).Send()
+			}
 
 			// Validate the operation
 			switch cmd.UserOperation {
 			case "add", "remove", "modify", "checkIfExists", "delete", "password":
 				cmd.userMan, err = usermanager.NewUserManager(cmd.OS)
-				if cmd.Host != nil {
+
-					host, ok := opts.Hosts[*cmd.Host]
+				if cmd.UserOperation == "password" {
+					opts.Logger.Debug().Msg("changing password for user: " + cmd.Username)
+					cmd.UserPassword = getExternalConfigDirectiveValue(cmd.UserPassword, opts, AllowedExternalDirectiveAll)
+				}
+
+				if !IsHostLocal(cmd.Host) {
+
+					host, ok := opts.Hosts[cmd.Host]
 					if ok {
 						cmd.userMan, err = usermanager.NewUserManager(host.OS)
 					}
 				}
+				for indx, key := range cmd.UserSshPubKeys {
+					opts.Logger.Debug().Msg("adding SSH Keys")
+					key = getExternalConfigDirectiveValue(key, opts, AllowedExternalDirectiveAll)
+					cmd.UserSshPubKeys[indx] = key
+				}
 				if err != nil {
 					return err
 				}
@@ -634,7 +645,7 @@ func processCmds(opts *ConfigOpts) error {
 
 		}
 
-		if cmd.Type == RemoteScriptCT {
+		if cmd.Type == RemoteScriptCommandType {
 			var fetchErr error
 			if !isRemoteURL(cmd.Cmd) {
 				return fmt.Errorf("remoteScript command %s must be a remote resource", cmdName)
@@ -645,9 +656,9 @@ func processCmds(opts *ConfigOpts) error {
 			}
 
 		}
-		if cmd.OutputFile != "" {
+		if cmd.Output.File != "" {
 			var err error
-			cmd.OutputFile, err = getFullPathWithHomeDir(cmd.OutputFile)
+			cmd.Output.File, err = getFullPathWithHomeDir(cmd.Output.File)
 			if err != nil {
 				return err
 			}
@@ -656,14 +667,8 @@ func processCmds(opts *ConfigOpts) error {
 	return nil
 }
 
-// processHooks evaluates if hooks are valid Commands
-//
-// The cmd.hookRefs[hookType] is created with any hooks found.
-//
-// Returns an error, if any, if the hook command is not found
 func processHooks(cmd *Command, hooks []string, opts *ConfigOpts, hookType string) error {
 
-	// initialize hook type
 	var hookCmdFound bool
 	cmd.hookRefs = map[string]map[string]*Command{}
 	cmd.hookRefs[hookType] = map[string]*Command{}
@@ -690,13 +695,18 @@ func processHooks(cmd *Command, hooks []string, opts *ConfigOpts, hookType strin
 }
 
 func detectOSType(cmd *Command, opts *ConfigOpts) error {
-	if cmd.Host == nil {
+
-		if runtime.GOOS == "linux" { // also can be specified to FreeBSD
+	if IsHostLocal(cmd.Host) {
+
+		if runtime.GOOS == "linux" {
 			cmd.OS = "linux"
 			opts.Logger.Info().Msg("Unix/Linux type OS detected")
+			return nil
 		}
+		return fmt.Errorf("using an os that is not yet supported for user commands")
 	}
-	host, ok := opts.Hosts[*cmd.Host]
+
+	host, ok := opts.Hosts[cmd.Host]
 	if ok {
 		if host.OS != "" {
 			return nil
@@ -728,3 +738,25 @@ func keyNotSupported(oldKey, newKey string, koanf *koanf.Koanf, opts *ConfigOpts
 		}
 	}
 }
+
+func replaceVarInString(vars map[string]string, str string, logger zerolog.Logger) string {
+	if strings.Contains(str, "%{var:") && strings.Contains(str, "}%") {
+		logger.Debug().Msgf("replacing vars in string %s", str)
+		for k, v := range vars {
+			if strings.Contains(str, "%{var:"+k+"}%") {
+				str = strings.ReplaceAll(str, "%{var:"+k+"}%", v)
+			}
+		}
+		if strings.Contains(str, "%{var:") && strings.Contains(str, "}%") {
+			logger.Warn().Msg("could not replace all vars in string")
+		}
+	}
+	return str
+}
+
+func (c *Command) GetVariablesFromConf(opts *ConfigOpts) {
+	c.ScriptEnvFile = replaceVarInString(opts.Vars, c.ScriptEnvFile, opts.Logger)
+	c.Name = replaceVarInString(opts.Vars, c.Name, opts.Logger)
+	c.Output.File = replaceVarInString(opts.Vars, c.Output.File, opts.Logger)
+	c.Host = replaceVarInString(opts.Vars, c.Host, opts.Logger)
+}

pkg/backy/lineinfile.go

@@ -0,0 +1,133 @@
+package backy
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"regexp"
+	"strings"
+
+	"golang.org/x/crypto/ssh"
+)
+
+func sshConnect(user, password, host string, port int) (*ssh.Client, error) {
+	config := &ssh.ClientConfig{
+		User: user,
+		Auth: []ssh.AuthMethod{
+			ssh.Password(password),
+		},
+		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
+	}
+	addr := fmt.Sprintf("%s:%d", host, port)
+	return ssh.Dial("tcp", addr, config)
+}
+
+func sshReadFile(client *ssh.Client, remotePath string) (string, error) {
+	session, err := client.NewSession()
+	if err != nil {
+		return "", err
+	}
+	defer session.Close()
+
+	var b bytes.Buffer
+	session.Stdout = &b
+	if err := session.Run(fmt.Sprintf("cat %s", remotePath)); err != nil {
+		return "", err
+	}
+	return b.String(), nil
+}
+
+func sshWriteFile(client *ssh.Client, remotePath, content string) error {
+	session, err := client.NewSession()
+	if err != nil {
+		return err
+	}
+	defer session.Close()
+
+	stdin, err := session.StdinPipe()
+	if err != nil {
+		return err
+	}
+
+	go func() {
+		defer stdin.Close()
+		io.WriteString(stdin, content)
+	}()
+
+	cmd := fmt.Sprintf("cat > %s", remotePath)
+	return session.Run(cmd)
+}
+
+func lineInString(content, regexpPattern, line string) string {
+	scanner := bufio.NewScanner(strings.NewReader(content))
+	var lines []string
+	found := false
+	re := regexp.MustCompile(regexpPattern)
+
+	for scanner.Scan() {
+		l := scanner.Text()
+		if re.MatchString(l) {
+			found = true
+			lines = append(lines, line)
+		} else {
+			lines = append(lines, l)
+		}
+	}
+	if !found {
+		lines = append(lines, line)
+	}
+	return strings.Join(lines, "\n") + "\n"
+}
+
+func Call() {
+	user := "youruser"
+	password := "yourpassword"
+	host := "yourhost"
+	port := 22
+	remotePath := "/path/to/remote/file"
+
+	client, err := sshConnect(user, password, host, port)
+	if err != nil {
+		fmt.Println("SSH connection error:", err)
+		return
+	}
+	defer client.Close()
+
+	content, err := sshReadFile(client, remotePath)
+	if err != nil {
+		fmt.Println("Read error:", err)
+		return
+	}
+
+	newContent := lineInString(content, "^foo=", "foo=bar")
+
+	if err := sshWriteFile(client, remotePath, newContent); err != nil {
+		fmt.Println("Write error:", err)
+		return
+	}
+
+	fmt.Println("Line updated successfully over SSH.")
+}
+
+type LineInFile struct {
+	RemotePath    string         // Path to the remote file
+	Pattern       string         // Regex pattern to match lines
+	Line          string         // Line to insert or replace
+	InsertAfter   bool           // If true, insert after matched line; else replace
+	User          string         // SSH username
+	Password      string         // SSH password (use key for production)
+	Host          string         // SSH host
+	Port          int            // SSH port
+	regexCompiled *regexp.Regexp // Compiled regex (internal use)
+}
+
+// CompileRegex compiles the regex pattern for later use
+func (l *LineInFile) CompileRegex() error {
+	re, err := regexp.Compile(l.Pattern)
+	if err != nil {
+		return err
+	}
+	l.regexCompiled = re
+	return nil
+}

pkg/backy/list.go

@@ -46,9 +46,9 @@ func (opts *ConfigOpts) ListCommand(cmd string) {
 		}
 
 		// is it remote or local
-		if cmdInfo.Host != nil {
+		if !IsHostLocal(cmdInfo.Host) {
 			println()
-			print("Host: ", *cmdInfo.Host)
+			print("Host: ", cmdInfo.Host)
 			println()
 
 		} else {

pkg/backy/metrics.go

@@ -0,0 +1,57 @@
+package backy
+
+import (
+	"encoding/json"
+	"os"
+)
+
+type Metrics struct {
+	SuccessfulExecutions uint64  `json:"successful_executions"`
+	FailedExecutions     uint64  `json:"failed_executions"`
+	TotalExecutions      uint64  `json:"total_executions"`
+	ExecutionTime        float64 `json:"execution_time"`         // in seconds
+	AverageExecutionTime float64 `json:"average_execution_time"` // in seconds
+	SuccessRate          float64 `json:"success_rate"`           // percentage of successful executions
+	FailureRate          float64 `json:"failure_rate"`           // percentage of failed executions
+}
+
+func NewMetrics() *Metrics {
+	return &Metrics{
+		SuccessfulExecutions: 0,
+		FailedExecutions:     0,
+		TotalExecutions:      0,
+		ExecutionTime:        0.0,
+		AverageExecutionTime: 0.0,
+		SuccessRate:          0.0,
+		FailureRate:          0.0,
+	}
+}
+
+func (m *Metrics) Update(success bool, executionTime float64) {
+	m.TotalExecutions++
+	if success {
+		m.SuccessfulExecutions++
+	} else {
+		m.FailedExecutions++
+	}
+
+	m.ExecutionTime += executionTime
+	m.AverageExecutionTime = m.ExecutionTime / float64(m.TotalExecutions)
+
+	if m.TotalExecutions > 0 {
+		m.SuccessRate = float64(m.SuccessfulExecutions) / float64(m.TotalExecutions) * 100
+		m.FailureRate = float64(m.FailedExecutions) / float64(m.TotalExecutions) * 100
+	}
+}
+
+func SaveToFile(metrics *Metrics, filename string) error {
+	data, err := json.MarshalIndent(metrics, "", "  ")
+	if err != nil {
+		return err
+	}
+	return os.WriteFile(filename, data, 0644)
+}
+
+func LoadFromFile(filename string) (*Metrics, error) {
+	return nil, nil
+}

pkg/backy/metrics_test.go

@@ -0,0 +1,7 @@
+package backy
+
+import "testing"
+
+func TestAddingMetricsForCommand(t *testing.T) {
+
+}

pkg/backy/notification.go

@@ -9,6 +9,7 @@ import (
 
 	"git.andrewnw.xyz/CyberShell/backy/pkg/logging"
 	"github.com/nikoksr/notify"
+	"github.com/nikoksr/notify/service/http"
 	"github.com/nikoksr/notify/service/mail"
 	"github.com/nikoksr/notify/service/matrix"
 	"maunium.net/go/mautrix/id"
@@ -30,6 +31,12 @@ type MailConfig struct {
 	Password      string   `yaml:"password"`
 }
 
+type HttpConfig struct {
+	URL     string              `yaml:"url"`
+	Method  string              `yaml:"method"`
+	Headers map[string][]string `yaml:"headers"`
+}
+
 // SetupNotify sets up notify instances for each command list.
 func (opts *ConfigOpts) SetupNotify() {
 
@@ -58,6 +65,8 @@ func (opts *ConfigOpts) SetupNotify() {
 					opts.Logger.Info().Err(fmt.Errorf("error: ID %s not found in mail object", confId)).Str("list", confName).Send()
 					continue
 				}
+				conf.Password = getExternalConfigDirectiveValue(conf.Password, opts, AllowedExternalDirectiveAll)
+				opts.Logger.Debug().Str("list", confName).Str("id", confId).Msg("adding mail notification service")
 				mailConf := setupMail(conf)
 				services = append(services, mailConf)
 			case "matrix":
@@ -66,14 +75,23 @@ func (opts *ConfigOpts) SetupNotify() {
 					opts.Logger.Info().Err(fmt.Errorf("error: ID %s not found in matrix object", confId)).Str("list", confName).Send()
 					continue
 				}
+				conf.AccessToken = getExternalConfigDirectiveValue(conf.AccessToken, opts, AllowedExternalDirectiveAll)
+				opts.Logger.Debug().Str("list", confName).Str("id", confId).Msg("adding matrix notification service")
 				mtrxConf, mtrxErr := setupMatrix(conf)
 				if mtrxErr != nil {
 					opts.Logger.Info().Str("list", confName).Err(fmt.Errorf("error: configuring matrix id %s failed during setup: %w", id, mtrxErr))
 					continue
 				}
-				// append the services
 				services = append(services, mtrxConf)
-			// service is not recognized
+			case "http":
+				conf, ok := opts.NotificationConf.HttpConfig[confId]
+				if !ok {
+					opts.Logger.Info().Err(fmt.Errorf("error: ID %s not found in http object", confId)).Str("list", confName).Send()
+					continue
+				}
+				opts.Logger.Debug().Str("list", confName).Str("id", confId).Msg("adding http notification service")
+				httpConf := setupHttp(conf)
+				services = append(services, httpConf)
 			default:
 				opts.Logger.Info().Err(fmt.Errorf("id %s not found", id)).Str("list", confName).Send()
 			}
@@ -99,3 +117,19 @@ func setupMail(config MailConfig) *mail.Mail {
 	mailClient.BodyFormat(mail.PlainText)
 	return mailClient
 }
+
+func setupHttp(httpConf HttpConfig) *http.Service {
+
+	httpService := http.New()
+	httpService.AddReceivers(&http.Webhook{
+		URL:         httpConf.URL,
+		Header:      httpConf.Headers,
+		ContentType: "text/plain",
+		Method:      httpConf.Method,
+		BuildPayload: func(subject, message string) (payload any) {
+			return subject + "\n\n" + message
+		},
+	})
+
+	return httpService
+}

pkg/backy/packageoperation_enumer.go

@@ -26,31 +26,31 @@ func (i PackageOperation) String() string {
 func _PackageOperationNoOp() {
 	var x [1]struct{}
 	_ = x[DefaultPO-(0)]
-	_ = x[PackOpInstall-(1)]
+	_ = x[PackageOperationInstall-(1)]
-	_ = x[PackOpUpgrade-(2)]
+	_ = x[PackageOperationUpgrade-(2)]
-	_ = x[PackOpPurge-(3)]
+	_ = x[PackageOperationPurge-(3)]
-	_ = x[PackOpRemove-(4)]
+	_ = x[PackageOperationRemove-(4)]
-	_ = x[PackOpCheckVersion-(5)]
+	_ = x[PackageOperationCheckVersion-(5)]
-	_ = x[PackOpIsInstalled-(6)]
+	_ = x[PackageOperationIsInstalled-(6)]
 }
 
-var _PackageOperationValues = []PackageOperation{DefaultPO, PackOpInstall, PackOpUpgrade, PackOpPurge, PackOpRemove, PackOpCheckVersion, PackOpIsInstalled}
+var _PackageOperationValues = []PackageOperation{DefaultPO, PackageOperationInstall, PackageOperationUpgrade, PackageOperationPurge, PackageOperationRemove, PackageOperationCheckVersion, PackageOperationIsInstalled}
 
 var _PackageOperationNameToValueMap = map[string]PackageOperation{
 	_PackageOperationName[0:0]:        DefaultPO,
 	_PackageOperationLowerName[0:0]:   DefaultPO,
-	_PackageOperationName[0:7]:        PackOpInstall,
+	_PackageOperationName[0:7]:        PackageOperationInstall,
-	_PackageOperationLowerName[0:7]:   PackOpInstall,
+	_PackageOperationLowerName[0:7]:   PackageOperationInstall,
-	_PackageOperationName[7:14]:       PackOpUpgrade,
+	_PackageOperationName[7:14]:       PackageOperationUpgrade,
-	_PackageOperationLowerName[7:14]:  PackOpUpgrade,
+	_PackageOperationLowerName[7:14]:  PackageOperationUpgrade,
-	_PackageOperationName[14:19]:      PackOpPurge,
+	_PackageOperationName[14:19]:      PackageOperationPurge,
-	_PackageOperationLowerName[14:19]: PackOpPurge,
+	_PackageOperationLowerName[14:19]: PackageOperationPurge,
-	_PackageOperationName[19:25]:      PackOpRemove,
+	_PackageOperationName[19:25]:      PackageOperationRemove,
-	_PackageOperationLowerName[19:25]: PackOpRemove,
+	_PackageOperationLowerName[19:25]: PackageOperationRemove,
-	_PackageOperationName[25:37]:      PackOpCheckVersion,
+	_PackageOperationName[25:37]:      PackageOperationCheckVersion,
-	_PackageOperationLowerName[25:37]: PackOpCheckVersion,
+	_PackageOperationLowerName[25:37]: PackageOperationCheckVersion,
-	_PackageOperationName[37:48]:      PackOpIsInstalled,
+	_PackageOperationName[37:48]:      PackageOperationIsInstalled,
-	_PackageOperationLowerName[37:48]: PackOpIsInstalled,
+	_PackageOperationLowerName[37:48]: PackageOperationIsInstalled,
 }
 
 var _PackageOperationNames = []string{

pkg/backy/ssh.go

@@ -15,50 +15,52 @@ import (
 	"strings"
 	"time"
 
+	"github.com/google/uuid"
 	"github.com/kevinburke/ssh_config"
 	"github.com/pkg/errors"
+	"github.com/pkg/sftp"
 	"github.com/rs/zerolog"
 	"golang.org/x/crypto/ssh"
 	"golang.org/x/crypto/ssh/knownhosts"
 )
 
-var PrivateKeyExtraInfoErr = errors.New("Private key may be encrypted. \nIf encrypted, make sure the password is specified correctly in the correct section. This may be done in one of three ways: \n privatekeypassword: env:PR_KEY_PASS \n privatekeypassword: file:/path/to/password-file \n privatekeypassword: password (not recommended). \n ")
+var PrivateKeyExtraInfoErr = errors.New("Private key may be encrypted. \nIf encrypted, make sure the password is specified correctly in the correct section. This may be done in one of two ways: \n Using external directives - see docs \n privatekeypassword: password (not recommended). \n ")
 var TS = strings.TrimSpace
 
 // ConnectToHost connects to a host by looking up the config values in the file ~/.ssh/config
 // It uses any set values and looks up an unset values in the config files
-// remoteConfig is modified directly. The *ssh.Client is returned as part of remoteConfig,
+// remoteHost is modified directly. The *ssh.Client is returned as part of remoteHost,
 // If configFile is empty, any required configuration is looked up in the default config files
 // If any value is not found, defaults are used
-func (remoteConfig *Host) ConnectToHost(opts *ConfigOpts) error {
+func (remoteHost *Host) ConnectToHost(opts *ConfigOpts) error {
 
 	var connectErr error
 
-	if TS(remoteConfig.ConfigFilePath) == "" {
+	if TS(remoteHost.ConfigFilePath) == "" {
-		remoteConfig.useDefaultConfig = true
+		remoteHost.useDefaultConfig = true
 	}
 
-	khPathErr := remoteConfig.GetKnownHosts()
+	khPathErr := remoteHost.GetKnownHosts()
 
 	if khPathErr != nil {
 		return khPathErr
 	}
 
-	if remoteConfig.ClientConfig == nil {
+	if remoteHost.ClientConfig == nil {
-		remoteConfig.ClientConfig = &ssh.ClientConfig{}
+		remoteHost.ClientConfig = &ssh.ClientConfig{}
 	}
 
 	var configFile *os.File
 
 	var sshConfigFileOpenErr error
 
-	if !remoteConfig.useDefaultConfig {
+	if !remoteHost.useDefaultConfig {
 		var err error
-		remoteConfig.ConfigFilePath, err = getFullPathWithHomeDir(remoteConfig.ConfigFilePath)
+		remoteHost.ConfigFilePath, err = getFullPathWithHomeDir(remoteHost.ConfigFilePath)
 		if err != nil {
 			return err
 		}
-		configFile, sshConfigFileOpenErr = os.Open(remoteConfig.ConfigFilePath)
+		configFile, sshConfigFileOpenErr = os.Open(remoteHost.ConfigFilePath)
 		if sshConfigFileOpenErr != nil {
 			return sshConfigFileOpenErr
 		}
@@ -69,22 +71,22 @@ func (remoteConfig *Host) ConnectToHost(opts *ConfigOpts) error {
 			return sshConfigFileOpenErr
 		}
 	}
-	remoteConfig.SSHConfigFile = &sshConfigFile{}
+	remoteHost.SSHConfigFile = &sshConfigFile{}
-	remoteConfig.SSHConfigFile.DefaultUserSettings = ssh_config.DefaultUserSettings
+	remoteHost.SSHConfigFile.DefaultUserSettings = ssh_config.DefaultUserSettings
 	var decodeErr error
-	remoteConfig.SSHConfigFile.SshConfigFile, decodeErr = ssh_config.Decode(configFile)
+	remoteHost.SSHConfigFile.SshConfigFile, decodeErr = ssh_config.Decode(configFile)
 	if decodeErr != nil {
 		return decodeErr
 	}
 
-	err := remoteConfig.GetProxyJumpFromConfig(opts.Hosts)
+	err := remoteHost.GetProxyJumpFromConfig(opts.Hosts)
 
 	if err != nil {
 		return err
 	}
 
-	if remoteConfig.ProxyHost != nil {
+	if remoteHost.ProxyHost != nil {
-		for _, proxyHost := range remoteConfig.ProxyHost {
+		for _, proxyHost := range remoteHost.ProxyHost {
 			err := proxyHost.GetProxyJumpConfig(opts.Hosts, opts)
 			opts.Logger.Info().Msgf("Proxy host: %s", proxyHost.Host)
 			if err != nil {
@@ -93,50 +95,49 @@ func (remoteConfig *Host) ConnectToHost(opts *ConfigOpts) error {
 		}
 	}
 
-	remoteConfig.ClientConfig.Timeout = time.Second * 30
+	remoteHost.ClientConfig.Timeout = time.Second * 30
 
-	remoteConfig.GetPrivateKeyFileFromConfig()
+	remoteHost.GetPrivateKeyFileFromConfig()
 
-	remoteConfig.GetPort()
+	remoteHost.GetPort()
 
-	remoteConfig.GetHostName()
+	remoteHost.GetHostName()
 
-	remoteConfig.CombineHostNameWithPort()
+	remoteHost.CombineHostNameWithPort()
 
-	remoteConfig.GetSshUserFromConfig()
+	remoteHost.GetSshUserFromConfig()
 
-	if remoteConfig.HostName == "" {
+	if remoteHost.HostName == "" {
-		return errors.Errorf("No hostname found or specified for host %s", remoteConfig.Host)
+		return errors.Errorf("No hostname found or specified for host %s", remoteHost.Host)
 	}
 
-	err = remoteConfig.GetAuthMethods(opts)
+	err = remoteHost.GetAuthMethods(opts)
 	if err != nil {
 		return err
 	}
 
-	hostKeyCallback, err := knownhosts.New(remoteConfig.KnownHostsFile)
+	hostKeyCallback, err := knownhosts.New(remoteHost.KnownHostsFile)
 	if err != nil {
 		return errors.Wrap(err, "could not create hostkeycallback function")
 	}
-	remoteConfig.ClientConfig.HostKeyCallback = hostKeyCallback
+	remoteHost.ClientConfig.HostKeyCallback = hostKeyCallback
-	// opts.Logger.Info().Str("user", remoteConfig.ClientConfig.User).Send()
 
-	remoteConfig.SshClient, connectErr = remoteConfig.ConnectThroughBastion(opts.Logger)
+	remoteHost.SshClient, connectErr = remoteHost.ConnectThroughBastion(opts.Logger)
 	if connectErr != nil {
 		return connectErr
 	}
-	if remoteConfig.SshClient != nil {
+	if remoteHost.SshClient != nil {
-		opts.Hosts[remoteConfig.Host] = remoteConfig
+		opts.Hosts[remoteHost.Host] = remoteHost
 		return nil
 	}
 
-	opts.Logger.Info().Msgf("Connecting to host %s", remoteConfig.HostName)
+	opts.Logger.Info().Msgf("Connecting to host %s", remoteHost.HostName)
-	remoteConfig.SshClient, connectErr = ssh.Dial("tcp", remoteConfig.HostName, remoteConfig.ClientConfig)
+	remoteHost.SshClient, connectErr = ssh.Dial("tcp", remoteHost.HostName, remoteHost.ClientConfig)
 	if connectErr != nil {
 		return connectErr
 	}
 
-	opts.Hosts[remoteConfig.Host] = remoteConfig
+	opts.Hosts[remoteHost.Host] = remoteHost
 	return nil
 }
 
@@ -180,11 +181,7 @@ func (remoteHost *Host) GetAuthMethods(opts *ConfigOpts) error {
 			return err
 		}
 
-		remoteHost.PrivateKeyPassword, err = GetPrivateKeyPassword(remoteHost.PrivateKeyPassword, opts, opts.Logger)
+		remoteHost.PrivateKeyPassword = GetPrivateKeyPassword(remoteHost.PrivateKeyPassword, opts)
-
-		if err != nil {
-			return err
-		}
 
 		if remoteHost.PrivateKeyPassword == "" {
 
@@ -207,14 +204,13 @@ func (remoteHost *Host) GetAuthMethods(opts *ConfigOpts) error {
 		}
 	}
 
-	if remoteHost.Password == "" {
+	if remoteHost.Password != "" {
 
-		remoteHost.Password, err = GetPassword(remoteHost.Password, opts, opts.Logger)
+		opts.Logger.Debug().Str("password", remoteHost.Password).Str("Host", remoteHost.Host).Send()
 
-		if err != nil {
+		remoteHost.Password = GetPassword(remoteHost.Password, opts)
 
-			return err
+		// opts.Logger.Debug().Str("actual password", remoteHost.Password).Str("Host", remoteHost.Host).Send()
-		}
 
 		remoteHost.ClientConfig.Auth = append(remoteHost.ClientConfig.Auth, ssh.Password(remoteHost.Password))
 	}
@@ -231,6 +227,8 @@ func (remoteHost *Host) GetPrivateKeyFileFromConfig() {
 	var identityFile string
 	if remoteHost.PrivateKeyPath == "" {
 		identityFile, _ = remoteHost.SSHConfigFile.SshConfigFile.Get(remoteHost.Host, "IdentityFile")
+		// println("Identity file:", identityFile)
+		// println("Host:", remoteHost.Host)
 		if identityFile == "" {
 			identityFile, _ = remoteHost.SSHConfigFile.DefaultUserSettings.GetStrict(remoteHost.Host, "IdentityFile")
 			if identityFile == "" {
@@ -242,6 +240,7 @@ func (remoteHost *Host) GetPrivateKeyFileFromConfig() {
 		identityFile = remoteHost.PrivateKeyPath
 	}
 
+	// println("Identity file:", identityFile)
 	remoteHost.PrivateKeyPath, _ = getFullPathWithHomeDir(identityFile)
 }
 
@@ -249,14 +248,13 @@ func (remoteHost *Host) GetPrivateKeyFileFromConfig() {
 // If it is the port is searched in the SSH config file(s)
 func (remoteHost *Host) GetPort() {
 	port := fmt.Sprintf("%d", remoteHost.Port)
-	// port specifed?
+	// port specified?
 	// port will be 0 if missing from backy config
 	if port == "0" {
 		port, _ = remoteHost.SSHConfigFile.SshConfigFile.Get(remoteHost.Host, "Port")
 
 		if port == "" {
 
-			// get port from default SSH config file
 			port = remoteHost.SSHConfigFile.DefaultUserSettings.Get(remoteHost.Host, "Port")
 
 			// set port to be default
@@ -271,7 +269,6 @@ func (remoteHost *Host) GetPort() {
 
 func (remoteHost *Host) CombineHostNameWithPort() {
 
-	// if the port is already in the HostName, leave it
 	if strings.HasSuffix(remoteHost.HostName, fmt.Sprintf(":%d", remoteHost.Port)) {
 		return
 	}
@@ -321,95 +318,44 @@ func (remoteHost *Host) ConnectThroughBastion(log zerolog.Logger) (*ssh.Client,
 
 // GetKnownHosts resolves the host's KnownHosts file if it is defined
 // if not defined, the default location for this file is used
-func (remotehHost *Host) GetKnownHosts() error {
+func (remoteHost *Host) GetKnownHosts() error {
 	var knownHostsFileErr error
-	if TS(remotehHost.KnownHostsFile) != "" {
+	if TS(remoteHost.KnownHostsFile) != "" {
-		remotehHost.KnownHostsFile, knownHostsFileErr = getFullPathWithHomeDir(remotehHost.KnownHostsFile)
+		remoteHost.KnownHostsFile, knownHostsFileErr = getFullPathWithHomeDir(remoteHost.KnownHostsFile)
 		return knownHostsFileErr
 	}
-	remotehHost.KnownHostsFile, knownHostsFileErr = getFullPathWithHomeDir("~/.ssh/known_hosts")
+	remoteHost.KnownHostsFile, knownHostsFileErr = getFullPathWithHomeDir("~/.ssh/known_hosts")
 	return knownHostsFileErr
 }
 
-func GetPrivateKeyPassword(key string, opts *ConfigOpts, log zerolog.Logger) (string, error) {
+func GetPrivateKeyPassword(key string, opts *ConfigOpts) string {
-
+	return getExternalConfigDirectiveValue(key, opts, AllowedExternalDirectiveAll)
-	var prKeyPassword string
-	if strings.HasPrefix(key, "file:") {
-		privKeyPassFilePath := strings.TrimPrefix(key, "file:")
-		privKeyPassFilePath, _ = getFullPathWithHomeDir(privKeyPassFilePath)
-		keyFile, keyFileErr := os.Open(privKeyPassFilePath)
-		if keyFileErr != nil {
-			return "", errors.Errorf("Private key password file %s failed to open. \n Make sure it is accessible and correct.", privKeyPassFilePath)
-		}
-		passwordScanner := bufio.NewScanner(keyFile)
-		for passwordScanner.Scan() {
-			prKeyPassword = passwordScanner.Text()
-		}
-	} else if strings.HasPrefix(key, "env:") {
-		privKey := strings.TrimPrefix(key, "env:")
-		privKey = strings.TrimPrefix(privKey, "${")
-		privKey = strings.TrimSuffix(privKey, "}")
-		privKey = strings.TrimPrefix(privKey, "$")
-		prKeyPassword = os.Getenv(privKey)
-	} else {
-		prKeyPassword = key
-	}
-	prKeyPassword = GetVaultKey(prKeyPassword, opts, opts.Logger)
-	return prKeyPassword, nil
 }
 
 // GetPassword gets any password
-func GetPassword(pass string, opts *ConfigOpts, log zerolog.Logger) (string, error) {
+func GetPassword(pass string, opts *ConfigOpts) string {
-
+	return getExternalConfigDirectiveValue(pass, opts, AllowedExternalDirectiveAll)
-	pass = strings.TrimSpace(pass)
-	if pass == "" {
-		return "", nil
-	}
-	var password string
-	if strings.HasPrefix(pass, "file:") {
-		passFilePath := strings.TrimPrefix(pass, "file:")
-		passFilePath, _ = getFullPathWithHomeDir(passFilePath)
-		keyFile, keyFileErr := os.Open(passFilePath)
-		if keyFileErr != nil {
-			return "", errors.New("Password file failed to open")
-		}
-		passwordScanner := bufio.NewScanner(keyFile)
-		for passwordScanner.Scan() {
-			password = passwordScanner.Text()
-		}
-	} else if strings.HasPrefix(pass, "env:") {
-		passEnv := strings.TrimPrefix(pass, "env:")
-		passEnv = strings.TrimPrefix(passEnv, "${")
-		passEnv = strings.TrimSuffix(passEnv, "}")
-		passEnv = strings.TrimPrefix(passEnv, "$")
-		password = os.Getenv(passEnv)
-	} else {
-		password = pass
-	}
-	password = GetVaultKey(password, opts, opts.Logger)
-
-	return password, nil
 }
 
-func (remoteConfig *Host) GetProxyJumpFromConfig(hosts map[string]*Host) error {
+func (remoteHost *Host) GetProxyJumpFromConfig(hosts map[string]*Host) error {
 
-	proxyJump, _ := remoteConfig.SSHConfigFile.SshConfigFile.Get(remoteConfig.Host, "ProxyJump")
+	proxyJump, _ := remoteHost.SSHConfigFile.SshConfigFile.Get(remoteHost.Host, "ProxyJump")
 	if proxyJump == "" {
-		proxyJump = remoteConfig.SSHConfigFile.DefaultUserSettings.Get(remoteConfig.Host, "ProxyJump")
+		proxyJump = remoteHost.SSHConfigFile.DefaultUserSettings.Get(remoteHost.Host, "ProxyJump")
 	}
-	if remoteConfig.ProxyJump == "" && proxyJump != "" {
+	if remoteHost.ProxyJump == "" && proxyJump != "" {
-		remoteConfig.ProxyJump = proxyJump
+		remoteHost.ProxyJump = proxyJump
 	}
-	proxyJumpHosts := strings.Split(remoteConfig.ProxyJump, ",")
+	proxyJumpHosts := strings.Split(remoteHost.ProxyJump, ",")
-	if remoteConfig.ProxyHost == nil && len(proxyJumpHosts) == 1 {
+	if remoteHost.ProxyHost == nil && len(proxyJumpHosts) == 1 {
-		remoteConfig.ProxyJump = proxyJump
+		remoteHost.ProxyJump = proxyJump
 		proxyHost, proxyHostFound := hosts[proxyJump]
 		if proxyHostFound {
-			remoteConfig.ProxyHost = append(remoteConfig.ProxyHost, proxyHost)
+			remoteHost.ProxyHost = append(remoteHost.ProxyHost, proxyHost)
 		} else {
 			if proxyJump != "" {
 				newProxy := &Host{Host: proxyJump}
-				remoteConfig.ProxyHost = append(remoteConfig.ProxyHost, newProxy)
+				remoteHost.ProxyHost = append(remoteHost.ProxyHost, newProxy)
 			}
 		}
 	}
@@ -417,25 +363,25 @@ func (remoteConfig *Host) GetProxyJumpFromConfig(hosts map[string]*Host) error {
 	return nil
 }
 
-func (remoteConfig *Host) GetProxyJumpConfig(hosts map[string]*Host, opts *ConfigOpts) error {
+func (remoteHost *Host) GetProxyJumpConfig(hosts map[string]*Host, opts *ConfigOpts) error {
 
-	if TS(remoteConfig.ConfigFilePath) == "" {
+	if TS(remoteHost.ConfigFilePath) == "" {
-		remoteConfig.useDefaultConfig = true
+		remoteHost.useDefaultConfig = true
 	}
 
-	khPathErr := remoteConfig.GetKnownHosts()
+	khPathErr := remoteHost.GetKnownHosts()
 
 	if khPathErr != nil {
 		return khPathErr
 	}
-	if remoteConfig.ClientConfig == nil {
+	if remoteHost.ClientConfig == nil {
-		remoteConfig.ClientConfig = &ssh.ClientConfig{}
+		remoteHost.ClientConfig = &ssh.ClientConfig{}
 	}
 	var configFile *os.File
 	var sshConfigFileOpenErr error
-	if !remoteConfig.useDefaultConfig {
+	if !remoteHost.useDefaultConfig {
 
-		configFile, sshConfigFileOpenErr = os.Open(remoteConfig.ConfigFilePath)
+		configFile, sshConfigFileOpenErr = os.Open(remoteHost.ConfigFilePath)
 		if sshConfigFileOpenErr != nil {
 			return sshConfigFileOpenErr
 		}
@@ -446,39 +392,39 @@ func (remoteConfig *Host) GetProxyJumpConfig(hosts map[string]*Host, opts *Confi
 			return sshConfigFileOpenErr
 		}
 	}
-	remoteConfig.SSHConfigFile = &sshConfigFile{}
+	remoteHost.SSHConfigFile = &sshConfigFile{}
-	remoteConfig.SSHConfigFile.DefaultUserSettings = ssh_config.DefaultUserSettings
+	remoteHost.SSHConfigFile.DefaultUserSettings = ssh_config.DefaultUserSettings
 	var decodeErr error
-	remoteConfig.SSHConfigFile.SshConfigFile, decodeErr = ssh_config.Decode(configFile)
+	remoteHost.SSHConfigFile.SshConfigFile, decodeErr = ssh_config.Decode(configFile)
 	if decodeErr != nil {
 		return decodeErr
 	}
-	remoteConfig.GetPrivateKeyFileFromConfig()
+	remoteHost.GetPrivateKeyFileFromConfig()
-	remoteConfig.GetPort()
+	remoteHost.GetPort()
-	remoteConfig.GetHostName()
+	remoteHost.GetHostName()
-	remoteConfig.CombineHostNameWithPort()
+	remoteHost.CombineHostNameWithPort()
-	remoteConfig.GetSshUserFromConfig()
+	remoteHost.GetSshUserFromConfig()
-	remoteConfig.isProxyHost = true
+	remoteHost.isProxyHost = true
-	if remoteConfig.HostName == "" {
+	if remoteHost.HostName == "" {
-		return errors.Errorf("No hostname found or specified for host %s", remoteConfig.Host)
+		return errors.Errorf("No hostname found or specified for host %s", remoteHost.Host)
 	}
-	err := remoteConfig.GetAuthMethods(opts)
+	err := remoteHost.GetAuthMethods(opts)
 	if err != nil {
 		return err
 	}
 
 	// TODO: Add value/option to config for host key and add bool to check for host key
-	hostKeyCallback, err := knownhosts.New(remoteConfig.KnownHostsFile)
+	hostKeyCallback, err := knownhosts.New(remoteHost.KnownHostsFile)
 	if err != nil {
 		return fmt.Errorf("could not create hostkeycallback function: %v", err)
 	}
-	remoteConfig.ClientConfig.HostKeyCallback = hostKeyCallback
+	remoteHost.ClientConfig.HostKeyCallback = hostKeyCallback
-	hosts[remoteConfig.Host] = remoteConfig
+	hosts[remoteHost.Host] = remoteHost
 
 	return nil
 }
 
-func (command *Command) RunCmdSSH(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([]string, error) {
+func (command *Command) RunCmdOnHost(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([]string, error) {
 	var (
 		ArgsStr       string
 		cmdOutBuf     bytes.Buffer
@@ -489,7 +435,6 @@ func (command *Command) RunCmdSSH(cmdCtxLogger zerolog.Logger, opts *ConfigOpts)
 			env:  command.Environment,
 		}
 	)
-	// Getting the command type must be done before concatenating the arguments
 	command = getCommandTypeAndSetCommandInfo(command)
 
 	// Prepare command arguments
@@ -499,8 +444,8 @@ func (command *Command) RunCmdSSH(cmdCtxLogger zerolog.Logger, opts *ConfigOpts)
 
 	cmdCtxLogger.Info().
 		Str("Command", command.Name).
-		Str("Host", *command.Host).
+		Str("Host", command.Host).
-		Msgf("Running %s on host %s", getCommandTypeAndSetCommandInfoLabel(command.Type), *command.Host)
+		Msgf("Running %s on host %s", getCommandTypeAndSetCommandInfoLabel(command.Type), command.Host)
 
 	// cmdCtxLogger.Debug().Str("cmd", command.Cmd).Strs("args", command.Args).Send()
 
@@ -531,14 +476,14 @@ func (command *Command) RunCmdSSH(cmdCtxLogger zerolog.Logger, opts *ConfigOpts)
 
 	// Handle command execution based on type
 	switch command.Type {
-	case ScriptCT:
+	case ScriptCommandType:
 		return command.runScript(commandSession, cmdCtxLogger, &cmdOutBuf)
-	case RemoteScriptCT:
+	case RemoteScriptCommandType:
 		return command.runRemoteScript(commandSession, cmdCtxLogger, &cmdOutBuf)
-	case ScriptFileCT:
+	case ScriptFileCommandType:
 		return command.runScriptFile(commandSession, cmdCtxLogger, &cmdOutBuf)
-	case PackageCT:
+	case PackageCommandType:
-		if command.PackageOperation == PackOpCheckVersion {
+		if command.PackageOperation == PackageOperationCheckVersion {
 			commandSession.Stderr = nil
 			// Execute the package version command remotely
 			// Parse the output of package version command
@@ -555,7 +500,7 @@ func (command *Command) RunCmdSSH(cmdCtxLogger zerolog.Logger, opts *ConfigOpts)
 			cmdCtxLogger.Debug().Str("cmd + args", ArgsStr).Send()
 			// Run simple command
 			if err := commandSession.Run(ArgsStr); err != nil {
-				return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error running command: %w", err)
+				return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.Output.ToLog), fmt.Errorf("error running command: %w", err)
 			}
 		}
 	default:
@@ -565,17 +510,108 @@ func (command *Command) RunCmdSSH(cmdCtxLogger zerolog.Logger, opts *ConfigOpts)
 			ArgsStr = fmt.Sprintf("%s %s", command.Cmd, ArgsStr)
 		}
 		cmdCtxLogger.Debug().Str("cmd + args", ArgsStr).Send()
-		// Run simple command
+
+		if command.Type == UserCommandType && command.UserOperation == "password" {
+			// cmdCtxLogger.Debug().Msgf("adding stdin")
+
+			userNamePass := fmt.Sprintf("%s:%s", command.Username, command.UserPassword)
+			client, err := sftp.NewClient(command.RemoteHost.SshClient)
+			if err != nil {
+				return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.Output.ToLog), fmt.Errorf("error creating sftp client: %v", err)
+			}
+			uuidFile := uuid.New()
+			passFilePath := fmt.Sprintf("/tmp/%s", uuidFile.String())
+			passFile, passFileErr := client.Create(passFilePath)
+			if passFileErr != nil {
+				return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.Output.ToLog), fmt.Errorf("error creating file /tmp/%s: %v", uuidFile.String(), passFileErr)
+			}
+
+			_, err = passFile.Write([]byte(userNamePass))
+			if err != nil {
+				return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.Output.ToLog), fmt.Errorf("error writing to file /tmp/%s: %v", uuidFile.String(), err)
+			}
+
+			ArgsStr = fmt.Sprintf("cat %s | chpasswd", passFilePath)
+			defer passFile.Close()
+
+			rmFileFunc := func() {
+				_ = client.Remove(passFilePath)
+			}
+
+			defer rmFileFunc()
+			// commandSession.Stdin = command.stdin
+		}
 		if err := commandSession.Run(ArgsStr); err != nil {
-			return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error running command: %w", err)
+			return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.Output.ToLog), fmt.Errorf("error running command: %w", err)
+		}
+
+		if command.Type == UserCommandType {
+
+			// REFACTOR IF/WHEN WINDOWS SUPPORT IS ADDED
+
+			if command.UserOperation == "add" {
+				if command.UserSshPubKeys != nil {
+					var (
+						f        *sftp.File
+						err      error
+						userHome []byte
+						client   *sftp.Client
+					)
+
+					cmdCtxLogger.Info().Msg("adding SSH Keys")
+
+					commandSession, _ = command.RemoteHost.createSSHSession(opts)
+					userHome, err = commandSession.CombinedOutput(fmt.Sprintf("grep \"%s\" /etc/passwd | cut -d: -f6", command.Username))
+					if err != nil {
+						return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.Output.ToLog), fmt.Errorf("error finding user home from /etc/passwd: %v", err)
+					}
+
+					command.UserHome = strings.TrimSpace(string(userHome))
+					userSshDir := fmt.Sprintf("%s/.ssh", command.UserHome)
+					client, err = sftp.NewClient(command.RemoteHost.SshClient)
+					if err != nil {
+						return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.Output.ToLog), fmt.Errorf("error creating sftp client: %v", err)
+					}
+
+					err = client.MkdirAll(userSshDir)
+					if err != nil {
+						return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.Output.ToLog), fmt.Errorf("error creating directory %s: %v", userSshDir, err)
+					}
+					_, err = client.Create(fmt.Sprintf("%s/authorized_keys", userSshDir))
+					if err != nil {
+						return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.Output.ToLog), fmt.Errorf("error opening file %s/authorized_keys: %v", userSshDir, err)
+					}
+					f, err = client.OpenFile(fmt.Sprintf("%s/authorized_keys", userSshDir), os.O_APPEND|os.O_CREATE|os.O_WRONLY)
+					if err != nil {
+						return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.Output.ToLog), fmt.Errorf("error opening file %s/authorized_keys: %v", userSshDir, err)
+					}
+					defer f.Close()
+					for _, k := range command.UserSshPubKeys {
+						buf := bytes.NewBufferString(k)
+						cmdCtxLogger.Info().Str("key", k).Msg("adding SSH key")
+						if _, err := f.ReadFrom(buf); err != nil {
+							return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.Output.ToLog), fmt.Errorf("error adding to authorized keys: %v", err)
+						}
+					}
+
+					commandSession, _ = command.RemoteHost.createSSHSession(opts)
+					_, err = commandSession.CombinedOutput(fmt.Sprintf("chown -R %s:%s %s", command.Username, command.Username, userHome))
+					if err != nil {
+						return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.Output.ToLog), err
+					}
+
+				}
+			}
 		}
 	}
 
-	return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), nil
+	return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.Output.ToLog), nil
 }
 
 func checkPackageVersion(cmdCtxLogger zerolog.Logger, command *Command, commandSession *ssh.Session, cmdOutBuf bytes.Buffer) ([]string, error) {
-	cmdCtxLogger.Info().Str("package", command.PackageName).Msg("Checking package versions")
+	for _, p := range command.Packages {
+		cmdCtxLogger.Info().Str("package", p.Name).Msg("Checking package versions")
+	}
 	// Prepare command arguments
 	ArgsStr := command.Cmd
 	for _, v := range command.Args {
@@ -590,9 +626,9 @@ func checkPackageVersion(cmdCtxLogger zerolog.Logger, command *Command, commandS
 
 		_, parseErr := parsePackageVersion(string(cmdOut), cmdCtxLogger, command, cmdOutBuf)
 		if parseErr != nil {
-			return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error: package %s not listed: %w", command.PackageName, err)
+			return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.Output.ToLog), fmt.Errorf("error: packages %v not listed: %w", command.Packages, err)
 		}
-		return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error running %s: %w", ArgsStr, err)
+		return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.Output.ToLog), fmt.Errorf("error running %s: %w", ArgsStr, err)
 	}
 
 	return parsePackageVersion(string(cmdOut), cmdCtxLogger, command, cmdOutBuf)
@@ -622,7 +658,7 @@ func (command *Command) runScript(session *ssh.Session, cmdCtxLogger zerolog.Log
 		return collectOutput(outputBuf, command.Name, cmdCtxLogger, true), fmt.Errorf("error waiting for shell: %w", err)
 	}
 
-	return collectOutput(outputBuf, command.Name, cmdCtxLogger, command.GetOutput), nil
+	return collectOutput(outputBuf, command.Name, cmdCtxLogger, command.Output.ToLog), nil
 }
 
 // runScriptFile handles the execution of script files.
@@ -641,7 +677,7 @@ func (command *Command) runScriptFile(session *ssh.Session, cmdCtxLogger zerolog
 		return collectOutput(outputBuf, command.Name, cmdCtxLogger, true), fmt.Errorf("error waiting for shell: %w", err)
 	}
 
-	return collectOutput(outputBuf, command.Name, cmdCtxLogger, command.OutputToLog), nil
+	return collectOutput(outputBuf, command.Name, cmdCtxLogger, command.Output.ToLog), nil
 }
 
 // prepareScriptBuffer prepares a buffer for inline scripts.
@@ -698,10 +734,10 @@ func (command *Command) runRemoteScript(session *ssh.Session, cmdCtxLogger zerol
 	err = session.Run(command.Shell)
 
 	if err != nil {
-		return collectOutput(outputBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error running remote script: %w", err)
+		return collectOutput(outputBuf, command.Name, cmdCtxLogger, command.Output.ToLog), fmt.Errorf("error running remote script: %w", err)
 	}
 
-	return collectOutput(outputBuf, command.Name, cmdCtxLogger, command.OutputToLog), nil
+	return collectOutput(outputBuf, command.Name, cmdCtxLogger, command.Output.ToLog), nil
 }
 
 // readFileToBuffer reads a file into a buffer.
@@ -774,7 +810,7 @@ func (h *Host) DetectOS(opts *ConfigOpts) (string, error) {
 	return osName, nil
 }
 
-func CheckIfHostHasHostName(host string) (bool, string) {
+func DoesHostHaveHostName(host string) (bool, string) {
 	HostName, err := ssh_config.DefaultUserSettings.GetStrict(host, "HostName")
 	if err != nil {
 		return false, ""
@@ -782,3 +818,8 @@ func CheckIfHostHasHostName(host string) (bool, string) {
 	println(HostName)
 	return HostName != "", HostName
 }
+
+func IsHostLocal(host string) bool {
+	host = strings.ToLower(host)
+	return host == "127.0.0.1" || host == "localhost" || host == ""
+}

pkg/backy/types.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 
 	"git.andrewnw.xyz/CyberShell/backy/pkg/pkgman"
+	packagemanagercommon "git.andrewnw.xyz/CyberShell/backy/pkg/pkgman/common"
 	"git.andrewnw.xyz/CyberShell/backy/pkg/remotefetcher"
 	"git.andrewnw.xyz/CyberShell/backy/pkg/usermanager"
 	vaultapi "github.com/hashicorp/vault/api"
@@ -26,15 +27,15 @@ type (
 		ConfigFilePath     string `yaml:"config,omitempty"`
 		Host               string `yaml:"host,omitempty"`
 		HostName           string `yaml:"hostname,omitempty"`
-		KnownHostsFile     string `yaml:"knownhostsfile,omitempty"`
+		KnownHostsFile     string `yaml:"knownHostsFile,omitempty"`
 		ClientConfig       *ssh.ClientConfig
 		SSHConfigFile      *sshConfigFile
 		SshClient          *ssh.Client
 		Port               uint16 `yaml:"port,omitempty"`
 		ProxyJump          string `yaml:"proxyjump,omitempty"`
 		Password           string `yaml:"password,omitempty"`
-		PrivateKeyPath     string `yaml:"privatekeypath,omitempty"`
+		PrivateKeyPath     string `yaml:"IdentityFile,omitempty"`
-		PrivateKeyPassword string `yaml:"privatekeypassword,omitempty"`
+		PrivateKeyPassword string `yaml:"privateKeyPassword,omitempty"`
 		useDefaultConfig   bool
 		User               string `yaml:"user,omitempty"`
 		isProxyHost        bool
@@ -51,61 +52,45 @@ type (
 	Command struct {
 		Name string `yaml:"name,omitempty"`
 
-		// command to run
 		Cmd string `yaml:"cmd"`
 
 		// See CommandType enum further down the page for acceptable values
 		Type CommandType `yaml:"type,omitempty"`
 
-		// host on which to run cmd
+		Host string `yaml:"host,omitempty"`
-		Host *string `yaml:"host,omitempty"`
 
-		// Hooks are for running commands on certain events
 		Hooks *Hooks `yaml:"hooks,omitempty"`
 
-		// hook refs are internal references of commands for each hook type
 		hookRefs map[string]map[string]*Command
 
-		// Shell specifies which shell to run the command in, if any.
 		Shell string `yaml:"shell,omitempty"`
 
 		RemoteHost *Host `yaml:"-"`
 
-		// Args is an array that holds the arguments to cmd
 		Args []string `yaml:"args,omitempty"`
 
-		/*
-			Dir specifies a directory in which to run the command.
-		*/
 		Dir *string `yaml:"dir,omitempty"`
 
-		// Env points to a file containing env variables to be used with the command
 		Env string `yaml:"env,omitempty"`
 
-		// Environment holds env variables to be used with the command
 		Environment []string `yaml:"environment,omitempty"`
 
-		// Output determines if output is requested.
-		//
-		// Only for when command is in a list.
-		GetOutput bool `yaml:"getOutput,omitempty"`
-
 		ScriptEnvFile string `yaml:"scriptEnvFile"`
 
-		OutputToLog bool `yaml:"outputToLog,omitempty"`
+		Output struct {
-
+			File   string `yaml:"file,omitempty"`
-		OutputFile string `yaml:"outputFile,omitempty"`
+			ToLog  bool   `yaml:"toLog,omitempty"`
+			InList bool   `yaml:"inList,omitempty"`
+		} `yaml:"output"`
 
 		// BEGIN PACKAGE COMMAND FIELDS
 
 		PackageManager string `yaml:"packageManager,omitempty"`
 
-		PackageName string `yaml:"packageName,omitempty"`
+		Packages []packagemanagercommon.Package `yaml:"packages,omitempty"`
 
-		// Version specifies the desired version for package execution
 		PackageVersion string `yaml:"packageVersion,omitempty"`
 
-		// PackageOperation specifies the action for package-related commands (e.g., "install" or "remove")
 		PackageOperation PackageOperation `yaml:"packageOperation,omitempty"`
 
 		pkgMan pkgman.PackageManager
@@ -113,42 +98,37 @@ type (
 		packageCmdSet bool
 		// END PACKAGE COMMAND FIELDS
 
-		// RemoteSource specifies a URL to fetch the command or configuration remotely
 		RemoteSource string `yaml:"remoteSource,omitempty"`
 
-		// FetchBeforeExecution determines if the remoteSource should be fetched before running
 		FetchBeforeExecution bool `yaml:"fetchBeforeExecution,omitempty"`
 
 		Fetcher remotefetcher.RemoteFetcher
 
 		// BEGIN USER COMMAND FIELDS
 
-		// Username specifies the username for user creation or related operations
 		Username string `yaml:"userName,omitempty"`
 
 		UserID string `yaml:"userID,omitempty"`
 
-		// UserGroups specifies the groups to add the user to
 		UserGroups []string `yaml:"userGroups,omitempty"`
 
-		// UserHome specifies the home directory for the user
 		UserHome string `yaml:"userHome,omitempty"`
 
-		// UserShell specifies the shell for the user
 		UserShell string `yaml:"userShell,omitempty"`
 
-		// SystemUser specifies whether the user is a system account
+		UserCreateHome bool `yaml:"userCreateHome,omitempty"`
-		SystemUser bool `yaml:"systemUser,omitempty"`
+
+		UserIsSystem bool `yaml:"userIsSystem,omitempty"`
 
-		// UserPassword specifies the password for the user (can be file: or plain text)
 		UserPassword string `yaml:"userPassword,omitempty"`
 
+		UserSshPubKeys []string `yaml:"userSshPubKeys,omitempty"`
+
 		userMan usermanager.UserManager
 
 		// OS for the command, only used when type is user
 		OS string `yaml:"OS,omitempty"`
 
-		// UserOperation specifies the action for user-related commands (e.g., "create" or "remove")
 		UserOperation string `yaml:"userOperation,omitempty"`
 
 		userCmdSet bool
@@ -156,7 +136,7 @@ type (
 		// stdin only for userOperation = password (for now)
 		stdin *strings.Reader
 
-		// END USER STRUCT FIELDS
+		// END USER STRUCommandType FIELDS
 	}
 
 	RemoteSource struct {
@@ -171,13 +151,17 @@ type (
 	BackyOptionFunc func(*ConfigOpts)
 
 	CmdList struct {
-		Name            string   `yaml:"name,omitempty"`
+		Name                                     string   `yaml:"name,omitempty"`
-		Cron            string   `yaml:"cron,omitempty"`
+		Cron                                     string   `yaml:"cron,omitempty"`
-		RunCmdOnFailure string   `yaml:"runCmdOnFailure,omitempty"`
+		RunCmdOnFailure                          string   `yaml:"runCmdOnFailure,omitempty"`
-		Order           []string `yaml:"order,omitempty"`
+		Order                                    []string `yaml:"order,omitempty"`
-		Notifications   []string `yaml:"notifications,omitempty"`
+		Notifications                            []string `yaml:"notifications,omitempty"`
-		GetOutput       bool     `yaml:"getOutput,omitempty"`
+		GetCommandOutputInNotificationsOnSuccess bool     `yaml:"sendNotificationOnSuccess,omitempty"`
-		NotifyOnSuccess bool     `yaml:"notifyOnSuccess,omitempty"`
+
+		Notify struct {
+			OnFailure bool `yaml:"onFailure,omitempty"`
+			OnSuccess bool `yaml:"onSuccess,omitempty"`
+		} `yaml:"notify,omitempty"`
 
 		NotifyConfig *notify.Notify
 		Source       string `yaml:"source"` // URL to fetch remote commands
@@ -206,6 +190,8 @@ type (
 
 		ConfigFilePath string
 
+		HostsFilePath string
+
 		ConfigDir string
 
 		LogFilePath string
@@ -226,6 +212,8 @@ type (
 
 		List ListConfig
 
+		Vars map[string]string `yaml:"variables"`
+
 		VaultKeys []*VaultKey `yaml:"keys"`
 
 		koanf *koanf.Koanf
@@ -244,6 +232,7 @@ type (
 
 	VaultKey struct {
 		Name      string `yaml:"name"`
+		Key       string `yaml:"key"`
 		Path      string `yaml:"path"`
 		ValueType string `yaml:"type"`
 		MountPath string `yaml:"mountpath"`
@@ -259,6 +248,7 @@ type (
 	Notifications struct {
 		MailConfig   map[string]MailConfig   `yaml:"mail,omitempty"`
 		MatrixConfig map[string]MatrixStruct `yaml:"matrix,omitempty"`
+		HttpConfig   map[string]HttpConfig   `yaml:"http,omitempty"`
 	}
 
 	CmdOutput struct {
@@ -294,28 +284,55 @@ type (
 		Error    error  // Error encountered, if any
 	}
 
+	ListMetrics struct {
+		Name                 string
+		SuccessfulExecutions uint64
+		FailedExecutions     uint64
+		TotalExecutions      uint64
+	}
+
+	CommandMetrics struct {
+		Name                 string
+		SuccessfulExecutions uint64
+		FailedExecutions     uint64
+		TotalExecutions      uint64
+	}
+
 	// use ints so we can use enums
-	CommandType      int
+	CommandType               int
-	PackageOperation int
+	PackageOperation          int
+	AllowedExternalDirectives int
 )
 
 //go:generate go run github.com/dmarkham/enumer -linecomment -yaml -text -json -type=CommandType
 const (
-	DefaultCT      CommandType = iota //
+	DefaultCommandType      CommandType = iota //
-	ScriptCT                          // script
+	ScriptCommandType                          // script
-	ScriptFileCT                      // scriptFile
+	ScriptFileCommandType                      // scriptFile
-	RemoteScriptCT                    // remoteScript
+	RemoteScriptCommandType                    // remoteScript
-	PackageCT                         // package
+	PackageCommandType                         // package
-	UserCT                            // user
+	UserCommandType                            // user
 )
 
 //go:generate go run github.com/dmarkham/enumer -linecomment -yaml -text -json -type=PackageOperation
 const (
-	DefaultPO          PackageOperation = iota //
+	DefaultPO                    PackageOperation = iota //
-	PackOpInstall                              // install
+	PackageOperationInstall                              // install
-	PackOpUpgrade                              // upgrade
+	PackageOperationUpgrade                              // upgrade
-	PackOpPurge                                // purge
+	PackageOperationPurge                                // purge
-	PackOpRemove                               // remove
+	PackageOperationRemove                               // remove
-	PackOpCheckVersion                         // checkVersion
+	PackageOperationCheckVersion                         // checkVersion
-	PackOpIsInstalled                          // isInstalled
+	PackageOperationIsInstalled                          // isInstalled
+)
+
+//go:generate go run github.com/dmarkham/enumer -linecomment -yaml -text -json -type=AllowedExternalDirectives
+const (
+	DefaultExternalDir                AllowedExternalDirectives = iota
+	AllowedExternalDirectiveVault                               // vault
+	AllowedExternalDirectiveVaultEnv                            // vault-env
+	AllowedExternalDirectiveVaultFile                           // vault-file
+	AllowedExternalDirectiveAll                                 // vault-file-env
+	AllowedExternalDirectiveFileEnv                             // file-env
+	AllowedExternalDirectiveFile                                // file
+	AllowedExternalDirectiveEnv                                 // env
 )

pkg/backy/utils.go

@@ -6,16 +6,19 @@ package backy
 
 import (
 	"bytes"
+	"context"
 	"errors"
 	"fmt"
 	"os"
 	"os/exec"
 	"path"
 	"path/filepath"
+	"regexp"
 	"strings"
 
 	"git.andrewnw.xyz/CyberShell/backy/pkg/logging"
 	"git.andrewnw.xyz/CyberShell/backy/pkg/remotefetcher"
+	vault "github.com/hashicorp/vault/api"
 	"github.com/joho/godotenv"
 	"github.com/knadh/koanf/v2"
 	"github.com/rs/zerolog"
@@ -65,8 +68,14 @@ func SetLogFile(logFile string) BackyOptionFunc {
 	}
 }
 
-// SetCmdStdOut forces the command output to stdout
+func SetHostsConfigFile(hostsConfigFile string) BackyOptionFunc {
-func SetCmdStdOut(setStdOut bool) BackyOptionFunc {
+	return func(bco *ConfigOpts) {
+		bco.HostsFilePath = hostsConfigFile
+	}
+}
+
+// EnableCommandStdOut forces the command output to stdout
+func EnableCommandStdOut(setStdOut bool) BackyOptionFunc {
 	return func(bco *ConfigOpts) {
 		bco.CmdStdOut = setStdOut
 	}
@@ -79,7 +88,7 @@ func EnableCron() BackyOptionFunc {
 	}
 }
 
-func NewOpts(configFilePath string, opts ...BackyOptionFunc) *ConfigOpts {
+func NewConfigOptions(configFilePath string, opts ...BackyOptionFunc) *ConfigOpts {
 	b := &ConfigOpts{}
 	b.ConfigFilePath = configFilePath
 	for _, opt := range opts {
@@ -108,7 +117,11 @@ func injectEnvIntoSSH(envVarsToInject environmentVars, process *ssh.Session, opt
 			goto errEnvFile
 		}
 		for key, val := range envMap {
-			process.Setenv(key, GetVaultKey(val, opts, log))
+			err = process.Setenv(key, getExternalConfigDirectiveValue(val, opts, AllowedExternalDirectiveVault))
+			if err != nil {
+				log.Error().Err(err).Send()
+
+			}
 		}
 	}
 
@@ -119,12 +132,15 @@ errEnvFile:
 		if strings.Contains(envVal, "=") {
 			envVarArr := strings.Split(envVal, "=")
 
-			process.Setenv(envVarArr[0], GetVaultKey(envVarArr[1], opts, log))
+			err := process.Setenv(envVarArr[0], getExternalConfigDirectiveValue(envVarArr[1], opts, AllowedExternalDirectiveVaultFile))
+			if err != nil {
+				log.Error().Err(err).Send()
+			}
 		}
 	}
 }
 
-func injectEnvIntoLocalCMD(envVarsToInject environmentVars, process *exec.Cmd, log zerolog.Logger) {
+func injectEnvIntoLocalCMD(envVarsToInject environmentVars, process *exec.Cmd, log zerolog.Logger, opts *ConfigOpts) {
 	if envVarsToInject.file != "" {
 		envPath, _ := getFullPathWithHomeDir(envVarsToInject.file)
 
@@ -148,7 +164,8 @@ errEnvFile:
 
 	for _, envVal := range envVarsToInject.env {
 		if strings.Contains(envVal, "=") {
-			process.Env = append(process.Env, envVal)
+			envVarArr := strings.Split(envVal, "=")
+			process.Env = append(process.Env, fmt.Sprintf("%s=%s", envVarArr[0], getExternalConfigDirectiveValue(envVarArr[1], opts, AllowedExternalDirectiveVault)))
 		}
 	}
 	process.Env = append(process.Env, os.Environ()...)
@@ -181,7 +198,6 @@ func testFile(c string) error {
 			return fileOpenErr
 		}
 	}
-
 	return nil
 }
 
@@ -247,11 +263,9 @@ func (opts *ConfigOpts) loadEnv() {
 	opts.backyEnv = backyEnv
 }
 
-// expandEnvVars expands environment variables with the env used in the config
 func expandEnvVars(backyEnv map[string]string, envVars []string) {
 
 	env := func(name string) string {
-		name = strings.ToUpper(name)
 		envVar, found := backyEnv[name]
 		if found {
 			return envVar
@@ -259,36 +273,35 @@ func expandEnvVars(backyEnv map[string]string, envVars []string) {
 		return ""
 	}
 
-	// parse env variables using new macros
 	for indx, v := range envVars {
-		if strings.HasPrefix(v, macroStart) && strings.HasSuffix(v, macroEnd) {
+
-			if strings.HasPrefix(v, envMacroStart) {
+		if strings.HasPrefix(v, envExternDirectiveStart) && strings.HasSuffix(v, externDirectiveEnd) {
-				v = strings.TrimPrefix(v, envMacroStart)
+			v = strings.TrimPrefix(v, envExternDirectiveStart)
-				v = strings.TrimRight(v, macroEnd)
+			v = strings.TrimRight(v, externDirectiveEnd)
-				out, _ := shell.Expand(v, env)
+			out, _ := shell.Expand(v, env)
-				envVars[indx] = out
+			envVars[indx] = out
-			}
 		}
+
 	}
 }
 
 func getCommandTypeAndSetCommandInfo(command *Command) *Command {
 
-	if command.Type == PackageCT && !command.packageCmdSet {
+	if command.Type == PackageCommandType && !command.packageCmdSet {
 		command.packageCmdSet = true
 		switch command.PackageOperation {
-		case PackOpInstall:
+		case PackageOperationInstall:
-			command.Cmd, command.Args = command.pkgMan.Install(command.PackageName, command.PackageVersion, command.Args)
+			command.Cmd, command.Args = command.pkgMan.Install(command.Packages, command.Args)
-		case PackOpRemove:
+		case PackageOperationRemove:
-			command.Cmd, command.Args = command.pkgMan.Remove(command.PackageName, command.Args)
+			command.Cmd, command.Args = command.pkgMan.Remove(command.Packages, command.Args)
-		case PackOpUpgrade:
+		case PackageOperationUpgrade:
-			command.Cmd, command.Args = command.pkgMan.Upgrade(command.PackageName, command.PackageVersion)
+			command.Cmd, command.Args = command.pkgMan.Upgrade(command.Packages)
-		case PackOpCheckVersion:
+		case PackageOperationCheckVersion:
-			command.Cmd, command.Args = command.pkgMan.CheckVersion(command.PackageName, command.PackageVersion)
+			command.Cmd, command.Args = command.pkgMan.CheckVersion(command.Packages)
 		}
 	}
 
-	if command.Type == UserCT && !command.userCmdSet {
+	if command.Type == UserCommandType && !command.userCmdSet {
 		command.userCmdSet = true
 		switch command.UserOperation {
 		case "add":
@@ -296,7 +309,8 @@ func getCommandTypeAndSetCommandInfo(command *Command) *Command {
 				command.Username,
 				command.UserHome,
 				command.UserShell,
-				command.SystemUser,
+				command.UserIsSystem,
+				command.UserCreateHome,
 				command.UserGroups,
 				command.Args)
 		case "modify":
@@ -319,33 +333,188 @@ func getCommandTypeAndSetCommandInfo(command *Command) *Command {
 
 func parsePackageVersion(output string, cmdCtxLogger zerolog.Logger, command *Command, cmdOutBuf bytes.Buffer) ([]string, error) {
 
-	var err error
+	var errs []error
-	pkgVersion, err := command.pkgMan.Parse(output)
+	pkgVersionOnSystem, errs := command.pkgMan.ParseRemotePackageManagerVersionOutput(output)
-	// println(output)
+	if errs != nil {
-	if err != nil {
+		cmdCtxLogger.Error().Errs("Error parsing package version output", errs).Send()
-		cmdCtxLogger.Error().Err(err).Str("package", command.PackageName).Msg("Error parsing package version output")
+		return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.Output.ToLog), fmt.Errorf("error parsing package version output: %v", errs)
-		return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.GetOutput), err
 	}
 
-	cmdCtxLogger.Info().
+	for _, p := range pkgVersionOnSystem {
-		Str("Installed", pkgVersion.Installed).
+		packageIndex := getPackageIndexFromCommand(command, p.Name)
-		Str("Candidate", pkgVersion.Candidate).
+		if packageIndex == -1 {
-		Msg("Package version comparison")
+			cmdCtxLogger.Error().Str("package", p.Name).Msg("Package not found in command")
-
+			continue
-	if command.PackageVersion != "" {
-		if pkgVersion.Installed == command.PackageVersion {
-			cmdCtxLogger.Info().Msgf("Installed version matches specified version: %s", command.PackageVersion)
-		} else {
-			cmdCtxLogger.Info().Msgf("Installed version does not match specified version: %s", command.PackageVersion)
-			err = fmt.Errorf("Installed version does not match specified version: %s", command.PackageVersion)
 		}
-	} else {
+		command.Packages[packageIndex].VersionCheck = p.VersionCheck
-		if pkgVersion.Installed == pkgVersion.Candidate {
+		packageFromCommand := command.Packages[packageIndex]
-			cmdCtxLogger.Info().Msg("Installed and Candidate versions match")
+		cmdCtxLogger.Info().
+			Str("Installed", packageFromCommand.VersionCheck.Installed).
+			Msg("Package version comparison")
+
+		versionLogger := cmdCtxLogger.With().Str("package", packageFromCommand.Name).Logger()
+
+		if packageFromCommand.Version != "" {
+			versionLogger := cmdCtxLogger.With().Str("package", packageFromCommand.Name).Str("Specified Version", packageFromCommand.Version).Logger()
+			packageVersionRegex, PkgRegexErr := regexp.Compile(packageFromCommand.Version)
+			if PkgRegexErr != nil {
+				versionLogger.Error().Err(PkgRegexErr).Msg("Error compiling package version regex")
+				errs = append(errs, PkgRegexErr)
+				continue
+			}
+			if p.Version == packageFromCommand.Version {
+				versionLogger.Info().Msgf("Installed version matches specified version: %s", packageFromCommand.Version)
+			} else if packageVersionRegex.MatchString(p.VersionCheck.Installed) {
+				versionLogger.Info().Msgf("Installed version contains specified version: %s", packageFromCommand.Version)
+			} else {
+				versionLogger.Info().Msg("Installed version does not match specified version")
+				errs = append(errs, fmt.Errorf("installed version of %s does not match specified version: %s", packageFromCommand.Name, packageFromCommand.Version))
+			}
 		} else {
-			cmdCtxLogger.Info().Msg("Installed and Candidate versions differ")
+			if p.VersionCheck.Installed == p.VersionCheck.Candidate {
-			err = errors.New("Installed and Candidate versions differ")
+				versionLogger.Info().Msg("Installed and Candidate versions match")
+			} else {
+				cmdCtxLogger.Info().Msg("Installed and Candidate versions differ")
+				errs = append(errs, errors.New("installed and Candidate versions differ"))
+			}
 		}
 	}
-	return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, false), err
+	if errs == nil {
+		return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.Output.ToLog), nil
+	}
+	return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.Output.ToLog), fmt.Errorf("error parsing package version output: %v", errs)
+}
+
+func getPackageIndexFromCommand(command *Command, name string) int {
+	for i, v := range command.Packages {
+		if name == v.Name {
+			return i
+		}
+	}
+
+	return -1
+}
+
+func getExternalConfigDirectiveValue(key string, opts *ConfigOpts, allowedDirectives AllowedExternalDirectives) string {
+	if !(strings.HasPrefix(key, externDirectiveStart) && strings.HasSuffix(key, externDirectiveEnd)) {
+		return key
+	}
+	key = replaceVarInString(opts.Vars, key, opts.Logger)
+	opts.Logger.Debug().Str("expanding external key", key).Send()
+
+	if strings.HasPrefix(key, envExternDirectiveStart) {
+		if IsExternalDirectiveEnv(allowedDirectives) {
+
+			key = strings.TrimPrefix(key, envExternDirectiveStart)
+			key = strings.TrimSuffix(key, externDirectiveEnd)
+			key = os.Getenv(key)
+		} else {
+			opts.Logger.Error().Msgf("Config key with value %s does not support env directive", key)
+		}
+	}
+
+	if strings.HasPrefix(key, externFileDirectiveStart) {
+		if IsExternalDirectiveFile(allowedDirectives) {
+
+			var err error
+			var keyValue []byte
+			key = strings.TrimPrefix(key, externFileDirectiveStart)
+			key = strings.TrimSuffix(key, externDirectiveEnd)
+			key, err = getFullPathWithHomeDir(key)
+			if err != nil {
+				opts.Logger.Err(err).Send()
+				return ""
+			}
+			if !path.IsAbs(key) {
+				key = path.Join(opts.ConfigDir, key)
+			}
+			keyValue, err = os.ReadFile(key)
+			if err != nil {
+				opts.Logger.Err(err).Send()
+				return ""
+			}
+			key = string(keyValue)
+		} else {
+			opts.Logger.Error().Msgf("Config key with value %s does not support file directive", key)
+		}
+	}
+
+	if strings.HasPrefix(key, vaultExternDirectiveStart) {
+		if IsExternalDirectiveVault(allowedDirectives) {
+
+			key = strings.TrimPrefix(key, vaultExternDirectiveStart)
+			key = strings.TrimSuffix(key, externDirectiveEnd)
+			key = GetVaultKey(key, opts, opts.Logger)
+		} else {
+			opts.Logger.Error().Msgf("Config key with value %s does not support vault directive", key)
+		}
+	}
+
+	return key
+}
+
+func getVaultSecret(vaultClient *vault.Client, key *VaultKey) (string, error) {
+	var (
+		secret *vault.KVSecret
+		err    error
+	)
+
+	if key.ValueType == "KVv2" {
+		secret, err = vaultClient.KVv2(key.MountPath).Get(context.Background(), key.Path)
+	} else if key.ValueType == "KVv1" {
+		secret, err = vaultClient.KVv1(key.MountPath).Get(context.Background(), key.Path)
+	} else if key.ValueType != "" {
+		return "", fmt.Errorf("type %s for key %s not known. Valid types are KVv1 or KVv2", key.ValueType, key.Name)
+	} else {
+		return "", fmt.Errorf("type for key %s must be specified. Valid types are KVv1 or KVv2", key.Name)
+	}
+	if err != nil {
+		return "", fmt.Errorf("unable to read secret: %v", err)
+	}
+
+	value, ok := secret.Data[key.Key].(string)
+	if !ok {
+		return "", fmt.Errorf("value type assertion failed for vault key %s: %T %#v", key.Name, secret.Data[key.Name], secret.Data[key.Name])
+	}
+
+	return value, nil
+}
+
+func getVaultKeyData(keyName string, keys []*VaultKey) (*VaultKey, error) {
+	for _, k := range keys {
+		if k.Name == keyName {
+			return k, nil
+		}
+	}
+	return nil, fmt.Errorf("key %s not found in vault keys", keyName)
+}
+
+func GetVaultKey(str string, opts *ConfigOpts, log zerolog.Logger) string {
+	key, err := getVaultKeyData(str, opts.VaultKeys)
+	if key == nil && err == nil {
+		return str
+	}
+	if err != nil && key == nil {
+		log.Err(err).Send()
+		return ""
+	}
+
+	value, secretErr := getVaultSecret(opts.vaultClient, key)
+	if secretErr != nil {
+		log.Err(secretErr).Send()
+		return value
+	}
+	return value
+}
+
+func IsExternalDirectiveFile(allowedExternalDirectives AllowedExternalDirectives) bool {
+	return strings.Contains(allowedExternalDirectives.String(), "file")
+}
+
+func IsExternalDirectiveEnv(allowedExternalDirectives AllowedExternalDirectives) bool {
+	return strings.Contains(allowedExternalDirectives.String(), "env")
+}
+
+func IsExternalDirectiveVault(allowedExternalDirectives AllowedExternalDirectives) bool {
+	return strings.Contains(allowedExternalDirectives.String(), "vault")
 }

pkg/pkgman/apt/apt.go

@@ -1,18 +1,19 @@
 package apt
 
 import (
+	"bufio"
+	"bytes"
 	"fmt"
-	"regexp"
 	"strings"
 
-	"git.andrewnw.xyz/CyberShell/backy/pkg/pkgman/pkgcommon"
+	packagemanagercommon "git.andrewnw.xyz/CyberShell/backy/pkg/pkgman/common"
 )
 
 // AptManager implements PackageManager for systems using APT.
 type AptManager struct {
 	useAuth     bool   // Whether to use an authentication command
 	authCommand string // The authentication command, e.g., "sudo"
-	Parser      pkgcommon.PackageParser
+	Parser      packagemanagercommon.PackageParser
 }
 
 // DefaultAuthCommand is the default command used for authentication.
@@ -29,14 +30,13 @@ func NewAptManager() *AptManager {
 }
 
 // Install returns the command and arguments for installing a package.
-func (a *AptManager) Install(pkg, version string, args []string) (string, []string) {
+func (a *AptManager) Install(pkgs []packagemanagercommon.Package, args []string) (string, []string) {
 	baseCmd := a.prependAuthCommand(DefaultPackageCommand)
 	baseArgs := []string{"update", "&&", baseCmd, "install", "-y"}
-	if version != "" {
+	for _, p := range pkgs {
-		baseArgs = append(baseArgs, fmt.Sprintf("%s=%s", pkg, version))
+		baseArgs = append(baseArgs, p.Name)
-	} else {
-		baseArgs = append(baseArgs, pkg)
 	}
+
 	if args != nil {
 		baseArgs = append(baseArgs, args...)
 	}
@@ -44,31 +44,34 @@ func (a *AptManager) Install(pkg, version string, args []string) (string, []stri
 }
 
 // Remove returns the command and arguments for removing a package.
-func (a *AptManager) Remove(pkg string, args []string) (string, []string) {
+func (a *AptManager) Remove(pkgs []packagemanagercommon.Package, args []string) (string, []string) {
 	baseCmd := a.prependAuthCommand(DefaultPackageCommand)
-	baseArgs := []string{"remove", "-y", pkg}
+	baseArgs := []string{"remove", "-y"}
+	for _, p := range pkgs {
+		baseArgs = append(baseArgs, p.Name)
+	}
 	if args != nil {
 		baseArgs = append(baseArgs, args...)
 	}
 	return baseCmd, baseArgs
 }
 
-// Upgrade returns the command and arguments for upgrading a specific package.
+func (a *AptManager) Upgrade(pkgs []packagemanagercommon.Package) (string, []string) {
-func (a *AptManager) Upgrade(pkg, version string) (string, []string) {
 	baseCmd := a.prependAuthCommand(DefaultPackageCommand)
 	baseArgs := []string{"update", "&&", baseCmd, "install", "--only-upgrade", "-y"}
-	if version != "" {
+	for _, p := range pkgs {
-		baseArgs = append(baseArgs, fmt.Sprintf("%s=%s", pkg, version))
+		baseArgs = append(baseArgs, p.Name)
-	} else {
-		baseArgs = append(baseArgs, pkg)
 	}
+
 	return baseCmd, baseArgs
 }
 
-// CheckVersion returns the command and arguments for checking the info of a specific package.
+func (a *AptManager) CheckVersion(pkgs []packagemanagercommon.Package) (string, []string) {
-func (a *AptManager) CheckVersion(pkg, version string) (string, []string) {
 	baseCmd := a.prependAuthCommand("apt-cache")
-	baseArgs := []string{"policy", pkg}
+	baseArgs := []string{"policy"}
+	for _, p := range pkgs {
+		baseArgs = append(baseArgs, p.Name)
+	}
 
 	return baseCmd, baseArgs
 }
@@ -81,7 +84,7 @@ func (a *AptManager) UpgradeAll() (string, []string) {
 }
 
 // Configure applies functional options to customize the package manager.
-func (a *AptManager) Configure(options ...pkgcommon.PackageManagerOption) {
+func (a *AptManager) Configure(options ...packagemanagercommon.PackageManagerOption) {
 	for _, opt := range options {
 		opt(a)
 	}
@@ -106,25 +109,56 @@ func (a *AptManager) SetAuthCommand(authCommand string) {
 }
 
 // Parse parses the apt-cache policy output to extract Installed and Candidate versions.
-func (a *AptManager) Parse(output string) (*pkgcommon.PackageVersion, error) {
+func (a *AptManager) ParseRemotePackageManagerVersionOutput(output string) ([]packagemanagercommon.Package, []error) {
+	var (
+		packageName        string
+		installedString    string
+		candidateString    string
+		countRelevantLines int
+	)
 	// Check for error message in the output
 	if strings.Contains(output, "Unable to locate package") {
-		return nil, fmt.Errorf("error: %s", strings.TrimSpace(output))
+		return nil, []error{fmt.Errorf("error: %s", strings.TrimSpace(output))}
+	}
+	packages := []packagemanagercommon.Package{}
+	outputBuf := bytes.NewBufferString(output)
+	outputScan := bufio.NewScanner(outputBuf)
+	for outputScan.Scan() {
+		line := outputScan.Text()
+		if !strings.HasPrefix(line, "  ") && strings.HasSuffix(line, ":") {
+			// count++
+			packageName = strings.TrimSpace(strings.TrimSuffix(line, ":"))
+		}
+		if strings.Contains(line, "Installed:") {
+			countRelevantLines++
+			installedString = strings.TrimPrefix(strings.TrimSpace(line), "Installed:")
+		}
+
+		if strings.Contains(line, "Candidate:") {
+			countRelevantLines++
+			candidateString = strings.TrimPrefix(strings.TrimSpace(line), "Candidate:")
+		}
+
+		if countRelevantLines == 2 {
+			countRelevantLines = 0
+			packages = append(packages, packagemanagercommon.Package{
+				Name: packageName,
+				VersionCheck: packagemanagercommon.PackageVersion{
+					Installed: strings.TrimSpace(installedString),
+					Candidate: strings.TrimSpace(candidateString),
+					Match:     installedString == candidateString,
+				}},
+			)
+		}
 	}
 
-	reInstalled := regexp.MustCompile(`Installed:\s*([^\s]+)`)
+	return packages, nil
-	reCandidate := regexp.MustCompile(`Candidate:\s*([^\s]+)`)
+}
-
+
-	installedMatch := reInstalled.FindStringSubmatch(output)
+func SearchPackages(pkgs []string, version string) (string, []string) {
-	candidateMatch := reCandidate.FindStringSubmatch(output)
+	baseCommand := "dpkg-query"
-
+	baseArgs := []string{"-W", "-f='${Package}\t${Architecture}\t${db:Status-Status}\t${Version}\t${Installed-Size}\t${Binary:summary}\n'"}
-	if len(installedMatch) < 2 || len(candidateMatch) < 2 {
+	baseArgs = append(baseArgs, pkgs...)
-		return nil, fmt.Errorf("failed to parse Installed or Candidate versions from apt output. check package name")
+
-	}
+	return baseCommand, baseArgs
-
-	return &pkgcommon.PackageVersion{
-		Installed: strings.TrimSpace(installedMatch[1]),
-		Candidate: strings.TrimSpace(candidateMatch[1]),
-		Match:     installedMatch[1] == candidateMatch[1],
-	}, nil
 }

pkg/pkgman/common/options.go

@@ -1,4 +1,4 @@
-package pkgcommon
+package packagemanagercommon
 
 // PackageManagerOption defines a functional option for configuring a PackageManager.
 type PackageManagerOption func(interface{})
@@ -15,3 +15,9 @@ type PackageVersion struct {
 	Match     bool
 	Message   string
 }
+
+type Package struct {
+	Name         string `yaml:"name"`
+	Version      string `yaml:"version,omitempty"`
+	VersionCheck PackageVersion
+}

pkg/pkgman/dnf/dnf.go

@@ -5,7 +5,7 @@ import (
 	"regexp"
 	"strings"
 
-	"git.andrewnw.xyz/CyberShell/backy/pkg/pkgman/pkgcommon"
+	packagemanagercommon "git.andrewnw.xyz/CyberShell/backy/pkg/pkgman/common"
 )
 
 // DnfManager implements PackageManager for systems using YUM.
@@ -26,21 +26,21 @@ func NewDnfManager() *DnfManager {
 }
 
 // Configure applies functional options to customize the package manager.
-func (y *DnfManager) Configure(options ...pkgcommon.PackageManagerOption) {
+func (y *DnfManager) Configure(options ...packagemanagercommon.PackageManagerOption) {
 	for _, opt := range options {
 		opt(y)
 	}
 }
 
 // Install returns the command and arguments for installing a package.
-func (y *DnfManager) Install(pkg, version string, args []string) (string, []string) {
+func (y *DnfManager) Install(pkgs []packagemanagercommon.Package, args []string) (string, []string) {
 	baseCmd := y.prependAuthCommand("dnf")
 	baseArgs := []string{"install", "-y"}
-	if version != "" {
+
-		baseArgs = append(baseArgs, fmt.Sprintf("%s-%s", pkg, version))
+	for _, p := range pkgs {
-	} else {
+		baseArgs = append(baseArgs, p.Name)
-		baseArgs = append(baseArgs, pkg)
 	}
+
 	if args != nil {
 		baseArgs = append(baseArgs, args...)
 	}
@@ -48,9 +48,13 @@ func (y *DnfManager) Install(pkg, version string, args []string) (string, []stri
 }
 
 // Remove returns the command and arguments for removing a package.
-func (y *DnfManager) Remove(pkg string, args []string) (string, []string) {
+func (y *DnfManager) Remove(pkgs []packagemanagercommon.Package, args []string) (string, []string) {
 	baseCmd := y.prependAuthCommand("dnf")
-	baseArgs := []string{"remove", "-y", pkg}
+	baseArgs := []string{"remove", "-y"}
+	for _, p := range pkgs {
+		baseArgs = append(baseArgs, p.Name)
+	}
+
 	if args != nil {
 		baseArgs = append(baseArgs, args...)
 	}
@@ -58,38 +62,41 @@ func (y *DnfManager) Remove(pkg string, args []string) (string, []string) {
 }
 
 // Upgrade returns the command and arguments for upgrading a specific package.
-func (y *DnfManager) Upgrade(pkg, version string) (string, []string) {
+func (y *DnfManager) Upgrade(pkgs []packagemanagercommon.Package) (string, []string) {
 	baseCmd := y.prependAuthCommand("dnf")
 	baseArgs := []string{"update", "-y"}
-	if version != "" {
+
-		baseArgs = append(baseArgs, fmt.Sprintf("%s-%s", pkg, version))
+	for _, p := range pkgs {
-	} else {
+		baseArgs = append(baseArgs, p.Name)
-		baseArgs = append(baseArgs, pkg)
 	}
+
 	return baseCmd, baseArgs
 }
 
 // UpgradeAll returns the command and arguments for upgrading all packages.
 func (y *DnfManager) UpgradeAll() (string, []string) {
 	baseCmd := y.prependAuthCommand("dnf")
-	baseArgs := []string{"update", "-y"}
+	baseArgs := []string{"upgrade", "-y"}
 	return baseCmd, baseArgs
 }
 
 // CheckVersion returns the command and arguments for checking the info of a specific package.
-func (d *DnfManager) CheckVersion(pkg, version string) (string, []string) {
+func (d *DnfManager) CheckVersion(pkgs []packagemanagercommon.Package) (string, []string) {
 	baseCmd := d.prependAuthCommand("dnf")
-	baseArgs := []string{"info", pkg}
+	baseArgs := []string{"info"}
+	for _, p := range pkgs {
+		baseArgs = append(baseArgs, p.Name)
+	}
 
 	return baseCmd, baseArgs
 }
 
 // Parse parses the dnf info output to extract Installed and Candidate versions.
-func (d DnfManager) Parse(output string) (*pkgcommon.PackageVersion, error) {
+func (d DnfManager) ParseRemotePackageManagerVersionOutput(output string) ([]packagemanagercommon.Package, []error) {
 
 	// Check for error message in the output
 	if strings.Contains(output, "No matching packages to list") {
-		return nil, fmt.Errorf("error: package not listed")
+		return nil, []error{fmt.Errorf("error: package not listed")}
 	}
 
 	// Define regular expressions to capture installed and available versions
@@ -111,13 +118,10 @@ func (d DnfManager) Parse(output string) (*pkgcommon.PackageVersion, error) {
 	}
 
 	if installedVersion == "" && candidateVersion == "" {
-		return nil, fmt.Errorf("failed to parse versions from dnf output")
+		return nil, []error{fmt.Errorf("failed to parse versions from dnf output")}
 	}
 
-	return &pkgcommon.PackageVersion{
+	return nil, nil
-		Installed: installedVersion,
-		Candidate: candidateVersion,
-	}, nil
 }
 
 // prependAuthCommand prepends the authentication command if UseAuth is true.

pkg/pkgman/pkgman.go

@@ -4,26 +4,26 @@ import (
 	"fmt"
 
 	"git.andrewnw.xyz/CyberShell/backy/pkg/pkgman/apt"
+	packagemanagercommon "git.andrewnw.xyz/CyberShell/backy/pkg/pkgman/common"
 	"git.andrewnw.xyz/CyberShell/backy/pkg/pkgman/dnf"
-	"git.andrewnw.xyz/CyberShell/backy/pkg/pkgman/pkgcommon"
 	"git.andrewnw.xyz/CyberShell/backy/pkg/pkgman/yum"
 )
 
 // PackageManager is an interface used to define common package commands. This shall be implemented by every package.
 type PackageManager interface {
-	Install(pkg, version string, args []string) (string, []string)
+	Install(pkgs []packagemanagercommon.Package, args []string) (string, []string)
-	Remove(pkg string, args []string) (string, []string)
+	Remove(pkgs []packagemanagercommon.Package, args []string) (string, []string)
-	Upgrade(pkg, version string) (string, []string) // Upgrade a specific package
+	Upgrade(pkgs []packagemanagercommon.Package) (string, []string) // Upgrade a specific package
 	UpgradeAll() (string, []string)
-	CheckVersion(pkg, version string) (string, []string)
+	CheckVersion(pkgs []packagemanagercommon.Package) (string, []string)
-	Parse(output string) (*pkgcommon.PackageVersion, error)
+	ParseRemotePackageManagerVersionOutput(output string) ([]packagemanagercommon.Package, []error)
 	// Configure applies functional options to customize the package manager.
-	Configure(options ...pkgcommon.PackageManagerOption)
+	Configure(options ...packagemanagercommon.PackageManagerOption)
 }
 
 // PackageManagerFactory returns the appropriate PackageManager based on the package tool.
 // Takes variable number of options.
-func PackageManagerFactory(managerType string, options ...pkgcommon.PackageManagerOption) (PackageManager, error) {
+func PackageManagerFactory(managerType string, options ...packagemanagercommon.PackageManagerOption) (PackageManager, error) {
 	var manager PackageManager
 
 	switch managerType {
@@ -43,7 +43,7 @@ func PackageManagerFactory(managerType string, options ...pkgcommon.PackageManag
 }
 
 // WithAuth enables authentication and sets the authentication command.
-func WithAuth(authCommand string) pkgcommon.PackageManagerOption {
+func WithAuth(authCommand string) packagemanagercommon.PackageManagerOption {
 	return func(manager interface{}) {
 		if configurable, ok := manager.(interface {
 			SetUseAuth(bool)
@@ -56,7 +56,7 @@ func WithAuth(authCommand string) pkgcommon.PackageManagerOption {
 }
 
 // WithoutAuth disables authentication.
-func WithoutAuth() pkgcommon.PackageManagerOption {
+func WithoutAuth() packagemanagercommon.PackageManagerOption {
 	return func(manager interface{}) {
 		if configurable, ok := manager.(interface {
 			SetUseAuth(bool)
@@ -68,8 +68,8 @@ func WithoutAuth() pkgcommon.PackageManagerOption {
 
 // ConfigurablePackageManager defines methods for setting configuration options.
 type ConfigurablePackageManager interface {
-	pkgcommon.PackageParser
+	packagemanagercommon.PackageParser
 	SetUseAuth(useAuth bool)
 	SetAuthCommand(authCommand string)
-	SetPackageParser(parser pkgcommon.PackageParser)
+	SetPackageParser(parser packagemanagercommon.PackageParser)
 }

pkg/pkgman/yum/yum.go

@@ -3,8 +3,9 @@ package yum
 import (
 	"fmt"
 	"regexp"
+	"strings"
 
-	"git.andrewnw.xyz/CyberShell/backy/pkg/pkgman/pkgcommon"
+	packagemanagercommon "git.andrewnw.xyz/CyberShell/backy/pkg/pkgman/common"
 )
 
 // YumManager implements PackageManager for systems using YUM.
@@ -25,21 +26,20 @@ func NewYumManager() *YumManager {
 }
 
 // Configure applies functional options to customize the package manager.
-func (y *YumManager) Configure(options ...pkgcommon.PackageManagerOption) {
+func (y *YumManager) Configure(options ...packagemanagercommon.PackageManagerOption) {
 	for _, opt := range options {
 		opt(y)
 	}
 }
 
 // Install returns the command and arguments for installing a package.
-func (y *YumManager) Install(pkg, version string, args []string) (string, []string) {
+func (y *YumManager) Install(pkgs []packagemanagercommon.Package, args []string) (string, []string) {
 	baseCmd := y.prependAuthCommand("yum")
 	baseArgs := []string{"install", "-y"}
-	if version != "" {
+	for _, p := range pkgs {
-		baseArgs = append(baseArgs, fmt.Sprintf("%s-%s", pkg, version))
+		baseArgs = append(baseArgs, p.Name)
-	} else {
-		baseArgs = append(baseArgs, pkg)
 	}
+
 	if args != nil {
 		baseArgs = append(baseArgs, args...)
 	}
@@ -47,9 +47,13 @@ func (y *YumManager) Install(pkg, version string, args []string) (string, []stri
 }
 
 // Remove returns the command and arguments for removing a package.
-func (y *YumManager) Remove(pkg string, args []string) (string, []string) {
+func (y *YumManager) Remove(pkgs []packagemanagercommon.Package, args []string) (string, []string) {
 	baseCmd := y.prependAuthCommand("yum")
-	baseArgs := []string{"remove", "-y", pkg}
+	baseArgs := []string{"remove", "-y"}
+	for _, p := range pkgs {
+		baseArgs = append(baseArgs, p.Name)
+	}
+
 	if args != nil {
 		baseArgs = append(baseArgs, args...)
 	}
@@ -57,14 +61,13 @@ func (y *YumManager) Remove(pkg string, args []string) (string, []string) {
 }
 
 // Upgrade returns the command and arguments for upgrading a specific package.
-func (y *YumManager) Upgrade(pkg, version string) (string, []string) {
+func (y *YumManager) Upgrade(pkgs []packagemanagercommon.Package) (string, []string) {
 	baseCmd := y.prependAuthCommand("yum")
 	baseArgs := []string{"update", "-y"}
-	if version != "" {
+	for _, p := range pkgs {
-		baseArgs = append(baseArgs, fmt.Sprintf("%s-%s", pkg, version))
+		baseArgs = append(baseArgs, p.Name)
-	} else {
-		baseArgs = append(baseArgs, pkg)
 	}
+
 	return baseCmd, baseArgs
 }
 
@@ -76,17 +79,27 @@ func (y *YumManager) UpgradeAll() (string, []string) {
 }
 
 // CheckVersion returns the command and arguments for checking the info of a specific package.
-func (y *YumManager) CheckVersion(pkg, version string) (string, []string) {
+func (y *YumManager) CheckVersion(pkgs []packagemanagercommon.Package) (string, []string) {
 	baseCmd := y.prependAuthCommand("yum")
-	baseArgs := []string{"info", pkg}
+	baseArgs := []string{"info"}
+	for _, p := range pkgs {
+		baseArgs = append(baseArgs, p.Name)
+	}
 
 	return baseCmd, baseArgs
 }
 
 // Parse parses the dnf info output to extract Installed and Candidate versions.
-func (y YumManager) Parse(output string) (*pkgcommon.PackageVersion, error) {
+func (y YumManager) ParseRemotePackageManagerVersionOutput(output string) ([]packagemanagercommon.Package, []error) {
-	reInstalled := regexp.MustCompile(`(?m)^Installed Packages\s*Name\s*:\s*\S+\s*Version\s*:\s*([^\s]+)\s*Release\s*:\s*([^\s]+)`)
+
-	reAvailable := regexp.MustCompile(`(?m)^Available Packages\s*Name\s*:\s*\S+\s*Version\s*:\s*([^\s]+)\s*Release\s*:\s*([^\s]+)`)
+	// Check for error message in the output
+	if strings.Contains(output, "No matching packages to list") {
+		return nil, []error{fmt.Errorf("error: package not listed")}
+	}
+
+	// Define regular expressions to capture installed and available versions
+	reInstalled := regexp.MustCompile(`(?m)^Installed packages\s*Name\s*:\s*\S+\s*Epoch\s*:\s*\S+\s*Version\s*:\s*([^\s]+)\s*Release\s*:\s*([^\s]+)`)
+	reAvailable := regexp.MustCompile(`(?m)^Available packages\s*Name\s*:\s*\S+\s*Epoch\s*:\s*\S+\s*Version\s*:\s*([^\s]+)\s*Release\s*:\s*([^\s]+)`)
 
 	installedMatch := reInstalled.FindStringSubmatch(output)
 	candidateMatch := reAvailable.FindStringSubmatch(output)
@@ -103,13 +116,10 @@ func (y YumManager) Parse(output string) (*pkgcommon.PackageVersion, error) {
 	}
 
 	if installedVersion == "" && candidateVersion == "" {
-		return nil, fmt.Errorf("failed to parse versions from dnf output")
+		return nil, []error{fmt.Errorf("failed to parse versions from dnf output")}
 	}
 
-	return &pkgcommon.PackageVersion{
+	return nil, nil
-		Installed: installedVersion,
-		Candidate: candidateVersion,
-	}, nil
 }
 
 // prependAuthCommand prepends the authentication command if UseAuth is true.

pkg/remotefetcher/cache.go

@@ -116,7 +116,7 @@ func (c *Cache) Set(source, hash string, data []byte, dataType string) (CacheDat
 	path := filepath.Join(c.dir, fmt.Sprintf("%s-%s", fileName, sourceHash))
 
 	if _, err := os.Stat(path); os.IsNotExist(err) {
-		os.MkdirAll(c.dir, 0700)
+		_ = os.MkdirAll(c.dir, 0700)
 	}
 
 	err := os.WriteFile(path, data, 0644)
@@ -171,7 +171,7 @@ func (cf *CachedFetcher) Hash(data []byte) string {
 func LoadMetadataFromFile(filePath string) ([]*CacheData, error) {
 	if _, err := os.Stat(filePath); os.IsNotExist(err) {
 		// Create the file if it does not exist
-		os.MkdirAll(path.Dir(filePath), 0700)
+		_ = os.MkdirAll(path.Dir(filePath), 0700)
 		emptyData := []byte("[]")
 		err := os.WriteFile(filePath, emptyData, 0644)
 		if err != nil {

pkg/remotefetcher/configfetcher.go

@@ -29,7 +29,7 @@ func NewRemoteFetcher(source string, cache *Cache, options ...FetcherOption) (Re
 		option(&config)
 	}
 
-	// If FileType is empty (i.e. WithFileType was not called), yaml is the default file type
+	// WithFileType was not called. yaml is the default file type
 	if strings.TrimSpace(config.FileType) == "" {
 		config.FileType = "yaml"
 	}

pkg/remotefetcher/local.go

@@ -20,7 +20,7 @@ func (l *LocalFetcher) Fetch(source string) ([]byte, error) {
 		if l.config.IgnoreFileNotFound {
 			return nil, ErrIgnoreFileNotFound
 		}
-		return nil, nil
+		return nil, err
 	}
 	file, err := os.Open(source)
 	if err != nil {

pkg/usermanager/linux/linux.go

@@ -15,7 +15,7 @@ func (l LinuxUserManager) NewLinuxManager() *LinuxUserManager {
 }
 
 // AddUser adds a new user to the system.
-func (l LinuxUserManager) AddUser(username, homeDir, shell string, isSystem bool, groups, args []string) (string, []string) {
+func (l LinuxUserManager) AddUser(username, homeDir, shell string, isSystem, createHome bool, groups, args []string) (string, []string) {
 	baseArgs := []string{}
 
 	if isSystem {
@@ -38,6 +38,10 @@ func (l LinuxUserManager) AddUser(username, homeDir, shell string, isSystem bool
 		baseArgs = append(baseArgs, args...)
 	}
 
+	if createHome {
+		baseArgs = append(baseArgs, "-m")
+	}
+
 	args = append(baseArgs, username)
 
 	cmd := "useradd"

pkg/usermanager/userman.go

@@ -10,7 +10,7 @@ import (
 // UserManager defines the interface for user management operations.
 // All functions but one return a string for the command and any args.
 type UserManager interface {
-	AddUser(username, homeDir, shell string, isSystem bool, groups, args []string) (string, []string)
+	AddUser(username, homeDir, shell string, createHome, isSystem bool, groups, args []string) (string, []string)
 	RemoveUser(username string) (string, []string)
 	ModifyUser(username, homeDir, shell string, groups []string) (string, []string)
 	// Modify password uses chpasswd for Linux systems to build the command to change the password

release

@@ -1,5 +1,6 @@
 #!/bin/bash
 set -eou pipefail
+go mod tidy
 go generate ./...
 CURRENT_TAG="$(go run backy.go version -V)"
 goreleaser -f .goreleaser/github.yml check

tests/.gitignore

@@ -0,0 +1 @@
+private

tests/ErrorHook.yml

@@ -0,0 +1,32 @@
+commands:
+  echoTestFail:
+    cmd: ech
+    shell: bash
+    Args: 
+      - hello world
+    hooks:
+      error:
+        - errorCmd
+      final:
+        - finalCmd
+  
+  finalCmd:
+    cmd: echo
+    Args:
+      - "echo test fail"
+
+  errorCmd:
+    name: get docker version
+    cmd: docker
+    getOutput: true
+    outputToLog: true
+    Args:
+      - "-v"
+    host: email-svr
+
+cmdLists:
+  TestHooks:
+    output:
+      onFailure: true
+    order:
+      - echoTestFail

tests/HookNotInFile.yaml

@@ -0,0 +1,14 @@
+commands:
+  echoTestFail:
+    cmd: ech
+    shell: bash
+    Args: hello world
+    hooks:
+      error:
+        - errorCm # 
+
+  errorCmd:
+    name: get docker version
+    cmd: docker
+    Args:
+      - "-v"

tests/SuccessHook.yml

@@ -0,0 +1,16 @@
+commands:
+  echoTestSuccess:
+    cmd: echo
+    shell: bash
+    Args: hello world
+    hooks:
+      success:
+        - successCmd
+
+  errorCmd:
+    name: get docker version
+    cmd: docker
+    getOutput: true
+    outputToLog: true
+    Args:
+      - "-v"

tests/VaultTest.yml

@@ -0,0 +1,27 @@
+commands:
+  vaultEnvVar:
+    cmd: echo
+    shell: /bin/zsh
+    Args:
+      - ${VAULT_VAR}
+    environment:
+      "VAULT_VAR=%{vault:vaultTestSecret}%"
+
+logging:
+  verbose: true
+
+vault:
+  token: root
+  address: http://127.0.0.1:8200
+  enabled: true
+  keys:
+    - name: vaultTestSecret
+      key: data
+      mountpath: secret
+      path: test/var
+      type: KVv2 # KVv1 or KVv2
+
+cmdLists:
+  addUsers:
+    order:
+      - vaultEnvVar

tests/backy.yaml

@@ -1,18 +0,0 @@
-commands:
-  echoTestPass:
-    cmd: echo
-    shell: bash
-    Args: hello world
-
-  runRemoteShellScriptSuccess:
-    cmd: 
-
-
-  packageCommandSuccess:
-    packageName: docker-ce
-    Args:
-      - docker-ce-cli
-    packageManager: apt
-    packageOperation: install
-
-  

tests/docker/Dockerfile

@@ -0,0 +1,26 @@
+FROM debian:buster
+
+# Install SSH server
+RUN apt-get update && \
+    apt-get install -y openssh-server && \
+    apt-get clean
+
+RUN sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config
+
+RUN sed -i 's/#PasswordAuthentication yes/PasswordAuthentication yes/' /etc/ssh/sshd_config
+
+RUN useradd -m -s /bin/bash backy
+RUN echo "backy:backy" | chpasswd
+RUN echo "root:test" | chpasswd
+COPY --chown=backy:backy backytest.pub /home/backy/.ssh/authorized_keys
+COPY --chown=root:root backytest.pub /root/.ssh/authorized_keys
+
+EXPOSE 22
+RUN mkdir /var/run/sshd
+RUN chmod 0755 /var/run/sshd
+
+# RUN apt-get update && apt-get install -y 
+
+# Start SSH service
+CMD ["/usr/sbin/sshd", "-D"]
+# ENTRYPOINT service ssh start && bash

tests/docker/backytest

@@ -0,0 +1,7 @@
+-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
+QyNTUxOQAAACBtBASN+HMx/EVUIs5ThK9Nlw0wPFVt2rXsiNZlDN+CswAAAKAfFc5AHxXO
+QAAAAAtzc2gtZWQyNTUxOQAAACBtBASN+HMx/EVUIs5ThK9Nlw0wPFVt2rXsiNZlDN+Csw
+AAAEAxs6uRkenVbXPrjgbIv/1THXL6dUdgr5KaCd7uBVm0PW0EBI34czH8RVQizlOEr02X
+DTA8VW3ateyI1mUM34KzAAAAGU1lZGlhIHVzZXIgc3RvcmFnZSBzZXJ2ZXIBAgME
+-----END OPENSSH PRIVATE KEY-----

tests/docker/backytest.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIG0EBI34czH8RVQizlOEr02XDTA8VW3ateyI1mUM34Kz Backy test

tests/docker/buildDocker.sh

@@ -0,0 +1,3 @@
+docker container rm -f ssh_server_container
+docker build -t ssh_server_image .
+docker run -d -p 2222:22 --name ssh_server_container ssh_server_image

tests/hosts.yml

@@ -0,0 +1,6 @@
+hosts:
+  docker:
+    port: 2222
+    Hostname: localhost
+    user: backy
+    IdentityFile: ./docker/backytest

tests/packageCommands.yml

@@ -0,0 +1,53 @@
+commands:
+  checkDockerNoVersion:
+    type: package
+    shell: zsh
+    packages: 
+      - name: "docker-ce-cli"
+      - name: "docker-ce"
+    packageManager: apt
+    packageOperation: checkVersion
+
+  checkDockerPartialVersionWithoutRegex:
+    type: package
+    shell: zsh
+    packages: 
+      - name: "docker-ce-cli"
+        version: "5:28.0.4-1"
+      - name: "docker-ce"
+        version: "5:28.0.4-1"
+    packageManager: apt
+    packageOperation: checkVersion
+
+  checkDockerPartialVersionWithRegex:
+    type: package
+    shell: zsh
+    packages: 
+      - name: "docker-ce-cli"
+        version: "5:28.0-*"
+      - name: "docker-ce"
+        version: '5:28\.0\.4-1~([A-Za-z0-9]+(\.[A-Za-z0-9]+)+)*'
+    packageManager: apt
+    packageOperation: checkVersion
+    
+  checkDockerVersionWithInvalidRegex:
+    type: package
+    shell: zsh
+    packages: 
+      - name: "docker-ce-cli"
+        version: "5:28.0-**"
+      - name: "docker-ce"
+        version: '5:28\.0\K.4-1~([A-Za-z0-9]+(\.[A-Za-z0-9]+)+)*'
+    packageManager: apt
+    packageOperation: checkVersion
+    # host: mail.andrewnw.com
+
+cmdLists:
+  packageCommands:
+    output:
+      onFailure: true
+    order:
+      - checkDockerPartialVersionWithoutRegex
+      - checkDockerPartialVersionWithRegex
+      - checkDockerVersionWithInvalidRegex
+      - checkDockerNoVersion

tests/packageCommandsDNF.yml

@@ -0,0 +1,52 @@
+commands:
+  checkDockerNoVersion:
+    type: package
+    shell: zsh
+    packages: 
+      - name: "docker-ce-cli"
+      - name: "docker-ce"
+    packageManager: dnf
+    packageOperation: checkVersion
+
+  checkDockerPartialVersionWithoutRegex:
+    type: package
+    shell: zsh
+    packages: 
+      - name: "docker-ce-cli"
+        version: "5:28.0.4-1"
+      - name: "docker-ce"
+        version: "5:28.0.4-1"
+    packageManager: dnf
+    packageOperation: checkVersion
+
+  checkDockerPartialVersionWithRegex:
+    type: package
+    shell: zsh
+    packages: 
+      - name: "docker-ce-cli"
+        version: "5:28.0-*"
+      - name: "docker-ce"
+        version: '5:28\.0\.4-1~([A-Za-z0-9]+(\.[A-Za-z0-9]+)+)*'
+    packageManager: dnf
+    packageOperation: checkVersion
+    
+  checkDockerVersionWithInvalidRegex:
+    type: package
+    shell: zsh
+    packages: 
+      - name: "docker-ce-cli"
+        version: "5:28.0-**"
+      - name: "docker-ce"
+        version: '5:28\.0\K.4-1~([A-Za-z0-9]+(\.[A-Za-z0-9]+)+)*'
+    packageManager: dnf
+    packageOperation: checkVersion
+
+cmdLists:
+  packageCommands:
+    output:
+      onFailure: true
+    order:
+      - checkDockerPartialVersionWithoutRegex
+      - checkDockerPartialVersionWithRegex
+      - checkDockerVersionWithInvalidRegex
+      - checkDockerNoVersion