add files

.changes/unreleased/Added-20250409-174528.yaml

@@ -0,0 +1,3 @@
+kind: Added
+body: 'feat: Package operation `versionCheck` supports regular expressions (see [regexp](https://pkg.go.dev/regexp) package for docs)'
+time: 2025-04-09T17:45:28.836497149-05:00

.changes/unreleased/Added-20250501-110745.yaml

@@ -0,0 +1,3 @@
+kind: Added
+body: 'Command lists: added `cmdLists.[name].notify` object'
+time: 2025-05-01T11:07:45.96164753-05:00

.changes/unreleased/Added-20250704-085917.yaml

@@ -0,0 +1,3 @@
+kind: Added
+body: Testing setup with Docker
+time: 2025-07-04T08:59:17.430373451-05:00

.changes/unreleased/Added-20250704-102126.yaml

@@ -0,0 +1,3 @@
+kind: Added
+body: 'CLI: add global flag --hostsConfig that allows hosts to be dynamic in relation to the main config'
+time: 2025-07-04T10:21:26.864635558-05:00

.changes/unreleased/Added-20250715-202303.yaml

@@ -0,0 +1,3 @@
+kind: Added
+body: 'CLI: Exec subcommand `hosts`. See documentation for more details.'
+time: 2025-07-15T20:23:03.647128713-05:00

.changes/unreleased/Added-20250723-220340.yaml

@@ -0,0 +1,3 @@
+kind: Added
+body: 'CLI: added `exec hosts` subcommand `list`'
+time: 2025-07-23T22:03:40.24191927-05:00

.changes/unreleased/Changed-20250321-090849.yaml

@@ -0,0 +1,3 @@
+kind: Changed
+body: 'Commands: `host` can now be `localhost` or `127.0.0.1` to run commands locally'
+time: 2025-03-21T09:08:49.871021144-05:00

.changes/unreleased/Changed-20250325-003357.yaml

@@ -0,0 +1,3 @@
+kind: Changed
+body: lists loaded from external files only if no list config present in current file
+time: 2025-03-25T00:33:57.039431409-05:00

.changes/unreleased/Changed-20250407-223020.yaml

@@ -0,0 +1,3 @@
+kind: Changed
+body: "`PackageManager.Parse` renamed to `ParseRemotePackageManagerVersionOutput`. This now returns arrays of PackageManagerCommon.Package and errors."
+time: 2025-04-07T22:30:20.342177323-05:00

.changes/unreleased/Changed-20250418-133440.yaml

@@ -0,0 +1,3 @@
+kind: Changed
+body: 'Internal: refactoring and renaming functions'
+time: 2025-04-18T13:34:40.842541658-05:00

.changes/unreleased/Changed-20250501-110534.yaml

@@ -0,0 +1,3 @@
+kind: Changed
+body: 'Commands: moved output-prefixed keys to the `commands.[name].output` object'
+time: 2025-05-01T11:05:34.90130087-05:00

.changes/unreleased/Changed-20250609-072601.yaml

@@ -0,0 +1,3 @@
+kind: Changed
+body: Change internal method name for better understanding
+time: 2025-06-09T07:26:01.819927627-05:00

.changes/unreleased/Changed-20250709-231919.yaml

@@ -0,0 +1,3 @@
+kind: Changed
+body: Improved error message for remote version package output
+time: 2025-07-09T23:19:19.431960446-05:00

.changes/unreleased/Fixed-20250418-095747.yaml

@@ -0,0 +1,3 @@
+kind: Fixed
+body: 'Command Lists: hooks now run correctly when commands finish'
+time: 2025-04-18T09:57:47.39035092-05:00

.changes/unreleased/Fixed-20250424-225711.yaml

@@ -0,0 +1,3 @@
+kind: Fixed
+body: Log file passed using `--log-file` correctly used
+time: 2025-04-24T22:57:11.592829277-05:00

.changes/v0.10.0.md

@@ -0,0 +1,16 @@
+## v0.10.0 - 2025-03-08
+### Added
+* Hooks: improved logging when executing
+* User commands: adding SSH keys using config key `userSshPubKeys`
+* directives: added support for fetching values using directive `%{externalSource:key}%`
+### Changed
+* Commands: if dir is not specified, run in config dir
+* FileDirective: use the config directory if path is not absolute
+* Host: changes to case of some keys
+* Notifications: added external directive to sensitive keys
+### Fixed
+* LocalFetcher: return fetch error
+* Lists: load file key before attempting to load from current file
+* fix: host not in config file, but in ssh config, properly added to hosts struct
+* SSH: password authentication bugs
+* User commands: change user password works

.changes/v0.10.1.md

@@ -0,0 +1,8 @@
+## v0.10.1 - 2025-03-11
+### Added
+* UserCommands: add ssh public keys when running locally
+* UserCommands: add field CreateUserHome
+### Changed
+* UserCommands: create temp file when modifing password over SSH
+* UserCommands: change field name
+* Vault: keys are now referenced by `name`, and the actual data by `data`

.changes/v0.10.2.md

@@ -0,0 +1,6 @@
+## v0.10.2 - 2025-03-19
+### Added
+* Notifications: http service added
+* Variable support. Can be referenced with `%{var:nameOfVar}%` in select string fields.
+### Changed
+* vault: initialize vault before validating config

.gitignore

@@ -5,6 +5,6 @@ dist/
 .codegpt
 
 *.log
-*.sh
+/*.sh
 /*.yaml
 /*.yml

.vscode/settings.json

@@ -9,5 +9,6 @@
         "mautrix",
         "nikoksr",
         "Strs"
-    ]
+    ],
+    "CodeGPT.apiKey": "CodeGPT Plus Beta"
 }

.woodpecker/gitea.yml

@@ -1,9 +1,7 @@
-name: goreleaser release
 steps:
   golang:
     image: golang:1.23
     commands:
-      - go mod tidy
       - go install github.com/goreleaser/goreleaser/v2@v2.7.0
       - goreleaser release -f .goreleaser/gitea.yml --release-notes=".changes/$(go run backy.go version -V).md"
     environment:

.woodpecker/go-lint.yml

@@ -5,7 +5,7 @@ steps:
       - go build
       - go test
   release:
-    image: golangci/golangci-lint:v1.53.3
+    image: golangci/golangci-lint:v1.64.7
     commands:
       - golangci-lint run -v --timeout 5m
 

CHANGELOG.md

@@ -6,6 +6,39 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html),
 and is generated by [Changie](https://github.com/miniscruff/changie).
 
 
+## v0.10.2 - 2025-03-19
+### Added
+* Notifications: http service added
+* Variable support. Can be referenced with `%{var:nameOfVar}%` in select string fields.
+### Changed
+* vault: initialize vault before validating config
+
+## v0.10.1 - 2025-03-11
+### Added
+* UserCommands: add ssh public keys when running locally
+* UserCommands: add field CreateUserHome
+### Changed
+* UserCommands: create temp file when modifing password over SSH
+* UserCommands: change field name
+* Vault: keys are now referenced by `name`, and the actual data by `data`
+
+## v0.10.0 - 2025-03-08
+### Added
+* Hooks: improved logging when executing
+* User commands: adding SSH keys using config key `userSshPubKeys`
+* directives: added support for fetching values using directive `%{externalSource:key}%`
+### Changed
+* Commands: if dir is not specified, run in config dir
+* FileDirective: use the config directory if path is not absolute
+* Host: changes to case of some keys
+* Notifications: added external directive to sensitive keys
+### Fixed
+* LocalFetcher: return fetch error
+* Lists: load file key before attempting to load from current file
+* fix: host not in config file, but in ssh config, properly added to hosts struct
+* SSH: password authentication bugs
+* User commands: change user password works
+
 ## v0.9.1 - 2025-03-01
 ### Changed
 * Use EnvVar AWS_PROFILE to get S3 profile

backy.code-workspace

@@ -18,6 +18,8 @@
 			"maunium",
 			"mautrix",
 			"nikoksr",
+			"packagemanagercommon",
+			"rawbytes",
 			"remotefetcher",
 			"Strs"
 		]

cmd/backup.go

@@ -30,9 +30,14 @@ func init() {
 }
 
 func Backup(cmd *cobra.Command, args []string) {
-	backyConfOpts := backy.NewOpts(cfgFile, backy.AddCommandLists(cmdLists), backy.SetLogFile(logFile), backy.SetCmdStdOut(cmdStdOut))
+	backyConfOpts := backy.NewConfigOptions(configFile,
+		backy.AddCommandLists(cmdLists),
+		backy.SetLogFile(logFile),
+		backy.EnableCommandStdOut(cmdStdOut),
+		backy.SetHostsConfigFile(hostsConfigFile))
+
 	backyConfOpts.InitConfig()
-	backyConfOpts.ReadConfig()
+	backyConfOpts.ParseConfigurationFile()
 
 	backyConfOpts.RunListConfig("")
 	for _, host := range backyConfOpts.Hosts {

cmd/backup_test.go

@@ -0,0 +1,54 @@
+package cmd
+
+// import (
+// 	"bufio"
+// 	"encoding/json"
+// 	"os"
+// 	"os/exec"
+// 	"strings"
+// 	"testing"
+
+// 	"github.com/stretchr/testify/assert"
+// )
+
+// // TestConfigOptions tests the configuration options for the backy package.
+// func Test_ErrorHook(t *testing.T) {
+// 	configFile := "-f ../../tests/ErrorHook.yml"
+// 	logFile := "--log-file=ErrorHook.log"
+// 	backyCommand := exec.Command("go", "run", "../../backy.go", configFile, logFile, "backup")
+// 	backyCommand.Stderr = os.Stdout
+// 	backyCommand.Stdout = os.Stdout
+// 	err := backyCommand.Run()
+// 	assert.Nil(t, err)
+// 	os.Remove("ErrorHook.log")
+// 	logFileData, logFileErr := os.ReadFile("ErrorHook.log")
+// 	if logFileErr != nil {
+// 		assert.FailNow(t, logFileErr.Error())
+
+// 	}
+// 	var JsonData []map[string]interface{}
+// 	jsonScanner := bufio.NewScanner(strings.NewReader(string(logFileData)))
+
+// 	for jsonScanner.Scan() {
+// 		var jsonDataLine map[string]interface{}
+// 		err = json.Unmarshal(jsonScanner.Bytes(), &jsonDataLine)
+// 		assert.Nil(t, err)
+// 		JsonData = append(JsonData, jsonDataLine)
+// 	}
+// 	for _, v := range JsonData {
+// 		_, ok := v["error"]
+// 		if !ok {
+// 			assert.FailNow(t, "error does not exist\n")
+// 			// return
+// 		}
+// 	}
+// 	// t.Logf("%s", logFileData)
+// 	// t.Logf("%v", JsonData)
+// }
+
+// func TestBackupErrorHook(t *testing.T) {
+// 	logFile = "ErrorHook.log"
+
+// 	configFile = "../tests/ErrorHook.yml"
+
+// }

cmd/config.go

@@ -20,7 +20,7 @@ package cmd
 
 // func config(cmd *cobra.Command, args []string) {
 
-// 	opts := backy.NewOpts(cfgFile, backy.cronEnabled())
+// 	opts := backy.NewConfigOptions(configFile, backy.cronEnabled())
 // 	opts.InitConfig()
 
 // }

cmd/cron.go

@@ -18,13 +18,14 @@ var (
 func cron(cmd *cobra.Command, args []string) {
 	parseS3Config()
 
-	opts := backy.NewOpts(cfgFile,
+	opts := backy.NewConfigOptions(configFile,
 		backy.EnableCron(),
 		backy.SetLogFile(logFile),
-		backy.SetCmdStdOut(cmdStdOut))
+		backy.EnableCommandStdOut(cmdStdOut),
+		backy.SetHostsConfigFile(hostsConfigFile))
 
 	opts.InitConfig()
-	opts.ReadConfig()
+	opts.ParseConfigurationFile()
 
 	opts.Cron()
 }

cmd/exec.go

@@ -21,7 +21,7 @@ var (
 )
 
 func init() {
-	execCmd.AddCommand(hostExecCommand)
+	execCmd.AddCommand(hostExecCommand, hostsExecCommand)
 
 }
 
@@ -32,8 +32,12 @@ func execute(cmd *cobra.Command, args []string) {
 		logging.ExitWithMSG("Please provide a command to run. Pass --help to see options.", 1, nil)
 	}
 
-	opts := backy.NewOpts(cfgFile, backy.AddCommands(args), backy.SetLogFile(logFile), backy.SetCmdStdOut(cmdStdOut))
+	opts := backy.NewConfigOptions(configFile,
+		backy.AddCommands(args),
+		backy.SetLogFile(logFile),
+		backy.EnableCommandStdOut(cmdStdOut),
+		backy.SetHostsConfigFile(hostsConfigFile))
 	opts.InitConfig()
-	opts.ReadConfig()
+	opts.ParseConfigurationFile()
 	opts.ExecuteCmds()
 }

cmd/host.go

@@ -35,10 +35,13 @@ func init() {
 //    2. stdin (on command line) (TODO)
 
 func Host(cmd *cobra.Command, args []string) {
-	backyConfOpts := backy.NewOpts(cfgFile, backy.SetLogFile(logFile), backy.SetCmdStdOut(cmdStdOut))
+	backyConfOpts := backy.NewConfigOptions(configFile,
+		backy.SetLogFile(logFile),
+		backy.EnableCommandStdOut(cmdStdOut),
+		backy.SetHostsConfigFile(hostsConfigFile))
 	backyConfOpts.InitConfig()
 
-	backyConfOpts.ReadConfig()
+	backyConfOpts.ParseConfigurationFile()
 
 	// check CLI input
 	if hostsList == nil {
@@ -46,14 +49,20 @@ func Host(cmd *cobra.Command, args []string) {
 	}
 
 	for _, h := range hostsList {
+		if backy.IsHostLocal(h) {
+			continue
+		}
 		// check if h exists in the config file
 		_, hostFound := backyConfOpts.Hosts[h]
 		if !hostFound {
 			// check if h exists in the SSH config file
-			hostFoundInConfig, s := backy.CheckIfHostHasHostName(h)
+			hostFoundInConfig, s := backy.DoesHostHaveHostName(h)
 			if !hostFoundInConfig {
 				logging.ExitWithMSG("host "+h+" not found", 1, &backyConfOpts.Logger)
 			}
+			if backyConfOpts.Hosts == nil {
+				backyConfOpts.Hosts = make(map[string]*backy.Host)
+			}
 			// create host with hostname and host
 			backyConfOpts.Hosts[h] = &backy.Host{Host: h, HostName: s}
 		}
@@ -68,5 +77,5 @@ func Host(cmd *cobra.Command, args []string) {
 		}
 	}
 
-	backyConfOpts.ExecCmdsSSH(cmdList, hostsList)
+	backyConfOpts.ExecCmdsOnHosts(cmdList, hostsList)
 }

cmd/hosts.go

@@ -0,0 +1,93 @@
+package cmd
+
+import (
+	"maps"
+	"slices"
+
+	"git.andrewnw.xyz/CyberShell/backy/pkg/backy"
+	"git.andrewnw.xyz/CyberShell/backy/pkg/logging"
+	"github.com/spf13/cobra"
+)
+
+var (
+	runCommandsInParallel bool
+
+	hostsExecCommand = &cobra.Command{
+		Use:   "hosts [--command=command1 --command=command2 ... | -c command1 -c command2 ...]",
+		Short: "Runs command defined in config file on the hosts in order specified.",
+		Long:  "Hosts executes specified commands on all the hosts defined in config file.\nUse the --commands or -c flag to choose the commands.",
+		Run:   Hosts,
+	}
+
+	hostsListExecCommand = &cobra.Command{
+		Use:   "list list1 list2 ...",
+		Short: "Runs lists in order specified defined in config file on all hosts.",
+		Long:  "Lists executes specified lists on all the hosts defined in hosts config.\nPass the names of lists as arguments after command.",
+		Run:   HostsList,
+	}
+)
+
+func init() {
+	hostsExecCommand.AddCommand(hostsListExecCommand)
+	hostsListExecCommand.Flags().BoolVarP(&runCommandsInParallel, "parallel", "p", false, "Run commands in parallel on hosts")
+	parseS3Config()
+}
+
+// cli input should be hosts and commands. Hosts are defined in config files.
+// commands can be passed by the following mutually exclusive options:
+//    1. as a list of commands defined in the config file
+//    2. stdin (on command line) (TODO)
+
+func Hosts(cmd *cobra.Command, args []string) {
+	backyConfOpts := backy.NewConfigOptions(configFile,
+		backy.SetLogFile(logFile),
+		backy.EnableCommandStdOut(cmdStdOut),
+		backy.SetHostsConfigFile(hostsConfigFile))
+	backyConfOpts.InitConfig()
+
+	backyConfOpts.ParseConfigurationFile()
+
+	for _, h := range backyConfOpts.Hosts {
+
+		hostsList = append(hostsList, h.Host)
+	}
+
+	if cmdList == nil {
+		logging.ExitWithMSG("error: commands must be specified", 1, &backyConfOpts.Logger)
+	}
+	for _, c := range cmdList {
+		_, cmdFound := backyConfOpts.Cmds[c]
+		if !cmdFound {
+			logging.ExitWithMSG("cmd "+c+" not found", 1, &backyConfOpts.Logger)
+		}
+	}
+
+	backyConfOpts.ExecCmdsOnHosts(cmdList, hostsList)
+}
+
+func HostsList(cmd *cobra.Command, args []string) {
+	backyConfOpts := backy.NewConfigOptions(configFile,
+		backy.SetLogFile(logFile),
+		backy.EnableCommandStdOut(cmdStdOut),
+		backy.SetHostsConfigFile(hostsConfigFile))
+	backyConfOpts.InitConfig()
+
+	backyConfOpts.ParseConfigurationFile()
+
+	if len(args) == 0 {
+		logging.ExitWithMSG("error: no lists specified", 1, &backyConfOpts.Logger)
+	}
+
+	for _, l := range args {
+		_, listFound := backyConfOpts.CmdConfigLists[l]
+		if !listFound {
+			logging.ExitWithMSG("list "+l+" not found", 1, &backyConfOpts.Logger)
+		}
+	}
+
+	maps.DeleteFunc(backyConfOpts.CmdConfigLists, func(k string, v *backy.CmdList) bool {
+		return !slices.Contains(args, k)
+	})
+
+	backyConfOpts.ExecuteListOnHosts(args, runCommandsInParallel)
+}

cmd/list.go

@@ -22,13 +22,13 @@ var (
 		Use:   "cmds [cmd1 cmd2 cmd3...]",
 		Short: "List commands defined in config file.",
 		Long:  "List commands defined in config file",
-		Run:   ListCmds,
+		Run:   ListCommands,
 	}
 	listCmdLists = &cobra.Command{
 		Use:   "lists [list1 list2 ...]",
 		Short: "List lists defined in config file.",
 		Long:  "List lists defined in config file",
-		Run:   ListCmdLists,
+		Run:   ListCommandLists,
 	}
 )
 
@@ -40,7 +40,7 @@ func init() {
 
 }
 
-func ListCmds(cmd *cobra.Command, args []string) {
+func ListCommands(cmd *cobra.Command, args []string) {
 
 	// setup based on whats passed in:
 	//   - cmds
@@ -54,17 +54,19 @@ func ListCmds(cmd *cobra.Command, args []string) {
 
 	parseS3Config()
 
-	opts := backy.NewOpts(cfgFile, backy.SetLogFile(logFile))
+	opts := backy.NewConfigOptions(configFile,
+		backy.SetLogFile(logFile),
+		backy.SetHostsConfigFile(hostsConfigFile))
 
 	opts.InitConfig()
-	opts.ReadConfig()
+	opts.ParseConfigurationFile()
 
 	for _, v := range cmdsToList {
 		opts.ListCommand(v)
 	}
 }
 
-func ListCmdLists(cmd *cobra.Command, args []string) {
+func ListCommandLists(cmd *cobra.Command, args []string) {
 
 	parseS3Config()
 
@@ -74,10 +76,12 @@ func ListCmdLists(cmd *cobra.Command, args []string) {
 		logging.ExitWithMSG("Error: lists subcommand needs lists", 1, nil)
 	}
 
-	opts := backy.NewOpts(cfgFile, backy.SetLogFile(logFile))
+	opts := backy.NewConfigOptions(configFile,
+		backy.SetLogFile(logFile),
+		backy.SetHostsConfigFile(hostsConfigFile))
 
 	opts.InitConfig()
-	opts.ReadConfig()
+	opts.ParseConfigurationFile()
 
 	for _, v := range listsToList {
 		opts.ListCommandList(v)

cmd/root.go

@@ -13,7 +13,8 @@ import (
 
 var (
 	// Used for flags.
-	cfgFile    string
+	configFile      string
+	hostsConfigFile string
 	verbose         bool
 	cmdStdOut       bool
 	logFile         string
@@ -35,12 +36,13 @@ func Execute() {
 }
 
 func init() {
-	rootCmd.PersistentFlags().StringVar(&logFile, "log-file", "", "log file to write to")
+	rootCmd.PersistentFlags().StringVar(&logFile, "logFile", "", "log file to write to")
 	rootCmd.PersistentFlags().BoolVar(&cmdStdOut, "cmdStdOut", false, "Pass to print command output to stdout")
 
-	rootCmd.PersistentFlags().StringVarP(&cfgFile, "config", "f", "", "config file to read from")
+	rootCmd.PersistentFlags().StringVarP(&configFile, "config", "f", "", "config file to read from")
+	rootCmd.PersistentFlags().StringVar(&hostsConfigFile, "hostsConfig", "", "yaml hosts file to read from")
 	rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "Sets verbose level")
-	rootCmd.PersistentFlags().StringVar(&s3Endpoint, "s3-endpoint", "", "Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable.")
+	rootCmd.PersistentFlags().StringVar(&s3Endpoint, "s3Endpoint", "", "Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable.")
 	rootCmd.AddCommand(backupCmd, execCmd, cronCmd, versionCmd, listCmd)
 }
 

cmd/version.go

@@ -7,7 +7,7 @@ import (
 	"github.com/spf13/cobra"
 )
 
-const versionStr = "0.9.1"
+const versionStr = "0.10.2"
 
 var (
 	versionCmd = &cobra.Command{

docs/content/cli/_index.md

@@ -26,8 +26,9 @@ Flags:
       --cmdStdOut            Pass to print command output to stdout
   -f, --config string        config file to read from
   -h, --help                 help for backy
-      --log-file string      log file to write to
+      --hostsConfig string   yaml hosts file to read from
-      --s3-endpoint string   Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable.
+      --logFile string       log file to write to
+      --s3Endpoint string    Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable.
   -v, --verbose              Sets verbose level
 
 Use "backy [command] --help" for more information about a command.
@@ -51,8 +52,9 @@ Flags:
 Global Flags:
       --cmdStdOut            Pass to print command output to stdout
   -f, --config string        config file to read from
-      --log-file string      log file to write to
+      --hostsConfig string   yaml hosts file to read from
-      --s3-endpoint string   Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable.
+      --logFile string       log file to write to
+      --s3Endpoint string    Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable.
   -v, --verbose              Sets verbose level
 ```
 
@@ -70,8 +72,9 @@ Flags:
 Global Flags:
       --cmdStdOut            Pass to print command output to stdout
   -f, --config string        config file to read from
-      --log-file string      log file to write to
+      --hostsConfig string   yaml hosts file to read from
-      --s3-endpoint string   Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable.
+      --logFile string       log file to write to
+      --s3Endpoint string    Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable.
   -v, --verbose              Sets verbose level
 ```
 
@@ -86,6 +89,7 @@ Usage:
 
 Available Commands:
   host        Runs command defined in config file on the hosts in order specified.
+  hosts       Runs command defined in config file on the hosts in order specified.
 
 Flags:
   -h, --help   help for exec
@@ -93,8 +97,9 @@ Flags:
 Global Flags:
       --cmdStdOut            Pass to print command output to stdout
   -f, --config string        config file to read from
-      --log-file string      log file to write to
+      --hostsConfig string   yaml hosts file to read from
-      --s3-endpoint string   Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable.
+      --logFile string       log file to write to
+      --s3Endpoint string    Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable.
   -v, --verbose              Sets verbose level
 
 Use "backy exec [command] --help" for more information about a command.
@@ -117,8 +122,9 @@ Flags:
 Global Flags:
       --cmdStdOut            Pass to print command output to stdout
   -f, --config string        config file to read from
-      --log-file string      log file to write to
+      --hostsConfig string   yaml hosts file to read from
-      --s3-endpoint string   Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable.
+      --logFile string       log file to write to
+      --s3Endpoint string    Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable.
   -v, --verbose              Sets verbose level
 ```
 
@@ -138,8 +144,9 @@ Flags:
 Global Flags:
       --cmdStdOut            Pass to print command output to stdout
   -f, --config string        config file to read from
-      --log-file string      log file to write to
+      --hostsConfig string   yaml hosts file to read from
-      --s3-endpoint string   Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable.
+      --logFile string       log file to write to
+      --s3Endpoint string    Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable.
   -v, --verbose              Sets verbose level
 ```
 
@@ -161,8 +168,9 @@ Flags:
 Global Flags:
       --cmdStdOut            Pass to print command output to stdout
   -f, --config string        config file to read from
-      --log-file string      log file to write to
+      --hostsConfig string   yaml hosts file to read from
-      --s3-endpoint string   Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable.
+      --logFile string       log file to write to
+      --s3Endpoint string    Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable.
   -v, --verbose              Sets verbose level
 
 Use "backy list [command] --help" for more information about a command.
@@ -181,8 +189,9 @@ Flags:
 Global Flags:
       --cmdStdOut            Pass to print command output to stdout
   -f, --config string        config file to read from
-      --log-file string      log file to write to
+      --hostsConfig string   yaml hosts file to read from
-      --s3-endpoint string   Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable.
+      --logFile string       log file to write to
+      --s3Endpoint string    Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable.
   -v, --verbose              Sets verbose level
 ```
 ## list lists
@@ -199,7 +208,8 @@ Flags:
 Global Flags:
       --cmdStdOut            Pass to print command output to stdout
   -f, --config string        config file to read from
-      --log-file string      log file to write to
+      --hostsConfig string   yaml hosts file to read from
-      --s3-endpoint string   Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable.
+      --logFile string       log file to write to
+      --s3Endpoint string    Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable.
   -v, --verbose              Sets verbose level
 ```

docs/content/cli/exec.md

@@ -15,5 +15,5 @@ The `exec` subcommand can do some things that the configuration file can't do ye
 The commands have to be defined in the config file. The hosts need to at least be in the ssh_config(5) file.
 
 ```sh
-backy exec host [--commands=command1 -commands=command2 ... | -c command1 -c command2 ...] [--hosts=host1 --hosts=hosts2 ... | -m host1 -c host2 ...]  [flags]
+backy exec host [--commands=command1 -commands=command2 ... | -c command1 -c command2 ...] [--hosts=host1 --hosts=hosts2 ... | -m host1 -m host2 ...]  [flags]
 ```

docs/content/cli/list.md

@@ -0,0 +1,29 @@
+---
+title: List
+---
+
+
+List commands, lists, or hosts defined in config file
+
+Usage:
+```
+  backy list [command]
+```
+
+Available Commands:
+  cmds        List commands defined in config file.
+  lists       List lists defined in config file.
+
+Flags:
+```
+  -h, --help   help for list
+```
+
+Global Flags:
+```
+      --cmdStdOut            Pass to print command output to stdout
+  -f, --config string        config file to read from
+      --log-file string      log file to write to
+      --s3-endpoint string   Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable.
+  -v, --verbose              Sets verbose level
+```

docs/content/config/command-lists.md

@@ -35,12 +35,12 @@ If a remote config file is specified (on the command-line using `-f`) and the li
 ```
 
 | key | description | type | required
-| --- | --- | --- | --- |
+| --- | --- | --- | ---
-| `order` | Defines the sequence of commands to execute | `[]string` | yes |
+| `order` | Defines the sequence of commands to execute | `[]string` | yes
-| `getOutput` | Command(s) output is in the notification(s) | `bool` | no |
+| `sendNotificationOnSuccess` | Whether to send notification on list success with the commands' output | `bool` | no
-| `notifications` | The notification service(s) and ID(s) to use on success and failure. Must be *`service.id`*. See the [notifications documentation page](/config/notifications/) for more | `[]string` | no |
+| `notifications` | The notification service(s) and ID(s) to use on success and failure. Must be *`service.id`*. See the [notifications documentation page](/config/notifications/) for more | `[]string` | no
-| `name` | Optional name of the list | `string` | no |
+| `name` | Optional name of the list | `string` | no
-| `cron` | Time at which to schedule the list. Only has affect when cron subcommand is run. | `string` | no |
+| `cron` | Time at which to schedule the list. Only has affect when cron subcommand is run. | `string` | no
 
 ### Order
 

docs/content/config/commands/_index.md

@@ -8,46 +8,21 @@ weight: 1
 
 ### Example Config
 
-```yaml
+{{% code file="/examples/example.yml" language="yaml" %}}
-commands:
-  stop-docker-container:
-    cmd: docker
-    Args:
-      - compose
-      - -f /some/path/to/docker-compose.yaml
-      - down
-    # if host is not defined, command will be run locally
-    # The host has to be defined in either the config file or the SSH Config files
-    host: some-host
-    hooks
-      error:
-        - some-other-command-when-failing
-      success:
-        - success-command
-      final:
-        - final-command
-  backup-docker-container-script:
-    cmd: /path/to/local/script
-    # script file is input as stdin to SSH
-    type: scriptFile # also can be script
-    environment:
-      - FOO=BAR
-      - APP=$VAR
-```
 
 Values available for this section **(case-sensitive)**:
 
-| name | notes | type | required
+| name            | notes                                                                                                   | type                  | required | External directive support |
-| --- | --- | --- | --- |
+| ----------------| ------------------------------------------------------------------------------------------------------- | --------------------- | -------- |----------------------------|
-| `cmd` | Defines the command to execute | `string` | yes |
+| `cmd`           | Defines the command to execute                                                                          | `string`              | yes      | No                         |
-| `Args` | Defines the arguments to the command | `[]string` | no |
+| `Args`          | Defines the arguments to the command                                                                    | `[]string`            | no       | No                         |
-| `environment` | Defines environment variables for the command | `[]string` | no |
+| `environment`   | Defines environment variables for the command                                                           | `[]string`            | no       | Partial                    |
-| `type` | See documentation further down the page. Additional fields may be required. | `string` | no |
+| `type`          | See documentation further down the page. Additional fields may be required.                             | `string`              | no       | No                         |
-| `getOutput` | Command(s) output is in the notification(s) | `bool` | no |
+| `getOutput`     | Command(s) output is in the notification(s)                                                             | `bool`                | no       | No                         |
-| `host` | If not specified, the command will execute locally. | `string` | no |
+| `host`          | If not specified, the command will execute locally.                                                     | `string`              | no       | No                         |
-| `scriptEnvFile` | When type is `scriptFile` or `script`, this file is prepended to the input. | `string` | no |
+| `scriptEnvFile` | When type is `scriptFile` or `script`, this file is prepended to the input.                             | `string`              | no       | No                         |
-| `shell` | Run the command in the shell | `string` | no |
+| `shell`         | Run the command in the shell                                                                            | `string`              | no       | No                         |
-| `hooks` | Hooks are used at the end of the individual command. Must have at least `error`, `success`, or `final`. | `map[string][]string` | no |
+| `hooks`         | Hooks are used at the end of the individual command. Must have at least `error`, `success`, or `final`. | `map[string][]string` | no       | No                         |
 
 #### cmd
 
@@ -120,8 +95,9 @@ The following options are available:
 The environment variables support expansion:
 
 - using escaped values `$VAR` or `${VAR}`
+- using any external directive, and if using the env directive, the variable will be read from a `.env` file
 
-For now, the variables have to be defined in an `.env` file in the same directory that the program is run from.
+<!-- For now, the variables expanded have to be defined in an `.env` file in the same directory that the program is run from. -->
 
 If using it with host specified, the SSH server has to be configured to accept those env variables.
 

docs/content/config/commands/user-commands.md

@@ -6,14 +6,18 @@ description: This is dedicated to user commands.
 
 This is dedicated to `user` commands. The command `type` field must be `user`. User is a type that allows one to perform user operations. There are several additional options available when `type` is `user`:
 
-| name | notes | type | required |
+| name            | notes                                                        | type       | required | External directive support
-| --- | --- | --- | --- |
+| ----------------| -------------------------------------------------------------| ---------- | ---------| --------------------------|
-| `userName` | The name of a user to be configured. | `string` | yes |
+| `userName`      | The name of a user to be configured.                         | `string`   | yes      | no 						 |
-| `userOperation` | The type of operation to perform. | `string` | yes |
+| `userOperation` | The type of operation to perform.                            | `string`   | yes      | no 						 |
-| `userID` | The user ID to use. | `string` | yes |
+| `userID`        | The user ID to use.                                          | `string`   | no       | no 						 |
-| `userGroups` | The groups the user should be added to. | `[]string` | yes |
+| `userGroups`    | The groups the user should be added to.                      | `[]string` | no       | no 						 |
-| `userShell` | The shell for the user. | `string` | yes |
+| `systemUser`    | Create a system user.                                        | `bool`     | no       | no 						 |
-| `userHome` | The user's home directory. | `string` | no |
+| `userCreateHome`| Create the home directory.                                   | `bool`     | no       | no 						 |
+| `userSshPubKeys`| The keys to add to the user's authorized keys.               | `[]string` | no       | yes 						 |
+| `userShell`     | The shell for the user.                                      | `string`   | no       | no 						 |
+| `userHome`      | The user's home directory.                                   | `string`   | no       | no 						 |
+| `userPassword`  | The new password value when using the `password` operation.  | `string`   | no       | yes						 |
 
 
 #### example

docs/content/config/directives.md

@@ -0,0 +1,15 @@
+---
+title: "External Directives"
+weight: 2
+description: How to set up external directives.
+---
+
+External directives are for including data that should not be in the config file. The following directives are supported:
+
+- `%{file:path/to/file}%`
+- `%{env:ENV_VAR}%`
+- `%{vault:vault-key}%`
+
+See the docs of each command if the field is supported.
+
+If the file path does not begin with a `/`, the config file's directory will be used as the starting point.

docs/content/config/hosts.md

@@ -5,20 +5,20 @@ description: >
   This page tells you how to use hosts.
 ---
 
-| Key                  | Description                                                   | Type     | Required |
+| Key                  | Description                                                   | Type     | Required | External directive support |
-|----------------------|---------------------------------------------------------------|----------|----------|
+|----------------------|---------------------------------------------------------------|----------|----------|----------------------------|
-| `OS`                 | Operating system of the host (used for package commands)      | `string` | no       |
+| `OS`                 | Operating system of the host (used for package commands)      | `string` | no       | No                         |
-| `config`             | Path to the SSH config file                                   | `string` | no       |
+| `config`             | Path to the SSH config file                                   | `string` | no       | No                         |
-| `host`               | Specifies the `Host` ssh_config(5) directive                  | `string` | yes      |
+| `host`               | Specifies the `Host` ssh_config(5) directive                  | `string` | yes      | No                         |
-| `hostname`           | Hostname of the host                                          | `string` | no       |
+| `hostname`           | Hostname of the host                                          | `string` | no       | No                         |
-| `knownhostsfile`     | Path to the known hosts file                                  | `string` | no       |
+| `knownHostsFile`     | Path to the known hosts file                                  | `string` | no       | No                         |
-| `port`               | Port number to connect to                                     | `uint16` | no       |
+| `port`               | Port number to connect to                                     | `uint16` | no       | No                         |
-| `proxyjump`          | Proxy jump hosts, comma-separated                             | `string` | no       |
+| `proxyjump`          | Proxy jump hosts, comma-separated                             | `string` | no       | No                         |
-| `password`           | Password for SSH authentication                               | `string` | no       |
+| `password`           | Password for SSH authentication                               | `string` | no       | No                         |
-| `privatekeypath`     | Path to the private key file                                  | `string` | no       |
+| `privateKeyPath`     | Path to the private key file                                  | `string` | no       | No                         |
-| `privatekeypassword` | Password for the private key file                             | `string` | no       |
+| `privateKeyPassword` | Password for the private key file                             | `string` | no       | Yes                        |
-| `user`               | Username for SSH authentication                               | `string` | no       |
+| `user`               | Username for SSH authentication                               | `string` | no       | No                         |
 
 ## exec host subcommand
 
-Backy has a subcommand `exec host`. This subcommand takes the flags of `-m host1 -m host2`. For now these hosts need to be defined in the config file.
+Backy has a subcommand `exec host`. This subcommand takes the flags of `-m host1 -m host2`. The commands can also be specified by `-c command1 -c command2`.

docs/content/config/notifications.md

@@ -39,23 +39,23 @@ There must be a section with an id (eg. `mail.test-svr`) following one of these
 
 ### mail
 
-| key | description | type
+| key | description | type | External directive support |
-| --- | --- | ---
+| --- | --- | --- | --- |
-| `host` | Specifies the SMTP host to connect to | `string`
+| `host` | Specifies the SMTP host to connect to | `string` | no
-| `port` | Specifies the SMTP port | `uint16`
+| `port` | Specifies the SMTP port | `uint16` | no
-| `senderaddress` | Address from which to send mail | `string`
+| `senderaddress` | Address from which to send mail | `string` | no
-| `to` | Recipients to send emails to | `[]string`
+| `to` | Recipients to send emails to | `[]string` | no
-| `username` | SMTP username | `string`
+| `username` | SMTP username | `string` | no
-| `password` | SMTP password | `string`
+| `password` | SMTP password | `string` | yes
 
 ### matrix
 
-| key | description | type
+| key | description | type | External directive support |
-| --- | --- | ---
+| --- | --- | ---| ---- |
-| `home-server` | Specifies the Matrix server connect to | `string`
+| `home-server` | Specifies the Matrix server connect to | `string` | no
-| `room-id` | Specifies the room ID of the room to send messages to | `string`
+| `room-id` | Specifies the room ID of the room to send messages to | `string` | no
-| `access-token` | Matrix access token | `string`
+| `access-token` | Matrix access token  | `string` | yes
-| `user-id` | Matrix user ID | `string`
+| `user-id` | Matrix user ID | `string` | no
 
 To get your access token (assumes you are using [Element](https://element.io/)) :
 

docs/content/config/vault.md

@@ -6,7 +6,7 @@ description: Set up and configure vault.
 
 [Vault](https://www.vaultproject.io/) is a tool for storing secrets and other data securely.
 
-Vault config can be used by prefixing `vault:` in front of a password or ENV var.
+A Vault key can be used by prefixing `%{vault:vault.keys.name}%` in a field that supports external directives.
 
 This is the object in the config file:
 
@@ -18,10 +18,12 @@ vault:
   keys:
     - name: mongourl
       mountpath: secret
+      key: data
       path: mongo/url
-      type:  # KVv1 or KVv2
+      type: KVv2  # KVv1 or KVv2
-    - name:
+    - name: someKeyName
-      path:
+      mountpath: secret
-      type:
+      key: keyData
-      mountpath:
+      type: KVv2
+      path: some/path
 ```

docs/content/examples/backy.yaml

@@ -0,0 +1,108 @@
+commands:
+  stop-docker-container:
+    cmd: docker
+    Args:
+      - compose
+      - -f /some/path/to/docker-compose.yaml
+      - down
+    # if host is not defined, cmd will be run locally
+    host: some-host
+    hooks:
+      final:
+        - hostname
+      error:
+        - hostname
+  backup-docker-container-script:
+    cmd: /path/to/script
+    # The host has to be defined in the config file
+    host: some-host
+    environment:
+      - FOO=BAR
+      - APP=$VAR
+  shell-cmd:
+    cmd: rsync
+    shell: bash
+    Args:
+      - -av some-host:/path/to/data ~/Docker/Backups/docker-data
+  hostname:
+    cmd: hostname
+  update-docker:
+    type: package
+    shell: zsh # best to run package commands in a shell
+    packageName: docker-ce
+    Args:
+      - docker-ce-cli
+    packageManager: apt
+    packageOperation: install
+  update-dockerApt:
+    # type: package
+    shell: zsh
+    cmd: apt
+    Args:
+      - update
+      - "&&"
+      - apt install -y docker-ce
+      - docker-ce-cli
+    packageManager: apt
+    packageOperation: install
+
+cmd-lists:
+  cmds-to-run: # this can be any name you want
+    # all commands have to be defined
+    order:
+      - stop-docker-container
+      - backup-docker-container-script
+      - shell-cmd
+      - hostname
+    notifications:
+      - matrix.matrix
+    name: backup-some-server
+    cron: "0 0 1 * * *"
+  hostname:
+    name: hostname
+    order:
+      - hostname
+    notifications:
+      - mail.prod-email
+
+hosts:
+  # any ssh_config(5) keys/values not listed here will be looked up in the config file or the default config file
+  some-host:
+    hostname: some-hostname
+    config: ~/.ssh/config
+    user: user
+    privateKeyPath: /path/to/private/key
+    port: 22
+    # can also be env:VAR
+    password: file:/path/to/file
+    # only one is supported for now
+    proxyjump: some-proxy-host
+
+# optional
+logging:
+  verbose: true
+  file: ./backy.log
+  console: false
+  cmd-std-out: false
+
+
+notifications:
+  mail:
+    prod-email:
+      id: prod-email
+      type: mail
+      host: yourhost.tld
+      port: 587
+      senderAddress: email@domain.tld
+      to:
+        - admin@domain.tld
+      username: smtp-username@domain.tld
+      password: your-password-here
+  matrix:
+    matrix:
+      id: matrix
+      type: matrix
+      home-server: your-home-server.tld
+      room-id: room-id
+      access-token: your-access-token
+      user-id: your-user-id

docs/content/examples/example.yml

@@ -0,0 +1,24 @@
+commands:
+  stop-docker-container:
+    cmd: docker
+    Args:
+      - compose
+      - -f /some/path/to/docker-compose.yaml
+      - down
+    # if host is not defined, command will be run locally
+    # The host has to be defined in either the config file or the SSH Config files
+    host: some-host
+    hooks:
+      error:
+        - some-other-command-when-failing
+      success:
+        - success-command
+      final:
+        - final-command
+  backup-docker-container-script:
+    cmd: /path/to/local/script
+    # script file is input as stdin to SSH
+    type: scriptFile # also can be script
+    environment:
+      - FOO=BAR
+      - APP=$VAR

docs/content/getting-started/config.md

@@ -124,13 +124,13 @@ notifications:
 
 ### Logging
 
-cmd-std-out controls whether commands output is echoed to StdOut.
+`cmd-std-out` controls whether commands output is echoed to StdOut.
 
-If logfile is not defined, the log file will be written to the config directory in the file `backy.log`.
+If `logfile` is not defined, the log file will be written to the config directory in the file `backy.log`.
 
 `console-disabled` controls whether the logging messages are echoed to StdOut. Default is false.
 
-`verbose` basically does nothing as all necessary info is already output.
+`verbose` prints out debugging messages.
 
 ```yaml
 logging:
@@ -144,7 +144,7 @@ logging:
 
 [Vault](https://www.vaultproject.io/) can be used to get some configuration values and ENV variables securely.
 
-```
+```yaml
 vault:
   token: hvs.tXqcASvTP8wg92f7riyvGyuf
   address: http://127.0.0.1:8200

docs/layouts/shortcodes/code.html

@@ -0,0 +1,3 @@
+{{ $file := .Get "file" | readFile }}
+{{ $lang := .Get "language" }}
+{{ (print "```" $lang "\n" $file "\n```") }}

examples/example.yml

@@ -0,0 +1,24 @@
+commands:
+  stop-docker-container:
+    cmd: docker
+    Args:
+      - compose
+      - -f /some/path/to/docker-compose.yaml
+      - down
+    # if host is not defined, command will be run locally
+    # The host has to be defined in either the config file or the SSH Config files
+    host: some-host
+    hooks:
+      error:
+        - some-other-command-when-failing
+      success:
+        - success-command
+      final:
+        - final-command
+  backup-docker-container-script:
+    cmd: /path/to/local/script
+    # script file is input as stdin to SSH
+    type: scriptFile # also can be script
+    environment:
+      - FOO=BAR
+      - APP=$VAR

go.mod

@@ -1,76 +1,78 @@
 module git.andrewnw.xyz/CyberShell/backy
 
-go 1.23
+go 1.23.0
 
-toolchain go1.23.6
+toolchain go1.23.7
-
-replace git.andrewnw.xyz/CyberShell/backy => /home/andrew/Projects/backy
 
 require (
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.76.0
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.83.0
 	github.com/dmarkham/enumer v1.5.11
 	github.com/go-co-op/gocron v1.37.0
-	github.com/hashicorp/vault/api v1.15.0
+	github.com/google/uuid v1.6.0
+	github.com/hashicorp/vault/api v1.20.0
 	github.com/joho/godotenv v1.5.1
 	github.com/kevinburke/ssh_config v1.2.0
-	github.com/knadh/koanf/parsers/yaml v0.1.0
+	github.com/knadh/koanf/parsers/yaml v1.1.0
-	github.com/knadh/koanf/providers/rawbytes v0.1.0
+	github.com/knadh/koanf/providers/rawbytes v1.0.0
-	github.com/knadh/koanf/v2 v2.1.2
+	github.com/knadh/koanf/v2 v2.2.2
 	github.com/mattn/go-isatty v0.0.20
-	github.com/minio/minio-go/v7 v7.0.84
+	github.com/minio/minio-go/v7 v7.0.94
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/nikoksr/notify v1.3.0
 	github.com/pkg/errors v0.9.1
-	github.com/rs/zerolog v1.33.0
+	github.com/pkg/sftp v1.13.9
+	github.com/rs/zerolog v1.34.0
 	github.com/sethvargo/go-password v0.3.1
-	github.com/spf13/cobra v1.8.1
+	github.com/spf13/cobra v1.9.1
-	golang.org/x/crypto v0.33.0
+	golang.org/x/crypto v0.40.0
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1
 	gopkg.in/yaml.v3 v3.0.1
-	maunium.net/go/mautrix v0.23.0
+	maunium.net/go/mautrix v0.24.1
-	mvdan.cc/sh/v3 v3.10.0
+	mvdan.cc/sh/v3 v3.12.0
 )
 
 require (
 	filippo.io/edwards25519 v1.1.0 // indirect
-	github.com/aws/aws-sdk-go-v2 v1.36.1 // indirect
+	github.com/aws/aws-sdk-go-v2 v1.36.5 // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.11 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.32 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.36 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.32 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.36 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.32 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.36 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.13 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.17 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.13 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.17 // indirect
-	github.com/aws/smithy-go v1.22.2 // indirect
+	github.com/aws/smithy-go v1.22.4 // indirect
 	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/go-ini/ini v1.67.0 // indirect
-	github.com/go-jose/go-jose/v4 v4.0.1 // indirect
+	github.com/go-jose/go-jose/v4 v4.1.1 // indirect
-	github.com/go-viper/mapstructure/v2 v2.2.1 // indirect
+	github.com/go-viper/mapstructure/v2 v2.3.0 // indirect
 	github.com/goccy/go-json v0.10.5 // indirect
-	github.com/google/uuid v1.6.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
-	github.com/hashicorp/go-retryablehttp v0.7.7 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.8 // indirect
 	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
-	github.com/hashicorp/go-secure-stdlib/parseutil v0.1.9 // indirect
+	github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0 // indirect
 	github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
 	github.com/hashicorp/go-sockaddr v1.0.7 // indirect
-	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/hashicorp/hcl v1.0.1-vault-7 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jordan-wright/email v4.0.1-0.20210109023952-943e75fe5223+incompatible // indirect
-	github.com/klauspost/compress v1.17.11 // indirect
+	github.com/klauspost/compress v1.18.0 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.9 // indirect
+	github.com/klauspost/cpuid/v2 v2.3.0 // indirect
-	github.com/knadh/koanf/maps v0.1.1 // indirect
+	github.com/knadh/koanf/maps v0.1.2 // indirect
+	github.com/kr/fs v0.1.0 // indirect
 	github.com/mattn/go-colorable v0.1.14 // indirect
+	github.com/minio/crc64nvme v1.0.2 // indirect
 	github.com/minio/md5-simd v1.1.2 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
-	github.com/pascaldekloe/name v1.0.0 // indirect
+	github.com/pascaldekloe/name v1.0.1 // indirect
+	github.com/philhofer/fwd v1.2.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/robfig/cron/v3 v3.0.1 // indirect
 	github.com/rs/xid v1.6.0 // indirect
@@ -82,14 +84,16 @@ require (
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.1 // indirect
 	github.com/tidwall/sjson v1.2.5 // indirect
-	go.mau.fi/util v0.8.4 // indirect
+	github.com/tinylib/msgp v1.3.0 // indirect
+	go.mau.fi/util v0.8.8 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
-	golang.org/x/exp v0.0.0-20250207012021-f9890c6ad9f3 // indirect
+	go.yaml.in/yaml/v3 v3.0.4 // indirect
-	golang.org/x/mod v0.23.0 // indirect
+	golang.org/x/exp v0.0.0-20250711185948-6ae5c78190dc // indirect
-	golang.org/x/net v0.35.0 // indirect
+	golang.org/x/mod v0.26.0 // indirect
-	golang.org/x/sync v0.11.0 // indirect
+	golang.org/x/net v0.42.0 // indirect
-	golang.org/x/sys v0.30.0 // indirect
+	golang.org/x/sync v0.16.0 // indirect
-	golang.org/x/text v0.22.0 // indirect
+	golang.org/x/sys v0.34.0 // indirect
-	golang.org/x/time v0.10.0 // indirect
+	golang.org/x/text v0.27.0 // indirect
-	golang.org/x/tools v0.30.0 // indirect
+	golang.org/x/time v0.12.0 // indirect
+	golang.org/x/tools v0.35.0 // indirect
 )

go.sum

@@ -1,31 +1,31 @@
 filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
 filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
-github.com/aws/aws-sdk-go-v2 v1.36.1 h1:iTDl5U6oAhkNPba0e1t1hrwAo02ZMqbrGq4k5JBWM5E=
+github.com/aws/aws-sdk-go-v2 v1.36.5 h1:0OF9RiEMEdDdZEMqF9MRjevyxAQcf6gY+E7vwBILFj0=
-github.com/aws/aws-sdk-go-v2 v1.36.1/go.mod h1:5PMILGVKiW32oDzjj6RU52yrNrDPUHcbZQYr1sM7qmM=
+github.com/aws/aws-sdk-go-v2 v1.36.5/go.mod h1:EYrzvCCN9CMUTa5+6lf6MM4tq3Zjp8UhSGR/cBsjai0=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8 h1:zAxi9p3wsZMIaVCdoiQp2uZ9k1LsZvmAnoTBeZPXom0=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.11 h1:12SpdwU8Djs+YGklkinSSlcrPyj3H4VifVsKf78KbwA=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8/go.mod h1:3XkePX5dSaxveLAYY7nsbsZZrKxCyEuE5pM4ziFxyGg=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.11/go.mod h1:dd+Lkp6YmMryke+qxW/VnKyhMBDTYP41Q2Bb+6gNZgY=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.32 h1:BjUcr3X3K0wZPGFg2bxOWW3VPN8rkE3/61zhP+IHviA=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.36 h1:SsytQyTMHMDPspp+spo7XwXTP44aJZZAC7fBV2C5+5s=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.32/go.mod h1:80+OGC/bgzzFFTUmcuwD0lb4YutwQeKLFpmt6hoWapU=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.36/go.mod h1:Q1lnJArKRXkenyog6+Y+zr7WDpk4e6XlR6gs20bbeNo=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.32 h1:m1GeXHVMJsRsUAqG6HjZWx9dj7F5TR+cF1bjyfYyBd4=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.36 h1:i2vNHQiXUvKhs3quBR6aqlgJaiaexz/aNvdCktW/kAM=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.32/go.mod h1:IitoQxGfaKdVLNg0hD8/DXmAqNy0H4K2H2Sf91ti8sI=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.36/go.mod h1:UdyGa7Q91id/sdyHPwth+043HhmP6yP9MBHgbZM0xo8=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.32 h1:OIHj/nAhVzIXGzbAE+4XmZ8FPvro3THr6NlqErJc3wY=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.36 h1:GMYy2EOWfzdP3wfVAGXBNKY5vK4K8vMET4sYOYltmqs=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.32/go.mod h1:LiBEsDo34OJXqdDlRGsilhlIiXR7DL+6Cx2f4p1EgzI=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.36/go.mod h1:gDhdAV6wL3PmPqBhiPbnlS447GoWs8HTTOYef9/9Inw=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 h1:D4oz8/CzT9bAEYtVhSBmFj2dNOtaHOtMKc2vHBwYizA=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.4 h1:CXV68E2dNqhuynZJPB80bhPQwAKqBWVer887figW6Jc=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2/go.mod h1:Za3IHqTQ+yNcRHxu1OFucBh0ACZT4j4VQFF0BqpZcLY=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.4/go.mod h1:/xFi9KtvBXP97ppCz1TAEvU1Uf66qvid89rbem3wCzQ=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.6 h1:cCBJaT7EeEojpJ4s7wTDbhZlHVJOgNHN7iw6qVurGaw=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.4 h1:nAP2GYbfh8dd2zGZqFRSMlq+/F6cMPBUuCsGAMkN074=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.6/go.mod h1:WYH1ABybY7JK9TITPnk6ZlP7gQB8psI4c9qDmMsnLSA=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.4/go.mod h1:LT10DsiGjLWh4GbjInf9LQejkYEhBgBCjLG5+lvk4EE=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.13 h1:SYVGSFQHlchIcy6e7x12bsrxClCXSP5et8cqVhL8cuw=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.17 h1:t0E6FzREdtCsiLIoLCWsYliNsRBgyGD/MCK571qk4MI=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.13/go.mod h1:kizuDaLX37bG5WZaoxGPQR/LNFXpxp0vsUnqfkWXfNE=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.17/go.mod h1:ygpklyoaypuyDvOM5ujWGrYWpAK3h7ugnmKCU/76Ys4=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.13 h1:OBsrtam3rk8NfBEq7OLOMm5HtQ9Yyw32X4UQMya/wjw=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.17 h1:qcLWgdhq45sDM9na4cvXax9dyLitn8EYBRl8Ak4XtG4=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.13/go.mod h1:3U4gFA5pmoCOja7aq4nSaIAGbaOHv2Yl2ug018cmC+Q=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.17/go.mod h1:M+jkjBFZ2J6DJrjMv2+vkBbuht6kxJYtJiwoVgX4p4U=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.76.0 h1:ehvUZNVrGA1Usa6yYo8A8pUqrigRelWXSbcCqYpRLeI=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.83.0 h1:5Y75q0RPQoAbieyOuGLhjV9P3txvYgXv2lg0UwJOfmE=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.76.0/go.mod h1:KuLNrwYJFaC2AVZ+CVVc12k9NyqwgWsoNNHjwqF6QNk=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.83.0/go.mod h1:kUklwasNoCn5YpyAqC/97r6dzTA1SRKJfKq16SXeoDU=
-github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ=
+github.com/aws/smithy-go v1.22.4 h1:uqXzVZNuNexwc/xrh6Tb56u89WDlJY6HS+KC0S4QSjw=
-github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg=
+github.com/aws/smithy-go v1.22.4/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI=
 github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
 github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
-github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -40,19 +40,20 @@ github.com/go-co-op/gocron v1.37.0 h1:ZYDJGtQ4OMhTLKOKMIch+/CY70Brbb1dGdooLEhh7b
 github.com/go-co-op/gocron v1.37.0/go.mod h1:3L/n6BkO7ABj+TrfSVXLRzsP26zmikL4ISkLQ0O8iNY=
 github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
 github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
-github.com/go-jose/go-jose/v4 v4.0.1 h1:QVEPDE3OluqXBQZDcnNvQrInro2h0e4eqNbnZSWqS6U=
+github.com/go-jose/go-jose/v4 v4.1.1 h1:JYhSgy4mXXzAdF3nUx3ygx347LRXJRrpgyU3adRmkAI=
-github.com/go-jose/go-jose/v4 v4.0.1/go.mod h1:WVf9LFMHh/QVrmqrOfqun0C45tMe3RoiKJMPvgWwLfY=
+github.com/go-jose/go-jose/v4 v4.1.1/go.mod h1:BdsZGqgdO3b6tTc6LSE56wcDbMMLuPsw5d4ZD5f94kA=
 github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI=
 github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow=
 github.com/go-test/deep v1.0.2 h1:onZX1rnHT3Wv6cqNgYyFOOlgVKJrksuCMCRvJStbMYw=
 github.com/go-test/deep v1.0.2/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
-github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss=
+github.com/go-viper/mapstructure/v2 v2.3.0 h1:27XbWsHIqhbdR5TIC911OfYvgSaW93HM+dX7970Q7jk=
-github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
+github.com/go-viper/mapstructure/v2 v2.3.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
 github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
 github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
-github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -65,20 +66,20 @@ github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB1
 github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
 github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
 github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
-github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU=
+github.com/hashicorp/go-retryablehttp v0.7.8 h1:ylXZWnqa7Lhqpk0L1P1LzDtGcCR0rPVUrx/c8Unxc48=
-github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk=
+github.com/hashicorp/go-retryablehttp v0.7.8/go.mod h1:rjiScheydd+CxvumBsIrFKlx3iS0jrZ7LvzFGFmuKbw=
 github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc=
 github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
-github.com/hashicorp/go-secure-stdlib/parseutil v0.1.9 h1:FW0YttEnUNDJ2WL9XcrrfteS1xW8u+sh4ggM8pN5isQ=
+github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0 h1:U+kC2dOhMFQctRfhK0gRctKAPTloZdMU5ZJxaesJ/VM=
-github.com/hashicorp/go-secure-stdlib/parseutil v0.1.9/go.mod h1:Ll013mhdmsVDuoIXVfBtvgGJsXDYkTw1kooNcoCXuE0=
+github.com/hashicorp/go-secure-stdlib/parseutil v0.2.0/go.mod h1:Ll013mhdmsVDuoIXVfBtvgGJsXDYkTw1kooNcoCXuE0=
 github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9CdjCtrXrXGuOpxEA7Ts=
 github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4=
 github.com/hashicorp/go-sockaddr v1.0.7 h1:G+pTkSO01HpR5qCxg7lxfsFEZaG+C0VssTy/9dbT+Fw=
 github.com/hashicorp/go-sockaddr v1.0.7/go.mod h1:FZQbEYa1pxkQ7WLpyXJ6cbjpT8q0YgQaK/JakXqGyWw=
-github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
+github.com/hashicorp/hcl v1.0.1-vault-7 h1:ag5OxFVy3QYTFTJODRzTKVZ6xvdfLLCA1cy/Y6xGI0I=
-github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/hashicorp/hcl v1.0.1-vault-7/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM=
-github.com/hashicorp/vault/api v1.15.0 h1:O24FYQCWwhwKnF7CuSqP30S51rTV7vz1iACXE/pj5DA=
+github.com/hashicorp/vault/api v1.20.0 h1:KQMHElgudOsr+IbJgmbjHnCTxEpKs9LnozA1D3nozU4=
-github.com/hashicorp/vault/api v1.15.0/go.mod h1:+5YTO09JGn0u+b6ySD/LLVf8WkJCPLAL2Vkmrn2+CM8=
+github.com/hashicorp/vault/api v1.20.0/go.mod h1:GZ4pcjfzoOWpkJ3ijHNpEoAxKEsBJnVljyTe3jM2Sms=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
@@ -87,19 +88,21 @@ github.com/jordan-wright/email v4.0.1-0.20210109023952-943e75fe5223+incompatible
 github.com/jordan-wright/email v4.0.1-0.20210109023952-943e75fe5223+incompatible/go.mod h1:1c7szIrayyPPB/987hsnvNzLushdWf4o/79s3P08L8A=
 github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=
 github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
-github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
+github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
-github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
+github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
 github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
-github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY=
+github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
-github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8=
+github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
-github.com/knadh/koanf/maps v0.1.1 h1:G5TjmUh2D7G2YWf5SQQqSiHRJEjaicvU0KpypqB3NIs=
+github.com/knadh/koanf/maps v0.1.2 h1:RBfmAW5CnZT+PJ1CVc1QSJKf4Xu9kxfQgYVQSu8hpbo=
-github.com/knadh/koanf/maps v0.1.1/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI=
+github.com/knadh/koanf/maps v0.1.2/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI=
-github.com/knadh/koanf/parsers/yaml v0.1.0 h1:ZZ8/iGfRLvKSaMEECEBPM1HQslrZADk8fP1XFUxVI5w=
+github.com/knadh/koanf/parsers/yaml v1.1.0 h1:3ltfm9ljprAHt4jxgeYLlFPmUaunuCgu1yILuTXRdM4=
-github.com/knadh/koanf/parsers/yaml v0.1.0/go.mod h1:cvbUDC7AL23pImuQP0oRw/hPuccrNBS2bps8asS0CwY=
+github.com/knadh/koanf/parsers/yaml v1.1.0/go.mod h1:HHmcHXUrp9cOPcuC+2wrr44GTUB0EC+PyfN3HZD9tFg=
-github.com/knadh/koanf/providers/rawbytes v0.1.0 h1:dpzgu2KO6uf6oCb4aP05KDmKmAmI51k5pe8RYKQ0qME=
+github.com/knadh/koanf/providers/rawbytes v1.0.0 h1:MrKDh/HksJlKJmaZjgs4r8aVBb/zsJyc/8qaSnzcdNI=
-github.com/knadh/koanf/providers/rawbytes v0.1.0/go.mod h1:mMTB1/IcJ/yE++A2iEZbY1MLygX7vttU+C+S/YmPu9c=
+github.com/knadh/koanf/providers/rawbytes v1.0.0/go.mod h1:KxwYJf1uezTKy6PBtfE+m725NGp4GPVA7XoNTJ/PtLo=
-github.com/knadh/koanf/v2 v2.1.2 h1:I2rtLRqXRy1p01m/utEtpZSSA6dcJbgGVuE27kW2PzQ=
+github.com/knadh/koanf/v2 v2.2.2 h1:ghbduIkpFui3L587wavneC9e3WIliCgiCgdxYO/wd7A=
-github.com/knadh/koanf/v2 v2.1.2/go.mod h1:Gphfaen0q1Fc1HTgJgSTC4oRX9R2R5ErYMZJy8fLJBo=
+github.com/knadh/koanf/v2 v2.2.2/go.mod h1:abWQc0cBXLSF/PSOMCB/SK+T13NXDsPvOksbpi5e/9Q=
+github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
+github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
@@ -116,10 +119,12 @@ github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/
 github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/minio/crc64nvme v1.0.2 h1:6uO1UxGAD+kwqWWp7mBFsi5gAse66C4NXO8cmcVculg=
+github.com/minio/crc64nvme v1.0.2/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
 github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
 github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
-github.com/minio/minio-go/v7 v7.0.84 h1:D1HVmAF8JF8Bpi6IU4V9vIEj+8pc+xU88EWMs2yed0E=
+github.com/minio/minio-go/v7 v7.0.94 h1:1ZoksIKPyaSt64AVOyaQvhDOgVC3MfZsWM6mZXRUGtM=
-github.com/minio/minio-go/v7 v7.0.84/go.mod h1:57YXpvc5l3rjPdhqNrDsvVlY0qPI6UTk1bflAe+9doY=
+github.com/minio/minio-go/v7 v7.0.94/go.mod h1:71t2CqDt3ThzESgZUlU1rBN54mksGGlkLcFgguDnnAc=
 github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
 github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
@@ -130,32 +135,34 @@ github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zx
 github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
 github.com/nikoksr/notify v1.3.0 h1:UxzfxzAYGQD9a5JYLBTVx0lFMxeHCke3rPCkfWdPgLs=
 github.com/nikoksr/notify v1.3.0/go.mod h1:Xor2hMmkvrCfkCKvXGbcrESez4brac2zQjhd6U2BbeM=
-github.com/pascaldekloe/name v1.0.0 h1:n7LKFgHixETzxpRv2R77YgPUFo85QHGZKrdaYm7eY5U=
+github.com/pascaldekloe/name v1.0.1 h1:9lnXOHeqeHHnWLbKfH6X98+4+ETVqFqxN09UXSjcMb0=
-github.com/pascaldekloe/name v1.0.0/go.mod h1:Z//MfYJnH4jVpQ9wkclwu2I2MkHmXTlT9wR5UZScttM=
+github.com/pascaldekloe/name v1.0.1/go.mod h1:Z//MfYJnH4jVpQ9wkclwu2I2MkHmXTlT9wR5UZScttM=
+github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM=
+github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/sftp v1.13.9 h1:4NGkvGudBL7GteO3m6qnaQ4pC0Kvf0onSVc9gR3EWBw=
+github.com/pkg/sftp v1.13.9/go.mod h1:OBN7bVXdstkFFN/gdnHPUb5TE8eb8G1Rp9wCItqjkkA=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
 github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
 github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
 github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o=
-github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
+github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
-github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
+github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
-github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU=
 github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
-github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8=
+github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY=
-github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss=
+github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk=
 github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
 github.com/sethvargo/go-password v0.3.1 h1:WqrLTjo7X6AcVYfC6R7GtSyuUQR9hGyAj/f1PYQZCJU=
 github.com/sethvargo/go-password v0.3.1/go.mod h1:rXofC1zT54N7R8K/h1WDUdkf9BOx5OptoxrMBcrXzvs=
-github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM=
+github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo=
-github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y=
+github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0=
-github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
 github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -179,34 +186,100 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
 github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
 github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
 github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
-go.mau.fi/util v0.8.4 h1:mVKlJcXWfVo8ZW3f4vqtjGpqtZqJvX4ETekxawt2vnQ=
+github.com/tinylib/msgp v1.3.0 h1:ULuf7GPooDaIlbyvgAxBV/FI7ynli6LZ1/nVUNu+0ww=
-go.mau.fi/util v0.8.4/go.mod h1:MOfGTs1CBuK6ERTcSL4lb5YU7/ujz09eOPVEDckuazY=
+github.com/tinylib/msgp v1.3.0/go.mod h1:ykjzy2wzgrlvpDCRc4LA8UXy6D8bzMSuAF3WD57Gok0=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+go.mau.fi/util v0.8.8 h1:OnuEEc/sIJFhnq4kFggiImUpcmnmL/xpvQMRu5Fiy5c=
+go.mau.fi/util v0.8.8/go.mod h1:Y/kS3loxTEhy8Vill513EtPXr+CRDdae+Xj2BXXMy/c=
 go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
 go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
-golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus=
+go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
-golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M=
+go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
-golang.org/x/exp v0.0.0-20250207012021-f9890c6ad9f3 h1:qNgPs5exUA+G0C96DrPwNrvLSj7GT/9D+3WMWUcUg34=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/exp v0.0.0-20250207012021-f9890c6ad9f3/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/mod v0.23.0 h1:Zb7khfcRGKk+kqfxFaP5tZqCnDZMjC5VtUBs87Hr6QM=
+golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
-golang.org/x/mod v0.23.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
+golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
-golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8=
+golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
-golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk=
+golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
-golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w=
+golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM=
-golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY=
+golang.org/x/exp v0.0.0-20250711185948-6ae5c78190dc h1:TS73t7x3KarrNd5qAipmspBDS1rkMcgVG/fS1aRb4Rc=
+golang.org/x/exp v0.0.0-20250711185948-6ae5c78190dc/go.mod h1:A+z0yzpGtvnG90cToK5n2tu8UJVP2XUATh+r+sfOOOc=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/mod v0.26.0 h1:EGMPT//Ezu+ylkCijjPc+f4Aih7sZvaAr+O3EHBxvZg=
+golang.org/x/mod v0.26.0/go.mod h1:/j6NAhSk8iQ723BGAUyoAcn7SlD7s15Dp9Nd/SfeaFQ=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
+golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
+golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs=
+golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw=
+golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
+golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU=
+golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s=
+golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA=
-golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM=
+golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
-golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
+golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
-golang.org/x/time v0.10.0 h1:3usCWA8tQn0L8+hFJQNgzpWbd89begxN66o1Ojdn5L4=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/time v0.10.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/tools v0.30.0 h1:BgcpHewrV5AUp2G9MebG4XPFI1E2W41zU1SaqVA9vJY=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
-golang.org/x/tools v0.30.0/go.mod h1:c347cR/OJfw5TI+GfX7RUPNMdDRRbjvYTS0jPyvsVtY=
+golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
+golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
+golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
+golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
+golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
+golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg=
+golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
+golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4=
+golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU=
+golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE=
+golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
+golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
+golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0=
+golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
@@ -217,7 +290,7 @@ gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYs
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-maunium.net/go/mautrix v0.23.0 h1:HNlR19eew5lvrNSL2muhExaGhYdaGk5FfEiA82QqUP4=
+maunium.net/go/mautrix v0.24.1 h1:09/xi4qTeA03g1n/DPmmqAlT8Cx4QrgwiPlmLVzA9AU=
-maunium.net/go/mautrix v0.23.0/go.mod h1:AGnnaz3ylGikUo1I1MJVn9QLsl2No1/ZNnGDyO0QD5s=
+maunium.net/go/mautrix v0.24.1/go.mod h1:Xy6o+pXmbqmgWsUWh15EQ1eozjC+k/VT/7kloByv9PI=
-mvdan.cc/sh/v3 v3.10.0 h1:v9z7N1DLZ7owyLM/SXZQkBSXcwr2IGMm2LY2pmhVXj4=
+mvdan.cc/sh/v3 v3.12.0 h1:ejKUR7ONP5bb+UGHGEG/k9V5+pRVIyD+LsZz7o8KHrI=
-mvdan.cc/sh/v3 v3.10.0/go.mod h1:z/mSSVyLFGZzqb3ZIKojjyqIx/xbmz/UHdCSv9HmqXY=
+mvdan.cc/sh/v3 v3.12.0/go.mod h1:Se6Cj17eYSn+sNooLZiEUnNNmNxg0imoYlTu4CyaGyg=

pkg/backy/allowedexternaldirectives_enumer.go

@@ -0,0 +1,149 @@
+// Code generated by "enumer -linecomment -yaml -text -json -type=AllowedExternalDirectives"; DO NOT EDIT.
+
+package backy
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+)
+
+const _AllowedExternalDirectivesName = "DefaultExternalDirvaultvault-envvault-filevault-file-envfile-envfileenv"
+
+var _AllowedExternalDirectivesIndex = [...]uint8{0, 18, 23, 32, 42, 56, 64, 68, 71}
+
+const _AllowedExternalDirectivesLowerName = "defaultexternaldirvaultvault-envvault-filevault-file-envfile-envfileenv"
+
+func (i AllowedExternalDirectives) String() string {
+	if i < 0 || i >= AllowedExternalDirectives(len(_AllowedExternalDirectivesIndex)-1) {
+		return fmt.Sprintf("AllowedExternalDirectives(%d)", i)
+	}
+	return _AllowedExternalDirectivesName[_AllowedExternalDirectivesIndex[i]:_AllowedExternalDirectivesIndex[i+1]]
+}
+
+// An "invalid array index" compiler error signifies that the constant values have changed.
+// Re-run the stringer command to generate them again.
+func _AllowedExternalDirectivesNoOp() {
+	var x [1]struct{}
+	_ = x[DefaultExternalDir-(0)]
+	_ = x[AllowedExternalDirectiveVault-(1)]
+	_ = x[AllowedExternalDirectiveVaultEnv-(2)]
+	_ = x[AllowedExternalDirectiveVaultFile-(3)]
+	_ = x[AllowedExternalDirectiveAll-(4)]
+	_ = x[AllowedExternalDirectiveFileEnv-(5)]
+	_ = x[AllowedExternalDirectiveFile-(6)]
+	_ = x[AllowedExternalDirectiveEnv-(7)]
+}
+
+var _AllowedExternalDirectivesValues = []AllowedExternalDirectives{DefaultExternalDir, AllowedExternalDirectiveVault, AllowedExternalDirectiveVaultEnv, AllowedExternalDirectiveVaultFile, AllowedExternalDirectiveAll, AllowedExternalDirectiveFileEnv, AllowedExternalDirectiveFile, AllowedExternalDirectiveEnv}
+
+var _AllowedExternalDirectivesNameToValueMap = map[string]AllowedExternalDirectives{
+	_AllowedExternalDirectivesName[0:18]:       DefaultExternalDir,
+	_AllowedExternalDirectivesLowerName[0:18]:  DefaultExternalDir,
+	_AllowedExternalDirectivesName[18:23]:      AllowedExternalDirectiveVault,
+	_AllowedExternalDirectivesLowerName[18:23]: AllowedExternalDirectiveVault,
+	_AllowedExternalDirectivesName[23:32]:      AllowedExternalDirectiveVaultEnv,
+	_AllowedExternalDirectivesLowerName[23:32]: AllowedExternalDirectiveVaultEnv,
+	_AllowedExternalDirectivesName[32:42]:      AllowedExternalDirectiveVaultFile,
+	_AllowedExternalDirectivesLowerName[32:42]: AllowedExternalDirectiveVaultFile,
+	_AllowedExternalDirectivesName[42:56]:      AllowedExternalDirectiveAll,
+	_AllowedExternalDirectivesLowerName[42:56]: AllowedExternalDirectiveAll,
+	_AllowedExternalDirectivesName[56:64]:      AllowedExternalDirectiveFileEnv,
+	_AllowedExternalDirectivesLowerName[56:64]: AllowedExternalDirectiveFileEnv,
+	_AllowedExternalDirectivesName[64:68]:      AllowedExternalDirectiveFile,
+	_AllowedExternalDirectivesLowerName[64:68]: AllowedExternalDirectiveFile,
+	_AllowedExternalDirectivesName[68:71]:      AllowedExternalDirectiveEnv,
+	_AllowedExternalDirectivesLowerName[68:71]: AllowedExternalDirectiveEnv,
+}
+
+var _AllowedExternalDirectivesNames = []string{
+	_AllowedExternalDirectivesName[0:18],
+	_AllowedExternalDirectivesName[18:23],
+	_AllowedExternalDirectivesName[23:32],
+	_AllowedExternalDirectivesName[32:42],
+	_AllowedExternalDirectivesName[42:56],
+	_AllowedExternalDirectivesName[56:64],
+	_AllowedExternalDirectivesName[64:68],
+	_AllowedExternalDirectivesName[68:71],
+}
+
+// AllowedExternalDirectivesString retrieves an enum value from the enum constants string name.
+// Throws an error if the param is not part of the enum.
+func AllowedExternalDirectivesString(s string) (AllowedExternalDirectives, error) {
+	if val, ok := _AllowedExternalDirectivesNameToValueMap[s]; ok {
+		return val, nil
+	}
+
+	if val, ok := _AllowedExternalDirectivesNameToValueMap[strings.ToLower(s)]; ok {
+		return val, nil
+	}
+	return 0, fmt.Errorf("%s does not belong to AllowedExternalDirectives values", s)
+}
+
+// AllowedExternalDirectivesValues returns all values of the enum
+func AllowedExternalDirectivesValues() []AllowedExternalDirectives {
+	return _AllowedExternalDirectivesValues
+}
+
+// AllowedExternalDirectivesStrings returns a slice of all String values of the enum
+func AllowedExternalDirectivesStrings() []string {
+	strs := make([]string, len(_AllowedExternalDirectivesNames))
+	copy(strs, _AllowedExternalDirectivesNames)
+	return strs
+}
+
+// IsAAllowedExternalDirectives returns "true" if the value is listed in the enum definition. "false" otherwise
+func (i AllowedExternalDirectives) IsAAllowedExternalDirectives() bool {
+	for _, v := range _AllowedExternalDirectivesValues {
+		if i == v {
+			return true
+		}
+	}
+	return false
+}
+
+// MarshalJSON implements the json.Marshaler interface for AllowedExternalDirectives
+func (i AllowedExternalDirectives) MarshalJSON() ([]byte, error) {
+	return json.Marshal(i.String())
+}
+
+// UnmarshalJSON implements the json.Unmarshaler interface for AllowedExternalDirectives
+func (i *AllowedExternalDirectives) UnmarshalJSON(data []byte) error {
+	var s string
+	if err := json.Unmarshal(data, &s); err != nil {
+		return fmt.Errorf("AllowedExternalDirectives should be a string, got %s", data)
+	}
+
+	var err error
+	*i, err = AllowedExternalDirectivesString(s)
+	return err
+}
+
+// MarshalText implements the encoding.TextMarshaler interface for AllowedExternalDirectives
+func (i AllowedExternalDirectives) MarshalText() ([]byte, error) {
+	return []byte(i.String()), nil
+}
+
+// UnmarshalText implements the encoding.TextUnmarshaler interface for AllowedExternalDirectives
+func (i *AllowedExternalDirectives) UnmarshalText(text []byte) error {
+	var err error
+	*i, err = AllowedExternalDirectivesString(string(text))
+	return err
+}
+
+// MarshalYAML implements a YAML Marshaler for AllowedExternalDirectives
+func (i AllowedExternalDirectives) MarshalYAML() (interface{}, error) {
+	return i.String(), nil
+}
+
+// UnmarshalYAML implements a YAML Unmarshaler for AllowedExternalDirectives
+func (i *AllowedExternalDirectives) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var s string
+	if err := unmarshal(&s); err != nil {
+		return err
+	}
+
+	var err error
+	*i, err = AllowedExternalDirectivesString(s)
+	return err
+}

pkg/backy/backy.go

@@ -11,6 +11,8 @@ import (
 	"io"
 	"os"
 	"os/exec"
+	"strings"
+	"sync"
 	"text/template"
 
 	"embed"
@@ -25,6 +27,124 @@ var requiredKeys = []string{"commands"}
 
 var Sprintf = fmt.Sprintf
 
+type CommandExecutor interface {
+	Run(cmd *Command, opts *ConfigOpts, logger zerolog.Logger) ([]string, error)
+}
+
+type OutputHandler interface {
+	CollectOutput(buf *bytes.Buffer, commandName string, logger zerolog.Logger, wantOutput bool) []string
+}
+
+type EnvInjector interface {
+	Inject(cmd *Command, opts *ConfigOpts)
+}
+
+type PackageCommandExecutor struct{}
+
+func (e *PackageCommandExecutor) Run(cmd *Command, opts *ConfigOpts, logger zerolog.Logger) ([]string, error) {
+	var (
+		ArgsStr       string
+		cmdOutBuf     bytes.Buffer
+		outputArr     []string
+		cmdOutWriters io.Writer
+	)
+
+	for _, v := range cmd.Args {
+		ArgsStr += fmt.Sprintf(" %s", v)
+	}
+
+	// Example: Check version operation
+	if cmd.PackageOperation == PackageOperationCheckVersion {
+		logger.Info().Msg("Checking package versions")
+
+		logger.Info().Msg("")
+		for _, p := range cmd.Packages {
+			logger.Info().Str("package", p.Name).Msg("Checking installed and remote package versions")
+		}
+		opts.Logger.Info().Msg("")
+
+		// Execute the package version command
+		execCmd := exec.Command(cmd.Cmd, cmd.Args...)
+		cmdOutWriters = io.MultiWriter(&cmdOutBuf)
+
+		if IsCmdStdOutEnabled() {
+			cmdOutWriters = io.MultiWriter(os.Stdout, &cmdOutBuf)
+		}
+		execCmd.Stdout = cmdOutWriters
+		execCmd.Stderr = cmdOutWriters
+
+		if err := execCmd.Run(); err != nil {
+			return nil, fmt.Errorf("error running command %s: %w", ArgsStr, err)
+		}
+
+		return parsePackageVersion(cmdOutBuf.String(), logger, cmd, cmdOutBuf)
+	}
+
+	// Other package operations (install, upgrade, etc.) can be handled here
+
+	// Default: run as a shell command
+	execCmd := exec.Command(cmd.Cmd, cmd.Args...)
+	execCmd.Stdout = &cmdOutBuf
+	execCmd.Stderr = &cmdOutBuf
+	err := execCmd.Run()
+	outputArr = logCommandOutput(cmd, cmdOutBuf, logger, outputArr)
+	if err != nil {
+		logger.Error().Err(fmt.Errorf("error running package command %s: %w", cmd.Name, err)).Send()
+		return outputArr, err
+	}
+	return outputArr, nil
+}
+
+type LocalCommandExecutor struct{}
+
+func (e *LocalCommandExecutor) Run(cmd *Command, opts *ConfigOpts, logger zerolog.Logger) ([]string, error) {
+	var (
+		ArgsStr   string
+		cmdOutBuf bytes.Buffer
+		outputArr []string
+	)
+
+	for _, v := range cmd.Args {
+		ArgsStr += fmt.Sprintf(" %s", v)
+	}
+
+	// Build the command
+	var localCMD *exec.Cmd
+	if cmd.Shell != "" {
+		logger.Info().Str("Command", fmt.Sprintf("Running command %s on local machine in %s", cmd.Name, cmd.Shell)).Send()
+		ArgsStr = fmt.Sprintf("%s %s", cmd.Cmd, ArgsStr)
+		localCMD = exec.Command(cmd.Shell, "-c", ArgsStr)
+	} else {
+		localCMD = exec.Command(cmd.Cmd, cmd.Args...)
+	}
+
+	// Set working directory
+	if cmd.Dir != nil {
+		localCMD.Dir = *cmd.Dir
+	}
+
+	// Inject environment variables (extract this to an EnvInjector if desired)
+	// injectEnvIntoLocalCMD(...)
+
+	// Set output writers
+	cmdOutWriters := io.MultiWriter(&cmdOutBuf)
+	if IsCmdStdOutEnabled() {
+		cmdOutWriters = io.MultiWriter(os.Stdout, &cmdOutBuf)
+	}
+	localCMD.Stdout = cmdOutWriters
+	localCMD.Stderr = cmdOutWriters
+
+	// Run the command
+	err := localCMD.Run()
+	outputArr = logCommandOutput(cmd, cmdOutBuf, logger, outputArr)
+	if err != nil {
+		logger.Error().Err(fmt.Errorf("error when running cmd %s: %w", cmd.Name, err)).Send()
+		return outputArr, err
+	}
+
+	return outputArr, nil
+}
+
 // RunCmd runs a Command.
 // The environment of local commands will be the machine's environment plus any extra
 // variables specified in the Env file or Environment.
@@ -34,7 +154,7 @@ var Sprintf = fmt.Sprintf
 func (command *Command) RunCmd(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([]string, error) {
 
 	var (
-		ArgsStr       string // concatenating the arguments
+		ArgsStr       string
 		cmdOutBuf     bytes.Buffer
 		cmdOutWriters io.Writer
 		errSSH        error
@@ -54,38 +174,29 @@ func (command *Command) RunCmd(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([
 		ArgsStr += fmt.Sprintf(" %s", v)
 	}
 
-	if command.Type == UserCT {
+	if command.Type == UserCommandType {
 		if command.UserOperation == "password" {
 			cmdCtxLogger.Info().Str("password", command.UserPassword).Msg("user password to be updated")
 		}
 	}
 
-	if command.Host != nil {
+	if !IsHostLocal(command.Host) {
-		outputArr, errSSH = command.RunCmdSSH(cmdCtxLogger, opts)
+
+		outputArr, errSSH = command.RunCmdOnHost(cmdCtxLogger, opts)
 		if errSSH != nil {
 			return outputArr, errSSH
 		}
 	} else {
 
-		// Handle package operations
+		switch command.Type {
-		if command.Type == PackageCT && command.PackageOperation == PackOpCheckVersion {
+		case PackageCommandType:
-			cmdCtxLogger.Info().Str("package", command.PackageName).Msg("Checking package versions")
+			var executor PackageCommandExecutor
-
+			return executor.Run(command, opts, cmdCtxLogger)
-			// Execute the package version command
-			cmd := exec.Command(command.Cmd, command.Args...)
-			cmdOutWriters = io.MultiWriter(&cmdOutBuf)
-			cmd.Stdout = cmdOutWriters
-			cmd.Stderr = cmdOutWriters
-
-			if err := cmd.Run(); err != nil {
-				return nil, fmt.Errorf("error running command %s: %w", ArgsStr, err)
-			}
-
-			return parsePackageVersion(cmdOutBuf.String(), cmdCtxLogger, command, cmdOutBuf)
 		}
 
 		var localCMD *exec.Cmd
-		if command.Type == RemoteScriptCT {
+
+		if command.Type == RemoteScriptCommandType {
 			script, err := command.Fetcher.Fetch(command.Cmd)
 			if err != nil {
 				return nil, err
@@ -95,15 +206,15 @@ func (command *Command) RunCmd(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([
 				command.Shell = "sh"
 			}
 			localCMD = exec.Command(command.Shell, command.Args...)
-			injectEnvIntoLocalCMD(envVars, localCMD, cmdCtxLogger)
+			injectEnvIntoLocalCMD(envVars, localCMD, cmdCtxLogger, opts)
 
 			cmdOutWriters = io.MultiWriter(&cmdOutBuf)
 
 			if IsCmdStdOutEnabled() {
 				cmdOutWriters = io.MultiWriter(os.Stdout, &cmdOutBuf)
 			}
-			if command.OutputFile != "" {
+			if command.Output.File != "" {
-				file, err := os.Create(command.OutputFile)
+				file, err := os.Create(command.Output.File)
 				if err != nil {
 					return nil, fmt.Errorf("error creating output file: %w", err)
 				}
@@ -136,7 +247,7 @@ func (command *Command) RunCmd(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([
 				if str, ok := outMap["output"].(string); ok {
 					outputArr = append(outputArr, str)
 				}
-				if command.OutputToLog {
+				if command.Output.ToLog {
 					cmdCtxLogger.Info().Fields(outMap).Send()
 				}
 			}
@@ -144,6 +255,7 @@ func (command *Command) RunCmd(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([
 		}
 
 		var err error
+
 		if command.Shell != "" {
 			cmdCtxLogger.Info().Str("Command", fmt.Sprintf("Running command %s on local machine in %s", command.Name, command.Shell)).Send()
 
@@ -156,8 +268,10 @@ func (command *Command) RunCmd(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([
 			cmdCtxLogger.Info().Str("Command", fmt.Sprintf("Running command %s on local machine", command.Name)).Send()
 
 			// execute package commands in a shell
-			if command.Type == PackageCT {
+			if command.Type == PackageCommandType {
-				cmdCtxLogger.Info().Str("package", command.PackageName).Msg("Executing package command")
+				for _, p := range command.Packages {
+					cmdCtxLogger.Info().Str("packages", p.Name).Msg("Executing package command")
+				}
 				ArgsStr = fmt.Sprintf("%s %s", command.Cmd, ArgsStr)
 				localCMD = exec.Command("/bin/sh", "-c", ArgsStr)
 			} else {
@@ -165,11 +279,17 @@ func (command *Command) RunCmd(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([
 			}
 		}
 
+		if command.Type == UserCommandType {
+			if command.UserOperation == "password" {
+				localCMD.Stdin = command.stdin
+				cmdCtxLogger.Info().Str("password", command.UserPassword).Msg("user password to be updated")
+			}
+		}
 		if command.Dir != nil {
 			localCMD.Dir = *command.Dir
 		}
 
-		injectEnvIntoLocalCMD(envVars, localCMD, cmdCtxLogger)
+		injectEnvIntoLocalCMD(envVars, localCMD, cmdCtxLogger, opts)
 
 		cmdOutWriters = io.MultiWriter(&cmdOutBuf)
 
@@ -182,38 +302,84 @@ func (command *Command) RunCmd(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([
 
 		err = localCMD.Run()
 
-		outScanner := bufio.NewScanner(&cmdOutBuf)
+		outputArr = logCommandOutput(command, cmdOutBuf, cmdCtxLogger, outputArr)
-
-		for outScanner.Scan() {
-			outMap := make(map[string]interface{})
-			outMap["cmd"] = command.Cmd
-			outMap["output"] = outScanner.Text()
-
-			if str, ok := outMap["output"].(string); ok {
-				outputArr = append(outputArr, str)
-			}
-			// if command.GetOutput {
-			cmdCtxLogger.Info().Fields(outMap).Send()
-			// }
-		}
 		if err != nil {
 			cmdCtxLogger.Error().Err(fmt.Errorf("error when running cmd %s: %w", command.Name, err)).Send()
 			return outputArr, err
 		}
+
+		if command.Type == UserCommandType {
+
+			if command.UserOperation == "add" {
+				if command.UserSshPubKeys != nil {
+					var (
+						authorizedKeysFile *os.File
+						err                error
+						userHome           []byte
+					)
+
+					cmdCtxLogger.Info().Msg("adding SSH Keys")
+
+					localCMD := exec.Command(fmt.Sprintf("grep \"%s\" /etc/passwd | cut -d: -f6", command.Username))
+					userHome, err = localCMD.CombinedOutput()
+					if err != nil {
+						return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.Output.ToLog), fmt.Errorf("error finding user home from /etc/passwd: %v", err)
+					}
+
+					command.UserHome = strings.TrimSpace(string(userHome))
+					userSshDir := fmt.Sprintf("%s/.ssh", command.UserHome)
+
+					if _, err := os.Stat(userSshDir); os.IsNotExist(err) {
+						err := os.MkdirAll(userSshDir, 0700)
+						if err != nil {
+							return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.Output.ToLog), fmt.Errorf("error creating directory %s %v", userSshDir, err)
+						}
+					}
+
+					if _, err := os.Stat(fmt.Sprintf("%s/authorized_keys", userSshDir)); os.IsNotExist(err) {
+						_, err := os.Create(fmt.Sprintf("%s/authorized_keys", userSshDir))
+						if err != nil {
+							return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.Output.ToLog), fmt.Errorf("error creating file %s/authorized_keys: %v", userSshDir, err)
+						}
+					}
+
+					authorizedKeysFile, err = os.OpenFile(fmt.Sprintf("%s/authorized_keys", userSshDir), 0700, os.ModeAppend)
+					if err != nil {
+						return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.Output.ToLog), fmt.Errorf("error opening file %s/authorized_keys: %v", userSshDir, err)
+					}
+					defer authorizedKeysFile.Close()
+					for _, k := range command.UserSshPubKeys {
+						buf := bytes.NewBufferString(k)
+						cmdCtxLogger.Info().Str("key", k).Msg("adding SSH key")
+						if _, err := authorizedKeysFile.ReadFrom(buf); err != nil {
+							return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.Output.ToLog), fmt.Errorf("error adding to authorized keys: %v", err)
+						}
+					}
+					localCMD = exec.Command(fmt.Sprintf("chown -R %s:%s %s", command.Username, command.Username, userHome))
+					_, err = localCMD.CombinedOutput()
+					if err != nil {
+						return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.Output.ToLog), err
+					}
+
+				}
+			}
+		}
 	}
 	return outputArr, nil
 }
 
-func cmdListWorker(msgTemps *msgTemplates, jobs <-chan *CmdList, results chan<- CmdResult, opts *ConfigOpts) {
+func cmdListWorker(msgTemps *msgTemplates, jobs <-chan *CmdList, results chan<- string, opts *ConfigOpts) {
 	for list := range jobs {
 		fieldsMap := map[string]interface{}{"list": list.Name}
 		var cmdLogger zerolog.Logger
+		var commandExecuted *Command
 		var cmdsRan []string
 		var outStructArr []outStruct
 		var hasError bool // Tracks if any command in the list failed
 
 		for _, cmd := range list.Order {
 			cmdToRun := opts.Cmds[cmd]
+			commandExecuted = cmdToRun
 			currentCmd := cmdToRun.Name
 			fieldsMap["cmd"] = currentCmd
 			cmdLogger = cmdToRun.GenerateLogger(opts)
@@ -224,23 +390,21 @@ func cmdListWorker(msgTemps *msgTemplates, jobs <-chan *CmdList, results chan<-
 
 			if runErr != nil {
 
-				// Log the error and send a failed result
 				cmdLogger.Err(runErr).Send()
-				results <- CmdResult{CmdName: cmd, ListName: list.Name, Error: runErr}
 
-				// Execute error hooks for the failed command
 				cmdToRun.ExecuteHooks("error", opts)
 
 				// Notify failure
 				if list.NotifyConfig != nil {
 					notifyError(cmdLogger, msgTemps, list, cmdsRan, outStructArr, runErr, cmdToRun)
 				}
+
+				// Execute error hooks for the failed command
 				hasError = true
 				break
 			}
 
-			// Collect output if required
+			if list.GetCommandOutputInNotificationsOnSuccess || cmdToRun.Output.InList {
-			if list.GetOutput || cmdToRun.GetOutput {
 				outStructArr = append(outStructArr, outStruct{
 					CmdName:     currentCmd,
 					CmdExecuted: currentCmd,
@@ -249,44 +413,242 @@ func cmdListWorker(msgTemps *msgTemplates, jobs <-chan *CmdList, results chan<-
 			}
 		}
 
-		// Notify success if no errors occurred
+		if !hasError && list.NotifyConfig != nil && list.Notify.OnFailure {
-		if !hasError && list.NotifyConfig != nil && (list.NotifyOnSuccess || list.GetOutput) {
 			notifySuccess(cmdLogger, msgTemps, list, cmdsRan, outStructArr)
 		}
 
-		// Execute success and final hooks for all commands
+		if !hasError {
+			commandExecuted.ExecuteHooks("success", opts)
+		}
+
+		commandExecuted.ExecuteHooks("final", opts)
+
+		results <- "done"
+	}
+}
+func cmdListWorkerWithHosts(msgTemps *msgTemplates, jobs <-chan *CmdList, hosts <-chan *Host, results chan<- string, opts *ConfigOpts) {
+	for list := range jobs {
+		fieldsMap := map[string]interface{}{"list": list.Name}
+		var cmdLogger zerolog.Logger
+		var commandExecuted *Command
+		var cmdsRan []string
+		var outStructArr []outStruct
+		var hasError bool // Tracks if any command in the list failed
+
+		for host := range hosts {
+
 			for _, cmd := range list.Order {
 				cmdToRun := opts.Cmds[cmd]
+				if cmdToRun.Host != host.Host {
+					cmdToRun.Host = host.Host
+					cmdToRun.RemoteHost = host
+				}
+				commandExecuted = cmdToRun
+				currentCmd := cmdToRun.Name
+				fieldsMap["cmd"] = currentCmd
+				cmdLogger = cmdToRun.GenerateLogger(opts)
+				cmdLogger.Info().Fields(fieldsMap).Send()
 
-			// Execute success hooks if the command succeeded
+				outputArr, runErr := cmdToRun.RunCmd(cmdLogger, opts)
-			if !hasError || cmdsRanContains(cmd, cmdsRan) {
+				cmdsRan = append(cmdsRan, cmd)
-				cmdToRun.ExecuteHooks("success", opts)
+
+				if runErr != nil {
+
+					cmdLogger.Err(runErr).Send()
+
+					cmdToRun.ExecuteHooks("error", opts)
+
+					// Notify failure
+					if list.NotifyConfig != nil {
+						notifyError(cmdLogger, msgTemps, list, cmdsRan, outStructArr, runErr, cmdToRun)
 					}
 
-			// Execute final hooks for every command
+					// Execute error hooks for the failed command
-			cmdToRun.ExecuteHooks("final", opts)
+					hasError = true
+					break
 				}
 
-		// Send the final result for the list
+				if list.GetCommandOutputInNotificationsOnSuccess || cmdToRun.Output.InList {
-		if hasError {
+					outStructArr = append(outStructArr, outStruct{
-			results <- CmdResult{CmdName: cmdsRan[len(cmdsRan)-1], ListName: list.Name, Error: fmt.Errorf("list execution failed")}
+						CmdName:     currentCmd,
-		} else {
+						CmdExecuted: currentCmd,
-			results <- CmdResult{CmdName: cmdsRan[len(cmdsRan)-1], ListName: list.Name, Error: nil}
+						Output:      outputArr,
+					})
 				}
 			}
+
+			if !hasError && list.NotifyConfig != nil && list.Notify.OnFailure {
+				notifySuccess(cmdLogger, msgTemps, list, cmdsRan, outStructArr)
+			}
+
+			if !hasError {
+				commandExecuted.ExecuteHooks("success", opts)
+			}
+
+			commandExecuted.ExecuteHooks("final", opts)
+
+		}
+		results <- "done"
+	}
 }
 
-// Helper to check if a command is in the list of executed commands
+// func cmdListWorkerExecuteCommandsInParallel(msgTemps *msgTemplates, jobs <-chan *CmdList, hosts <-chan *Host, results chan<- string, opts *ConfigOpts) {
-func cmdsRanContains(cmd string, cmdsRan []string) bool {
+// 	opts.Logger.Info().Msg("Running commands in parallel")
-	for _, c := range cmdsRan {
+// 	for list := range jobs {
-		if c == cmd {
+// 		fieldsMap := map[string]interface{}{"list": list.Name}
-			return true
+// 		var cmdLogger zerolog.Logger
+// 		var commandExecuted *Command
+// 		var cmdsRan []string
+// 		var outStructArr []outStruct
+// 		var hasError bool // Tracks if any command in the list failed
+
+// 		for _, cmd := range list.Order {
+// 			for host := range hosts {
+// 				cmdToRun := opts.Cmds[cmd]
+// 				if cmdToRun.Host != host.Host {
+// 					cmdToRun.Host = host.Host
+// 					cmdToRun.RemoteHost = host
+// 				}
+// 				commandExecuted = cmdToRun
+// 				currentCmd := cmdToRun.Name
+// 				fieldsMap["cmd"] = currentCmd
+// 				cmdLogger = cmdToRun.GenerateLogger(opts)
+// 				cmdLogger.Info().Fields(fieldsMap).Send()
+
+// 				outputArr, runErr := cmdToRun.RunCmd(cmdLogger, opts)
+// 				cmdsRan = append(cmdsRan, cmd)
+
+// 				if runErr != nil {
+
+// 					cmdLogger.Err(runErr).Send()
+
+// 					cmdToRun.ExecuteHooks("error", opts)
+
+// 					// Notify failure
+// 					if list.NotifyConfig != nil {
+// 						notifyError(cmdLogger, msgTemps, list, cmdsRan, outStructArr, runErr, cmdToRun)
+// 					}
+
+// 					// Execute error hooks for the failed command
+// 					hasError = true
+// 					break
+// 				}
+
+// 				if list.GetCommandOutputInNotificationsOnSuccess || cmdToRun.Output.InList {
+// 					outStructArr = append(outStructArr, outStruct{
+// 						CmdName:     currentCmd,
+// 						CmdExecuted: currentCmd,
+// 						Output:      outputArr,
+// 					})
+// 				}
+// 			}
+
+// 			if !hasError && list.NotifyConfig != nil && list.Notify.OnFailure {
+// 				notifySuccess(cmdLogger, msgTemps, list, cmdsRan, outStructArr)
+// 			}
+
+// 			if !hasError {
+// 				commandExecuted.ExecuteHooks("success", opts)
+// 			}
+
+// 			commandExecuted.ExecuteHooks("final", opts)
+
+// 		}
+// 		results <- "done"
+// 	}
+// }
+
+func cmdListWorkerExecuteCommandsInParallel(msgTemps *msgTemplates, jobs <-chan *CmdList, hosts <-chan *Host, results chan<- string, opts *ConfigOpts) {
+	opts.Logger.Info().Msg("Running commands in parallel")
+	for list := range jobs {
+		fieldsMap := map[string]interface{}{"list": list.Name}
+		var cmdLogger zerolog.Logger
+		var commandExecuted *Command
+		var cmdsRan []string
+		var outStructArr []outStruct
+		var hasError bool // Tracks if any command in the list failed
+
+		var wg sync.WaitGroup
+		hostList := []*Host{}
+		for host := range hosts {
+			hostList = append(hostList, host)
+		}
+		println("Total hosts to run commands on:", len(hostList))
+		println("Total commands to run:", len(list.Order))
+
+		for _, cmd := range list.Order {
+			cmdsRan = append(cmdsRan, cmd)
+			println("Running cmd:", cmd, "on", len(hostList), "hosts")
+			outputChan := make(chan outStruct, len(hostList))
+			errorChan := make(chan error, len(hostList))
+			// cmdToRun := opts.Cmds[cmd]
+			origCmd := opts.Cmds[cmd]
+
+			for _, host := range hostList {
+				wg.Add(1)
+				cmdToRun := *origCmd // shallow copy
+				commandExecuted = origCmd
+				if cmdToRun.Host != host.Host {
+					cmdToRun.Host = host.Host
+					cmdToRun.RemoteHost = host
+				}
+				cmdLogger = cmdToRun.GenerateLogger(opts)
+				cmdLogger.Info().Fields(fieldsMap).Send()
+				print("Running cmd on: ", host.Host, "\n")
+
+				go func(cmd string, host *Host) {
+					defer wg.Done()
+					currentCmd := cmdToRun.Name
+					fieldsMap["cmd"] = currentCmd
+
+					outputArr, runErr := cmdToRun.RunCmd(cmdLogger, opts)
+					if runErr != nil {
+						cmdLogger.Err(runErr).Send()
+						cmdToRun.ExecuteHooks("error", opts)
+						errorChan <- runErr
+						return
+					}
+
+					if list.GetCommandOutputInNotificationsOnSuccess || cmdToRun.Output.InList {
+						outputChan <- outStruct{
+							CmdName:     currentCmd,
+							CmdExecuted: currentCmd,
+							Output:      outputArr,
 						}
 					}
-	return false
+				}(cmd, host)
+			}
+
+			wg.Wait()
+			close(outputChan)
+			close(errorChan)
+
+			for out := range outputChan {
+				outStructArr = append(outStructArr, out)
+			}
+			if len(errorChan) > 0 {
+				hasError = true
+				runErr := <-errorChan
+				if list.NotifyConfig != nil {
+					notifyError(cmdLogger, msgTemps, list, cmdsRan, outStructArr, runErr, commandExecuted)
+				}
+				break
+			}
+
+			if !hasError && list.NotifyConfig != nil && list.Notify.OnFailure {
+				notifySuccess(cmdLogger, msgTemps, list, cmdsRan, outStructArr)
+			}
+
+			if !hasError {
+				commandExecuted.ExecuteHooks("success", opts)
+			}
+
+		}
+		commandExecuted.ExecuteHooks("final", opts)
+		results <- "done"
+	}
 }
 
-// Helper to notify errors
 func notifyError(logger zerolog.Logger, templates *msgTemplates, list *CmdList, cmdsRan []string, outStructArr []outStruct, err error, cmd *Command) {
 	errStruct := map[string]interface{}{
 		"listName":  list.Name,
@@ -332,7 +694,7 @@ func (opts *ConfigOpts) RunListConfig(cron string) {
 	}
 	configListsLen := len(opts.CmdConfigLists)
 	listChan := make(chan *CmdList, configListsLen)
-	results := make(chan CmdResult, configListsLen)
+	results := make(chan string, configListsLen)
 
 	// Start workers
 	for w := 1; w <= configListsLen; w++ {
@@ -352,14 +714,66 @@ func (opts *ConfigOpts) RunListConfig(cron string) {
 
 	// Process results
 	for a := 1; a <= configListsLen; a++ {
-		result := <-results
+		<-results
-		opts.Logger.Debug().Msgf("Processing result for list %s, command %s", result.ListName, result.CmdName)
-
-		// Process final hooks for the list (already handled in worker)
 	}
 	opts.closeHostConnections()
 }
 
+func (opts *ConfigOpts) ExecuteListOnHosts(lists []string, parallel bool) {
+
+	mTemps := &msgTemplates{
+		err:     template.Must(template.New("error.txt").ParseFS(templates, "templates/error.txt")),
+		success: template.Must(template.New("success.txt").ParseFS(templates, "templates/success.txt")),
+	}
+	// for _, l := range opts.CmdConfigLists {
+	// 	if !slices.Contains(lists, l.Name) {
+	// 		delete(opts.CmdConfigLists, l.Name)
+	// 	}
+	// }
+	configListsLen := len(opts.CmdConfigLists)
+	listChan := make(chan *CmdList, configListsLen)
+	hostChan := make(chan *Host, len(opts.Hosts))
+	results := make(chan string, configListsLen)
+
+	// Start workers
+	for w := 1; w <= configListsLen; w++ {
+		if parallel {
+			go cmdListWorkerExecuteCommandsInParallel(mTemps, listChan, hostChan, results, opts)
+		} else {
+			go cmdListWorkerWithHosts(mTemps, listChan, hostChan, results, opts)
+		}
+	}
+
+	// Enqueue jobs
+	for listName, cmdConfig := range opts.CmdConfigLists {
+		if cmdConfig.Name == "" {
+			cmdConfig.Name = listName
+		}
+		listChan <- cmdConfig
+	}
+	for _, h := range opts.Hosts {
+		if h.isProxyHost {
+			continue
+		}
+		hostChan <- h
+		// for _, proxyHost := range h.ProxyHost {
+		// 	if proxyHost.isProxyHost {
+		// 		continue
+		// 	}
+		// 	hostChan <- proxyHost
+		// }
+	}
+	close(listChan)
+	close(hostChan)
+
+	// Process results
+	for a := 1; a <= configListsLen; a++ {
+		<-results
+	}
+	opts.closeHostConnections()
+
+}
+
 func (opts *ConfigOpts) ExecuteCmds() {
 	for _, cmd := range opts.executeCmds {
 		cmdToRun := opts.Cmds[cmd]
@@ -367,10 +781,8 @@ func (opts *ConfigOpts) ExecuteCmds() {
 		_, runErr := cmdToRun.RunCmd(cmdLogger, opts)
 		if runErr != nil {
 			opts.Logger.Err(runErr).Send()
-
 			cmdToRun.ExecuteHooks("error", opts)
 		} else {
-
 			cmdToRun.ExecuteHooks("success", opts)
 		}
 
@@ -378,7 +790,6 @@ func (opts *ConfigOpts) ExecuteCmds() {
 	}
 
 	opts.closeHostConnections()
-
 }
 
 func (c *ConfigOpts) closeHostConnections() {
@@ -426,26 +837,31 @@ func (cmd *Command) ExecuteHooks(hookType string, opts *ConfigOpts) {
 		for _, v := range cmd.Hooks.Error {
 			errCmd := opts.Cmds[v]
 			cmdLogger := opts.Logger.With().
-				Str("backy-cmd", v).
+				Str("backy-cmd", v).Str("hookType", "error").
 				Logger()
-			errCmd.RunCmd(cmdLogger, opts)
+			cmdLogger.Info().Msgf("Running error hook command %s", v)
+			// URGENT: Never returns
+			_, _ = errCmd.RunCmd(cmdLogger, opts)
+			return
 		}
 
 	case "success":
 		for _, v := range cmd.Hooks.Success {
 			successCmd := opts.Cmds[v]
 			cmdLogger := opts.Logger.With().
-				Str("backy-cmd", v).
+				Str("backy-cmd", v).Str("hookType", "success").
 				Logger()
-			successCmd.RunCmd(cmdLogger, opts)
+			cmdLogger.Info().Msgf("Running success hook command %s", v)
+			_, _ = successCmd.RunCmd(cmdLogger, opts)
 		}
 	case "final":
 		for _, v := range cmd.Hooks.Final {
 			finalCmd := opts.Cmds[v]
 			cmdLogger := opts.Logger.With().
-				Str("backy-cmd", v).
+				Str("backy-cmd", v).Str("hookType", "final").
 				Logger()
-			finalCmd.RunCmd(cmdLogger, opts)
+			cmdLogger.Info().Msgf("Running final hook command %s", v)
+			_, _ = finalCmd.RunCmd(cmdLogger, opts)
 		}
 	}
 }
@@ -455,30 +871,84 @@ func (cmd *Command) GenerateLogger(opts *ConfigOpts) zerolog.Logger {
 		Str("Backy-cmd", cmd.Name).Str("Host", "local machine").
 		Logger()
 
-	if cmd.Host != nil {
+	if !IsHostLocal(cmd.Host) {
 		cmdLogger = opts.Logger.With().
-			Str("Backy-cmd", cmd.Name).Str("Host", *cmd.Host).
+			Str("Backy-cmd", cmd.Name).Str("Host", cmd.Host).
 			Logger()
-
 	}
 	return cmdLogger
 }
 
-func (opts *ConfigOpts) ExecCmdsSSH(cmdList []string, hostsList []string) {
+func (opts *ConfigOpts) ExecCmdsOnHosts(cmdList []string, hostsList []string) {
 	// Iterate over hosts and exec commands
 	for _, h := range hostsList {
 		host := opts.Hosts[h]
 		for _, c := range cmdList {
 			cmd := opts.Cmds[c]
 			cmd.RemoteHost = host
-			cmd.Host = &host.Host
+			cmd.Host = h
+			if IsHostLocal(h) {
+				_, err := cmd.RunCmd(cmd.GenerateLogger(opts), opts)
+				if err != nil {
+					opts.Logger.Err(err).Str("host", h).Str("cmd", c).Send()
+				}
+			} else {
+
+				cmd.Host = host.Host
 				opts.Logger.Info().Str("host", h).Str("cmd", c).Send()
-			_, err := cmd.RunCmdSSH(cmd.GenerateLogger(opts), opts)
+				_, err := cmd.RunCmdOnHost(cmd.GenerateLogger(opts), opts)
 				if err != nil {
 					opts.Logger.Err(err).Str("host", h).Str("cmd", c).Send()
 				}
 			}
 		}
+	}
+}
+
+func (opts *ConfigOpts) ExecCmdsOnHostsInParallel(cmdList []string, hostsList []string) {
+	opts.Logger.Info().Msg("Executing commands in parallel on hosts")
+	// Iterate over hosts and exec commands
+	for _, c := range cmdList {
+		for _, h := range hostsList {
+			host := opts.Hosts[h]
+			cmd := opts.Cmds[c]
+			cmd.RemoteHost = host
+			cmd.Host = h
+			if IsHostLocal(h) {
+				_, err := cmd.RunCmd(cmd.GenerateLogger(opts), opts)
+				if err != nil {
+					opts.Logger.Err(err).Str("host", h).Str("cmd", c).Send()
+				}
+			} else {
+
+				cmd.Host = host.Host
+				opts.Logger.Info().Str("host", h).Str("cmd", c).Send()
+				_, err := cmd.RunCmdOnHost(cmd.GenerateLogger(opts), opts)
+				if err != nil {
+					opts.Logger.Err(err).Str("host", h).Str("cmd", c).Send()
+				}
+			}
+		}
+	}
+}
+
+func logCommandOutput(command *Command, cmdOutBuf bytes.Buffer, cmdCtxLogger zerolog.Logger, outputArr []string) []string {
+
+	outScanner := bufio.NewScanner(&cmdOutBuf)
+
+	for outScanner.Scan() {
+		outMap := make(map[string]interface{})
+		outMap["cmd"] = command.Name
+		outMap["output"] = outScanner.Text()
+
+		if str, ok := outMap["output"].(string); ok {
+			outputArr = append(outputArr, str)
+		}
+		if command.Output.ToLog {
+			cmdCtxLogger.Info().Fields(outMap).Send()
+		}
+	}
+	return outputArr
 }
 
 // func executeUserCommands() []string {

pkg/backy/backy_test.go

@@ -0,0 +1,83 @@
+package backy
+
+// import (
+// 	"context"
+// 	"fmt"
+// 	"io"
+// 	"log"
+// 	"testing"
+
+// 	"git.andrewnw.xyz/CyberShell/backy/pkg/pkgman"
+// 	packagemanagercommon "git.andrewnw.xyz/CyberShell/backy/pkg/pkgman/common"
+// 	"github.com/stretchr/testify/assert"
+// 	"github.com/stretchr/testify/require"
+
+// 	"github.com/testcontainers/testcontainers-go"
+// )
+
+// // TestConfigOptions tests the configuration options for the backy package.
+// func Test_ErrorHook(t *testing.T) {
+
+// 	configFile := "../../tests/ErrorHook.yml"
+// 	logFile := "ErrorHook.log"
+// 	backyConfigOptions := NewConfigOptions(configFile, SetLogFile(logFile))
+// 	backyConfigOptions.InitConfig()
+// 	backyConfigOptions.ParseConfigurationFile()
+// 	backyConfigOptions.RunListConfig("")
+
+// }
+
+// func TestSettingCommandInfoPackageCommandDnf(t *testing.T) {
+
+// 	packagecommand := &Command{
+// 		Type:             PackageCommandType,
+// 		PackageManager:   "dnf",
+// 		Shell:            "zsh",
+// 		PackageOperation: PackageOperationCheckVersion,
+// 		Packages:         []packagemanagercommon.Package{{Name: "docker-ce"}},
+// 	}
+// 	dnfPackage, _ := pkgman.PackageManagerFactory("dnf", pkgman.WithoutAuth())
+
+// 	packagecommand.pkgMan = dnfPackage
+// 	PackageCommand := getCommandTypeAndSetCommandInfo(packagecommand)
+
+// 	assert.Equal(t, "dnf", PackageCommand.Cmd)
+
+// }
+
+// func TestWithDockerFile(t *testing.T) {
+// 	ctx := context.Background()
+
+// 	docker, err := testcontainers.Run(ctx, "",
+// 		testcontainers.WithDockerfile(testcontainers.FromDockerfile{
+// 			Context:    "../../tests/docker",
+// 			Dockerfile: "Dockerfile",
+// 			KeepImage:  false,
+// 			// BuildOptionsModifier: func(buildOptions *types.ImageBuildOptions) {
+// 			// 	buildOptions.Target = "target2"
+// 			// },
+// 		}),
+// 	)
+// 	// docker.
+
+// 	if err != nil {
+// 		log.Printf("failed to start container: %v", err)
+// 		return
+// 	}
+
+// 	r, err := docker.Logs(ctx)
+// 	if err != nil {
+// 		log.Printf("failed to get logs: %v", err)
+// 		return
+// 	}
+
+// 	logs, err := io.ReadAll(r)
+// 	if err != nil {
+// 		log.Printf("failed to read logs: %v", err)
+// 		return
+// 	}
+
+// 	fmt.Println(string(logs))
+
+// 	require.NoError(t, err)
+// }

pkg/backy/commandtype_enumer.go

@@ -25,29 +25,29 @@ func (i CommandType) String() string {
 // Re-run the stringer command to generate them again.
 func _CommandTypeNoOp() {
 	var x [1]struct{}
-	_ = x[DefaultCT-(0)]
+	_ = x[DefaultCommandType-(0)]
-	_ = x[ScriptCT-(1)]
+	_ = x[ScriptCommandType-(1)]
-	_ = x[ScriptFileCT-(2)]
+	_ = x[ScriptFileCommandType-(2)]
-	_ = x[RemoteScriptCT-(3)]
+	_ = x[RemoteScriptCommandType-(3)]
-	_ = x[PackageCT-(4)]
+	_ = x[PackageCommandType-(4)]
-	_ = x[UserCT-(5)]
+	_ = x[UserCommandType-(5)]
 }
 
-var _CommandTypeValues = []CommandType{DefaultCT, ScriptCT, ScriptFileCT, RemoteScriptCT, PackageCT, UserCT}
+var _CommandTypeValues = []CommandType{DefaultCommandType, ScriptCommandType, ScriptFileCommandType, RemoteScriptCommandType, PackageCommandType, UserCommandType}
 
 var _CommandTypeNameToValueMap = map[string]CommandType{
-	_CommandTypeName[0:0]:        DefaultCT,
+	_CommandTypeName[0:0]:        DefaultCommandType,
-	_CommandTypeLowerName[0:0]:   DefaultCT,
+	_CommandTypeLowerName[0:0]:   DefaultCommandType,
-	_CommandTypeName[0:6]:        ScriptCT,
+	_CommandTypeName[0:6]:        ScriptCommandType,
-	_CommandTypeLowerName[0:6]:   ScriptCT,
+	_CommandTypeLowerName[0:6]:   ScriptCommandType,
-	_CommandTypeName[6:16]:       ScriptFileCT,
+	_CommandTypeName[6:16]:       ScriptFileCommandType,
-	_CommandTypeLowerName[6:16]:  ScriptFileCT,
+	_CommandTypeLowerName[6:16]:  ScriptFileCommandType,
-	_CommandTypeName[16:28]:      RemoteScriptCT,
+	_CommandTypeName[16:28]:      RemoteScriptCommandType,
-	_CommandTypeLowerName[16:28]: RemoteScriptCT,
+	_CommandTypeLowerName[16:28]: RemoteScriptCommandType,
-	_CommandTypeName[28:35]:      PackageCT,
+	_CommandTypeName[28:35]:      PackageCommandType,
-	_CommandTypeLowerName[28:35]: PackageCT,
+	_CommandTypeLowerName[28:35]: PackageCommandType,
-	_CommandTypeName[35:39]:      UserCT,
+	_CommandTypeName[35:39]:      UserCommandType,
-	_CommandTypeLowerName[35:39]: UserCT,
+	_CommandTypeLowerName[35:39]: UserCommandType,
 }
 
 var _CommandTypeNames = []string{

pkg/backy/config.go

@@ -1,7 +1,6 @@
 package backy
 
 import (
-	"context"
 	"errors"
 	"fmt"
 	"net/url"
@@ -22,10 +21,13 @@ import (
 	"github.com/rs/zerolog"
 )
 
-const macroStart string = "%{"
+const (
-const macroEnd string = "}%"
+	externDirectiveStart      string = "%{"
-const envMacroStart string = "%{env:"
+	externDirectiveEnd        string = "}%"
-const vaultMacroStart string = "%{vault:"
+	externFileDirectiveStart  string = "%{file:"
+	envExternDirectiveStart   string = "%{env:"
+	vaultExternDirectiveStart string = "%{vault:"
+)
 
 func (opts *ConfigOpts) InitConfig() {
 	var err error
@@ -78,30 +80,117 @@ func (opts *ConfigOpts) InitConfig() {
 		logging.ExitWithMSG(fmt.Sprintf("error initializing cache: %v", err), 1, nil)
 	}
 
-	fetcher, err := remotefetcher.NewRemoteFetcher(opts.ConfigFilePath, opts.Cache)
-
 	if isRemoteURL(opts.ConfigFilePath) {
 		p, _ := getRemoteDir(opts.ConfigFilePath)
 		opts.ConfigDir = p
 	}
+
+	fetcher, err := remotefetcher.NewRemoteFetcher(opts.ConfigFilePath, opts.Cache)
 	if err != nil {
 		logging.ExitWithMSG(fmt.Sprintf("error initializing config fetcher: %v", err), 1, nil)
 	}
+
 	if opts.ConfigFilePath != "" {
 		loadConfigFile(fetcher, opts.ConfigFilePath, backyKoanf, opts)
 	} else {
 		loadDefaultConfigFiles(fetcher, configFiles, backyKoanf, opts)
 	}
+
 	opts.koanf = backyKoanf
 }
 
-func loadConfigFile(fetcher remotefetcher.RemoteFetcher, filePath string, k *koanf.Koanf, opts *ConfigOpts) {
+func (opts *ConfigOpts) ParseConfigurationFile() *ConfigOpts {
-	data, err := fetcher.Fetch(filePath)
+	setTerminalEnv()
-	if err != nil {
+
-		logging.ExitWithMSG(fmt.Sprintf("Could not fetch config file %s: %v", filePath, err), 1, nil)
+	backyKoanf := opts.koanf
+
+	if backyKoanf.Exists("variables") {
+		unmarshalConfigIntoStruct(backyKoanf, "variables", &opts.Vars, opts.Logger)
 	}
 
-	if err := k.Load(rawbytes.Provider(data), yaml.Parser()); err != nil {
+	getConfigDir(opts)
+
+	opts.loadEnv()
+
+	if backyKoanf.Bool(getNestedConfig("logging", "cmd-std-out")) {
+		os.Setenv("BACKY_CMDSTDOUT", "enabled")
+	}
+
+	// override the default value of cmd-std-out if flag is set
+	if opts.CmdStdOut {
+		os.Setenv("BACKY_CMDSTDOUT", "enabled")
+	}
+
+	CheckConfigValues(backyKoanf, opts.ConfigFilePath)
+
+	validateExecCommandsFromCLI(backyKoanf, opts)
+
+	setLoggingOptions(backyKoanf, opts)
+
+	log := setupLogger(opts)
+	opts.Logger = log
+
+	hostsFetcher, err := remotefetcher.NewRemoteFetcher(opts.HostsFilePath, opts.Cache)
+	opts.Logger.Info().Str("hosts file", opts.HostsFilePath).Send()
+	if err != nil {
+		logging.ExitWithMSG(fmt.Sprintf("error initializing config fetcher: %v", err), 1, nil)
+	}
+
+	var hostKoanf = koanf.New(".")
+	if opts.HostsFilePath != "" {
+		loadConfigFile(hostsFetcher, opts.HostsFilePath, hostKoanf, opts)
+		unmarshalConfigIntoStruct(hostKoanf, "hosts", &opts.Hosts, opts.Logger)
+	} else {
+		unmarshalConfigIntoStruct(backyKoanf, "hosts", &opts.Hosts, opts.Logger)
+	}
+
+	log.Info().Str("config file", opts.ConfigFilePath).Send()
+
+	if err := opts.initializeVault(); err != nil {
+		log.Err(err).Send()
+	}
+
+	unmarshalConfigIntoStruct(backyKoanf, "commands", &opts.Cmds, opts.Logger)
+
+	getCommandEnvironments(opts)
+
+	getHostConfigs(opts)
+
+	for k, v := range opts.Vars {
+		v = getExternalConfigDirectiveValue(v, opts, AllowedExternalDirectiveAll)
+		opts.Vars[k] = v
+	}
+
+	loadCommandLists(opts, backyKoanf)
+
+	validateCommandLists(opts)
+
+	if opts.cronEnabled && len(opts.CmdConfigLists) == 0 {
+		logging.ExitWithMSG("No cron fields detected in any command lists", 1, nil)
+	}
+
+	if err := processCmds(opts); err != nil {
+		logging.ExitWithMSG(err.Error(), 1, &opts.Logger)
+	}
+
+	filterExecuteLists(opts)
+
+	if backyKoanf.Exists("notifications") {
+		unmarshalConfigIntoStruct(backyKoanf, "notifications", &opts.NotificationConf, opts.Logger)
+	}
+
+	opts.SetupNotify()
+
+	return opts
+}
+
+func loadConfigFile(fetcher remotefetcher.RemoteFetcher, filePath string, koanfConfigParser *koanf.Koanf, opts *ConfigOpts) {
+	data, err := fetcher.Fetch(filePath)
+	if err != nil {
+		logging.ExitWithMSG(generateFileFetchErrorString(filePath, "config", err), 1, nil)
+	}
+
+	if err := koanfConfigParser.Load(rawbytes.Provider(data), yaml.Parser()); err != nil {
 		logging.ExitWithMSG(fmt.Sprintf("error loading config: %v", err), 1, &opts.Logger)
 	}
 }
@@ -130,68 +219,6 @@ func loadDefaultConfigFiles(fetcher remotefetcher.RemoteFetcher, configFiles []s
 	}
 }
 
-func (opts *ConfigOpts) ReadConfig() *ConfigOpts {
-	setTerminalEnv()
-
-	backyKoanf := opts.koanf
-
-	opts.loadEnv()
-
-	if backyKoanf.Bool(getNestedConfig("logging", "cmd-std-out")) {
-		os.Setenv("BACKY_CMDSTDOUT", "enabled")
-	}
-
-	// override the default value of cmd-std-out if flag is set
-	if opts.CmdStdOut {
-		os.Setenv("BACKY_CMDSTDOUT", "enabled")
-	}
-
-	CheckConfigValues(backyKoanf, opts.ConfigFilePath)
-
-	validateCommands(backyKoanf, opts)
-
-	setLoggingOptions(backyKoanf, opts)
-
-	log := setupLogger(opts)
-	opts.Logger = log
-
-	log.Info().Str("config file", opts.ConfigFilePath).Send()
-
-	unmarshalConfig(backyKoanf, "commands", &opts.Cmds, opts.Logger)
-
-	validateCommandEnvironments(opts)
-
-	unmarshalConfig(backyKoanf, "hosts", &opts.Hosts, opts.Logger)
-
-	resolveHostConfigs(opts)
-
-	loadCommandLists(opts, backyKoanf)
-
-	validateCommandLists(opts)
-
-	if opts.cronEnabled && len(opts.CmdConfigLists) == 0 {
-		logging.ExitWithMSG("No cron fields detected in any command lists", 1, nil)
-	}
-
-	if err := processCmds(opts); err != nil {
-		logging.ExitWithMSG(err.Error(), 1, &opts.Logger)
-	}
-
-	filterExecuteLists(opts)
-
-	if backyKoanf.Exists("notifications") {
-		unmarshalConfig(backyKoanf, "notifications", &opts.NotificationConf, opts.Logger)
-	}
-
-	opts.SetupNotify()
-
-	if err := opts.setupVault(); err != nil {
-		log.Err(err).Send()
-	}
-
-	return opts
-}
-
 func setTerminalEnv() {
 	if isatty.IsTerminal(os.Stdout.Fd()) || isatty.IsCygwinTerminal(os.Stdout.Fd()) {
 		os.Setenv("BACKY_TERM", "enabled")
@@ -200,7 +227,7 @@ func setTerminalEnv() {
 	}
 }
 
-func validateCommands(k *koanf.Koanf, opts *ConfigOpts) {
+func validateExecCommandsFromCLI(k *koanf.Koanf, opts *ConfigOpts) {
 	for _, c := range opts.executeCmds {
 		if !k.Exists(getCmdFromConfig(c)) {
 			logging.ExitWithMSG(fmt.Sprintf("command %s is not in config file %s", c, opts.ConfigFilePath), 1, nil)
@@ -208,23 +235,23 @@ func validateCommands(k *koanf.Koanf, opts *ConfigOpts) {
 	}
 }
 
-func setLoggingOptions(k *koanf.Koanf, opts *ConfigOpts) {
+func setLoggingOptions(backyKoanf *koanf.Koanf, opts *ConfigOpts) {
-	isLoggingVerbose := k.Bool(getLoggingKeyFromConfig("verbose"))
+	isVerboseLoggingSetInConfig := backyKoanf.Bool(getLoggingKeyFromConfig("verbose"))
 
 	// if log file is set in config file and not set on command line, use "./backy.log"
 	logFile := "./backy.log"
-	if opts.LogFilePath == "" && k.Exists(getLoggingKeyFromConfig("file")) {
+	if opts.LogFilePath == "" && backyKoanf.Exists(getLoggingKeyFromConfig("file")) {
-		logFile = k.String(getLoggingKeyFromConfig("file"))
+		logFile = backyKoanf.String(getLoggingKeyFromConfig("file"))
 		opts.LogFilePath = logFile
 	}
 
 	zerolog.SetGlobalLevel(zerolog.InfoLevel)
-	if isLoggingVerbose {
+	if isVerboseLoggingSetInConfig {
 		zerolog.SetGlobalLevel(zerolog.DebugLevel)
 		os.Setenv("BACKY_LOGLEVEL", fmt.Sprintf("%v", zerolog.GlobalLevel()))
 	}
 
-	if k.Bool(getLoggingKeyFromConfig("console-disabled")) {
+	if backyKoanf.Bool(getLoggingKeyFromConfig("console-disabled")) {
 		os.Setenv("BACKY_CONSOLE_LOGGING", "")
 	} else {
 		os.Setenv("BACKY_CONSOLE_LOGGING", "enabled")
@@ -236,34 +263,37 @@ func setupLogger(opts *ConfigOpts) zerolog.Logger {
 	return zerolog.New(writers).With().Timestamp().Logger()
 }
 
-func unmarshalConfig(k *koanf.Koanf, key string, target interface{}, log zerolog.Logger) {
+func unmarshalConfigIntoStruct(k *koanf.Koanf, key string, target interface{}, log zerolog.Logger) {
 	if err := k.UnmarshalWithConf(key, target, koanf.UnmarshalConf{Tag: "yaml"}); err != nil {
-		logging.ExitWithMSG(fmt.Sprintf("error unmarshalling key %s into struct: %v", key, err), 1, &log)
+		logging.ExitWithMSG(fmt.Sprintf("error unmarshaling key %s into struct: %v", key, err), 1, &log)
 	}
 }
 
-func validateCommandEnvironments(opts *ConfigOpts) {
+func getCommandEnvironments(opts *ConfigOpts) {
 	for cmdName, cmdConf := range opts.Cmds {
+		if cmdConf.Env == "" {
+			continue
+		}
+		opts.Logger.Debug().Str("env file", cmdConf.Env).Str("cmd", cmdName).Send()
 		if err := testFile(cmdConf.Env); err != nil {
-			opts.Logger.Info().Str("cmd", cmdName).Err(err).Send()
+			logging.ExitWithMSG("Could not open file"+cmdConf.Env+": "+err.Error(), 1, &opts.Logger)
-			os.Exit(1)
 		}
 		expandEnvVars(opts.backyEnv, cmdConf.Environment)
 	}
 }
 
-func resolveHostConfigs(opts *ConfigOpts) {
+func getHostConfigs(opts *ConfigOpts) {
 	for hostConfigName, host := range opts.Hosts {
 		if host.Host == "" {
 			host.Host = hostConfigName
 		}
 		if host.ProxyJump != "" {
-			resolveProxyHosts(host, opts)
+			getProxyHosts(host, opts)
 		}
 	}
 }
 
-func resolveProxyHosts(host *Host, opts *ConfigOpts) {
+func getProxyHosts(host *Host, opts *ConfigOpts) {
 	proxyHosts := strings.Split(host.ProxyJump, ",")
 	for _, h := range proxyHosts {
 		proxyHost, defined := opts.Hosts[h]
@@ -275,16 +305,22 @@ func resolveProxyHosts(host *Host, opts *ConfigOpts) {
 	}
 }
 
+func getConfigDir(opts *ConfigOpts) {
+	if isRemoteURL(opts.ConfigFilePath) {
+		p, _ := getRemoteDir(opts.ConfigFilePath)
+		opts.ConfigDir = p
+	} else {
+		opts.ConfigDir = path.Dir(opts.ConfigFilePath)
+	}
+}
+
 func loadCommandLists(opts *ConfigOpts, backyKoanf *koanf.Koanf) {
 	var listConfigFiles []string
 	var u *url.URL
 	var p string
-	// if config file is remote, use the directory of the remote file
 	if isRemoteURL(opts.ConfigFilePath) {
 		p, u = getRemoteDir(opts.ConfigFilePath)
 		opts.ConfigDir = p
-		println(p)
-		// // Still use local list files if a remote config file is used, but use them last
 		listConfigFiles = []string{u.JoinPath("lists.yml").String(), u.JoinPath("lists.yaml").String()}
 	} else {
 		opts.ConfigDir = path.Dir(opts.ConfigFilePath)
@@ -297,17 +333,20 @@ func loadCommandLists(opts *ConfigOpts, backyKoanf *koanf.Koanf) {
 
 	listsConfig := koanf.New(".")
 
+	if backyKoanf.Exists("cmdLists") {
+		if backyKoanf.Exists("cmdLists.file") {
+			loadCmdListsFile(backyKoanf, listsConfig, opts)
+		} else {
+			unmarshalConfigIntoStruct(backyKoanf, "cmdLists", &opts.CmdConfigLists, opts.Logger)
+		}
+	}
+
+	if opts.CmdConfigLists == nil {
 		for _, l := range listConfigFiles {
 			if loadListConfigFile(l, listsConfig, opts) {
 				break
 			}
 		}
-
-	if backyKoanf.Exists("cmdLists") {
-		unmarshalConfig(backyKoanf, "cmdLists", &opts.CmdConfigLists, opts.Logger)
-		if backyKoanf.Exists("cmdLists.file") {
-			loadCmdListsFile(backyKoanf, listsConfig, opts)
-		}
 	}
 }
 
@@ -346,7 +385,7 @@ func loadListConfigFile(filePath string, k *koanf.Koanf, opts *ConfigOpts) bool
 		return false
 	}
 
-	unmarshalConfig(k, "cmdLists", &opts.CmdConfigLists, opts.Logger)
+	unmarshalConfigIntoStruct(k, "cmdLists", &opts.CmdConfigLists, opts.Logger)
 	keyNotSupported("cmd-lists", "cmdLists", k, opts, true)
 	opts.CmdListFile = filePath
 	return true
@@ -355,6 +394,7 @@ func loadListConfigFile(filePath string, k *koanf.Koanf, opts *ConfigOpts) bool
 func loadCmdListsFile(backyKoanf *koanf.Koanf, listsConfig *koanf.Koanf, opts *ConfigOpts) {
 	opts.CmdListFile = strings.TrimSpace(backyKoanf.String("cmdLists.file"))
 	if !path.IsAbs(opts.CmdListFile) {
+		// TODO: Needs testing - might cause undefined/unexpected behavior if remote config path is used
 		opts.CmdListFile = path.Join(path.Dir(opts.ConfigFilePath), opts.CmdListFile)
 	}
 
@@ -366,7 +406,7 @@ func loadCmdListsFile(backyKoanf *koanf.Koanf, listsConfig *koanf.Koanf, opts *C
 
 	data, err := fetcher.Fetch(opts.CmdListFile)
 	if err != nil {
-		logging.ExitWithMSG(fmt.Sprintf("Could not fetch config file %s: %v", opts.CmdListFile, err), 1, nil)
+		logging.ExitWithMSG(generateFileFetchErrorString(opts.CmdListFile, "list config", err), 1, nil)
 	}
 
 	if err := listsConfig.Load(rawbytes.Provider(data), yaml.Parser()); err != nil {
@@ -374,10 +414,14 @@ func loadCmdListsFile(backyKoanf *koanf.Koanf, listsConfig *koanf.Koanf, opts *C
 	}
 
 	keyNotSupported("cmd-lists", "cmdLists", listsConfig, opts, true)
-	unmarshalConfig(listsConfig, "cmdLists", &opts.CmdConfigLists, opts.Logger)
+	unmarshalConfigIntoStruct(listsConfig, "cmdLists", &opts.CmdConfigLists, opts.Logger)
 	opts.Logger.Info().Str("using lists config file", opts.CmdListFile).Send()
 }
 
+func generateFileFetchErrorString(file, fileType string, err error) string {
+	return fmt.Sprintf("Could not fetch %s file %s: %v", file, fileType, err)
+}
+
 func validateCommandLists(opts *ConfigOpts) {
 	var cmdNotFoundSliceErr []error
 	for cmdListName, cmdList := range opts.CmdConfigLists {
@@ -427,7 +471,7 @@ func getLoggingKeyFromConfig(key string) string {
 // 	return fmt.Sprintf("cmdLists.%s", list)
 // }
 
-func (opts *ConfigOpts) setupVault() error {
+func (opts *ConfigOpts) initializeVault() error {
 	if !opts.koanf.Bool("vault.enabled") {
 		return nil
 	}
@@ -448,97 +492,41 @@ func (opts *ConfigOpts) setupVault() error {
 		token = os.Getenv("VAULT_TOKEN")
 	}
 	if strings.TrimSpace(token) == "" {
-		return fmt.Errorf("no token found, but one was required. \n\nSet the config key vault.token or the environment variable VAULT_TOKEN")
+		return fmt.Errorf("no token found. One is required. \n\nSet the config key vault.token or the environment variable VAULT_TOKEN")
 	}
 
 	client.SetToken(token)
 
 	unmarshalErr := opts.koanf.UnmarshalWithConf("vault.keys", &opts.VaultKeys, koanf.UnmarshalConf{Tag: "yaml"})
 	if unmarshalErr != nil {
-		logging.ExitWithMSG(fmt.Sprintf("error unmarshalling vault.keys into struct: %v", unmarshalErr), 1, &opts.Logger)
+		logging.ExitWithMSG(fmt.Sprintf("error unmarshaling vault.keys into struct: %v", unmarshalErr), 1, &opts.Logger)
 	}
 
 	opts.vaultClient = client
 
+	for _, v := range opts.VaultKeys {
+		v.Name = replaceVarInString(opts.Vars, v.Key, opts.Logger)
+		v.MountPath = replaceVarInString(opts.Vars, v.MountPath, opts.Logger)
+	}
+
 	return nil
 }
 
-func getVaultSecret(vaultClient *vault.Client, key *VaultKey) (string, error) {
-	var (
-		secret *vault.KVSecret
-		err    error
-	)
-
-	if key.ValueType == "KVv2" {
-		secret, err = vaultClient.KVv2(key.MountPath).Get(context.Background(), key.Path)
-	} else if key.ValueType == "KVv1" {
-		secret, err = vaultClient.KVv1(key.MountPath).Get(context.Background(), key.Path)
-	} else if key.ValueType != "" {
-		return "", fmt.Errorf("type %s for key %s not known. Valid types are KVv1 or KVv2", key.ValueType, key.Name)
-	} else {
-		return "", fmt.Errorf("type for key %s must be specified. Valid types are KVv1 or KVv2", key.Name)
-
-	}
-	if err != nil {
-		return "", fmt.Errorf("unable to read secret: %v", err)
-	}
-
-	value, ok := secret.Data[key.Name].(string)
-	if !ok {
-		return "", fmt.Errorf("value type assertion failed: %T %#v", secret.Data[key.Name], secret.Data[key.Name])
-	}
-
-	return value, nil
-}
-
-func isVaultKey(str string) (string, bool) {
-	str = strings.TrimSpace(str)
-	return strings.TrimPrefix(str, "vault:"), strings.HasPrefix(str, "vault:")
-}
-
-func parseVaultKey(str string, keys []*VaultKey) (*VaultKey, error) {
-	keyName, isKey := isVaultKey(str)
-	if !isKey {
-		return nil, nil
-	}
-
-	for _, k := range keys {
-		if k.Name == keyName {
-			return k, nil
-		}
-	}
-	return nil, fmt.Errorf("key %s not found in vault keys", keyName)
-}
-
-func GetVaultKey(str string, opts *ConfigOpts, log zerolog.Logger) string {
-	key, err := parseVaultKey(str, opts.VaultKeys)
-	if key == nil && err == nil {
-		return str
-	}
-	if err != nil && key == nil {
-		log.Err(err).Send()
-		return ""
-	}
-
-	value, secretErr := getVaultSecret(opts.vaultClient, key)
-	if secretErr != nil {
-		log.Err(secretErr).Send()
-		return value
-	}
-	return value
-}
-
 func processCmds(opts *ConfigOpts) error {
 
 	// process commands
 	for cmdName, cmd := range opts.Cmds {
-
+		cmd.GetVariablesFromConf(opts)
+		cmd.Cmd = replaceVarInString(opts.Vars, cmd.Cmd, opts.Logger)
+		for i, v := range cmd.Args {
+			v = replaceVarInString(opts.Vars, v, opts.Logger)
+			cmd.Args[i] = v
+		}
 		if cmd.Name == "" {
 			cmd.Name = cmdName
 		}
-		// println("Cmd.Name = " + cmd.Name)
+
 		hooks := cmd.Hooks
-		// resolve hooks
 		if hooks != nil {
 
 			processHookSuccess := processHooks(cmd, hooks.Error, opts, "error")
@@ -555,9 +543,13 @@ func processCmds(opts *ConfigOpts) error {
 			}
 		}
 
-		// resolve hosts
+		if !IsHostLocal(cmd.Host) {
-		if cmd.Host != nil {
+
-			host, hostFound := opts.Hosts[*cmd.Host]
+			cmdHost := replaceVarInString(opts.Vars, cmd.Host, opts.Logger)
+			if cmdHost != cmd.Host {
+				cmd.Host = cmdHost
+			}
+			host, hostFound := opts.Hosts[cmd.Host]
 			if hostFound {
 				cmd.RemoteHost = host
 				cmd.RemoteHost.Host = host.Host
@@ -565,8 +557,12 @@ func processCmds(opts *ConfigOpts) error {
 					cmd.RemoteHost.HostName = host.HostName
 				}
 			} else {
-				opts.Hosts[*cmd.Host] = &Host{Host: *cmd.Host}
+				opts.Logger.Info().Msgf("adding host %s to host list", cmd.Host)
-				cmd.RemoteHost = &Host{Host: *cmd.Host}
+				if opts.Hosts == nil {
+					opts.Hosts = make(map[string]*Host)
+				}
+				opts.Hosts[cmd.Host] = &Host{Host: cmd.Host}
+				cmd.RemoteHost = &Host{Host: cmd.Host}
 			}
 		} else {
 
@@ -577,19 +573,20 @@ func processCmds(opts *ConfigOpts) error {
 					return err
 				}
 				cmd.Dir = &cmdDir
+			} else {
+				cmd.Dir = &opts.ConfigDir
 			}
 		}
 
-		// Parse package commands
+		if cmd.Type == PackageCommandType {
-		if cmd.Type == PackageCT {
 			if cmd.PackageManager == "" {
-				return fmt.Errorf("package manager is required for package command %s", cmd.PackageName)
+				return fmt.Errorf("package manager is required for package command %s", cmd.Name)
 			}
 			if cmd.PackageOperation.String() == "" {
-				return fmt.Errorf("package operation is required for package command %s", cmd.PackageName)
+				return fmt.Errorf("package operation is required for package command %s", cmd.Name)
 			}
-			if cmd.PackageName == "" {
+			if cmd.Packages == nil {
-				return fmt.Errorf("package name is required for package command %s", cmd.PackageName)
+				return fmt.Errorf("package name is required for package command %s", cmd.Name)
 			}
 			var err error
 
@@ -607,24 +604,38 @@ func processCmds(opts *ConfigOpts) error {
 		}
 
 		// Parse user commands
-		if cmd.Type == UserCT {
+		if cmd.Type == UserCommandType {
 			if cmd.Username == "" {
 				return fmt.Errorf("username is required for user command %s", cmd.Name)
 			}
-
+			cmd.Username = replaceVarInString(opts.Vars, cmd.Username, opts.Logger)
-			detectOSType(cmd, opts)
+			err := detectOSType(cmd, opts)
-			var err error
+			if err != nil {
+				opts.Logger.Info().Err(err).Str("command", cmdName).Send()
+			}
 
 			// Validate the operation
 			switch cmd.UserOperation {
 			case "add", "remove", "modify", "checkIfExists", "delete", "password":
 				cmd.userMan, err = usermanager.NewUserManager(cmd.OS)
-				if cmd.Host != nil {
+
-					host, ok := opts.Hosts[*cmd.Host]
+				if cmd.UserOperation == "password" {
+					opts.Logger.Debug().Msg("changing password for user: " + cmd.Username)
+					cmd.UserPassword = getExternalConfigDirectiveValue(cmd.UserPassword, opts, AllowedExternalDirectiveAll)
+				}
+
+				if !IsHostLocal(cmd.Host) {
+
+					host, ok := opts.Hosts[cmd.Host]
 					if ok {
 						cmd.userMan, err = usermanager.NewUserManager(host.OS)
 					}
 				}
+				for indx, key := range cmd.UserSshPubKeys {
+					opts.Logger.Debug().Msg("adding SSH Keys")
+					key = getExternalConfigDirectiveValue(key, opts, AllowedExternalDirectiveAll)
+					cmd.UserSshPubKeys[indx] = key
+				}
 				if err != nil {
 					return err
 				}
@@ -634,7 +645,7 @@ func processCmds(opts *ConfigOpts) error {
 
 		}
 
-		if cmd.Type == RemoteScriptCT {
+		if cmd.Type == RemoteScriptCommandType {
 			var fetchErr error
 			if !isRemoteURL(cmd.Cmd) {
 				return fmt.Errorf("remoteScript command %s must be a remote resource", cmdName)
@@ -645,9 +656,9 @@ func processCmds(opts *ConfigOpts) error {
 			}
 
 		}
-		if cmd.OutputFile != "" {
+		if cmd.Output.File != "" {
 			var err error
-			cmd.OutputFile, err = getFullPathWithHomeDir(cmd.OutputFile)
+			cmd.Output.File, err = getFullPathWithHomeDir(cmd.Output.File)
 			if err != nil {
 				return err
 			}
@@ -656,14 +667,8 @@ func processCmds(opts *ConfigOpts) error {
 	return nil
 }
 
-// processHooks evaluates if hooks are valid Commands
-//
-// The cmd.hookRefs[hookType] is created with any hooks found.
-//
-// Returns an error, if any, if the hook command is not found
 func processHooks(cmd *Command, hooks []string, opts *ConfigOpts, hookType string) error {
 
-	// initialize hook type
 	var hookCmdFound bool
 	cmd.hookRefs = map[string]map[string]*Command{}
 	cmd.hookRefs[hookType] = map[string]*Command{}
@@ -690,13 +695,18 @@ func processHooks(cmd *Command, hooks []string, opts *ConfigOpts, hookType strin
 }
 
 func detectOSType(cmd *Command, opts *ConfigOpts) error {
-	if cmd.Host == nil {
+
-		if runtime.GOOS == "linux" { // also can be specified to FreeBSD
+	if IsHostLocal(cmd.Host) {
+
+		if runtime.GOOS == "linux" {
 			cmd.OS = "linux"
 			opts.Logger.Info().Msg("Unix/Linux type OS detected")
+			return nil
 		}
+		return fmt.Errorf("using an os that is not yet supported for user commands")
 	}
-	host, ok := opts.Hosts[*cmd.Host]
+
+	host, ok := opts.Hosts[cmd.Host]
 	if ok {
 		if host.OS != "" {
 			return nil
@@ -728,3 +738,25 @@ func keyNotSupported(oldKey, newKey string, koanf *koanf.Koanf, opts *ConfigOpts
 		}
 	}
 }
+
+func replaceVarInString(vars map[string]string, str string, logger zerolog.Logger) string {
+	if strings.Contains(str, "%{var:") && strings.Contains(str, "}%") {
+		logger.Debug().Msgf("replacing vars in string %s", str)
+		for k, v := range vars {
+			if strings.Contains(str, "%{var:"+k+"}%") {
+				str = strings.ReplaceAll(str, "%{var:"+k+"}%", v)
+			}
+		}
+		if strings.Contains(str, "%{var:") && strings.Contains(str, "}%") {
+			logger.Warn().Msg("could not replace all vars in string")
+		}
+	}
+	return str
+}
+
+func (c *Command) GetVariablesFromConf(opts *ConfigOpts) {
+	c.ScriptEnvFile = replaceVarInString(opts.Vars, c.ScriptEnvFile, opts.Logger)
+	c.Name = replaceVarInString(opts.Vars, c.Name, opts.Logger)
+	c.Output.File = replaceVarInString(opts.Vars, c.Output.File, opts.Logger)
+	c.Host = replaceVarInString(opts.Vars, c.Host, opts.Logger)
+}

pkg/backy/lineinfile.go

@@ -0,0 +1,133 @@
+package backy
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"regexp"
+	"strings"
+
+	"golang.org/x/crypto/ssh"
+)
+
+func sshConnect(user, password, host string, port int) (*ssh.Client, error) {
+	config := &ssh.ClientConfig{
+		User: user,
+		Auth: []ssh.AuthMethod{
+			ssh.Password(password),
+		},
+		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
+	}
+	addr := fmt.Sprintf("%s:%d", host, port)
+	return ssh.Dial("tcp", addr, config)
+}
+
+func sshReadFile(client *ssh.Client, remotePath string) (string, error) {
+	session, err := client.NewSession()
+	if err != nil {
+		return "", err
+	}
+	defer session.Close()
+
+	var b bytes.Buffer
+	session.Stdout = &b
+	if err := session.Run(fmt.Sprintf("cat %s", remotePath)); err != nil {
+		return "", err
+	}
+	return b.String(), nil
+}
+
+func sshWriteFile(client *ssh.Client, remotePath, content string) error {
+	session, err := client.NewSession()
+	if err != nil {
+		return err
+	}
+	defer session.Close()
+
+	stdin, err := session.StdinPipe()
+	if err != nil {
+		return err
+	}
+
+	go func() {
+		defer stdin.Close()
+		io.WriteString(stdin, content)
+	}()
+
+	cmd := fmt.Sprintf("cat > %s", remotePath)
+	return session.Run(cmd)
+}
+
+func lineInString(content, regexpPattern, line string) string {
+	scanner := bufio.NewScanner(strings.NewReader(content))
+	var lines []string
+	found := false
+	re := regexp.MustCompile(regexpPattern)
+
+	for scanner.Scan() {
+		l := scanner.Text()
+		if re.MatchString(l) {
+			found = true
+			lines = append(lines, line)
+		} else {
+			lines = append(lines, l)
+		}
+	}
+	if !found {
+		lines = append(lines, line)
+	}
+	return strings.Join(lines, "\n") + "\n"
+}
+
+func Call() {
+	user := "youruser"
+	password := "yourpassword"
+	host := "yourhost"
+	port := 22
+	remotePath := "/path/to/remote/file"
+
+	client, err := sshConnect(user, password, host, port)
+	if err != nil {
+		fmt.Println("SSH connection error:", err)
+		return
+	}
+	defer client.Close()
+
+	content, err := sshReadFile(client, remotePath)
+	if err != nil {
+		fmt.Println("Read error:", err)
+		return
+	}
+
+	newContent := lineInString(content, "^foo=", "foo=bar")
+
+	if err := sshWriteFile(client, remotePath, newContent); err != nil {
+		fmt.Println("Write error:", err)
+		return
+	}
+
+	fmt.Println("Line updated successfully over SSH.")
+}
+
+type LineInFile struct {
+	RemotePath    string         // Path to the remote file
+	Pattern       string         // Regex pattern to match lines
+	Line          string         // Line to insert or replace
+	InsertAfter   bool           // If true, insert after matched line; else replace
+	User          string         // SSH username
+	Password      string         // SSH password (use key for production)
+	Host          string         // SSH host
+	Port          int            // SSH port
+	regexCompiled *regexp.Regexp // Compiled regex (internal use)
+}
+
+// CompileRegex compiles the regex pattern for later use
+func (l *LineInFile) CompileRegex() error {
+	re, err := regexp.Compile(l.Pattern)
+	if err != nil {
+		return err
+	}
+	l.regexCompiled = re
+	return nil
+}

pkg/backy/list.go

@@ -46,9 +46,9 @@ func (opts *ConfigOpts) ListCommand(cmd string) {
 		}
 
 		// is it remote or local
-		if cmdInfo.Host != nil {
+		if !IsHostLocal(cmdInfo.Host) {
 			println()
-			print("Host: ", *cmdInfo.Host)
+			print("Host: ", cmdInfo.Host)
 			println()
 
 		} else {

pkg/backy/metrics.go

@@ -0,0 +1,88 @@
+package backy
+
+import (
+	"encoding/json"
+	"os"
+	"time"
+)
+
+type MetricFile struct {
+	Filename       string              `json:"filename"`
+	CommandMetrics map[string]*Metrics `json:"commandMetrics"`
+	ListMetrics    map[string]*Metrics `json:"listMetrics"`
+}
+
+type Metrics struct {
+	DateStartedLast              string  `json:"dateStartedLast"`
+	DateLastFinished             string  `json:"dateLastFinished"`
+	DateLastFinishedSuccessfully string  `json:"dateLastFinishedSuccessfully"`
+	SuccessfulExecutions         uint64  `json:"successfulExecutions"`
+	FailedExecutions             uint64  `json:"failedExecutions"`
+	TotalExecutions              uint64  `json:"totalExecutions"`
+	TotalExecutionTime           float64 `json:"lastExecutionTime"`  // in seconds
+	AverageExecutionTime         float64 `json:"totalExecutionTime"` // in seconds
+	SuccessRate                  float64 `json:"successRate"`        // percentage of successful executions
+	FailureRate                  float64 `json:"failureRate"`        // percentage of failed executions
+}
+
+func NewMetrics() *Metrics {
+	return &Metrics{
+		DateStartedLast:      time.Now().Format(time.RFC3339),
+		SuccessfulExecutions: 0,
+		FailedExecutions:     0,
+		TotalExecutions:      0,
+		TotalExecutionTime:   0.0,
+		AverageExecutionTime: 0.0,
+		SuccessRate:          0.0,
+		FailureRate:          0.0,
+	}
+}
+
+func NewMetricsFromFile(filename string) *MetricFile {
+	return &MetricFile{
+		Filename:       filename,
+		CommandMetrics: make(map[string]*Metrics),
+		ListMetrics:    make(map[string]*Metrics),
+	}
+
+}
+
+func (m *Metrics) Update(success bool, executionTime float64, dateLastFinished time.Time) {
+	m.TotalExecutions++
+	if success {
+		m.SuccessfulExecutions++
+	} else {
+		m.FailedExecutions++
+	}
+
+	m.DateLastFinished = dateLastFinished.Format(time.RFC3339)
+
+	m.TotalExecutionTime += executionTime
+	m.AverageExecutionTime = m.TotalExecutionTime / float64(m.TotalExecutions)
+
+	if m.TotalExecutions > 0 {
+		m.SuccessRate = float64(m.SuccessfulExecutions) / float64(m.TotalExecutions) * 100
+		m.FailureRate = float64(m.FailedExecutions) / float64(m.TotalExecutions) * 100
+	}
+}
+
+func (metricFile *MetricFile) SaveToFile() error {
+	data, err := json.MarshalIndent(metricFile, "", "  ")
+	if err != nil {
+		return err
+	}
+	return os.WriteFile(metricFile.Filename, data, 0644)
+}
+
+func LoadMetricsFromFile(filename string) (*MetricFile, error) {
+	jsonData, err := os.ReadFile(filename)
+	if err != nil {
+		return nil, err
+	}
+	var metrics MetricFile
+	err = json.Unmarshal(jsonData, &metrics)
+	if err != nil {
+		return nil, err
+	}
+	return &metrics, nil
+}

pkg/backy/metrics_test.go

@@ -0,0 +1,67 @@
+package backy
+
+import (
+	"testing"
+	"time"
+)
+
+func TestAddingMetricsForCommand(t *testing.T) {
+
+	// Create a new MetricFile
+	metricFile := NewMetricsFromFile("test_metrics.json")
+
+	metricFile, err := LoadMetricsFromFile(metricFile.Filename)
+	if err != nil {
+		t.Errorf("Failed to load metrics from file: %v", err)
+	}
+
+	// Add metrics for a command
+	commandName := "test_command"
+	if _, exists := metricFile.CommandMetrics[commandName]; !exists {
+		metricFile.CommandMetrics[commandName] = NewMetrics()
+	}
+
+	// Update the metrics for the command
+	executionTime := 1.8 // Example execution time in seconds
+	success := true      // Example success status
+	metricFile.CommandMetrics[commandName].Update(success, executionTime, time.Now())
+
+	// Check if the metrics were updated correctly
+	if metricFile.CommandMetrics[commandName].SuccessfulExecutions > 50 {
+		t.Errorf("Expected 1 successful execution, got %d", metricFile.CommandMetrics[commandName].SuccessfulExecutions)
+	}
+	if metricFile.CommandMetrics[commandName].TotalExecutions > 50 {
+		t.Errorf("Expected 1 total execution, got %d", metricFile.CommandMetrics[commandName].TotalExecutions)
+	}
+	// if metricFile.CommandMetrics[commandName].TotalExecutionTime != executionTime {
+	// 	t.Errorf("Expected execution time %f, got %f", executionTime, metricFile.CommandMetrics[commandName].TotalExecutionTime)
+	// }
+
+	err = metricFile.SaveToFile()
+	if err != nil {
+		t.Errorf("Failed to save metrics to file: %v", err)
+	}
+
+	listName := "test_list"
+	if _, exists := metricFile.ListMetrics[listName]; !exists {
+		metricFile.ListMetrics[listName] = NewMetrics()
+	}
+	// Update the metrics for the list
+	metricFile.ListMetrics[listName].Update(success, executionTime, time.Now())
+	if metricFile.ListMetrics[listName].SuccessfulExecutions > 50 {
+		t.Errorf("Expected 1 successful execution for list, got %d", metricFile.ListMetrics[listName].SuccessfulExecutions)
+	}
+	if metricFile.ListMetrics[listName].TotalExecutions > 50 {
+		t.Errorf("Expected 1 total execution for list, got %d", metricFile.ListMetrics[listName].TotalExecutions)
+	}
+	// if metricFile.ListMetrics[listName].TotalExecutionTime > executionTime {
+	// 	t.Errorf("Expected execution time %f for list, got %f", executionTime, metricFile.ListMetrics[listName].TotalExecutionTime)
+	// }
+
+	// Save the metrics to a file
+	err = metricFile.SaveToFile()
+	if err != nil {
+		t.Errorf("Failed to save metrics to file: %v", err)
+	}
+
+}

pkg/backy/notification.go

@@ -9,6 +9,7 @@ import (
 
 	"git.andrewnw.xyz/CyberShell/backy/pkg/logging"
 	"github.com/nikoksr/notify"
+	"github.com/nikoksr/notify/service/http"
 	"github.com/nikoksr/notify/service/mail"
 	"github.com/nikoksr/notify/service/matrix"
 	"maunium.net/go/mautrix/id"
@@ -30,6 +31,12 @@ type MailConfig struct {
 	Password      string   `yaml:"password"`
 }
 
+type HttpConfig struct {
+	URL     string              `yaml:"url"`
+	Method  string              `yaml:"method"`
+	Headers map[string][]string `yaml:"headers"`
+}
+
 // SetupNotify sets up notify instances for each command list.
 func (opts *ConfigOpts) SetupNotify() {
 
@@ -58,6 +65,8 @@ func (opts *ConfigOpts) SetupNotify() {
 					opts.Logger.Info().Err(fmt.Errorf("error: ID %s not found in mail object", confId)).Str("list", confName).Send()
 					continue
 				}
+				conf.Password = getExternalConfigDirectiveValue(conf.Password, opts, AllowedExternalDirectiveAll)
+				opts.Logger.Debug().Str("list", confName).Str("id", confId).Msg("adding mail notification service")
 				mailConf := setupMail(conf)
 				services = append(services, mailConf)
 			case "matrix":
@@ -66,14 +75,23 @@ func (opts *ConfigOpts) SetupNotify() {
 					opts.Logger.Info().Err(fmt.Errorf("error: ID %s not found in matrix object", confId)).Str("list", confName).Send()
 					continue
 				}
+				conf.AccessToken = getExternalConfigDirectiveValue(conf.AccessToken, opts, AllowedExternalDirectiveAll)
+				opts.Logger.Debug().Str("list", confName).Str("id", confId).Msg("adding matrix notification service")
 				mtrxConf, mtrxErr := setupMatrix(conf)
 				if mtrxErr != nil {
 					opts.Logger.Info().Str("list", confName).Err(fmt.Errorf("error: configuring matrix id %s failed during setup: %w", id, mtrxErr))
 					continue
 				}
-				// append the services
 				services = append(services, mtrxConf)
-			// service is not recognized
+			case "http":
+				conf, ok := opts.NotificationConf.HttpConfig[confId]
+				if !ok {
+					opts.Logger.Info().Err(fmt.Errorf("error: ID %s not found in http object", confId)).Str("list", confName).Send()
+					continue
+				}
+				opts.Logger.Debug().Str("list", confName).Str("id", confId).Msg("adding http notification service")
+				httpConf := setupHttp(conf)
+				services = append(services, httpConf)
 			default:
 				opts.Logger.Info().Err(fmt.Errorf("id %s not found", id)).Str("list", confName).Send()
 			}
@@ -99,3 +117,19 @@ func setupMail(config MailConfig) *mail.Mail {
 	mailClient.BodyFormat(mail.PlainText)
 	return mailClient
 }
+
+func setupHttp(httpConf HttpConfig) *http.Service {
+
+	httpService := http.New()
+	httpService.AddReceivers(&http.Webhook{
+		URL:         httpConf.URL,
+		Header:      httpConf.Headers,
+		ContentType: "text/plain",
+		Method:      httpConf.Method,
+		BuildPayload: func(subject, message string) (payload any) {
+			return subject + "\n\n" + message
+		},
+	})
+
+	return httpService
+}

pkg/backy/packageoperation_enumer.go

@@ -26,31 +26,31 @@ func (i PackageOperation) String() string {
 func _PackageOperationNoOp() {
 	var x [1]struct{}
 	_ = x[DefaultPO-(0)]
-	_ = x[PackOpInstall-(1)]
+	_ = x[PackageOperationInstall-(1)]
-	_ = x[PackOpUpgrade-(2)]
+	_ = x[PackageOperationUpgrade-(2)]
-	_ = x[PackOpPurge-(3)]
+	_ = x[PackageOperationPurge-(3)]
-	_ = x[PackOpRemove-(4)]
+	_ = x[PackageOperationRemove-(4)]
-	_ = x[PackOpCheckVersion-(5)]
+	_ = x[PackageOperationCheckVersion-(5)]
-	_ = x[PackOpIsInstalled-(6)]
+	_ = x[PackageOperationIsInstalled-(6)]
 }
 
-var _PackageOperationValues = []PackageOperation{DefaultPO, PackOpInstall, PackOpUpgrade, PackOpPurge, PackOpRemove, PackOpCheckVersion, PackOpIsInstalled}
+var _PackageOperationValues = []PackageOperation{DefaultPO, PackageOperationInstall, PackageOperationUpgrade, PackageOperationPurge, PackageOperationRemove, PackageOperationCheckVersion, PackageOperationIsInstalled}
 
 var _PackageOperationNameToValueMap = map[string]PackageOperation{
 	_PackageOperationName[0:0]:        DefaultPO,
 	_PackageOperationLowerName[0:0]:   DefaultPO,
-	_PackageOperationName[0:7]:        PackOpInstall,
+	_PackageOperationName[0:7]:        PackageOperationInstall,
-	_PackageOperationLowerName[0:7]:   PackOpInstall,
+	_PackageOperationLowerName[0:7]:   PackageOperationInstall,
-	_PackageOperationName[7:14]:       PackOpUpgrade,
+	_PackageOperationName[7:14]:       PackageOperationUpgrade,
-	_PackageOperationLowerName[7:14]:  PackOpUpgrade,
+	_PackageOperationLowerName[7:14]:  PackageOperationUpgrade,
-	_PackageOperationName[14:19]:      PackOpPurge,
+	_PackageOperationName[14:19]:      PackageOperationPurge,
-	_PackageOperationLowerName[14:19]: PackOpPurge,
+	_PackageOperationLowerName[14:19]: PackageOperationPurge,
-	_PackageOperationName[19:25]:      PackOpRemove,
+	_PackageOperationName[19:25]:      PackageOperationRemove,
-	_PackageOperationLowerName[19:25]: PackOpRemove,
+	_PackageOperationLowerName[19:25]: PackageOperationRemove,
-	_PackageOperationName[25:37]:      PackOpCheckVersion,
+	_PackageOperationName[25:37]:      PackageOperationCheckVersion,
-	_PackageOperationLowerName[25:37]: PackOpCheckVersion,
+	_PackageOperationLowerName[25:37]: PackageOperationCheckVersion,
-	_PackageOperationName[37:48]:      PackOpIsInstalled,
+	_PackageOperationName[37:48]:      PackageOperationIsInstalled,
-	_PackageOperationLowerName[37:48]: PackOpIsInstalled,
+	_PackageOperationLowerName[37:48]: PackageOperationIsInstalled,
 }
 
 var _PackageOperationNames = []string{

pkg/backy/planForHosts.md

@@ -0,0 +1,10 @@
+# Running commands on hosts
+
+I want all commands in a list to be able to be run on all hosts. The underlying solution will be using a function to run the list on a host, and therefore change the host on the commands. This can be done in several ways:
+
+1. The commands can have a `Hosts` field that will be a []string. This array can be populated several ways:
+    - From the config file
+    - using CLI options and commands
+   The commands can be run in succession on all hosts using functions
+
+2. The existing `Host` field can be modified in a function. The commands need to be added to a `[]*Command` slice so that all hosts can be run.

pkg/backy/ssh.go

@@ -15,50 +15,56 @@ import (
 	"strings"
 	"time"
 
+	"github.com/google/uuid"
 	"github.com/kevinburke/ssh_config"
 	"github.com/pkg/errors"
+	"github.com/pkg/sftp"
 	"github.com/rs/zerolog"
 	"golang.org/x/crypto/ssh"
 	"golang.org/x/crypto/ssh/knownhosts"
 )
 
-var PrivateKeyExtraInfoErr = errors.New("Private key may be encrypted. \nIf encrypted, make sure the password is specified correctly in the correct section. This may be done in one of three ways: \n privatekeypassword: env:PR_KEY_PASS \n privatekeypassword: file:/path/to/password-file \n privatekeypassword: password (not recommended). \n ")
+var PrivateKeyExtraInfoErr = errors.New("Private key may be encrypted. \nIf encrypted, make sure the password is specified correctly in the correct section. This may be done in one of two ways: \n Using external directives - see docs \n privatekeypassword: password (not recommended). \n ")
 var TS = strings.TrimSpace
 
+type RemoteHostCommandExecutor interface {
+	RunCmdOnHost(command *Command, commandSession *ssh.Session, cmdCtxLogger zerolog.Logger, cmdOutBuf bytes.Buffer) ([]string, error)
+}
+
 // ConnectToHost connects to a host by looking up the config values in the file ~/.ssh/config
 // It uses any set values and looks up an unset values in the config files
-// remoteConfig is modified directly. The *ssh.Client is returned as part of remoteConfig,
+// remoteHost is modified directly. The *ssh.Client is returned as part of remoteHost,
 // If configFile is empty, any required configuration is looked up in the default config files
 // If any value is not found, defaults are used
-func (remoteConfig *Host) ConnectToHost(opts *ConfigOpts) error {
+func (remoteHost *Host) ConnectToHost(opts *ConfigOpts) error {
 
 	var connectErr error
 
-	if TS(remoteConfig.ConfigFilePath) == "" {
+	if TS(remoteHost.ConfigFilePath) == "" {
-		remoteConfig.useDefaultConfig = true
+		remoteHost.useDefaultConfig = true
 	}
 
-	khPathErr := remoteConfig.GetKnownHosts()
+	khPathErr := remoteHost.GetKnownHosts()
 
 	if khPathErr != nil {
 		return khPathErr
 	}
 
-	if remoteConfig.ClientConfig == nil {
+	if remoteHost.ClientConfig == nil {
-		remoteConfig.ClientConfig = &ssh.ClientConfig{}
+		remoteHost.ClientConfig = &ssh.ClientConfig{}
 	}
 
 	var configFile *os.File
 
 	var sshConfigFileOpenErr error
 
-	if !remoteConfig.useDefaultConfig {
+	if !remoteHost.useDefaultConfig {
 		var err error
-		remoteConfig.ConfigFilePath, err = getFullPathWithHomeDir(remoteConfig.ConfigFilePath)
+		remoteHost.ConfigFilePath, err = getFullPathWithHomeDir(remoteHost.ConfigFilePath)
 		if err != nil {
 			return err
 		}
-		configFile, sshConfigFileOpenErr = os.Open(remoteConfig.ConfigFilePath)
+		configFile, sshConfigFileOpenErr = os.Open(remoteHost.ConfigFilePath)
 		if sshConfigFileOpenErr != nil {
 			return sshConfigFileOpenErr
 		}
@@ -69,22 +75,22 @@ func (remoteConfig *Host) ConnectToHost(opts *ConfigOpts) error {
 			return sshConfigFileOpenErr
 		}
 	}
-	remoteConfig.SSHConfigFile = &sshConfigFile{}
+	remoteHost.SSHConfigFile = &sshConfigFile{}
-	remoteConfig.SSHConfigFile.DefaultUserSettings = ssh_config.DefaultUserSettings
+	remoteHost.SSHConfigFile.DefaultUserSettings = ssh_config.DefaultUserSettings
 	var decodeErr error
-	remoteConfig.SSHConfigFile.SshConfigFile, decodeErr = ssh_config.Decode(configFile)
+	remoteHost.SSHConfigFile.SshConfigFile, decodeErr = ssh_config.Decode(configFile)
 	if decodeErr != nil {
 		return decodeErr
 	}
 
-	err := remoteConfig.GetProxyJumpFromConfig(opts.Hosts)
+	err := remoteHost.GetProxyJumpFromConfig(opts.Hosts)
 
 	if err != nil {
 		return err
 	}
 
-	if remoteConfig.ProxyHost != nil {
+	if remoteHost.ProxyHost != nil {
-		for _, proxyHost := range remoteConfig.ProxyHost {
+		for _, proxyHost := range remoteHost.ProxyHost {
 			err := proxyHost.GetProxyJumpConfig(opts.Hosts, opts)
 			opts.Logger.Info().Msgf("Proxy host: %s", proxyHost.Host)
 			if err != nil {
@@ -93,50 +99,49 @@ func (remoteConfig *Host) ConnectToHost(opts *ConfigOpts) error {
 		}
 	}
 
-	remoteConfig.ClientConfig.Timeout = time.Second * 30
+	remoteHost.ClientConfig.Timeout = time.Second * 30
 
-	remoteConfig.GetPrivateKeyFileFromConfig()
+	remoteHost.GetPrivateKeyFileFromConfig()
 
-	remoteConfig.GetPort()
+	remoteHost.GetPort()
 
-	remoteConfig.GetHostName()
+	remoteHost.GetHostName()
 
-	remoteConfig.CombineHostNameWithPort()
+	remoteHost.CombineHostNameWithPort()
 
-	remoteConfig.GetSshUserFromConfig()
+	remoteHost.GetSshUserFromConfig()
 
-	if remoteConfig.HostName == "" {
+	if remoteHost.HostName == "" {
-		return errors.Errorf("No hostname found or specified for host %s", remoteConfig.Host)
+		return errors.Errorf("No hostname found or specified for host %s", remoteHost.Host)
 	}
 
-	err = remoteConfig.GetAuthMethods(opts)
+	err = remoteHost.GetAuthMethods(opts)
 	if err != nil {
 		return err
 	}
 
-	hostKeyCallback, err := knownhosts.New(remoteConfig.KnownHostsFile)
+	hostKeyCallback, err := knownhosts.New(remoteHost.KnownHostsFile)
 	if err != nil {
 		return errors.Wrap(err, "could not create hostkeycallback function")
 	}
-	remoteConfig.ClientConfig.HostKeyCallback = hostKeyCallback
+	remoteHost.ClientConfig.HostKeyCallback = hostKeyCallback
-	// opts.Logger.Info().Str("user", remoteConfig.ClientConfig.User).Send()
 
-	remoteConfig.SshClient, connectErr = remoteConfig.ConnectThroughBastion(opts.Logger)
+	remoteHost.SshClient, connectErr = remoteHost.ConnectThroughBastion(opts.Logger)
 	if connectErr != nil {
 		return connectErr
 	}
-	if remoteConfig.SshClient != nil {
+	if remoteHost.SshClient != nil {
-		opts.Hosts[remoteConfig.Host] = remoteConfig
+		opts.Hosts[remoteHost.Host] = remoteHost
 		return nil
 	}
 
-	opts.Logger.Info().Msgf("Connecting to host %s", remoteConfig.HostName)
+	opts.Logger.Info().Msgf("Connecting to host %s", remoteHost.HostName)
-	remoteConfig.SshClient, connectErr = ssh.Dial("tcp", remoteConfig.HostName, remoteConfig.ClientConfig)
+	remoteHost.SshClient, connectErr = ssh.Dial("tcp", remoteHost.HostName, remoteHost.ClientConfig)
 	if connectErr != nil {
 		return connectErr
 	}
 
-	opts.Hosts[remoteConfig.Host] = remoteConfig
+	opts.Hosts[remoteHost.Host] = remoteHost
 	return nil
 }
 
@@ -180,11 +185,7 @@ func (remoteHost *Host) GetAuthMethods(opts *ConfigOpts) error {
 			return err
 		}
 
-		remoteHost.PrivateKeyPassword, err = GetPrivateKeyPassword(remoteHost.PrivateKeyPassword, opts, opts.Logger)
+		remoteHost.PrivateKeyPassword = GetPrivateKeyPassword(remoteHost.PrivateKeyPassword, opts)
-
-		if err != nil {
-			return err
-		}
 
 		if remoteHost.PrivateKeyPassword == "" {
 
@@ -207,14 +208,13 @@ func (remoteHost *Host) GetAuthMethods(opts *ConfigOpts) error {
 		}
 	}
 
-	if remoteHost.Password == "" {
+	if remoteHost.Password != "" {
 
-		remoteHost.Password, err = GetPassword(remoteHost.Password, opts, opts.Logger)
+		opts.Logger.Debug().Str("password", remoteHost.Password).Str("Host", remoteHost.Host).Send()
 
-		if err != nil {
+		remoteHost.Password = GetPassword(remoteHost.Password, opts)
 
-			return err
+		// opts.Logger.Debug().Str("actual password", remoteHost.Password).Str("Host", remoteHost.Host).Send()
-		}
 
 		remoteHost.ClientConfig.Auth = append(remoteHost.ClientConfig.Auth, ssh.Password(remoteHost.Password))
 	}
@@ -231,6 +231,8 @@ func (remoteHost *Host) GetPrivateKeyFileFromConfig() {
 	var identityFile string
 	if remoteHost.PrivateKeyPath == "" {
 		identityFile, _ = remoteHost.SSHConfigFile.SshConfigFile.Get(remoteHost.Host, "IdentityFile")
+		// println("Identity file:", identityFile)
+		// println("Host:", remoteHost.Host)
 		if identityFile == "" {
 			identityFile, _ = remoteHost.SSHConfigFile.DefaultUserSettings.GetStrict(remoteHost.Host, "IdentityFile")
 			if identityFile == "" {
@@ -242,6 +244,7 @@ func (remoteHost *Host) GetPrivateKeyFileFromConfig() {
 		identityFile = remoteHost.PrivateKeyPath
 	}
 
+	// println("Identity file:", identityFile)
 	remoteHost.PrivateKeyPath, _ = getFullPathWithHomeDir(identityFile)
 }
 
@@ -249,14 +252,13 @@ func (remoteHost *Host) GetPrivateKeyFileFromConfig() {
 // If it is the port is searched in the SSH config file(s)
 func (remoteHost *Host) GetPort() {
 	port := fmt.Sprintf("%d", remoteHost.Port)
-	// port specifed?
+	// port specified?
 	// port will be 0 if missing from backy config
 	if port == "0" {
 		port, _ = remoteHost.SSHConfigFile.SshConfigFile.Get(remoteHost.Host, "Port")
 
 		if port == "" {
 
-			// get port from default SSH config file
 			port = remoteHost.SSHConfigFile.DefaultUserSettings.Get(remoteHost.Host, "Port")
 
 			// set port to be default
@@ -271,7 +273,6 @@ func (remoteHost *Host) GetPort() {
 
 func (remoteHost *Host) CombineHostNameWithPort() {
 
-	// if the port is already in the HostName, leave it
 	if strings.HasSuffix(remoteHost.HostName, fmt.Sprintf(":%d", remoteHost.Port)) {
 		return
 	}
@@ -321,95 +322,44 @@ func (remoteHost *Host) ConnectThroughBastion(log zerolog.Logger) (*ssh.Client,
 
 // GetKnownHosts resolves the host's KnownHosts file if it is defined
 // if not defined, the default location for this file is used
-func (remotehHost *Host) GetKnownHosts() error {
+func (remoteHost *Host) GetKnownHosts() error {
 	var knownHostsFileErr error
-	if TS(remotehHost.KnownHostsFile) != "" {
+	if TS(remoteHost.KnownHostsFile) != "" {
-		remotehHost.KnownHostsFile, knownHostsFileErr = getFullPathWithHomeDir(remotehHost.KnownHostsFile)
+		remoteHost.KnownHostsFile, knownHostsFileErr = getFullPathWithHomeDir(remoteHost.KnownHostsFile)
 		return knownHostsFileErr
 	}
-	remotehHost.KnownHostsFile, knownHostsFileErr = getFullPathWithHomeDir("~/.ssh/known_hosts")
+	remoteHost.KnownHostsFile, knownHostsFileErr = getFullPathWithHomeDir("~/.ssh/known_hosts")
 	return knownHostsFileErr
 }
 
-func GetPrivateKeyPassword(key string, opts *ConfigOpts, log zerolog.Logger) (string, error) {
+func GetPrivateKeyPassword(key string, opts *ConfigOpts) string {
-
+	return getExternalConfigDirectiveValue(key, opts, AllowedExternalDirectiveAll)
-	var prKeyPassword string
-	if strings.HasPrefix(key, "file:") {
-		privKeyPassFilePath := strings.TrimPrefix(key, "file:")
-		privKeyPassFilePath, _ = getFullPathWithHomeDir(privKeyPassFilePath)
-		keyFile, keyFileErr := os.Open(privKeyPassFilePath)
-		if keyFileErr != nil {
-			return "", errors.Errorf("Private key password file %s failed to open. \n Make sure it is accessible and correct.", privKeyPassFilePath)
-		}
-		passwordScanner := bufio.NewScanner(keyFile)
-		for passwordScanner.Scan() {
-			prKeyPassword = passwordScanner.Text()
-		}
-	} else if strings.HasPrefix(key, "env:") {
-		privKey := strings.TrimPrefix(key, "env:")
-		privKey = strings.TrimPrefix(privKey, "${")
-		privKey = strings.TrimSuffix(privKey, "}")
-		privKey = strings.TrimPrefix(privKey, "$")
-		prKeyPassword = os.Getenv(privKey)
-	} else {
-		prKeyPassword = key
-	}
-	prKeyPassword = GetVaultKey(prKeyPassword, opts, opts.Logger)
-	return prKeyPassword, nil
 }
 
 // GetPassword gets any password
-func GetPassword(pass string, opts *ConfigOpts, log zerolog.Logger) (string, error) {
+func GetPassword(pass string, opts *ConfigOpts) string {
-
+	return getExternalConfigDirectiveValue(pass, opts, AllowedExternalDirectiveAll)
-	pass = strings.TrimSpace(pass)
-	if pass == "" {
-		return "", nil
-	}
-	var password string
-	if strings.HasPrefix(pass, "file:") {
-		passFilePath := strings.TrimPrefix(pass, "file:")
-		passFilePath, _ = getFullPathWithHomeDir(passFilePath)
-		keyFile, keyFileErr := os.Open(passFilePath)
-		if keyFileErr != nil {
-			return "", errors.New("Password file failed to open")
-		}
-		passwordScanner := bufio.NewScanner(keyFile)
-		for passwordScanner.Scan() {
-			password = passwordScanner.Text()
-		}
-	} else if strings.HasPrefix(pass, "env:") {
-		passEnv := strings.TrimPrefix(pass, "env:")
-		passEnv = strings.TrimPrefix(passEnv, "${")
-		passEnv = strings.TrimSuffix(passEnv, "}")
-		passEnv = strings.TrimPrefix(passEnv, "$")
-		password = os.Getenv(passEnv)
-	} else {
-		password = pass
-	}
-	password = GetVaultKey(password, opts, opts.Logger)
-
-	return password, nil
 }
 
-func (remoteConfig *Host) GetProxyJumpFromConfig(hosts map[string]*Host) error {
+func (remoteHost *Host) GetProxyJumpFromConfig(hosts map[string]*Host) error {
 
-	proxyJump, _ := remoteConfig.SSHConfigFile.SshConfigFile.Get(remoteConfig.Host, "ProxyJump")
+	proxyJump, _ := remoteHost.SSHConfigFile.SshConfigFile.Get(remoteHost.Host, "ProxyJump")
 	if proxyJump == "" {
-		proxyJump = remoteConfig.SSHConfigFile.DefaultUserSettings.Get(remoteConfig.Host, "ProxyJump")
+		proxyJump = remoteHost.SSHConfigFile.DefaultUserSettings.Get(remoteHost.Host, "ProxyJump")
 	}
-	if remoteConfig.ProxyJump == "" && proxyJump != "" {
+	if remoteHost.ProxyJump == "" && proxyJump != "" {
-		remoteConfig.ProxyJump = proxyJump
+		remoteHost.ProxyJump = proxyJump
 	}
-	proxyJumpHosts := strings.Split(remoteConfig.ProxyJump, ",")
+	proxyJumpHosts := strings.Split(remoteHost.ProxyJump, ",")
-	if remoteConfig.ProxyHost == nil && len(proxyJumpHosts) == 1 {
+	if remoteHost.ProxyHost == nil && len(proxyJumpHosts) == 1 {
-		remoteConfig.ProxyJump = proxyJump
+		remoteHost.ProxyJump = proxyJump
 		proxyHost, proxyHostFound := hosts[proxyJump]
 		if proxyHostFound {
-			remoteConfig.ProxyHost = append(remoteConfig.ProxyHost, proxyHost)
+			remoteHost.ProxyHost = append(remoteHost.ProxyHost, proxyHost)
 		} else {
 			if proxyJump != "" {
 				newProxy := &Host{Host: proxyJump}
-				remoteConfig.ProxyHost = append(remoteConfig.ProxyHost, newProxy)
+				remoteHost.ProxyHost = append(remoteHost.ProxyHost, newProxy)
 			}
 		}
 	}
@@ -417,25 +367,25 @@ func (remoteConfig *Host) GetProxyJumpFromConfig(hosts map[string]*Host) error {
 	return nil
 }
 
-func (remoteConfig *Host) GetProxyJumpConfig(hosts map[string]*Host, opts *ConfigOpts) error {
+func (remoteHost *Host) GetProxyJumpConfig(hosts map[string]*Host, opts *ConfigOpts) error {
 
-	if TS(remoteConfig.ConfigFilePath) == "" {
+	if TS(remoteHost.ConfigFilePath) == "" {
-		remoteConfig.useDefaultConfig = true
+		remoteHost.useDefaultConfig = true
 	}
 
-	khPathErr := remoteConfig.GetKnownHosts()
+	khPathErr := remoteHost.GetKnownHosts()
 
 	if khPathErr != nil {
 		return khPathErr
 	}
-	if remoteConfig.ClientConfig == nil {
+	if remoteHost.ClientConfig == nil {
-		remoteConfig.ClientConfig = &ssh.ClientConfig{}
+		remoteHost.ClientConfig = &ssh.ClientConfig{}
 	}
 	var configFile *os.File
 	var sshConfigFileOpenErr error
-	if !remoteConfig.useDefaultConfig {
+	if !remoteHost.useDefaultConfig {
 
-		configFile, sshConfigFileOpenErr = os.Open(remoteConfig.ConfigFilePath)
+		configFile, sshConfigFileOpenErr = os.Open(remoteHost.ConfigFilePath)
 		if sshConfigFileOpenErr != nil {
 			return sshConfigFileOpenErr
 		}
@@ -446,39 +396,39 @@ func (remoteConfig *Host) GetProxyJumpConfig(hosts map[string]*Host, opts *Confi
 			return sshConfigFileOpenErr
 		}
 	}
-	remoteConfig.SSHConfigFile = &sshConfigFile{}
+	remoteHost.SSHConfigFile = &sshConfigFile{}
-	remoteConfig.SSHConfigFile.DefaultUserSettings = ssh_config.DefaultUserSettings
+	remoteHost.SSHConfigFile.DefaultUserSettings = ssh_config.DefaultUserSettings
 	var decodeErr error
-	remoteConfig.SSHConfigFile.SshConfigFile, decodeErr = ssh_config.Decode(configFile)
+	remoteHost.SSHConfigFile.SshConfigFile, decodeErr = ssh_config.Decode(configFile)
 	if decodeErr != nil {
 		return decodeErr
 	}
-	remoteConfig.GetPrivateKeyFileFromConfig()
+	remoteHost.GetPrivateKeyFileFromConfig()
-	remoteConfig.GetPort()
+	remoteHost.GetPort()
-	remoteConfig.GetHostName()
+	remoteHost.GetHostName()
-	remoteConfig.CombineHostNameWithPort()
+	remoteHost.CombineHostNameWithPort()
-	remoteConfig.GetSshUserFromConfig()
+	remoteHost.GetSshUserFromConfig()
-	remoteConfig.isProxyHost = true
+	remoteHost.isProxyHost = true
-	if remoteConfig.HostName == "" {
+	if remoteHost.HostName == "" {
-		return errors.Errorf("No hostname found or specified for host %s", remoteConfig.Host)
+		return errors.Errorf("No hostname found or specified for host %s", remoteHost.Host)
 	}
-	err := remoteConfig.GetAuthMethods(opts)
+	err := remoteHost.GetAuthMethods(opts)
 	if err != nil {
 		return err
 	}
 
 	// TODO: Add value/option to config for host key and add bool to check for host key
-	hostKeyCallback, err := knownhosts.New(remoteConfig.KnownHostsFile)
+	hostKeyCallback, err := knownhosts.New(remoteHost.KnownHostsFile)
 	if err != nil {
 		return fmt.Errorf("could not create hostkeycallback function: %v", err)
 	}
-	remoteConfig.ClientConfig.HostKeyCallback = hostKeyCallback
+	remoteHost.ClientConfig.HostKeyCallback = hostKeyCallback
-	hosts[remoteConfig.Host] = remoteConfig
+	hosts[remoteHost.Host] = remoteHost
 
 	return nil
 }
 
-func (command *Command) RunCmdSSH(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([]string, error) {
+func (command *Command) RunCmdOnHost(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([]string, error) {
 	var (
 		ArgsStr       string
 		cmdOutBuf     bytes.Buffer
@@ -489,7 +439,6 @@ func (command *Command) RunCmdSSH(cmdCtxLogger zerolog.Logger, opts *ConfigOpts)
 			env:  command.Environment,
 		}
 	)
-	// Getting the command type must be done before concatenating the arguments
 	command = getCommandTypeAndSetCommandInfo(command)
 
 	// Prepare command arguments
@@ -499,8 +448,8 @@ func (command *Command) RunCmdSSH(cmdCtxLogger zerolog.Logger, opts *ConfigOpts)
 
 	cmdCtxLogger.Info().
 		Str("Command", command.Name).
-		Str("Host", *command.Host).
+		Str("Host", command.Host).
-		Msgf("Running %s on host %s", getCommandTypeAndSetCommandInfoLabel(command.Type), *command.Host)
+		Msgf("Running %s on host %s", getCommandTypeAndSetCommandInfoLabel(command.Type), command.Host)
 
 	// cmdCtxLogger.Debug().Str("cmd", command.Cmd).Strs("args", command.Args).Send()
 
@@ -531,33 +480,15 @@ func (command *Command) RunCmdSSH(cmdCtxLogger zerolog.Logger, opts *ConfigOpts)
 
 	// Handle command execution based on type
 	switch command.Type {
-	case ScriptCT:
+	case ScriptCommandType:
 		return command.runScript(commandSession, cmdCtxLogger, &cmdOutBuf)
-	case RemoteScriptCT:
+	case RemoteScriptCommandType:
 		return command.runRemoteScript(commandSession, cmdCtxLogger, &cmdOutBuf)
-	case ScriptFileCT:
+	case ScriptFileCommandType:
 		return command.runScriptFile(commandSession, cmdCtxLogger, &cmdOutBuf)
-	case PackageCT:
+	case PackageCommandType:
-		if command.PackageOperation == PackOpCheckVersion {
+		var remoteHostPackageExecutor RemoteHostPackageExecutor
-			commandSession.Stderr = nil
+		return remoteHostPackageExecutor.RunCmdOnHost(command, commandSession, cmdCtxLogger, cmdOutBuf)
-			// Execute the package version command remotely
-			// Parse the output of package version command
-			// Compare versions
-			// Check if a specific version is specified
-			commandSession.Stdout = nil
-			return checkPackageVersion(cmdCtxLogger, command, commandSession, cmdOutBuf)
-		} else {
-			if command.Shell != "" {
-				ArgsStr = fmt.Sprintf("%s -c '%s %s'", command.Shell, command.Cmd, ArgsStr)
-			} else {
-				ArgsStr = fmt.Sprintf("%s %s", command.Cmd, ArgsStr)
-			}
-			cmdCtxLogger.Debug().Str("cmd + args", ArgsStr).Send()
-			// Run simple command
-			if err := commandSession.Run(ArgsStr); err != nil {
-				return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error running command: %w", err)
-			}
-		}
 	default:
 		if command.Shell != "" {
 			ArgsStr = fmt.Sprintf("%s -c '%s %s'", command.Shell, command.Cmd, ArgsStr)
@@ -565,17 +496,107 @@ func (command *Command) RunCmdSSH(cmdCtxLogger zerolog.Logger, opts *ConfigOpts)
 			ArgsStr = fmt.Sprintf("%s %s", command.Cmd, ArgsStr)
 		}
 		cmdCtxLogger.Debug().Str("cmd + args", ArgsStr).Send()
-		// Run simple command
+
+		if command.Type == UserCommandType && command.UserOperation == "password" {
+			// cmdCtxLogger.Debug().Msgf("adding stdin")
+
+			userNamePass := fmt.Sprintf("%s:%s", command.Username, command.UserPassword)
+			client, err := sftp.NewClient(command.RemoteHost.SshClient)
+			if err != nil {
+				return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.Output.ToLog), fmt.Errorf("error creating sftp client: %v", err)
+			}
+			uuidFile := uuid.New()
+			passFilePath := fmt.Sprintf("/tmp/%s", uuidFile.String())
+			passFile, passFileErr := client.Create(passFilePath)
+			if passFileErr != nil {
+				return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.Output.ToLog), fmt.Errorf("error creating file /tmp/%s: %v", uuidFile.String(), passFileErr)
+			}
+
+			_, err = passFile.Write([]byte(userNamePass))
+			if err != nil {
+				return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.Output.ToLog), fmt.Errorf("error writing to file /tmp/%s: %v", uuidFile.String(), err)
+			}
+
+			ArgsStr = fmt.Sprintf("cat %s | chpasswd", passFilePath)
+			defer passFile.Close()
+
+			rmFileFunc := func() {
+				_ = client.Remove(passFilePath)
+			}
+
+			defer rmFileFunc()
+		}
 		if err := commandSession.Run(ArgsStr); err != nil {
-			return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error running command: %w", err)
+			return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.Output.ToLog), fmt.Errorf("error running command: %w", err)
+		}
+
+		if command.Type == UserCommandType {
+
+			// REFACTOR IF/WHEN WINDOWS SUPPORT IS ADDED
+
+			if command.UserOperation == "add" {
+				if command.UserSshPubKeys != nil {
+					var (
+						f        *sftp.File
+						err      error
+						userHome []byte
+						client   *sftp.Client
+					)
+
+					cmdCtxLogger.Info().Msg("adding SSH Keys")
+
+					commandSession, _ = command.RemoteHost.createSSHSession(opts)
+					userHome, err = commandSession.CombinedOutput(fmt.Sprintf("grep \"%s\" /etc/passwd | cut -d: -f6", command.Username))
+					if err != nil {
+						return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.Output.ToLog), fmt.Errorf("error finding user home from /etc/passwd: %v", err)
+					}
+
+					command.UserHome = strings.TrimSpace(string(userHome))
+					userSshDir := fmt.Sprintf("%s/.ssh", command.UserHome)
+					client, err = sftp.NewClient(command.RemoteHost.SshClient)
+					if err != nil {
+						return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.Output.ToLog), fmt.Errorf("error creating sftp client: %v", err)
+					}
+
+					err = client.MkdirAll(userSshDir)
+					if err != nil {
+						return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.Output.ToLog), fmt.Errorf("error creating directory %s: %v", userSshDir, err)
+					}
+					_, err = client.Create(fmt.Sprintf("%s/authorized_keys", userSshDir))
+					if err != nil {
+						return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.Output.ToLog), fmt.Errorf("error opening file %s/authorized_keys: %v", userSshDir, err)
+					}
+					f, err = client.OpenFile(fmt.Sprintf("%s/authorized_keys", userSshDir), os.O_APPEND|os.O_CREATE|os.O_WRONLY)
+					if err != nil {
+						return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.Output.ToLog), fmt.Errorf("error opening file %s/authorized_keys: %v", userSshDir, err)
+					}
+					defer f.Close()
+					for _, k := range command.UserSshPubKeys {
+						buf := bytes.NewBufferString(k)
+						cmdCtxLogger.Info().Str("key", k).Msg("adding SSH key")
+						if _, err := f.ReadFrom(buf); err != nil {
+							return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.Output.ToLog), fmt.Errorf("error adding to authorized keys: %v", err)
 						}
 					}
 
-	return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), nil
+					commandSession, _ = command.RemoteHost.createSSHSession(opts)
+					_, err = commandSession.CombinedOutput(fmt.Sprintf("chown -R %s:%s %s", command.Username, command.Username, userHome))
+					if err != nil {
+						return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.Output.ToLog), err
+					}
+
+				}
+			}
+		}
+	}
+
+	return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.Output.ToLog), nil
 }
 
 func checkPackageVersion(cmdCtxLogger zerolog.Logger, command *Command, commandSession *ssh.Session, cmdOutBuf bytes.Buffer) ([]string, error) {
-	cmdCtxLogger.Info().Str("package", command.PackageName).Msg("Checking package versions")
+	for _, p := range command.Packages {
+		cmdCtxLogger.Info().Str("package", p.Name).Msg("Checking package versions")
+	}
 	// Prepare command arguments
 	ArgsStr := command.Cmd
 	for _, v := range command.Args {
@@ -590,9 +611,9 @@ func checkPackageVersion(cmdCtxLogger zerolog.Logger, command *Command, commandS
 
 		_, parseErr := parsePackageVersion(string(cmdOut), cmdCtxLogger, command, cmdOutBuf)
 		if parseErr != nil {
-			return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error: package %s not listed: %w", command.PackageName, err)
+			return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.Output.ToLog), fmt.Errorf("error: packages %v not listed: %w", command.Packages, err)
 		}
-		return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error running %s: %w", ArgsStr, err)
+		return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.Output.ToLog), fmt.Errorf("error running %s: %w", ArgsStr, err)
 	}
 
 	return parsePackageVersion(string(cmdOut), cmdCtxLogger, command, cmdOutBuf)
@@ -622,7 +643,7 @@ func (command *Command) runScript(session *ssh.Session, cmdCtxLogger zerolog.Log
 		return collectOutput(outputBuf, command.Name, cmdCtxLogger, true), fmt.Errorf("error waiting for shell: %w", err)
 	}
 
-	return collectOutput(outputBuf, command.Name, cmdCtxLogger, command.GetOutput), nil
+	return collectOutput(outputBuf, command.Name, cmdCtxLogger, command.Output.ToLog), nil
 }
 
 // runScriptFile handles the execution of script files.
@@ -641,7 +662,7 @@ func (command *Command) runScriptFile(session *ssh.Session, cmdCtxLogger zerolog
 		return collectOutput(outputBuf, command.Name, cmdCtxLogger, true), fmt.Errorf("error waiting for shell: %w", err)
 	}
 
-	return collectOutput(outputBuf, command.Name, cmdCtxLogger, command.OutputToLog), nil
+	return collectOutput(outputBuf, command.Name, cmdCtxLogger, command.Output.ToLog), nil
 }
 
 // prepareScriptBuffer prepares a buffer for inline scripts.
@@ -698,10 +719,10 @@ func (command *Command) runRemoteScript(session *ssh.Session, cmdCtxLogger zerol
 	err = session.Run(command.Shell)
 
 	if err != nil {
-		return collectOutput(outputBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error running remote script: %w", err)
+		return collectOutput(outputBuf, command.Name, cmdCtxLogger, command.Output.ToLog), fmt.Errorf("error running remote script: %w", err)
 	}
 
-	return collectOutput(outputBuf, command.Name, cmdCtxLogger, command.OutputToLog), nil
+	return collectOutput(outputBuf, command.Name, cmdCtxLogger, command.Output.ToLog), nil
 }
 
 // readFileToBuffer reads a file into a buffer.
@@ -774,7 +795,7 @@ func (h *Host) DetectOS(opts *ConfigOpts) (string, error) {
 	return osName, nil
 }
 
-func CheckIfHostHasHostName(host string) (bool, string) {
+func DoesHostHaveHostName(host string) (bool, string) {
 	HostName, err := ssh_config.DefaultUserSettings.GetStrict(host, "HostName")
 	if err != nil {
 		return false, ""
@@ -782,3 +803,39 @@ func CheckIfHostHasHostName(host string) (bool, string) {
 	println(HostName)
 	return HostName != "", HostName
 }
+
+func IsHostLocal(host string) bool {
+	host = strings.ToLower(host)
+	return host == "127.0.0.1" || host == "localhost" || host == ""
+}
+
+type RemoteHostPackageExecutor struct{}
+
+func (r RemoteHostPackageExecutor) RunCmdOnHost(command *Command, commandSession *ssh.Session, cmdCtxLogger zerolog.Logger, cmdOutBuf bytes.Buffer) ([]string, error) {
+	var ArgsStr string
+	// Prepare command arguments
+	for _, v := range command.Args {
+		ArgsStr += fmt.Sprintf(" %s", v)
+	}
+
+	if command.PackageOperation == PackageOperationCheckVersion {
+		commandSession.Stderr = nil
+		// Execute the package version command remotely
+		// Parse the output of package version command
+		// Compare versions
+		// Check if a specific version is specified
+		commandSession.Stdout = nil
+		return checkPackageVersion(cmdCtxLogger, command, commandSession, cmdOutBuf)
+	}
+	if command.Shell != "" {
+		ArgsStr = fmt.Sprintf("%s -c '%s %s'", command.Shell, command.Cmd, ArgsStr)
+	} else {
+		ArgsStr = fmt.Sprintf("%s %s", command.Cmd, ArgsStr)
+	}
+	cmdCtxLogger.Debug().Str("cmd + args", ArgsStr).Send()
+	// Run simple command
+	if err := commandSession.Run(ArgsStr); err != nil {
+		return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.Output.ToLog), fmt.Errorf("error running command: %w", err)
+	}
+	return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.Output.ToLog), nil
+}

pkg/backy/types.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 
 	"git.andrewnw.xyz/CyberShell/backy/pkg/pkgman"
+	packagemanagercommon "git.andrewnw.xyz/CyberShell/backy/pkg/pkgman/common"
 	"git.andrewnw.xyz/CyberShell/backy/pkg/remotefetcher"
 	"git.andrewnw.xyz/CyberShell/backy/pkg/usermanager"
 	vaultapi "github.com/hashicorp/vault/api"
@@ -26,15 +27,15 @@ type (
 		ConfigFilePath     string `yaml:"config,omitempty"`
 		Host               string `yaml:"host,omitempty"`
 		HostName           string `yaml:"hostname,omitempty"`
-		KnownHostsFile     string `yaml:"knownhostsfile,omitempty"`
+		KnownHostsFile     string `yaml:"knownHostsFile,omitempty"`
 		ClientConfig       *ssh.ClientConfig
 		SSHConfigFile      *sshConfigFile
 		SshClient          *ssh.Client
 		Port               uint16 `yaml:"port,omitempty"`
 		ProxyJump          string `yaml:"proxyjump,omitempty"`
 		Password           string `yaml:"password,omitempty"`
-		PrivateKeyPath     string `yaml:"privatekeypath,omitempty"`
+		PrivateKeyPath     string `yaml:"IdentityFile,omitempty"`
-		PrivateKeyPassword string `yaml:"privatekeypassword,omitempty"`
+		PrivateKeyPassword string `yaml:"privateKeyPassword,omitempty"`
 		useDefaultConfig   bool
 		User               string `yaml:"user,omitempty"`
 		isProxyHost        bool
@@ -51,61 +52,45 @@ type (
 	Command struct {
 		Name string `yaml:"name,omitempty"`
 
-		// command to run
 		Cmd string `yaml:"cmd"`
 
 		// See CommandType enum further down the page for acceptable values
 		Type CommandType `yaml:"type,omitempty"`
 
-		// host on which to run cmd
+		Host string `yaml:"host,omitempty"`
-		Host *string `yaml:"host,omitempty"`
 
-		// Hooks are for running commands on certain events
 		Hooks *Hooks `yaml:"hooks,omitempty"`
 
-		// hook refs are internal references of commands for each hook type
 		hookRefs map[string]map[string]*Command
 
-		// Shell specifies which shell to run the command in, if any.
 		Shell string `yaml:"shell,omitempty"`
 
 		RemoteHost *Host `yaml:"-"`
 
-		// Args is an array that holds the arguments to cmd
 		Args []string `yaml:"args,omitempty"`
 
-		/*
-			Dir specifies a directory in which to run the command.
-		*/
 		Dir *string `yaml:"dir,omitempty"`
 
-		// Env points to a file containing env variables to be used with the command
 		Env string `yaml:"env,omitempty"`
 
-		// Environment holds env variables to be used with the command
 		Environment []string `yaml:"environment,omitempty"`
 
-		// Output determines if output is requested.
-		//
-		// Only for when command is in a list.
-		GetOutput bool `yaml:"getOutput,omitempty"`
-
 		ScriptEnvFile string `yaml:"scriptEnvFile"`
 
-		OutputToLog bool `yaml:"outputToLog,omitempty"`
+		Output struct {
-
+			File   string `yaml:"file,omitempty"`
-		OutputFile string `yaml:"outputFile,omitempty"`
+			ToLog  bool   `yaml:"toLog,omitempty"`
+			InList bool   `yaml:"inList,omitempty"`
+		} `yaml:"output"`
 
 		// BEGIN PACKAGE COMMAND FIELDS
 
 		PackageManager string `yaml:"packageManager,omitempty"`
 
-		PackageName string `yaml:"packageName,omitempty"`
+		Packages []packagemanagercommon.Package `yaml:"packages,omitempty"`
 
-		// Version specifies the desired version for package execution
 		PackageVersion string `yaml:"packageVersion,omitempty"`
 
-		// PackageOperation specifies the action for package-related commands (e.g., "install" or "remove")
 		PackageOperation PackageOperation `yaml:"packageOperation,omitempty"`
 
 		pkgMan pkgman.PackageManager
@@ -113,42 +98,37 @@ type (
 		packageCmdSet bool
 		// END PACKAGE COMMAND FIELDS
 
-		// RemoteSource specifies a URL to fetch the command or configuration remotely
 		RemoteSource string `yaml:"remoteSource,omitempty"`
 
-		// FetchBeforeExecution determines if the remoteSource should be fetched before running
 		FetchBeforeExecution bool `yaml:"fetchBeforeExecution,omitempty"`
 
 		Fetcher remotefetcher.RemoteFetcher
 
 		// BEGIN USER COMMAND FIELDS
 
-		// Username specifies the username for user creation or related operations
 		Username string `yaml:"userName,omitempty"`
 
 		UserID string `yaml:"userID,omitempty"`
 
-		// UserGroups specifies the groups to add the user to
 		UserGroups []string `yaml:"userGroups,omitempty"`
 
-		// UserHome specifies the home directory for the user
 		UserHome string `yaml:"userHome,omitempty"`
 
-		// UserShell specifies the shell for the user
 		UserShell string `yaml:"userShell,omitempty"`
 
-		// SystemUser specifies whether the user is a system account
+		UserCreateHome bool `yaml:"userCreateHome,omitempty"`
-		SystemUser bool `yaml:"systemUser,omitempty"`
+
+		UserIsSystem bool `yaml:"userIsSystem,omitempty"`
 
-		// UserPassword specifies the password for the user (can be file: or plain text)
 		UserPassword string `yaml:"userPassword,omitempty"`
 
+		UserSshPubKeys []string `yaml:"userSshPubKeys,omitempty"`
+
 		userMan usermanager.UserManager
 
 		// OS for the command, only used when type is user
 		OS string `yaml:"OS,omitempty"`
 
-		// UserOperation specifies the action for user-related commands (e.g., "create" or "remove")
 		UserOperation string `yaml:"userOperation,omitempty"`
 
 		userCmdSet bool
@@ -156,7 +136,7 @@ type (
 		// stdin only for userOperation = password (for now)
 		stdin *strings.Reader
 
-		// END USER STRUCT FIELDS
+		// END USER STRUCommandType FIELDS
 	}
 
 	RemoteSource struct {
@@ -176,8 +156,12 @@ type (
 		RunCmdOnFailure                          string   `yaml:"runCmdOnFailure,omitempty"`
 		Order                                    []string `yaml:"order,omitempty"`
 		Notifications                            []string `yaml:"notifications,omitempty"`
-		GetOutput       bool     `yaml:"getOutput,omitempty"`
+		GetCommandOutputInNotificationsOnSuccess bool     `yaml:"sendNotificationOnSuccess,omitempty"`
-		NotifyOnSuccess bool     `yaml:"notifyOnSuccess,omitempty"`
+
+		Notify struct {
+			OnFailure bool `yaml:"onFailure,omitempty"`
+			OnSuccess bool `yaml:"onSuccess,omitempty"`
+		} `yaml:"notify,omitempty"`
 
 		NotifyConfig *notify.Notify
 		Source       string `yaml:"source"` // URL to fetch remote commands
@@ -206,6 +190,8 @@ type (
 
 		ConfigFilePath string
 
+		HostsFilePath string
+
 		ConfigDir string
 
 		LogFilePath string
@@ -226,6 +212,8 @@ type (
 
 		List ListConfig
 
+		Vars map[string]string `yaml:"variables"`
+
 		VaultKeys []*VaultKey `yaml:"keys"`
 
 		koanf *koanf.Koanf
@@ -244,6 +232,7 @@ type (
 
 	VaultKey struct {
 		Name      string `yaml:"name"`
+		Key       string `yaml:"key"`
 		Path      string `yaml:"path"`
 		ValueType string `yaml:"type"`
 		MountPath string `yaml:"mountpath"`
@@ -259,6 +248,7 @@ type (
 	Notifications struct {
 		MailConfig   map[string]MailConfig   `yaml:"mail,omitempty"`
 		MatrixConfig map[string]MatrixStruct `yaml:"matrix,omitempty"`
+		HttpConfig   map[string]HttpConfig   `yaml:"http,omitempty"`
 	}
 
 	CmdOutput struct {
@@ -294,28 +284,55 @@ type (
 		Error    error  // Error encountered, if any
 	}
 
+	ListMetrics struct {
+		Name                 string
+		SuccessfulExecutions uint64
+		FailedExecutions     uint64
+		TotalExecutions      uint64
+	}
+
+	CommandMetrics struct {
+		Name                 string
+		SuccessfulExecutions uint64
+		FailedExecutions     uint64
+		TotalExecutions      uint64
+	}
+
 	// use ints so we can use enums
 	CommandType               int
 	PackageOperation          int
+	AllowedExternalDirectives int
 )
 
 //go:generate go run github.com/dmarkham/enumer -linecomment -yaml -text -json -type=CommandType
 const (
-	DefaultCT      CommandType = iota //
+	DefaultCommandType      CommandType = iota //
-	ScriptCT                          // script
+	ScriptCommandType                          // script
-	ScriptFileCT                      // scriptFile
+	ScriptFileCommandType                      // scriptFile
-	RemoteScriptCT                    // remoteScript
+	RemoteScriptCommandType                    // remoteScript
-	PackageCT                         // package
+	PackageCommandType                         // package
-	UserCT                            // user
+	UserCommandType                            // user
 )
 
 //go:generate go run github.com/dmarkham/enumer -linecomment -yaml -text -json -type=PackageOperation
 const (
 	DefaultPO                    PackageOperation = iota //
-	PackOpInstall                              // install
+	PackageOperationInstall                              // install
-	PackOpUpgrade                              // upgrade
+	PackageOperationUpgrade                              // upgrade
-	PackOpPurge                                // purge
+	PackageOperationPurge                                // purge
-	PackOpRemove                               // remove
+	PackageOperationRemove                               // remove
-	PackOpCheckVersion                         // checkVersion
+	PackageOperationCheckVersion                         // checkVersion
-	PackOpIsInstalled                          // isInstalled
+	PackageOperationIsInstalled                          // isInstalled
+)
+
+//go:generate go run github.com/dmarkham/enumer -linecomment -yaml -text -json -type=AllowedExternalDirectives
+const (
+	DefaultExternalDir                AllowedExternalDirectives = iota
+	AllowedExternalDirectiveVault                               // vault
+	AllowedExternalDirectiveVaultEnv                            // vault-env
+	AllowedExternalDirectiveVaultFile                           // vault-file
+	AllowedExternalDirectiveAll                                 // vault-file-env
+	AllowedExternalDirectiveFileEnv                             // file-env
+	AllowedExternalDirectiveFile                                // file
+	AllowedExternalDirectiveEnv                                 // env
 )

pkg/backy/utils.go

@@ -6,16 +6,19 @@ package backy
 
 import (
 	"bytes"
+	"context"
 	"errors"
 	"fmt"
 	"os"
 	"os/exec"
 	"path"
 	"path/filepath"
+	"regexp"
 	"strings"
 
 	"git.andrewnw.xyz/CyberShell/backy/pkg/logging"
 	"git.andrewnw.xyz/CyberShell/backy/pkg/remotefetcher"
+	vault "github.com/hashicorp/vault/api"
 	"github.com/joho/godotenv"
 	"github.com/knadh/koanf/v2"
 	"github.com/rs/zerolog"
@@ -65,8 +68,14 @@ func SetLogFile(logFile string) BackyOptionFunc {
 	}
 }
 
-// SetCmdStdOut forces the command output to stdout
+func SetHostsConfigFile(hostsConfigFile string) BackyOptionFunc {
-func SetCmdStdOut(setStdOut bool) BackyOptionFunc {
+	return func(bco *ConfigOpts) {
+		bco.HostsFilePath = hostsConfigFile
+	}
+}
+
+// EnableCommandStdOut forces the command output to stdout
+func EnableCommandStdOut(setStdOut bool) BackyOptionFunc {
 	return func(bco *ConfigOpts) {
 		bco.CmdStdOut = setStdOut
 	}
@@ -79,7 +88,7 @@ func EnableCron() BackyOptionFunc {
 	}
 }
 
-func NewOpts(configFilePath string, opts ...BackyOptionFunc) *ConfigOpts {
+func NewConfigOptions(configFilePath string, opts ...BackyOptionFunc) *ConfigOpts {
 	b := &ConfigOpts{}
 	b.ConfigFilePath = configFilePath
 	for _, opt := range opts {
@@ -108,7 +117,11 @@ func injectEnvIntoSSH(envVarsToInject environmentVars, process *ssh.Session, opt
 			goto errEnvFile
 		}
 		for key, val := range envMap {
-			process.Setenv(key, GetVaultKey(val, opts, log))
+			err = process.Setenv(key, getExternalConfigDirectiveValue(val, opts, AllowedExternalDirectiveVault))
+			if err != nil {
+				log.Error().Err(err).Send()
+
+			}
 		}
 	}
 
@@ -119,12 +132,15 @@ errEnvFile:
 		if strings.Contains(envVal, "=") {
 			envVarArr := strings.Split(envVal, "=")
 
-			process.Setenv(envVarArr[0], GetVaultKey(envVarArr[1], opts, log))
+			err := process.Setenv(envVarArr[0], getExternalConfigDirectiveValue(envVarArr[1], opts, AllowedExternalDirectiveVaultFile))
+			if err != nil {
+				log.Error().Err(err).Send()
+			}
 		}
 	}
 }
 
-func injectEnvIntoLocalCMD(envVarsToInject environmentVars, process *exec.Cmd, log zerolog.Logger) {
+func injectEnvIntoLocalCMD(envVarsToInject environmentVars, process *exec.Cmd, log zerolog.Logger, opts *ConfigOpts) {
 	if envVarsToInject.file != "" {
 		envPath, _ := getFullPathWithHomeDir(envVarsToInject.file)
 
@@ -148,7 +164,8 @@ errEnvFile:
 
 	for _, envVal := range envVarsToInject.env {
 		if strings.Contains(envVal, "=") {
-			process.Env = append(process.Env, envVal)
+			envVarArr := strings.Split(envVal, "=")
+			process.Env = append(process.Env, fmt.Sprintf("%s=%s", envVarArr[0], getExternalConfigDirectiveValue(envVarArr[1], opts, AllowedExternalDirectiveVault)))
 		}
 	}
 	process.Env = append(process.Env, os.Environ()...)
@@ -181,7 +198,6 @@ func testFile(c string) error {
 			return fileOpenErr
 		}
 	}
-
 	return nil
 }
 
@@ -247,11 +263,9 @@ func (opts *ConfigOpts) loadEnv() {
 	opts.backyEnv = backyEnv
 }
 
-// expandEnvVars expands environment variables with the env used in the config
 func expandEnvVars(backyEnv map[string]string, envVars []string) {
 
 	env := func(name string) string {
-		name = strings.ToUpper(name)
 		envVar, found := backyEnv[name]
 		if found {
 			return envVar
@@ -259,36 +273,35 @@ func expandEnvVars(backyEnv map[string]string, envVars []string) {
 		return ""
 	}
 
-	// parse env variables using new macros
 	for indx, v := range envVars {
-		if strings.HasPrefix(v, macroStart) && strings.HasSuffix(v, macroEnd) {
+
-			if strings.HasPrefix(v, envMacroStart) {
+		if strings.HasPrefix(v, envExternDirectiveStart) && strings.HasSuffix(v, externDirectiveEnd) {
-				v = strings.TrimPrefix(v, envMacroStart)
+			v = strings.TrimPrefix(v, envExternDirectiveStart)
-				v = strings.TrimRight(v, macroEnd)
+			v = strings.TrimRight(v, externDirectiveEnd)
 			out, _ := shell.Expand(v, env)
 			envVars[indx] = out
 		}
-		}
+
 	}
 }
 
 func getCommandTypeAndSetCommandInfo(command *Command) *Command {
 
-	if command.Type == PackageCT && !command.packageCmdSet {
+	if command.Type == PackageCommandType && !command.packageCmdSet {
 		command.packageCmdSet = true
 		switch command.PackageOperation {
-		case PackOpInstall:
+		case PackageOperationInstall:
-			command.Cmd, command.Args = command.pkgMan.Install(command.PackageName, command.PackageVersion, command.Args)
+			command.Cmd, command.Args = command.pkgMan.Install(command.Packages, command.Args)
-		case PackOpRemove:
+		case PackageOperationRemove:
-			command.Cmd, command.Args = command.pkgMan.Remove(command.PackageName, command.Args)
+			command.Cmd, command.Args = command.pkgMan.Remove(command.Packages, command.Args)
-		case PackOpUpgrade:
+		case PackageOperationUpgrade:
-			command.Cmd, command.Args = command.pkgMan.Upgrade(command.PackageName, command.PackageVersion)
+			command.Cmd, command.Args = command.pkgMan.Upgrade(command.Packages)
-		case PackOpCheckVersion:
+		case PackageOperationCheckVersion:
-			command.Cmd, command.Args = command.pkgMan.CheckVersion(command.PackageName, command.PackageVersion)
+			command.Cmd, command.Args = command.pkgMan.CheckVersion(command.Packages)
 		}
 	}
 
-	if command.Type == UserCT && !command.userCmdSet {
+	if command.Type == UserCommandType && !command.userCmdSet {
 		command.userCmdSet = true
 		switch command.UserOperation {
 		case "add":
@@ -296,7 +309,8 @@ func getCommandTypeAndSetCommandInfo(command *Command) *Command {
 				command.Username,
 				command.UserHome,
 				command.UserShell,
-				command.SystemUser,
+				command.UserIsSystem,
+				command.UserCreateHome,
 				command.UserGroups,
 				command.Args)
 		case "modify":
@@ -320,32 +334,188 @@ func getCommandTypeAndSetCommandInfo(command *Command) *Command {
 func parsePackageVersion(output string, cmdCtxLogger zerolog.Logger, command *Command, cmdOutBuf bytes.Buffer) ([]string, error) {
 
 	var err error
-	pkgVersion, err := command.pkgMan.Parse(output)
+	var errs []error
-	// println(output)
+	pkgVersionOnSystem, err := command.pkgMan.ParseRemotePackageManagerVersionOutput(output)
 	if err != nil {
-		cmdCtxLogger.Error().Err(err).Str("package", command.PackageName).Msg("Error parsing package version output")
+		cmdCtxLogger.Error().AnErr("Error parsing package version output", err).Send()
-		return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.GetOutput), err
+		return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.Output.ToLog), fmt.Errorf("error parsing package version output: %v", err)
 	}
 
+	for _, p := range pkgVersionOnSystem {
+		packageIndex := getPackageIndexFromCommand(command, p.Name)
+		if packageIndex == -1 {
+			cmdCtxLogger.Error().Str("package", p.Name).Msg("Package not found in command")
+			continue
+		}
+		command.Packages[packageIndex].VersionCheck = p.VersionCheck
+		packageFromCommand := command.Packages[packageIndex]
 		cmdCtxLogger.Info().
-		Str("Installed", pkgVersion.Installed).
+			Str("Installed", packageFromCommand.VersionCheck.Installed).
-		Str("Candidate", pkgVersion.Candidate).
 			Msg("Package version comparison")
 
-	if command.PackageVersion != "" {
+		versionLogger := cmdCtxLogger.With().Str("package", packageFromCommand.Name).Logger()
-		if pkgVersion.Installed == command.PackageVersion {
+
-			cmdCtxLogger.Info().Msgf("Installed version matches specified version: %s", command.PackageVersion)
+		if packageFromCommand.Version != "" {
+			versionLogger := cmdCtxLogger.With().Str("package", packageFromCommand.Name).Str("Specified Version", packageFromCommand.Version).Logger()
+			packageVersionRegex, PkgRegexErr := regexp.Compile(packageFromCommand.Version)
+			if PkgRegexErr != nil {
+				versionLogger.Error().Err(PkgRegexErr).Msg("Error compiling package version regex")
+				errs = append(errs, PkgRegexErr)
+				continue
+			}
+			if p.Version == packageFromCommand.Version {
+				versionLogger.Info().Msgf("Installed version matches specified version: %s", packageFromCommand.Version)
+			} else if packageVersionRegex.MatchString(p.VersionCheck.Installed) {
+				versionLogger.Info().Msgf("Installed version contains specified version: %s", packageFromCommand.Version)
 			} else {
-			cmdCtxLogger.Info().Msgf("Installed version does not match specified version: %s", command.PackageVersion)
+				versionLogger.Info().Msg("Installed version does not match specified version")
-			err = fmt.Errorf("Installed version does not match specified version: %s", command.PackageVersion)
+				errs = append(errs, fmt.Errorf("installed version of %s does not match specified version: %s", packageFromCommand.Name, packageFromCommand.Version))
 			}
 		} else {
-		if pkgVersion.Installed == pkgVersion.Candidate {
+			if p.VersionCheck.Installed == p.VersionCheck.Candidate {
-			cmdCtxLogger.Info().Msg("Installed and Candidate versions match")
+				versionLogger.Info().Msg("Installed and Candidate versions match")
 			} else {
 				cmdCtxLogger.Info().Msg("Installed and Candidate versions differ")
-			err = errors.New("Installed and Candidate versions differ")
+				errs = append(errs, errors.New("installed and Candidate versions differ"))
 			}
 		}
-	return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, false), err
+	}
+	if errs == nil {
+		return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.Output.ToLog), nil
+	}
+	return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.Output.ToLog), fmt.Errorf("error parsing package version output: %v", errs)
+}
+
+func getPackageIndexFromCommand(command *Command, name string) int {
+	for i, v := range command.Packages {
+		if name == v.Name {
+			return i
+		}
+	}
+
+	return -1
+}
+
+func getExternalConfigDirectiveValue(key string, opts *ConfigOpts, allowedDirectives AllowedExternalDirectives) string {
+	if !(strings.HasPrefix(key, externDirectiveStart) && strings.HasSuffix(key, externDirectiveEnd)) {
+		return key
+	}
+	key = replaceVarInString(opts.Vars, key, opts.Logger)
+	opts.Logger.Debug().Str("expanding external key", key).Send()
+
+	if strings.HasPrefix(key, envExternDirectiveStart) {
+		if IsExternalDirectiveEnv(allowedDirectives) {
+
+			key = strings.TrimPrefix(key, envExternDirectiveStart)
+			key = strings.TrimSuffix(key, externDirectiveEnd)
+			key = os.Getenv(key)
+		} else {
+			opts.Logger.Error().Msgf("Config key with value %s does not support env directive", key)
+		}
+	}
+
+	if strings.HasPrefix(key, externFileDirectiveStart) {
+		if IsExternalDirectiveFile(allowedDirectives) {
+
+			var err error
+			var keyValue []byte
+			key = strings.TrimPrefix(key, externFileDirectiveStart)
+			key = strings.TrimSuffix(key, externDirectiveEnd)
+			key, err = getFullPathWithHomeDir(key)
+			if err != nil {
+				opts.Logger.Err(err).Send()
+				return ""
+			}
+			if !path.IsAbs(key) {
+				key = path.Join(opts.ConfigDir, key)
+			}
+			keyValue, err = os.ReadFile(key)
+			if err != nil {
+				opts.Logger.Err(err).Send()
+				return ""
+			}
+			key = string(keyValue)
+		} else {
+			opts.Logger.Error().Msgf("Config key with value %s does not support file directive", key)
+		}
+	}
+
+	if strings.HasPrefix(key, vaultExternDirectiveStart) {
+		if IsExternalDirectiveVault(allowedDirectives) {
+
+			key = strings.TrimPrefix(key, vaultExternDirectiveStart)
+			key = strings.TrimSuffix(key, externDirectiveEnd)
+			key = GetVaultKey(key, opts, opts.Logger)
+		} else {
+			opts.Logger.Error().Msgf("Config key with value %s does not support vault directive", key)
+		}
+	}
+
+	return key
+}
+
+func getVaultSecret(vaultClient *vault.Client, key *VaultKey) (string, error) {
+	var (
+		secret *vault.KVSecret
+		err    error
+	)
+
+	if key.ValueType == "KVv2" {
+		secret, err = vaultClient.KVv2(key.MountPath).Get(context.Background(), key.Path)
+	} else if key.ValueType == "KVv1" {
+		secret, err = vaultClient.KVv1(key.MountPath).Get(context.Background(), key.Path)
+	} else if key.ValueType != "" {
+		return "", fmt.Errorf("type %s for key %s not known. Valid types are KVv1 or KVv2", key.ValueType, key.Name)
+	} else {
+		return "", fmt.Errorf("type for key %s must be specified. Valid types are KVv1 or KVv2", key.Name)
+	}
+	if err != nil {
+		return "", fmt.Errorf("unable to read secret: %v", err)
+	}
+
+	value, ok := secret.Data[key.Key].(string)
+	if !ok {
+		return "", fmt.Errorf("value type assertion failed for vault key %s: %T %#v", key.Name, secret.Data[key.Name], secret.Data[key.Name])
+	}
+
+	return value, nil
+}
+
+func getVaultKeyData(keyName string, keys []*VaultKey) (*VaultKey, error) {
+	for _, k := range keys {
+		if k.Name == keyName {
+			return k, nil
+		}
+	}
+	return nil, fmt.Errorf("key %s not found in vault keys", keyName)
+}
+
+func GetVaultKey(str string, opts *ConfigOpts, log zerolog.Logger) string {
+	key, err := getVaultKeyData(str, opts.VaultKeys)
+	if key == nil && err == nil {
+		return str
+	}
+	if err != nil && key == nil {
+		log.Err(err).Send()
+		return ""
+	}
+
+	value, secretErr := getVaultSecret(opts.vaultClient, key)
+	if secretErr != nil {
+		log.Err(secretErr).Send()
+		return value
+	}
+	return value
+}
+
+func IsExternalDirectiveFile(allowedExternalDirectives AllowedExternalDirectives) bool {
+	return strings.Contains(allowedExternalDirectives.String(), "file")
+}
+
+func IsExternalDirectiveEnv(allowedExternalDirectives AllowedExternalDirectives) bool {
+	return strings.Contains(allowedExternalDirectives.String(), "env")
+}
+
+func IsExternalDirectiveVault(allowedExternalDirectives AllowedExternalDirectives) bool {
+	return strings.Contains(allowedExternalDirectives.String(), "vault")
 }

pkg/pkgman/apt/apt.go

@@ -1,18 +1,19 @@
 package apt
 
 import (
+	"bufio"
+	"bytes"
 	"fmt"
-	"regexp"
 	"strings"
 
-	"git.andrewnw.xyz/CyberShell/backy/pkg/pkgman/pkgcommon"
+	packagemanagercommon "git.andrewnw.xyz/CyberShell/backy/pkg/pkgman/common"
 )
 
 // AptManager implements PackageManager for systems using APT.
 type AptManager struct {
 	useAuth     bool   // Whether to use an authentication command
 	authCommand string // The authentication command, e.g., "sudo"
-	Parser      pkgcommon.PackageParser
+	Parser      packagemanagercommon.PackageParser
 }
 
 // DefaultAuthCommand is the default command used for authentication.
@@ -29,14 +30,13 @@ func NewAptManager() *AptManager {
 }
 
 // Install returns the command and arguments for installing a package.
-func (a *AptManager) Install(pkg, version string, args []string) (string, []string) {
+func (a *AptManager) Install(pkgs []packagemanagercommon.Package, args []string) (string, []string) {
 	baseCmd := a.prependAuthCommand(DefaultPackageCommand)
 	baseArgs := []string{"update", "&&", baseCmd, "install", "-y"}
-	if version != "" {
+	for _, p := range pkgs {
-		baseArgs = append(baseArgs, fmt.Sprintf("%s=%s", pkg, version))
+		baseArgs = append(baseArgs, p.Name)
-	} else {
-		baseArgs = append(baseArgs, pkg)
 	}
+
 	if args != nil {
 		baseArgs = append(baseArgs, args...)
 	}
@@ -44,31 +44,34 @@ func (a *AptManager) Install(pkg, version string, args []string) (string, []stri
 }
 
 // Remove returns the command and arguments for removing a package.
-func (a *AptManager) Remove(pkg string, args []string) (string, []string) {
+func (a *AptManager) Remove(pkgs []packagemanagercommon.Package, args []string) (string, []string) {
 	baseCmd := a.prependAuthCommand(DefaultPackageCommand)
-	baseArgs := []string{"remove", "-y", pkg}
+	baseArgs := []string{"remove", "-y"}
+	for _, p := range pkgs {
+		baseArgs = append(baseArgs, p.Name)
+	}
 	if args != nil {
 		baseArgs = append(baseArgs, args...)
 	}
 	return baseCmd, baseArgs
 }
 
-// Upgrade returns the command and arguments for upgrading a specific package.
+func (a *AptManager) Upgrade(pkgs []packagemanagercommon.Package) (string, []string) {
-func (a *AptManager) Upgrade(pkg, version string) (string, []string) {
 	baseCmd := a.prependAuthCommand(DefaultPackageCommand)
 	baseArgs := []string{"update", "&&", baseCmd, "install", "--only-upgrade", "-y"}
-	if version != "" {
+	for _, p := range pkgs {
-		baseArgs = append(baseArgs, fmt.Sprintf("%s=%s", pkg, version))
+		baseArgs = append(baseArgs, p.Name)
-	} else {
-		baseArgs = append(baseArgs, pkg)
 	}
+
 	return baseCmd, baseArgs
 }
 
-// CheckVersion returns the command and arguments for checking the info of a specific package.
+func (a *AptManager) CheckVersion(pkgs []packagemanagercommon.Package) (string, []string) {
-func (a *AptManager) CheckVersion(pkg, version string) (string, []string) {
 	baseCmd := a.prependAuthCommand("apt-cache")
-	baseArgs := []string{"policy", pkg}
+	baseArgs := []string{"policy"}
+	for _, p := range pkgs {
+		baseArgs = append(baseArgs, p.Name)
+	}
 
 	return baseCmd, baseArgs
 }
@@ -81,7 +84,7 @@ func (a *AptManager) UpgradeAll() (string, []string) {
 }
 
 // Configure applies functional options to customize the package manager.
-func (a *AptManager) Configure(options ...pkgcommon.PackageManagerOption) {
+func (a *AptManager) Configure(options ...packagemanagercommon.PackageManagerOption) {
 	for _, opt := range options {
 		opt(a)
 	}
@@ -106,25 +109,64 @@ func (a *AptManager) SetAuthCommand(authCommand string) {
 }
 
 // Parse parses the apt-cache policy output to extract Installed and Candidate versions.
-func (a *AptManager) Parse(output string) (*pkgcommon.PackageVersion, error) {
+func (a *AptManager) ParseRemotePackageManagerVersionOutput(output string) ([]packagemanagercommon.Package, error) {
+	var (
+		packageName        string
+		installedString    string
+		candidateString    string
+		countRelevantLines int
+	)
 	// Check for error message in the output
 	if strings.Contains(output, "Unable to locate package") {
 		return nil, fmt.Errorf("error: %s", strings.TrimSpace(output))
 	}
-
+	packages := []packagemanagercommon.Package{}
-	reInstalled := regexp.MustCompile(`Installed:\s*([^\s]+)`)
+	outputBuf := bytes.NewBufferString(output)
-	reCandidate := regexp.MustCompile(`Candidate:\s*([^\s]+)`)
+	outputScan := bufio.NewScanner(outputBuf)
-
+	var packageCount uint
-	installedMatch := reInstalled.FindStringSubmatch(output)
+	for outputScan.Scan() {
-	candidateMatch := reCandidate.FindStringSubmatch(output)
+		line := outputScan.Text()
-
+		if !strings.HasPrefix(line, "  ") && strings.HasSuffix(line, ":") {
-	if len(installedMatch) < 2 || len(candidateMatch) < 2 {
+			// count++
-		return nil, fmt.Errorf("failed to parse Installed or Candidate versions from apt output. check package name")
+			packageName = strings.TrimSpace(strings.TrimSuffix(line, ":"))
+		}
+		if strings.Contains(line, "Installed:") {
+			countRelevantLines++
+			installedString = strings.TrimPrefix(strings.TrimSpace(line), "Installed:")
 		}
 
-	return &pkgcommon.PackageVersion{
+		if strings.Contains(line, "Candidate:") {
-		Installed: strings.TrimSpace(installedMatch[1]),
+			countRelevantLines++
-		Candidate: strings.TrimSpace(candidateMatch[1]),
+			candidateString = strings.TrimPrefix(strings.TrimSpace(line), "Candidate:")
-		Match:     installedMatch[1] == candidateMatch[1],
+		}
-	}, nil
+
+		if countRelevantLines == 2 {
+			countRelevantLines = 0
+			if !strings.Contains(installedString, " (none)") {
+				packageCount++
+				packages = append(packages, packagemanagercommon.Package{
+					Name: packageName,
+					VersionCheck: packagemanagercommon.PackageVersion{
+						Installed: strings.TrimSpace(installedString),
+						Candidate: strings.TrimSpace(candidateString),
+						Match:     installedString == candidateString,
+					}},
+				)
+			}
+		}
+	}
+
+	if packageCount == 0 {
+		return nil, fmt.Errorf("no packages found")
+	}
+
+	return packages, nil
+}
+
+func SearchPackages(pkgs []string, version string) (string, []string) {
+	baseCommand := "dpkg-query"
+	baseArgs := []string{"-W", "-f='${Package}\t${Architecture}\t${db:Status-Status}\t${Version}\t${Installed-Size}\t${Binary:summary}\n'"}
+	baseArgs = append(baseArgs, pkgs...)
+
+	return baseCommand, baseArgs
 }

pkg/pkgman/common/options.go

@@ -1,4 +1,4 @@
-package pkgcommon
+package packagemanagercommon
 
 // PackageManagerOption defines a functional option for configuring a PackageManager.
 type PackageManagerOption func(interface{})
@@ -15,3 +15,9 @@ type PackageVersion struct {
 	Match     bool
 	Message   string
 }
+
+type Package struct {
+	Name         string `yaml:"name"`
+	Version      string `yaml:"version,omitempty"`
+	VersionCheck PackageVersion
+}

pkg/pkgman/dnf/dnf.go

@@ -5,7 +5,7 @@ import (
 	"regexp"
 	"strings"
 
-	"git.andrewnw.xyz/CyberShell/backy/pkg/pkgman/pkgcommon"
+	packagemanagercommon "git.andrewnw.xyz/CyberShell/backy/pkg/pkgman/common"
 )
 
 // DnfManager implements PackageManager for systems using YUM.
@@ -26,21 +26,21 @@ func NewDnfManager() *DnfManager {
 }
 
 // Configure applies functional options to customize the package manager.
-func (y *DnfManager) Configure(options ...pkgcommon.PackageManagerOption) {
+func (y *DnfManager) Configure(options ...packagemanagercommon.PackageManagerOption) {
 	for _, opt := range options {
 		opt(y)
 	}
 }
 
 // Install returns the command and arguments for installing a package.
-func (y *DnfManager) Install(pkg, version string, args []string) (string, []string) {
+func (y *DnfManager) Install(pkgs []packagemanagercommon.Package, args []string) (string, []string) {
 	baseCmd := y.prependAuthCommand("dnf")
 	baseArgs := []string{"install", "-y"}
-	if version != "" {
+
-		baseArgs = append(baseArgs, fmt.Sprintf("%s-%s", pkg, version))
+	for _, p := range pkgs {
-	} else {
+		baseArgs = append(baseArgs, p.Name)
-		baseArgs = append(baseArgs, pkg)
 	}
+
 	if args != nil {
 		baseArgs = append(baseArgs, args...)
 	}
@@ -48,9 +48,13 @@ func (y *DnfManager) Install(pkg, version string, args []string) (string, []stri
 }
 
 // Remove returns the command and arguments for removing a package.
-func (y *DnfManager) Remove(pkg string, args []string) (string, []string) {
+func (y *DnfManager) Remove(pkgs []packagemanagercommon.Package, args []string) (string, []string) {
 	baseCmd := y.prependAuthCommand("dnf")
-	baseArgs := []string{"remove", "-y", pkg}
+	baseArgs := []string{"remove", "-y"}
+	for _, p := range pkgs {
+		baseArgs = append(baseArgs, p.Name)
+	}
+
 	if args != nil {
 		baseArgs = append(baseArgs, args...)
 	}
@@ -58,34 +62,37 @@ func (y *DnfManager) Remove(pkg string, args []string) (string, []string) {
 }
 
 // Upgrade returns the command and arguments for upgrading a specific package.
-func (y *DnfManager) Upgrade(pkg, version string) (string, []string) {
+func (y *DnfManager) Upgrade(pkgs []packagemanagercommon.Package) (string, []string) {
 	baseCmd := y.prependAuthCommand("dnf")
 	baseArgs := []string{"update", "-y"}
-	if version != "" {
+
-		baseArgs = append(baseArgs, fmt.Sprintf("%s-%s", pkg, version))
+	for _, p := range pkgs {
-	} else {
+		baseArgs = append(baseArgs, p.Name)
-		baseArgs = append(baseArgs, pkg)
 	}
+
 	return baseCmd, baseArgs
 }
 
 // UpgradeAll returns the command and arguments for upgrading all packages.
 func (y *DnfManager) UpgradeAll() (string, []string) {
 	baseCmd := y.prependAuthCommand("dnf")
-	baseArgs := []string{"update", "-y"}
+	baseArgs := []string{"upgrade", "-y"}
 	return baseCmd, baseArgs
 }
 
 // CheckVersion returns the command and arguments for checking the info of a specific package.
-func (d *DnfManager) CheckVersion(pkg, version string) (string, []string) {
+func (d *DnfManager) CheckVersion(pkgs []packagemanagercommon.Package) (string, []string) {
 	baseCmd := d.prependAuthCommand("dnf")
-	baseArgs := []string{"info", pkg}
+	baseArgs := []string{"info"}
+	for _, p := range pkgs {
+		baseArgs = append(baseArgs, p.Name)
+	}
 
 	return baseCmd, baseArgs
 }
 
 // Parse parses the dnf info output to extract Installed and Candidate versions.
-func (d DnfManager) Parse(output string) (*pkgcommon.PackageVersion, error) {
+func (d DnfManager) ParseRemotePackageManagerVersionOutput(output string) ([]packagemanagercommon.Package, error) {
 
 	// Check for error message in the output
 	if strings.Contains(output, "No matching packages to list") {
@@ -114,10 +121,7 @@ func (d DnfManager) Parse(output string) (*pkgcommon.PackageVersion, error) {
 		return nil, fmt.Errorf("failed to parse versions from dnf output")
 	}
 
-	return &pkgcommon.PackageVersion{
+	return nil, nil
-		Installed: installedVersion,
-		Candidate: candidateVersion,
-	}, nil
 }
 
 // prependAuthCommand prepends the authentication command if UseAuth is true.

pkg/pkgman/pkgman.go

@@ -4,26 +4,26 @@ import (
 	"fmt"
 
 	"git.andrewnw.xyz/CyberShell/backy/pkg/pkgman/apt"
+	packagemanagercommon "git.andrewnw.xyz/CyberShell/backy/pkg/pkgman/common"
 	"git.andrewnw.xyz/CyberShell/backy/pkg/pkgman/dnf"
-	"git.andrewnw.xyz/CyberShell/backy/pkg/pkgman/pkgcommon"
 	"git.andrewnw.xyz/CyberShell/backy/pkg/pkgman/yum"
 )
 
 // PackageManager is an interface used to define common package commands. This shall be implemented by every package.
 type PackageManager interface {
-	Install(pkg, version string, args []string) (string, []string)
+	Install(pkgs []packagemanagercommon.Package, args []string) (string, []string)
-	Remove(pkg string, args []string) (string, []string)
+	Remove(pkgs []packagemanagercommon.Package, args []string) (string, []string)
-	Upgrade(pkg, version string) (string, []string) // Upgrade a specific package
+	Upgrade(pkgs []packagemanagercommon.Package) (string, []string) // Upgrade a specific package
 	UpgradeAll() (string, []string)
-	CheckVersion(pkg, version string) (string, []string)
+	CheckVersion(pkgs []packagemanagercommon.Package) (string, []string)
-	Parse(output string) (*pkgcommon.PackageVersion, error)
+	ParseRemotePackageManagerVersionOutput(output string) ([]packagemanagercommon.Package, error)
 	// Configure applies functional options to customize the package manager.
-	Configure(options ...pkgcommon.PackageManagerOption)
+	Configure(options ...packagemanagercommon.PackageManagerOption)
 }
 
 // PackageManagerFactory returns the appropriate PackageManager based on the package tool.
 // Takes variable number of options.
-func PackageManagerFactory(managerType string, options ...pkgcommon.PackageManagerOption) (PackageManager, error) {
+func PackageManagerFactory(managerType string, options ...packagemanagercommon.PackageManagerOption) (PackageManager, error) {
 	var manager PackageManager
 
 	switch managerType {
@@ -43,7 +43,7 @@ func PackageManagerFactory(managerType string, options ...pkgcommon.PackageManag
 }
 
 // WithAuth enables authentication and sets the authentication command.
-func WithAuth(authCommand string) pkgcommon.PackageManagerOption {
+func WithAuth(authCommand string) packagemanagercommon.PackageManagerOption {
 	return func(manager interface{}) {
 		if configurable, ok := manager.(interface {
 			SetUseAuth(bool)
@@ -56,7 +56,7 @@ func WithAuth(authCommand string) pkgcommon.PackageManagerOption {
 }
 
 // WithoutAuth disables authentication.
-func WithoutAuth() pkgcommon.PackageManagerOption {
+func WithoutAuth() packagemanagercommon.PackageManagerOption {
 	return func(manager interface{}) {
 		if configurable, ok := manager.(interface {
 			SetUseAuth(bool)
@@ -68,8 +68,8 @@ func WithoutAuth() pkgcommon.PackageManagerOption {
 
 // ConfigurablePackageManager defines methods for setting configuration options.
 type ConfigurablePackageManager interface {
-	pkgcommon.PackageParser
+	packagemanagercommon.PackageParser
 	SetUseAuth(useAuth bool)
 	SetAuthCommand(authCommand string)
-	SetPackageParser(parser pkgcommon.PackageParser)
+	SetPackageParser(parser packagemanagercommon.PackageParser)
 }

pkg/pkgman/yum/yum.go

@@ -3,8 +3,9 @@ package yum
 import (
 	"fmt"
 	"regexp"
+	"strings"
 
-	"git.andrewnw.xyz/CyberShell/backy/pkg/pkgman/pkgcommon"
+	packagemanagercommon "git.andrewnw.xyz/CyberShell/backy/pkg/pkgman/common"
 )
 
 // YumManager implements PackageManager for systems using YUM.
@@ -25,21 +26,20 @@ func NewYumManager() *YumManager {
 }
 
 // Configure applies functional options to customize the package manager.
-func (y *YumManager) Configure(options ...pkgcommon.PackageManagerOption) {
+func (y *YumManager) Configure(options ...packagemanagercommon.PackageManagerOption) {
 	for _, opt := range options {
 		opt(y)
 	}
 }
 
 // Install returns the command and arguments for installing a package.
-func (y *YumManager) Install(pkg, version string, args []string) (string, []string) {
+func (y *YumManager) Install(pkgs []packagemanagercommon.Package, args []string) (string, []string) {
 	baseCmd := y.prependAuthCommand("yum")
 	baseArgs := []string{"install", "-y"}
-	if version != "" {
+	for _, p := range pkgs {
-		baseArgs = append(baseArgs, fmt.Sprintf("%s-%s", pkg, version))
+		baseArgs = append(baseArgs, p.Name)
-	} else {
-		baseArgs = append(baseArgs, pkg)
 	}
+
 	if args != nil {
 		baseArgs = append(baseArgs, args...)
 	}
@@ -47,9 +47,13 @@ func (y *YumManager) Install(pkg, version string, args []string) (string, []stri
 }
 
 // Remove returns the command and arguments for removing a package.
-func (y *YumManager) Remove(pkg string, args []string) (string, []string) {
+func (y *YumManager) Remove(pkgs []packagemanagercommon.Package, args []string) (string, []string) {
 	baseCmd := y.prependAuthCommand("yum")
-	baseArgs := []string{"remove", "-y", pkg}
+	baseArgs := []string{"remove", "-y"}
+	for _, p := range pkgs {
+		baseArgs = append(baseArgs, p.Name)
+	}
+
 	if args != nil {
 		baseArgs = append(baseArgs, args...)
 	}
@@ -57,14 +61,13 @@ func (y *YumManager) Remove(pkg string, args []string) (string, []string) {
 }
 
 // Upgrade returns the command and arguments for upgrading a specific package.
-func (y *YumManager) Upgrade(pkg, version string) (string, []string) {
+func (y *YumManager) Upgrade(pkgs []packagemanagercommon.Package) (string, []string) {
 	baseCmd := y.prependAuthCommand("yum")
 	baseArgs := []string{"update", "-y"}
-	if version != "" {
+	for _, p := range pkgs {
-		baseArgs = append(baseArgs, fmt.Sprintf("%s-%s", pkg, version))
+		baseArgs = append(baseArgs, p.Name)
-	} else {
-		baseArgs = append(baseArgs, pkg)
 	}
+
 	return baseCmd, baseArgs
 }
 
@@ -76,17 +79,27 @@ func (y *YumManager) UpgradeAll() (string, []string) {
 }
 
 // CheckVersion returns the command and arguments for checking the info of a specific package.
-func (y *YumManager) CheckVersion(pkg, version string) (string, []string) {
+func (y *YumManager) CheckVersion(pkgs []packagemanagercommon.Package) (string, []string) {
 	baseCmd := y.prependAuthCommand("yum")
-	baseArgs := []string{"info", pkg}
+	baseArgs := []string{"info"}
+	for _, p := range pkgs {
+		baseArgs = append(baseArgs, p.Name)
+	}
 
 	return baseCmd, baseArgs
 }
 
 // Parse parses the dnf info output to extract Installed and Candidate versions.
-func (y YumManager) Parse(output string) (*pkgcommon.PackageVersion, error) {
+func (y YumManager) ParseRemotePackageManagerVersionOutput(output string) ([]packagemanagercommon.Package, error) {
-	reInstalled := regexp.MustCompile(`(?m)^Installed Packages\s*Name\s*:\s*\S+\s*Version\s*:\s*([^\s]+)\s*Release\s*:\s*([^\s]+)`)
+
-	reAvailable := regexp.MustCompile(`(?m)^Available Packages\s*Name\s*:\s*\S+\s*Version\s*:\s*([^\s]+)\s*Release\s*:\s*([^\s]+)`)
+	// Check for error message in the output
+	if strings.Contains(output, "No matching packages to list") {
+		return nil, fmt.Errorf("error: package not listed")
+	}
+
+	// Define regular expressions to capture installed and available versions
+	reInstalled := regexp.MustCompile(`(?m)^Installed packages\s*Name\s*:\s*\S+\s*Epoch\s*:\s*\S+\s*Version\s*:\s*([^\s]+)\s*Release\s*:\s*([^\s]+)`)
+	reAvailable := regexp.MustCompile(`(?m)^Available packages\s*Name\s*:\s*\S+\s*Epoch\s*:\s*\S+\s*Version\s*:\s*([^\s]+)\s*Release\s*:\s*([^\s]+)`)
 
 	installedMatch := reInstalled.FindStringSubmatch(output)
 	candidateMatch := reAvailable.FindStringSubmatch(output)
@@ -106,10 +119,7 @@ func (y YumManager) Parse(output string) (*pkgcommon.PackageVersion, error) {
 		return nil, fmt.Errorf("failed to parse versions from dnf output")
 	}
 
-	return &pkgcommon.PackageVersion{
+	return nil, nil
-		Installed: installedVersion,
-		Candidate: candidateVersion,
-	}, nil
 }
 
 // prependAuthCommand prepends the authentication command if UseAuth is true.

pkg/remotefetcher/cache.go

@@ -116,7 +116,7 @@ func (c *Cache) Set(source, hash string, data []byte, dataType string) (CacheDat
 	path := filepath.Join(c.dir, fmt.Sprintf("%s-%s", fileName, sourceHash))
 
 	if _, err := os.Stat(path); os.IsNotExist(err) {
-		os.MkdirAll(c.dir, 0700)
+		_ = os.MkdirAll(c.dir, 0700)
 	}
 
 	err := os.WriteFile(path, data, 0644)
@@ -171,7 +171,7 @@ func (cf *CachedFetcher) Hash(data []byte) string {
 func LoadMetadataFromFile(filePath string) ([]*CacheData, error) {
 	if _, err := os.Stat(filePath); os.IsNotExist(err) {
 		// Create the file if it does not exist
-		os.MkdirAll(path.Dir(filePath), 0700)
+		_ = os.MkdirAll(path.Dir(filePath), 0700)
 		emptyData := []byte("[]")
 		err := os.WriteFile(filePath, emptyData, 0644)
 		if err != nil {

pkg/remotefetcher/configfetcher.go

@@ -12,7 +12,7 @@ type RemoteFetcher interface {
 
 	// Parse decodes the raw data into a Go structure (e.g., Commands, CommandLists)
 	// Takes the raw data as input and populates the target interface
-	Parse(data []byte, target interface{}) error
+	Parse(data []byte, target any) error
 
 	// Hash returns the hash of the configuration data
 	Hash(data []byte) string
@@ -29,7 +29,7 @@ func NewRemoteFetcher(source string, cache *Cache, options ...FetcherOption) (Re
 		option(&config)
 	}
 
-	// If FileType is empty (i.e. WithFileType was not called), yaml is the default file type
+	// WithFileType was not called. yaml is the default file type
 	if strings.TrimSpace(config.FileType) == "" {
 		config.FileType = "yaml"
 	}

pkg/remotefetcher/local.go

@@ -20,7 +20,7 @@ func (l *LocalFetcher) Fetch(source string) ([]byte, error) {
 		if l.config.IgnoreFileNotFound {
 			return nil, ErrIgnoreFileNotFound
 		}
-		return nil, nil
+		return nil, err
 	}
 	file, err := os.Open(source)
 	if err != nil {

pkg/usermanager/linux/linux.go

@@ -15,7 +15,7 @@ func (l LinuxUserManager) NewLinuxManager() *LinuxUserManager {
 }
 
 // AddUser adds a new user to the system.
-func (l LinuxUserManager) AddUser(username, homeDir, shell string, isSystem bool, groups, args []string) (string, []string) {
+func (l LinuxUserManager) AddUser(username, homeDir, shell string, isSystem, createHome bool, groups, args []string) (string, []string) {
 	baseArgs := []string{}
 
 	if isSystem {
@@ -38,6 +38,10 @@ func (l LinuxUserManager) AddUser(username, homeDir, shell string, isSystem bool
 		baseArgs = append(baseArgs, args...)
 	}
 
+	if createHome {
+		baseArgs = append(baseArgs, "-m")
+	}
+
 	args = append(baseArgs, username)
 
 	cmd := "useradd"

pkg/usermanager/userman.go

@@ -10,7 +10,7 @@ import (
 // UserManager defines the interface for user management operations.
 // All functions but one return a string for the command and any args.
 type UserManager interface {
-	AddUser(username, homeDir, shell string, isSystem bool, groups, args []string) (string, []string)
+	AddUser(username, homeDir, shell string, createHome, isSystem bool, groups, args []string) (string, []string)
 	RemoveUser(username string) (string, []string)
 	ModifyUser(username, homeDir, shell string, groups []string) (string, []string)
 	// Modify password uses chpasswd for Linux systems to build the command to change the password

release

@@ -1,5 +1,6 @@
 #!/bin/bash
 set -eou pipefail
+go mod tidy
 go generate ./...
 CURRENT_TAG="$(go run backy.go version -V)"
 goreleaser -f .goreleaser/github.yml check

tests/.gitignore

@@ -0,0 +1 @@
+private

tests/ErrorHook.yml

@@ -0,0 +1,31 @@
+commands:
+  echoTestFail:
+    cmd: ech
+    shell: bash
+    Args: 
+      - hello world
+    hooks:
+      error:
+        - errorCmd
+      final:
+        - finalCmd
+  
+  finalCmd:
+    cmd: echo
+    Args:
+      - "echo test fail"
+
+  errorCmd:
+    name: get docker version
+    cmd: docker
+    getOutput: true
+    outputToLog: true
+    Args:
+      - "-v"
+
+cmdLists:
+  TestHooks:
+    output:
+      onFailure: true
+    order:
+      - echoTestFail

tests/HookNotInFile.yaml

@@ -0,0 +1,14 @@
+commands:
+  echoTestFail:
+    cmd: ech
+    shell: bash
+    Args: hello world
+    hooks:
+      error:
+        - errorCm # 
+
+  errorCmd:
+    name: get docker version
+    cmd: docker
+    Args:
+      - "-v"

tests/SuccessHook.yml

@@ -0,0 +1,16 @@
+commands:
+  echoTestSuccess:
+    cmd: echo
+    shell: bash
+    Args: hello world
+    hooks:
+      success:
+        - successCmd
+
+  errorCmd:
+    name: get docker version
+    cmd: docker
+    getOutput: true
+    outputToLog: true
+    Args:
+      - "-v"

tests/VaultTest.yml

@@ -0,0 +1,27 @@
+commands:
+  vaultEnvVar:
+    cmd: echo
+    shell: /bin/zsh
+    Args:
+      - ${VAULT_VAR}
+    environment:
+      "VAULT_VAR=%{vault:vaultTestSecret}%"
+
+logging:
+  verbose: true
+
+vault:
+  token: root
+  address: http://127.0.0.1:8200
+  enabled: true
+  keys:
+    - name: vaultTestSecret
+      key: data
+      mountpath: secret
+      path: test/var
+      type: KVv2 # KVv1 or KVv2
+
+cmdLists:
+  addUsers:
+    order:
+      - vaultEnvVar

tests/backy.yaml

@@ -1,18 +0,0 @@
-commands:
-  echoTestPass:
-    cmd: echo
-    shell: bash
-    Args: hello world
-
-  runRemoteShellScriptSuccess:
-    cmd: 
-
-
-  packageCommandSuccess:
-    packageName: docker-ce
-    Args:
-      - docker-ce-cli
-    packageManager: apt
-    packageOperation: install
-
-  

tests/docker/Dockerfile

@@ -0,0 +1,26 @@
+FROM ubuntu:latest
+
+# Install SSH server
+RUN apt-get update && \
+    apt-get install -y openssh-server && \
+    apt-get clean
+
+RUN sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config
+
+RUN sed -i 's/#PasswordAuthentication yes/PasswordAuthentication yes/' /etc/ssh/sshd_config
+
+RUN useradd -m -s /bin/bash backy
+RUN echo "backy:backy" | chpasswd
+RUN echo "root:test" | chpasswd
+COPY --chown=backy:backy backytest.pub /home/backy/.ssh/authorized_keys
+COPY --chown=root:root backytest.pub /root/.ssh/authorized_keys
+
+EXPOSE 22
+RUN mkdir /var/run/sshd
+RUN chmod 0755 /var/run/sshd
+
+RUN apt-get update && apt-get install -y curl
+
+# Start SSH service
+CMD ["/usr/sbin/sshd", "-D"]
+# ENTRYPOINT service ssh start && bash

tests/docker/backytest

@@ -0,0 +1,7 @@
+-----BEGIN OPENSSH PRIVATE KEY-----
+b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAAAMwAAAAtzc2gtZW
+QyNTUxOQAAACBtBASN+HMx/EVUIs5ThK9Nlw0wPFVt2rXsiNZlDN+CswAAAKAfFc5AHxXO
+QAAAAAtzc2gtZWQyNTUxOQAAACBtBASN+HMx/EVUIs5ThK9Nlw0wPFVt2rXsiNZlDN+Csw
+AAAEAxs6uRkenVbXPrjgbIv/1THXL6dUdgr5KaCd7uBVm0PW0EBI34czH8RVQizlOEr02X
+DTA8VW3ateyI1mUM34KzAAAAGU1lZGlhIHVzZXIgc3RvcmFnZSBzZXJ2ZXIBAgME
+-----END OPENSSH PRIVATE KEY-----

tests/docker/backytest.pub

@@ -0,0 +1 @@
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIG0EBI34czH8RVQizlOEr02XDTA8VW3ateyI1mUM34Kz Backy test

tests/docker/buildDocker.sh

@@ -0,0 +1,4 @@
+cd ~/Projects/backy/tests/docker
+docker container rm -f ssh_server_container
+docker build -t ssh_server_image .
+docker run -d -p 2222:22 --name ssh_server_container ssh_server_image

tests/hosts.yml

@@ -0,0 +1,6 @@
+hosts:
+  docker:
+    port: 2222
+    Hostname: localhost
+    user: root
+    IdentityFile: ./docker/backytest

tests/packageCommands.yml

@@ -0,0 +1,99 @@
+commands:
+  checkDockerNoVersion:
+    type: package
+    shell: zsh
+    packages: 
+      - name: "docker-ce-cli"
+      - name: "docker-ce"
+    packageManager: apt
+    packageOperation: checkVersion
+
+  checkAptNoVersion:
+    type: package
+    shell: zsh
+    packages: 
+      - name: "apt"
+    packageManager: apt
+    packageOperation: checkVersion
+
+  checkDockerPartialVersionWithoutRegex:
+    type: package
+    shell: zsh
+    packages: 
+      - name: "docker-ce-cli"
+        version: "5:28.0.4-1"
+      - name: "docker-ce"
+        version: "5:28.0.4-1"
+    packageManager: apt
+    packageOperation: checkVersion
+
+  checkDockerPartialVersionWithRegex:
+    type: package
+    shell: zsh
+    packages: 
+      - name: "docker-ce-cli"
+        version: "5:28.0-*"
+      - name: "docker-ce"
+        version: '5:28\.0\.4-1~([A-Za-z0-9]+(\.[A-Za-z0-9]+)+)*'
+    packageManager: apt
+    packageOperation: checkVersion
+    
+  installJq:
+    type: package
+    shell: bash
+    packages: 
+      - name: "jq"
+    packageManager: apt
+    packageOperation: install
+    output:
+      toLog: true
+
+  testJq:
+    cmd: jq
+    shell: bash
+    Args: 
+      - '--version'
+    output:
+      toLog: true
+
+  checkDockerVersionWithInvalidRegex:
+    type: package
+    shell: zsh
+    packages: 
+      - name: "docker-ce-cli"
+        version: "5:28.0-**"
+      - name: "docker-ce"
+        version: '5:28\.0\K.4-1~([A-Za-z0-9]+(\.[A-Za-z0-9]+)+)*'
+    packageManager: apt
+    packageOperation: checkVersion
+
+  hostname:
+    shell: zsh
+    cmd: hostname
+    Args:
+      - '-f'
+    output:
+      toLog: true
+
+
+cmdLists:
+  packageCommands:
+    output:
+      onFailure: true
+    order:
+      - checkDockerPartialVersionWithoutRegex
+      - checkDockerPartialVersionWithRegex
+      - checkDockerVersionWithInvalidRegex
+      - checkDockerNoVersion
+      
+  aptCommands:
+    output:
+      onFailure: true
+    order:
+      - installJq
+      - checkDockerNoVersion
+      - hostname
+
+logging:
+  verbose: true
+  cmd-std-out: false

tests/packageCommandsDNF.yml

@@ -0,0 +1,52 @@
+commands:
+  checkDockerNoVersion:
+    type: package
+    shell: zsh
+    packages: 
+      - name: "docker-ce-cli"
+      - name: "docker-ce"
+    packageManager: dnf
+    packageOperation: checkVersion
+
+  checkDockerPartialVersionWithoutRegex:
+    type: package
+    shell: zsh
+    packages: 
+      - name: "docker-ce-cli"
+        version: "5:28.0.4-1"
+      - name: "docker-ce"
+        version: "5:28.0.4-1"
+    packageManager: dnf
+    packageOperation: checkVersion
+
+  checkDockerPartialVersionWithRegex:
+    type: package
+    shell: zsh
+    packages: 
+      - name: "docker-ce-cli"
+        version: "5:28.0-*"
+      - name: "docker-ce"
+        version: '5:28\.0\.4-1~([A-Za-z0-9]+(\.[A-Za-z0-9]+)+)*'
+    packageManager: dnf
+    packageOperation: checkVersion
+    
+  checkDockerVersionWithInvalidRegex:
+    type: package
+    shell: zsh
+    packages: 
+      - name: "docker-ce-cli"
+        version: "5:28.0-**"
+      - name: "docker-ce"
+        version: '5:28\.0\K.4-1~([A-Za-z0-9]+(\.[A-Za-z0-9]+)+)*'
+    packageManager: dnf
+    packageOperation: checkVersion
+
+cmdLists:
+  packageCommands:
+    output:
+      onFailure: true
+    order:
+      - checkDockerPartialVersionWithoutRegex
+      - checkDockerPartialVersionWithRegex
+      - checkDockerVersionWithInvalidRegex
+      - checkDockerNoVersion