version bump

.changes/unreleased/Added-20250301-132932.yaml

@@ -0,0 +1,3 @@
+kind: Added
+body: 'Hooks: improved logging when executing'
+time: 2025-03-01T13:29:32.195438013-06:00

.changes/unreleased/Added-20250303-234248.yaml

@@ -0,0 +1,3 @@
+kind: Added
+body: 'User commands: adding SSH keys using config key `userSshPubKeys`'
+time: 2025-03-03T23:42:48.009294808-06:00

.changes/unreleased/Added-20250303-234505.yaml

@@ -0,0 +1,3 @@
+kind: Added
+body: 'directives: added support for fetching values using directive `%{externalSource:key}%`'
+time: 2025-03-03T23:45:05.666939653-06:00

.changes/unreleased/Changed-20250301-194321.yaml

@@ -0,0 +1,3 @@
+kind: Changed
+body: 'Commands: if dir is not specified, run in config dir'
+time: 2025-03-01T19:43:21.323077376-06:00

.changes/unreleased/Changed-20250305-003415.yaml

@@ -0,0 +1,3 @@
+kind: Changed
+body: 'FileDirective: use the config directory if path is not absolute'
+time: 2025-03-05T00:34:15.689980075-06:00

.changes/unreleased/Changed-20250307-231946.yaml

@@ -0,0 +1,3 @@
+kind: Changed
+body: 'Host: changes to case of some keys'
+time: 2025-03-07T23:19:46.086408374-06:00

.changes/unreleased/Changed-20250308-001824.yaml

@@ -0,0 +1,3 @@
+kind: Changed
+body: 'Notifications: added external directive to sensitive keys'
+time: 2025-03-08T00:18:24.976897007-06:00

.changes/unreleased/Fixed-20250301-132600.yaml

@@ -0,0 +1,3 @@
+kind: Fixed
+body: 'LocalFetcher: return fetch error'
+time: 2025-03-01T13:26:00.330176712-06:00

.changes/unreleased/Fixed-20250301-132801.yaml

@@ -0,0 +1,3 @@
+kind: Fixed
+body: 'Lists: load file key before attempting to load from current file'
+time: 2025-03-01T13:28:01.739467944-06:00

.changes/unreleased/Fixed-20250301-182434.yaml

@@ -0,0 +1,3 @@
+kind: Fixed
+body: 'fix: host not in config file, but in ssh config, properly added to hosts struct'
+time: 2025-03-01T18:24:34.81395054-06:00

.changes/unreleased/Fixed-20250304-235706.yaml

@@ -0,0 +1,3 @@
+kind: Fixed
+body: 'SSH: password authentication bugs'
+time: 2025-03-04T23:57:06.326604774-06:00

.changes/unreleased/Fixed-20250306-233010.yaml

@@ -0,0 +1,3 @@
+kind: Fixed
+body: 'User commands: change user password works'
+time: 2025-03-06T23:30:10.791394853-06:00

backy.code-workspace

@@ -18,6 +18,7 @@
 			"maunium",
 			"mautrix",
 			"nikoksr",
+			"rawbytes",
 			"remotefetcher",
 			"Strs"
 		]

cmd/version.go

@@ -7,7 +7,7 @@ import (
 	"github.com/spf13/cobra"
 )
 
-const versionStr = "0.9.1"
+const versionStr = "0.10.0"
 
 var (
 	versionCmd = &cobra.Command{

docs/content/cli/exec.md

@@ -15,5 +15,5 @@ The `exec` subcommand can do some things that the configuration file can't do ye
 The commands have to be defined in the config file. The hosts need to at least be in the ssh_config(5) file.
 
 ```sh
-backy exec host [--commands=command1 -commands=command2 ... | -c command1 -c command2 ...] [--hosts=host1 --hosts=hosts2 ... | -m host1 -c host2 ...]  [flags]
+backy exec host [--commands=command1 -commands=command2 ... | -c command1 -c command2 ...] [--hosts=host1 --hosts=hosts2 ... | -m host1 -m host2 ...]  [flags]
 ```

docs/content/cli/list.md

@@ -0,0 +1,29 @@
+---
+title: List
+---
+
+
+List commands, lists, or hosts defined in config file
+
+Usage:
+```
+  backy list [command]
+```
+
+Available Commands:
+  cmds        List commands defined in config file.
+  lists       List lists defined in config file.
+
+Flags:
+```
+  -h, --help   help for list
+```
+
+Global Flags:
+```
+      --cmdStdOut            Pass to print command output to stdout
+  -f, --config string        config file to read from
+      --log-file string      log file to write to
+      --s3-endpoint string   Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable.
+  -v, --verbose              Sets verbose level
+```

docs/content/config/commands/_index.md

@@ -8,46 +8,21 @@ weight: 1
 
 ### Example Config
 
-```yaml
+{{% code file="/examples/example.yml" language="yaml" %}}
-commands:
-  stop-docker-container:
-    cmd: docker
-    Args:
-      - compose
-      - -f /some/path/to/docker-compose.yaml
-      - down
-    # if host is not defined, command will be run locally
-    # The host has to be defined in either the config file or the SSH Config files
-    host: some-host
-    hooks
-      error:
-        - some-other-command-when-failing
-      success:
-        - success-command
-      final:
-        - final-command
-  backup-docker-container-script:
-    cmd: /path/to/local/script
-    # script file is input as stdin to SSH
-    type: scriptFile # also can be script
-    environment:
-      - FOO=BAR
-      - APP=$VAR
-```
 
 Values available for this section **(case-sensitive)**:
 
-| name | notes | type | required
+| name            | notes                                                                                                   | type                  | required | External directive support |
-| --- | --- | --- | --- |
+| ----------------| ------------------------------------------------------------------------------------------------------- | --------------------- | -------- |----------------------------|
-| `cmd` | Defines the command to execute | `string` | yes |
+| `cmd`           | Defines the command to execute                                                                          | `string`              | yes      | No                         |
-| `Args` | Defines the arguments to the command | `[]string` | no |
+| `Args`          | Defines the arguments to the command                                                                    | `[]string`            | no       | No                         |
-| `environment` | Defines environment variables for the command | `[]string` | no |
+| `environment`   | Defines environment variables for the command                                                           | `[]string`            | no       | No                         |
-| `type` | See documentation further down the page. Additional fields may be required. | `string` | no |
+| `type`          | See documentation further down the page. Additional fields may be required.                             | `string`              | no       | No                         |
-| `getOutput` | Command(s) output is in the notification(s) | `bool` | no |
+| `getOutput`     | Command(s) output is in the notification(s)                                                             | `bool`                | no       | No                         |
-| `host` | If not specified, the command will execute locally. | `string` | no |
+| `host`          | If not specified, the command will execute locally.                                                     | `string`              | no       | No                         |
-| `scriptEnvFile` | When type is `scriptFile` or `script`, this file is prepended to the input. | `string` | no |
+| `scriptEnvFile` | When type is `scriptFile` or `script`, this file is prepended to the input.                             | `string`              | no       | No                         |
-| `shell` | Run the command in the shell | `string` | no |
+| `shell`         | Run the command in the shell                                                                            | `string`              | no       | No                         |
-| `hooks` | Hooks are used at the end of the individual command. Must have at least `error`, `success`, or `final`. | `map[string][]string` | no |
+| `hooks`         | Hooks are used at the end of the individual command. Must have at least `error`, `success`, or `final`. | `map[string][]string` | no       | No                         |
 
 #### cmd
 

docs/content/config/commands/user-commands.md

@@ -10,10 +10,12 @@ This is dedicated to `user` commands. The command `type` field must be `user`. U
 | --- | --- | --- | --- |
 | `userName` | The name of a user to be configured. | `string` | yes |
 | `userOperation` | The type of operation to perform. | `string` | yes |
-| `userID` | The user ID to use. | `string` | yes |
+| `userID` | The user ID to use. | `string` | no |
-| `userGroups` | The groups the user should be added to. | `[]string` | yes |
+| `userGroups` | The groups the user should be added to. | `[]string` | no |
-| `userShell` | The shell for the user. | `string` | yes |
+| `userSshPubKeys` | The keys to add to the user's authorized keys. | `[]string` | no |
+| `userShell` | The shell for the user. | `string` | no |
 | `userHome` | The user's home directory. | `string` | no |
+| `userPassword` | The new password value when using the `password` operation. Can be specified by using external directive. | `string` | no |
 
 
 #### example

docs/content/config/directives.md

@@ -0,0 +1,15 @@
+---
+title: "External Directives"
+weight: 2
+description: How to set up external directives.
+---
+
+External directives are for including data that should not be in the config file. The following directives are supported:
+
+- `%{file:path/to/file}%`
+- `%{env:ENV_VAR}%`
+- `%{vault:vault-key}%`
+
+See the docs of each command if the field is supported.
+
+If the file path does not begin with a `/`, the config file's directory will be used as the starting point.

docs/content/config/hosts.md

@@ -5,19 +5,19 @@ description: >
   This page tells you how to use hosts.
 ---
 
-| Key                  | Description                                                   | Type     | Required |
+| Key                  | Description                                                   | Type     | Required | External directive support |
-|----------------------|---------------------------------------------------------------|----------|----------|
+|----------------------|---------------------------------------------------------------|----------|----------|----------------------------|
-| `OS`                 | Operating system of the host (used for package commands)      | `string` | no       |
+| `OS`                 | Operating system of the host (used for package commands)      | `string` | no       | No                         |
-| `config`             | Path to the SSH config file                                   | `string` | no       |
+| `config`             | Path to the SSH config file                                   | `string` | no       | No                         |
-| `host`               | Specifies the `Host` ssh_config(5) directive                  | `string` | yes      |
+| `host`               | Specifies the `Host` ssh_config(5) directive                  | `string` | yes      | No                         |
-| `hostname`           | Hostname of the host                                          | `string` | no       |
+| `hostname`           | Hostname of the host                                          | `string` | no       | No                         |
-| `knownhostsfile`     | Path to the known hosts file                                  | `string` | no       |
+| `knownHostsFile`     | Path to the known hosts file                                  | `string` | no       | No                         |
-| `port`               | Port number to connect to                                     | `uint16` | no       |
+| `port`               | Port number to connect to                                     | `uint16` | no       | No                         |
-| `proxyjump`          | Proxy jump hosts, comma-separated                             | `string` | no       |
+| `proxyjump`          | Proxy jump hosts, comma-separated                             | `string` | no       | No                         |
-| `password`           | Password for SSH authentication                               | `string` | no       |
+| `password`           | Password for SSH authentication                               | `string` | no       | No                         |
-| `privatekeypath`     | Path to the private key file                                  | `string` | no       |
+| `privateKeyPath`     | Path to the private key file                                  | `string` | no       | No                         |
-| `privatekeypassword` | Password for the private key file                             | `string` | no       |
+| `privateKeyPassword` | Password for the private key file                             | `string` | no       | Yes                        |
-| `user`               | Username for SSH authentication                               | `string` | no       |
+| `user`               | Username for SSH authentication                               | `string` | no       | No                         |
 
 ## exec host subcommand
 

docs/content/config/notifications.md

@@ -39,23 +39,23 @@ There must be a section with an id (eg. `mail.test-svr`) following one of these
 
 ### mail
 
-| key | description | type
+| key | description | type | External directive support |
-| --- | --- | ---
+| --- | --- | --- | --- |
-| `host` | Specifies the SMTP host to connect to | `string`
+| `host` | Specifies the SMTP host to connect to | `string` | no
-| `port` | Specifies the SMTP port | `uint16`
+| `port` | Specifies the SMTP port | `uint16` | no
-| `senderaddress` | Address from which to send mail | `string`
+| `senderaddress` | Address from which to send mail | `string` | no
-| `to` | Recipients to send emails to | `[]string`
+| `to` | Recipients to send emails to | `[]string` | no
-| `username` | SMTP username | `string`
+| `username` | SMTP username | `string` | no
-| `password` | SMTP password | `string`
+| `password` | SMTP password | `string` | yes
 
 ### matrix
 
-| key | description | type
+| key | description | type | External directive support |
-| --- | --- | ---
+| --- | --- | ---| ---- |
-| `home-server` | Specifies the Matrix server connect to | `string`
+| `home-server` | Specifies the Matrix server connect to | `string` | no
-| `room-id` | Specifies the room ID of the room to send messages to | `string`
+| `room-id` | Specifies the room ID of the room to send messages to | `string` | no
-| `access-token` | Matrix access token | `string`
+| `access-token` | Matrix access token  | `string` | yes
-| `user-id` | Matrix user ID | `string`
+| `user-id` | Matrix user ID | `string` | no
 
 To get your access token (assumes you are using [Element](https://element.io/)) :
 

docs/content/examples/backy.yaml

@@ -0,0 +1,108 @@
+commands:
+  stop-docker-container:
+    cmd: docker
+    Args:
+      - compose
+      - -f /some/path/to/docker-compose.yaml
+      - down
+    # if host is not defined, cmd will be run locally
+    host: some-host
+    hooks:
+      final:
+        - hostname
+      error:
+        - hostname
+  backup-docker-container-script:
+    cmd: /path/to/script
+    # The host has to be defined in the config file
+    host: some-host
+    environment:
+      - FOO=BAR
+      - APP=$VAR
+  shell-cmd:
+    cmd: rsync
+    shell: bash
+    Args:
+      - -av some-host:/path/to/data ~/Docker/Backups/docker-data
+  hostname:
+    cmd: hostname
+  update-docker:
+    type: package
+    shell: zsh # best to run package commands in a shell
+    packageName: docker-ce
+    Args:
+      - docker-ce-cli
+    packageManager: apt
+    packageOperation: install
+  update-dockerApt:
+    # type: package
+    shell: zsh
+    cmd: apt
+    Args:
+      - update
+      - "&&"
+      - apt install -y docker-ce
+      - docker-ce-cli
+    packageManager: apt
+    packageOperation: install
+
+cmd-lists:
+  cmds-to-run: # this can be any name you want
+    # all commands have to be defined
+    order:
+      - stop-docker-container
+      - backup-docker-container-script
+      - shell-cmd
+      - hostname
+    notifications:
+      - matrix.matrix
+    name: backup-some-server
+    cron: "0 0 1 * * *"
+  hostname:
+    name: hostname
+    order:
+      - hostname
+    notifications:
+      - mail.prod-email
+
+hosts:
+  # any ssh_config(5) keys/values not listed here will be looked up in the config file or the default config file
+  some-host:
+    hostname: some-hostname
+    config: ~/.ssh/config
+    user: user
+    privateKeyPath: /path/to/private/key
+    port: 22
+    # can also be env:VAR
+    password: file:/path/to/file
+    # only one is supported for now
+    proxyjump: some-proxy-host
+
+# optional
+logging:
+  verbose: true
+  file: ./backy.log
+  console: false
+  cmd-std-out: false
+
+
+notifications:
+  mail:
+    prod-email:
+      id: prod-email
+      type: mail
+      host: yourhost.tld
+      port: 587
+      senderAddress: email@domain.tld
+      to:
+        - admin@domain.tld
+      username: smtp-username@domain.tld
+      password: your-password-here
+  matrix:
+    matrix:
+      id: matrix
+      type: matrix
+      home-server: your-home-server.tld
+      room-id: room-id
+      access-token: your-access-token
+      user-id: your-user-id

docs/content/examples/example.yml

@@ -0,0 +1,24 @@
+commands:
+  stop-docker-container:
+    cmd: docker
+    Args:
+      - compose
+      - -f /some/path/to/docker-compose.yaml
+      - down
+    # if host is not defined, command will be run locally
+    # The host has to be defined in either the config file or the SSH Config files
+    host: some-host
+    hooks:
+      error:
+        - some-other-command-when-failing
+      success:
+        - success-command
+      final:
+        - final-command
+  backup-docker-container-script:
+    cmd: /path/to/local/script
+    # script file is input as stdin to SSH
+    type: scriptFile # also can be script
+    environment:
+      - FOO=BAR
+      - APP=$VAR

docs/layouts/shortcodes/code.html

@@ -0,0 +1,3 @@
+{{ $file := .Get "file" | readFile }}
+{{ $lang := .Get "language" }}
+{{ (print "```" $lang "\n" $file "\n```") }}

examples/example.yml

@@ -0,0 +1,24 @@
+commands:
+  stop-docker-container:
+    cmd: docker
+    Args:
+      - compose
+      - -f /some/path/to/docker-compose.yaml
+      - down
+    # if host is not defined, command will be run locally
+    # The host has to be defined in either the config file or the SSH Config files
+    host: some-host
+    hooks:
+      error:
+        - some-other-command-when-failing
+      success:
+        - success-command
+      final:
+        - final-command
+  backup-docker-container-script:
+    cmd: /path/to/local/script
+    # script file is input as stdin to SSH
+    type: scriptFile # also can be script
+    environment:
+      - FOO=BAR
+      - APP=$VAR

go.mod

@@ -21,6 +21,7 @@ require (
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/nikoksr/notify v1.3.0
 	github.com/pkg/errors v0.9.1
+	github.com/pkg/sftp v1.13.7
 	github.com/rs/zerolog v1.33.0
 	github.com/sethvargo/go-password v0.3.1
 	github.com/spf13/cobra v1.8.1
@@ -65,6 +66,7 @@ require (
 	github.com/klauspost/compress v1.17.11 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.9 // indirect
 	github.com/knadh/koanf/maps v0.1.1 // indirect
+	github.com/kr/fs v0.1.0 // indirect
 	github.com/mattn/go-colorable v0.1.14 // indirect
 	github.com/minio/md5-simd v1.1.2 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect

go.sum

@@ -100,6 +100,8 @@ github.com/knadh/koanf/providers/rawbytes v0.1.0 h1:dpzgu2KO6uf6oCb4aP05KDmKmAmI
 github.com/knadh/koanf/providers/rawbytes v0.1.0/go.mod h1:mMTB1/IcJ/yE++A2iEZbY1MLygX7vttU+C+S/YmPu9c=
 github.com/knadh/koanf/v2 v2.1.2 h1:I2rtLRqXRy1p01m/utEtpZSSA6dcJbgGVuE27kW2PzQ=
 github.com/knadh/koanf/v2 v2.1.2/go.mod h1:Gphfaen0q1Fc1HTgJgSTC4oRX9R2R5ErYMZJy8fLJBo=
+github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
+github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
@@ -135,6 +137,8 @@ github.com/pascaldekloe/name v1.0.0/go.mod h1:Z//MfYJnH4jVpQ9wkclwu2I2MkHmXTlT9w
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/sftp v1.13.7 h1:uv+I3nNJvlKZIQGSr8JVQLNHFU9YhhNpvC14Y6KgmSM=
+github.com/pkg/sftp v1.13.7/go.mod h1:KMKI0t3T6hfA+lTR/ssZdunHo+uwq7ghoN09/FSu3DY=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
@@ -179,34 +183,72 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
 github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
 github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
 github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 go.mau.fi/util v0.8.4 h1:mVKlJcXWfVo8ZW3f4vqtjGpqtZqJvX4ETekxawt2vnQ=
 go.mau.fi/util v0.8.4/go.mod h1:MOfGTs1CBuK6ERTcSL4lb5YU7/ujz09eOPVEDckuazY=
 go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
 go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
 golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus=
 golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M=
 golang.org/x/exp v0.0.0-20250207012021-f9890c6ad9f3 h1:qNgPs5exUA+G0C96DrPwNrvLSj7GT/9D+3WMWUcUg34=
 golang.org/x/exp v0.0.0-20250207012021-f9890c6ad9f3/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.23.0 h1:Zb7khfcRGKk+kqfxFaP5tZqCnDZMjC5VtUBs87Hr6QM=
 golang.org/x/mod v0.23.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8=
 golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w=
 golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
 golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
+golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
 golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU=
 golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM=
 golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
 golang.org/x/time v0.10.0 h1:3usCWA8tQn0L8+hFJQNgzpWbd89begxN66o1Ojdn5L4=
 golang.org/x/time v0.10.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/tools v0.30.0 h1:BgcpHewrV5AUp2G9MebG4XPFI1E2W41zU1SaqVA9vJY=
 golang.org/x/tools v0.30.0/go.mod h1:c347cR/OJfw5TI+GfX7RUPNMdDRRbjvYTS0jPyvsVtY=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=

pkg/backy/backy.go

@@ -144,6 +144,7 @@ func (command *Command) RunCmd(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([
 		}
 
 		var err error
+
 		if command.Shell != "" {
 			cmdCtxLogger.Info().Str("Command", fmt.Sprintf("Running command %s on local machine in %s", command.Name, command.Shell)).Send()
 
@@ -165,6 +166,12 @@ func (command *Command) RunCmd(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([
 			}
 		}
 
+		if command.Type == UserCT {
+			if command.UserOperation == "password" {
+				localCMD.Stdin = command.stdin
+				cmdCtxLogger.Info().Str("password", command.UserPassword).Msg("user password to be updated")
+			}
+		}
 		if command.Dir != nil {
 			localCMD.Dir = *command.Dir
 		}
@@ -182,20 +189,7 @@ func (command *Command) RunCmd(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([
 
 		err = localCMD.Run()
 
-		outScanner := bufio.NewScanner(&cmdOutBuf)
+		outputArr = logCommandOutput(command, cmdOutBuf, cmdCtxLogger, outputArr)
-
-		for outScanner.Scan() {
-			outMap := make(map[string]interface{})
-			outMap["cmd"] = command.Cmd
-			outMap["output"] = outScanner.Text()
-
-			if str, ok := outMap["output"].(string); ok {
-				outputArr = append(outputArr, str)
-			}
-			// if command.GetOutput {
-			cmdCtxLogger.Info().Fields(outMap).Send()
-			// }
-		}
 		if err != nil {
 			cmdCtxLogger.Error().Err(fmt.Errorf("error when running cmd %s: %w", command.Name, err)).Send()
 			return outputArr, err
@@ -240,7 +234,7 @@ func cmdListWorker(msgTemps *msgTemplates, jobs <-chan *CmdList, results chan<-
 			}
 
 			// Collect output if required
-			if list.GetOutput || cmdToRun.GetOutput {
+			if list.GetOutput || cmdToRun.GetOutputInList {
 				outStructArr = append(outStructArr, outStruct{
 					CmdName:     currentCmd,
 					CmdExecuted: currentCmd,
@@ -249,17 +243,14 @@ func cmdListWorker(msgTemps *msgTemplates, jobs <-chan *CmdList, results chan<-
 			}
 		}
 
-		// Notify success if no errors occurred
 		if !hasError && list.NotifyConfig != nil && (list.NotifyOnSuccess || list.GetOutput) {
 			notifySuccess(cmdLogger, msgTemps, list, cmdsRan, outStructArr)
 		}
 
-		// Execute success and final hooks for all commands
 		for _, cmd := range list.Order {
 			cmdToRun := opts.Cmds[cmd]
 
-			// Execute success hooks if the command succeeded
+			if !hasError {
-			if !hasError || cmdsRanContains(cmd, cmdsRan) {
 				cmdToRun.ExecuteHooks("success", opts)
 			}
 
@@ -276,17 +267,6 @@ func cmdListWorker(msgTemps *msgTemplates, jobs <-chan *CmdList, results chan<-
 	}
 }
 
-// Helper to check if a command is in the list of executed commands
-func cmdsRanContains(cmd string, cmdsRan []string) bool {
-	for _, c := range cmdsRan {
-		if c == cmd {
-			return true
-		}
-	}
-	return false
-}
-
-// Helper to notify errors
 func notifyError(logger zerolog.Logger, templates *msgTemplates, list *CmdList, cmdsRan []string, outStructArr []outStruct, err error, cmd *Command) {
 	errStruct := map[string]interface{}{
 		"listName":  list.Name,
@@ -355,7 +335,6 @@ func (opts *ConfigOpts) RunListConfig(cron string) {
 		result := <-results
 		opts.Logger.Debug().Msgf("Processing result for list %s, command %s", result.ListName, result.CmdName)
 
-		// Process final hooks for the list (already handled in worker)
 	}
 	opts.closeHostConnections()
 }
@@ -367,10 +346,8 @@ func (opts *ConfigOpts) ExecuteCmds() {
 		_, runErr := cmdToRun.RunCmd(cmdLogger, opts)
 		if runErr != nil {
 			opts.Logger.Err(runErr).Send()
-
 			cmdToRun.ExecuteHooks("error", opts)
 		} else {
-
 			cmdToRun.ExecuteHooks("success", opts)
 		}
 
@@ -378,7 +355,6 @@ func (opts *ConfigOpts) ExecuteCmds() {
 	}
 
 	opts.closeHostConnections()
-
 }
 
 func (c *ConfigOpts) closeHostConnections() {
@@ -425,8 +401,9 @@ func (cmd *Command) ExecuteHooks(hookType string, opts *ConfigOpts) {
 	case "error":
 		for _, v := range cmd.Hooks.Error {
 			errCmd := opts.Cmds[v]
+			opts.Logger.Info().Msgf("Running error hook command %s", v)
 			cmdLogger := opts.Logger.With().
-				Str("backy-cmd", v).
+				Str("backy-cmd", v).Str("hookType", "error").
 				Logger()
 			errCmd.RunCmd(cmdLogger, opts)
 		}
@@ -434,16 +411,18 @@ func (cmd *Command) ExecuteHooks(hookType string, opts *ConfigOpts) {
 	case "success":
 		for _, v := range cmd.Hooks.Success {
 			successCmd := opts.Cmds[v]
+			opts.Logger.Info().Msgf("Running success hook command %s", v)
 			cmdLogger := opts.Logger.With().
-				Str("backy-cmd", v).
+				Str("backy-cmd", v).Str("hookType", "success").
 				Logger()
 			successCmd.RunCmd(cmdLogger, opts)
 		}
 	case "final":
 		for _, v := range cmd.Hooks.Final {
 			finalCmd := opts.Cmds[v]
+			opts.Logger.Info().Msgf("Running final hook command %s", v)
 			cmdLogger := opts.Logger.With().
-				Str("backy-cmd", v).
+				Str("backy-cmd", v).Str("hookType", "final").
 				Logger()
 			finalCmd.RunCmd(cmdLogger, opts)
 		}
@@ -481,6 +460,25 @@ func (opts *ConfigOpts) ExecCmdsSSH(cmdList []string, hostsList []string) {
 	}
 }
 
+func logCommandOutput(command *Command, cmdOutBuf bytes.Buffer, cmdCtxLogger zerolog.Logger, outputArr []string) []string {
+
+	outScanner := bufio.NewScanner(&cmdOutBuf)
+
+	for outScanner.Scan() {
+		outMap := make(map[string]interface{})
+		outMap["cmd"] = command.Name
+		outMap["output"] = outScanner.Text()
+
+		if str, ok := outMap["output"].(string); ok {
+			outputArr = append(outputArr, str)
+		}
+		if command.OutputToLog {
+			cmdCtxLogger.Info().Fields(outMap).Send()
+		}
+	}
+	return outputArr
+}
+
 // func executeUserCommands() []string {
 
 // }

pkg/backy/config.go

@@ -22,10 +22,13 @@ import (
 	"github.com/rs/zerolog"
 )
 
-const macroStart string = "%{"
+const (
-const macroEnd string = "}%"
+	externDirectiveStart      string = "%{"
-const envMacroStart string = "%{env:"
+	externDirectiveEnd        string = "}%"
-const vaultMacroStart string = "%{vault:"
+	externFileDirectiveStart  string = "%{file:"
+	envExternDirectiveStart   string = "%{env:"
+	vaultExternDirectiveStart string = "%{vault:"
+)
 
 func (opts *ConfigOpts) InitConfig() {
 	var err error
@@ -78,15 +81,16 @@ func (opts *ConfigOpts) InitConfig() {
 		logging.ExitWithMSG(fmt.Sprintf("error initializing cache: %v", err), 1, nil)
 	}
 
-	fetcher, err := remotefetcher.NewRemoteFetcher(opts.ConfigFilePath, opts.Cache)
-
 	if isRemoteURL(opts.ConfigFilePath) {
 		p, _ := getRemoteDir(opts.ConfigFilePath)
 		opts.ConfigDir = p
 	}
+
+	fetcher, err := remotefetcher.NewRemoteFetcher(opts.ConfigFilePath, opts.Cache)
 	if err != nil {
 		logging.ExitWithMSG(fmt.Sprintf("error initializing config fetcher: %v", err), 1, nil)
 	}
+
 	if opts.ConfigFilePath != "" {
 		loadConfigFile(fetcher, opts.ConfigFilePath, backyKoanf, opts)
 	} else {
@@ -95,41 +99,6 @@ func (opts *ConfigOpts) InitConfig() {
 	opts.koanf = backyKoanf
 }
 
-func loadConfigFile(fetcher remotefetcher.RemoteFetcher, filePath string, k *koanf.Koanf, opts *ConfigOpts) {
-	data, err := fetcher.Fetch(filePath)
-	if err != nil {
-		logging.ExitWithMSG(fmt.Sprintf("Could not fetch config file %s: %v", filePath, err), 1, nil)
-	}
-
-	if err := k.Load(rawbytes.Provider(data), yaml.Parser()); err != nil {
-		logging.ExitWithMSG(fmt.Sprintf("error loading config: %v", err), 1, &opts.Logger)
-	}
-}
-
-func loadDefaultConfigFiles(fetcher remotefetcher.RemoteFetcher, configFiles []string, k *koanf.Koanf, opts *ConfigOpts) {
-	cFileFailures := 0
-	for _, c := range configFiles {
-		opts.ConfigFilePath = c
-		data, err := fetcher.Fetch(c)
-		if err != nil {
-			cFileFailures++
-			continue
-		}
-
-		if data != nil {
-			if err := k.Load(rawbytes.Provider(data), yaml.Parser()); err == nil {
-				break
-			} else {
-				logging.ExitWithMSG(fmt.Sprintf("error loading config from file %s: %v", c, err), 1, &opts.Logger)
-			}
-		}
-	}
-
-	if cFileFailures == len(configFiles) {
-		logging.ExitWithMSG("Could not find any valid local config file", 1, nil)
-	}
-}
-
 func (opts *ConfigOpts) ReadConfig() *ConfigOpts {
 	setTerminalEnv()
 
@@ -148,7 +117,7 @@ func (opts *ConfigOpts) ReadConfig() *ConfigOpts {
 
 	CheckConfigValues(backyKoanf, opts.ConfigFilePath)
 
-	validateCommands(backyKoanf, opts)
+	validateExecCommandsFromCLI(backyKoanf, opts)
 
 	setLoggingOptions(backyKoanf, opts)
 
@@ -159,7 +128,7 @@ func (opts *ConfigOpts) ReadConfig() *ConfigOpts {
 
 	unmarshalConfig(backyKoanf, "commands", &opts.Cmds, opts.Logger)
 
-	validateCommandEnvironments(opts)
+	getCommandEnvironments(opts)
 
 	unmarshalConfig(backyKoanf, "hosts", &opts.Hosts, opts.Logger)
 
@@ -192,6 +161,41 @@ func (opts *ConfigOpts) ReadConfig() *ConfigOpts {
 	return opts
 }
 
+func loadConfigFile(fetcher remotefetcher.RemoteFetcher, filePath string, k *koanf.Koanf, opts *ConfigOpts) {
+	data, err := fetcher.Fetch(filePath)
+	if err != nil {
+		logging.ExitWithMSG(generateFileFetchErrorString(filePath, "config", err), 1, nil)
+	}
+
+	if err := k.Load(rawbytes.Provider(data), yaml.Parser()); err != nil {
+		logging.ExitWithMSG(fmt.Sprintf("error loading config: %v", err), 1, &opts.Logger)
+	}
+}
+
+func loadDefaultConfigFiles(fetcher remotefetcher.RemoteFetcher, configFiles []string, k *koanf.Koanf, opts *ConfigOpts) {
+	cFileFailures := 0
+	for _, c := range configFiles {
+		opts.ConfigFilePath = c
+		data, err := fetcher.Fetch(c)
+		if err != nil {
+			cFileFailures++
+			continue
+		}
+
+		if data != nil {
+			if err := k.Load(rawbytes.Provider(data), yaml.Parser()); err == nil {
+				break
+			} else {
+				logging.ExitWithMSG(fmt.Sprintf("error loading config from file %s: %v", c, err), 1, &opts.Logger)
+			}
+		}
+	}
+
+	if cFileFailures == len(configFiles) {
+		logging.ExitWithMSG("Could not find any valid local config file", 1, nil)
+	}
+}
+
 func setTerminalEnv() {
 	if isatty.IsTerminal(os.Stdout.Fd()) || isatty.IsCygwinTerminal(os.Stdout.Fd()) {
 		os.Setenv("BACKY_TERM", "enabled")
@@ -200,7 +204,7 @@ func setTerminalEnv() {
 	}
 }
 
-func validateCommands(k *koanf.Koanf, opts *ConfigOpts) {
+func validateExecCommandsFromCLI(k *koanf.Koanf, opts *ConfigOpts) {
 	for _, c := range opts.executeCmds {
 		if !k.Exists(getCmdFromConfig(c)) {
 			logging.ExitWithMSG(fmt.Sprintf("command %s is not in config file %s", c, opts.ConfigFilePath), 1, nil)
@@ -238,15 +242,15 @@ func setupLogger(opts *ConfigOpts) zerolog.Logger {
 
 func unmarshalConfig(k *koanf.Koanf, key string, target interface{}, log zerolog.Logger) {
 	if err := k.UnmarshalWithConf(key, target, koanf.UnmarshalConf{Tag: "yaml"}); err != nil {
-		logging.ExitWithMSG(fmt.Sprintf("error unmarshalling key %s into struct: %v", key, err), 1, &log)
+		logging.ExitWithMSG(fmt.Sprintf("error unmarshaling key %s into struct: %v", key, err), 1, &log)
 	}
 }
 
-func validateCommandEnvironments(opts *ConfigOpts) {
+func getCommandEnvironments(opts *ConfigOpts) {
 	for cmdName, cmdConf := range opts.Cmds {
+		opts.Logger.Debug().Str("env file", cmdConf.Env).Str("cmd", cmdName).Send()
 		if err := testFile(cmdConf.Env); err != nil {
-			opts.Logger.Info().Str("cmd", cmdName).Err(err).Send()
+			logging.ExitWithMSG("Could not open file"+cmdConf.Env+": "+err.Error(), 1, &opts.Logger)
-			os.Exit(1)
 		}
 		expandEnvVars(opts.backyEnv, cmdConf.Environment)
 	}
@@ -304,9 +308,10 @@ func loadCommandLists(opts *ConfigOpts, backyKoanf *koanf.Koanf) {
 	}
 
 	if backyKoanf.Exists("cmdLists") {
-		unmarshalConfig(backyKoanf, "cmdLists", &opts.CmdConfigLists, opts.Logger)
 		if backyKoanf.Exists("cmdLists.file") {
 			loadCmdListsFile(backyKoanf, listsConfig, opts)
+		} else {
+			unmarshalConfig(backyKoanf, "cmdLists", &opts.CmdConfigLists, opts.Logger)
 		}
 	}
 }
@@ -366,7 +371,7 @@ func loadCmdListsFile(backyKoanf *koanf.Koanf, listsConfig *koanf.Koanf, opts *C
 
 	data, err := fetcher.Fetch(opts.CmdListFile)
 	if err != nil {
-		logging.ExitWithMSG(fmt.Sprintf("Could not fetch config file %s: %v", opts.CmdListFile, err), 1, nil)
+		logging.ExitWithMSG(generateFileFetchErrorString(opts.CmdListFile, "list config", err), 1, nil)
 	}
 
 	if err := listsConfig.Load(rawbytes.Provider(data), yaml.Parser()); err != nil {
@@ -378,6 +383,10 @@ func loadCmdListsFile(backyKoanf *koanf.Koanf, listsConfig *koanf.Koanf, opts *C
 	opts.Logger.Info().Str("using lists config file", opts.CmdListFile).Send()
 }
 
+func generateFileFetchErrorString(file, fileType string, err error) string {
+	return fmt.Sprintf("Could not fetch %s file %s: %v", file, fileType, err)
+}
+
 func validateCommandLists(opts *ConfigOpts) {
 	var cmdNotFoundSliceErr []error
 	for cmdListName, cmdList := range opts.CmdConfigLists {
@@ -455,7 +464,7 @@ func (opts *ConfigOpts) setupVault() error {
 
 	unmarshalErr := opts.koanf.UnmarshalWithConf("vault.keys", &opts.VaultKeys, koanf.UnmarshalConf{Tag: "yaml"})
 	if unmarshalErr != nil {
-		logging.ExitWithMSG(fmt.Sprintf("error unmarshalling vault.keys into struct: %v", unmarshalErr), 1, &opts.Logger)
+		logging.ExitWithMSG(fmt.Sprintf("error unmarshaling vault.keys into struct: %v", unmarshalErr), 1, &opts.Logger)
 	}
 
 	opts.vaultClient = client
@@ -491,16 +500,7 @@ func getVaultSecret(vaultClient *vault.Client, key *VaultKey) (string, error) {
 	return value, nil
 }
 
-func isVaultKey(str string) (string, bool) {
+func parseVaultKey(keyName string, keys []*VaultKey) (*VaultKey, error) {
-	str = strings.TrimSpace(str)
-	return strings.TrimPrefix(str, "vault:"), strings.HasPrefix(str, "vault:")
-}
-
-func parseVaultKey(str string, keys []*VaultKey) (*VaultKey, error) {
-	keyName, isKey := isVaultKey(str)
-	if !isKey {
-		return nil, nil
-	}
 
 	for _, k := range keys {
 		if k.Name == keyName {
@@ -565,6 +565,10 @@ func processCmds(opts *ConfigOpts) error {
 					cmd.RemoteHost.HostName = host.HostName
 				}
 			} else {
+				opts.Logger.Info().Msgf("adding host %s to host list", *cmd.Host)
+				if opts.Hosts == nil {
+					opts.Hosts = make(map[string]*Host)
+				}
 				opts.Hosts[*cmd.Host] = &Host{Host: *cmd.Host}
 				cmd.RemoteHost = &Host{Host: *cmd.Host}
 			}
@@ -577,10 +581,11 @@ func processCmds(opts *ConfigOpts) error {
 					return err
 				}
 				cmd.Dir = &cmdDir
+			} else {
+				cmd.Dir = &opts.ConfigDir
 			}
 		}
 
-		// Parse package commands
 		if cmd.Type == PackageCT {
 			if cmd.PackageManager == "" {
 				return fmt.Errorf("package manager is required for package command %s", cmd.PackageName)
@@ -612,19 +617,31 @@ func processCmds(opts *ConfigOpts) error {
 				return fmt.Errorf("username is required for user command %s", cmd.Name)
 			}
 
-			detectOSType(cmd, opts)
+			err := detectOSType(cmd, opts)
-			var err error
+			if err != nil {
+				opts.Logger.Info().Err(err).Str("command", cmdName).Send()
+			}
 
 			// Validate the operation
 			switch cmd.UserOperation {
 			case "add", "remove", "modify", "checkIfExists", "delete", "password":
 				cmd.userMan, err = usermanager.NewUserManager(cmd.OS)
+
+				if cmd.UserOperation == "password" {
+					opts.Logger.Debug().Msg("changing password for user: " + cmd.Username)
+					cmd.UserPassword = getExternalConfigDirectiveValue(cmd.UserPassword, opts)
+				}
 				if cmd.Host != nil {
 					host, ok := opts.Hosts[*cmd.Host]
 					if ok {
 						cmd.userMan, err = usermanager.NewUserManager(host.OS)
 					}
 				}
+				for indx, key := range cmd.UserSshPubKeys {
+					opts.Logger.Debug().Msg("adding SSH Keys")
+					key = getExternalConfigDirectiveValue(key, opts)
+					cmd.UserSshPubKeys[indx] = key
+				}
 				if err != nil {
 					return err
 				}
@@ -656,14 +673,8 @@ func processCmds(opts *ConfigOpts) error {
 	return nil
 }
 
-// processHooks evaluates if hooks are valid Commands
-//
-// The cmd.hookRefs[hookType] is created with any hooks found.
-//
-// Returns an error, if any, if the hook command is not found
 func processHooks(cmd *Command, hooks []string, opts *ConfigOpts, hookType string) error {
 
-	// initialize hook type
 	var hookCmdFound bool
 	cmd.hookRefs = map[string]map[string]*Command{}
 	cmd.hookRefs[hookType] = map[string]*Command{}
@@ -691,11 +702,14 @@ func processHooks(cmd *Command, hooks []string, opts *ConfigOpts, hookType strin
 
 func detectOSType(cmd *Command, opts *ConfigOpts) error {
 	if cmd.Host == nil {
-		if runtime.GOOS == "linux" { // also can be specified to FreeBSD
+		if runtime.GOOS == "linux" {
 			cmd.OS = "linux"
 			opts.Logger.Info().Msg("Unix/Linux type OS detected")
+			return nil
 		}
+		return fmt.Errorf("using an os that is not yet supported for user commands")
 	}
+
 	host, ok := opts.Hosts[*cmd.Host]
 	if ok {
 		if host.OS != "" {

pkg/backy/notification.go

@@ -58,6 +58,7 @@ func (opts *ConfigOpts) SetupNotify() {
 					opts.Logger.Info().Err(fmt.Errorf("error: ID %s not found in mail object", confId)).Str("list", confName).Send()
 					continue
 				}
+				conf.Password = getExternalConfigDirectiveValue(conf.Password, opts)
 				mailConf := setupMail(conf)
 				services = append(services, mailConf)
 			case "matrix":
@@ -66,14 +67,14 @@ func (opts *ConfigOpts) SetupNotify() {
 					opts.Logger.Info().Err(fmt.Errorf("error: ID %s not found in matrix object", confId)).Str("list", confName).Send()
 					continue
 				}
+				conf.AccessToken = getExternalConfigDirectiveValue(conf.AccessToken, opts)
 				mtrxConf, mtrxErr := setupMatrix(conf)
 				if mtrxErr != nil {
 					opts.Logger.Info().Str("list", confName).Err(fmt.Errorf("error: configuring matrix id %s failed during setup: %w", id, mtrxErr))
 					continue
 				}
-				// append the services
 				services = append(services, mtrxConf)
-			// service is not recognized
+
 			default:
 				opts.Logger.Info().Err(fmt.Errorf("id %s not found", id)).Str("list", confName).Send()
 			}

pkg/backy/ssh.go

@@ -17,12 +17,13 @@ import (
 
 	"github.com/kevinburke/ssh_config"
 	"github.com/pkg/errors"
+	"github.com/pkg/sftp"
 	"github.com/rs/zerolog"
 	"golang.org/x/crypto/ssh"
 	"golang.org/x/crypto/ssh/knownhosts"
 )
 
-var PrivateKeyExtraInfoErr = errors.New("Private key may be encrypted. \nIf encrypted, make sure the password is specified correctly in the correct section. This may be done in one of three ways: \n privatekeypassword: env:PR_KEY_PASS \n privatekeypassword: file:/path/to/password-file \n privatekeypassword: password (not recommended). \n ")
+var PrivateKeyExtraInfoErr = errors.New("Private key may be encrypted. \nIf encrypted, make sure the password is specified correctly in the correct section. This may be done in one of two ways: \n Using external directives - see docs \n privatekeypassword: password (not recommended). \n ")
 var TS = strings.TrimSpace
 
 // ConnectToHost connects to a host by looking up the config values in the file ~/.ssh/config
@@ -119,7 +120,6 @@ func (remoteConfig *Host) ConnectToHost(opts *ConfigOpts) error {
 		return errors.Wrap(err, "could not create hostkeycallback function")
 	}
 	remoteConfig.ClientConfig.HostKeyCallback = hostKeyCallback
-	// opts.Logger.Info().Str("user", remoteConfig.ClientConfig.User).Send()
 
 	remoteConfig.SshClient, connectErr = remoteConfig.ConnectThroughBastion(opts.Logger)
 	if connectErr != nil {
@@ -180,11 +180,7 @@ func (remoteHost *Host) GetAuthMethods(opts *ConfigOpts) error {
 			return err
 		}
 
-		remoteHost.PrivateKeyPassword, err = GetPrivateKeyPassword(remoteHost.PrivateKeyPassword, opts, opts.Logger)
+		remoteHost.PrivateKeyPassword = GetPrivateKeyPassword(remoteHost.PrivateKeyPassword, opts)
-
-		if err != nil {
-			return err
-		}
 
 		if remoteHost.PrivateKeyPassword == "" {
 
@@ -207,14 +203,13 @@ func (remoteHost *Host) GetAuthMethods(opts *ConfigOpts) error {
 		}
 	}
 
-	if remoteHost.Password == "" {
+	if remoteHost.Password != "" {
 
-		remoteHost.Password, err = GetPassword(remoteHost.Password, opts, opts.Logger)
+		opts.Logger.Debug().Str("password", remoteHost.Password).Str("Host", remoteHost.Host).Send()
 
-		if err != nil {
+		remoteHost.Password = GetPassword(remoteHost.Password, opts)
 
-			return err
+		// opts.Logger.Debug().Str("actual password", remoteHost.Password).Str("Host", remoteHost.Host).Send()
-		}
 
 		remoteHost.ClientConfig.Auth = append(remoteHost.ClientConfig.Auth, ssh.Password(remoteHost.Password))
 	}
@@ -249,14 +244,13 @@ func (remoteHost *Host) GetPrivateKeyFileFromConfig() {
 // If it is the port is searched in the SSH config file(s)
 func (remoteHost *Host) GetPort() {
 	port := fmt.Sprintf("%d", remoteHost.Port)
-	// port specifed?
+	// port specified?
 	// port will be 0 if missing from backy config
 	if port == "0" {
 		port, _ = remoteHost.SSHConfigFile.SshConfigFile.Get(remoteHost.Host, "Port")
 
 		if port == "" {
 
-			// get port from default SSH config file
 			port = remoteHost.SSHConfigFile.DefaultUserSettings.Get(remoteHost.Host, "Port")
 
 			// set port to be default
@@ -271,7 +265,6 @@ func (remoteHost *Host) GetPort() {
 
 func (remoteHost *Host) CombineHostNameWithPort() {
 
-	// if the port is already in the HostName, leave it
 	if strings.HasSuffix(remoteHost.HostName, fmt.Sprintf(":%d", remoteHost.Port)) {
 		return
 	}
@@ -321,74 +314,23 @@ func (remoteHost *Host) ConnectThroughBastion(log zerolog.Logger) (*ssh.Client,
 
 // GetKnownHosts resolves the host's KnownHosts file if it is defined
 // if not defined, the default location for this file is used
-func (remotehHost *Host) GetKnownHosts() error {
+func (remoteHost *Host) GetKnownHosts() error {
 	var knownHostsFileErr error
-	if TS(remotehHost.KnownHostsFile) != "" {
+	if TS(remoteHost.KnownHostsFile) != "" {
-		remotehHost.KnownHostsFile, knownHostsFileErr = getFullPathWithHomeDir(remotehHost.KnownHostsFile)
+		remoteHost.KnownHostsFile, knownHostsFileErr = getFullPathWithHomeDir(remoteHost.KnownHostsFile)
 		return knownHostsFileErr
 	}
-	remotehHost.KnownHostsFile, knownHostsFileErr = getFullPathWithHomeDir("~/.ssh/known_hosts")
+	remoteHost.KnownHostsFile, knownHostsFileErr = getFullPathWithHomeDir("~/.ssh/known_hosts")
 	return knownHostsFileErr
 }
 
-func GetPrivateKeyPassword(key string, opts *ConfigOpts, log zerolog.Logger) (string, error) {
+func GetPrivateKeyPassword(key string, opts *ConfigOpts) string {
-
+	return getExternalConfigDirectiveValue(key, opts)
-	var prKeyPassword string
-	if strings.HasPrefix(key, "file:") {
-		privKeyPassFilePath := strings.TrimPrefix(key, "file:")
-		privKeyPassFilePath, _ = getFullPathWithHomeDir(privKeyPassFilePath)
-		keyFile, keyFileErr := os.Open(privKeyPassFilePath)
-		if keyFileErr != nil {
-			return "", errors.Errorf("Private key password file %s failed to open. \n Make sure it is accessible and correct.", privKeyPassFilePath)
-		}
-		passwordScanner := bufio.NewScanner(keyFile)
-		for passwordScanner.Scan() {
-			prKeyPassword = passwordScanner.Text()
-		}
-	} else if strings.HasPrefix(key, "env:") {
-		privKey := strings.TrimPrefix(key, "env:")
-		privKey = strings.TrimPrefix(privKey, "${")
-		privKey = strings.TrimSuffix(privKey, "}")
-		privKey = strings.TrimPrefix(privKey, "$")
-		prKeyPassword = os.Getenv(privKey)
-	} else {
-		prKeyPassword = key
-	}
-	prKeyPassword = GetVaultKey(prKeyPassword, opts, opts.Logger)
-	return prKeyPassword, nil
 }
 
 // GetPassword gets any password
-func GetPassword(pass string, opts *ConfigOpts, log zerolog.Logger) (string, error) {
+func GetPassword(pass string, opts *ConfigOpts) string {
-
+	return getExternalConfigDirectiveValue(pass, opts)
-	pass = strings.TrimSpace(pass)
-	if pass == "" {
-		return "", nil
-	}
-	var password string
-	if strings.HasPrefix(pass, "file:") {
-		passFilePath := strings.TrimPrefix(pass, "file:")
-		passFilePath, _ = getFullPathWithHomeDir(passFilePath)
-		keyFile, keyFileErr := os.Open(passFilePath)
-		if keyFileErr != nil {
-			return "", errors.New("Password file failed to open")
-		}
-		passwordScanner := bufio.NewScanner(keyFile)
-		for passwordScanner.Scan() {
-			password = passwordScanner.Text()
-		}
-	} else if strings.HasPrefix(pass, "env:") {
-		passEnv := strings.TrimPrefix(pass, "env:")
-		passEnv = strings.TrimPrefix(passEnv, "${")
-		passEnv = strings.TrimSuffix(passEnv, "}")
-		passEnv = strings.TrimPrefix(passEnv, "$")
-		password = os.Getenv(passEnv)
-	} else {
-		password = pass
-	}
-	password = GetVaultKey(password, opts, opts.Logger)
-
-	return password, nil
 }
 
 func (remoteConfig *Host) GetProxyJumpFromConfig(hosts map[string]*Host) error {
@@ -489,7 +431,6 @@ func (command *Command) RunCmdSSH(cmdCtxLogger zerolog.Logger, opts *ConfigOpts)
 			env:  command.Environment,
 		}
 	)
-	// Getting the command type must be done before concatenating the arguments
 	command = getCommandTypeAndSetCommandInfo(command)
 
 	// Prepare command arguments
@@ -565,10 +506,71 @@ func (command *Command) RunCmdSSH(cmdCtxLogger zerolog.Logger, opts *ConfigOpts)
 			ArgsStr = fmt.Sprintf("%s %s", command.Cmd, ArgsStr)
 		}
 		cmdCtxLogger.Debug().Str("cmd + args", ArgsStr).Send()
-		// Run simple command
+
+		if command.Type == UserCT && command.UserOperation == "password" {
+			// cmdCtxLogger.Debug().Msgf("adding stdin")
+			userNamePass := fmt.Sprintf("%s:%s", command.Username, command.UserPassword)
+			ArgsStr = fmt.Sprintf("echo %s | chpasswd", userNamePass)
+
+			// commandSession.Stdin = command.stdin
+		}
 		if err := commandSession.Run(ArgsStr); err != nil {
 			return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error running command: %w", err)
 		}
+
+		if command.Type == UserCT {
+
+			if command.UserOperation == "add" {
+				if command.UserSshPubKeys != nil {
+					var (
+						f        *sftp.File
+						err      error
+						userHome []byte
+						client   *sftp.Client
+					)
+
+					cmdCtxLogger.Info().Msg("adding SSH Keys")
+
+					commandSession, _ = command.RemoteHost.createSSHSession(opts)
+					userHome, err = commandSession.CombinedOutput(fmt.Sprintf("grep \"%s\" /etc/passwd | cut -d: -f6", command.Username))
+					if err != nil {
+						return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error finding user home from /etc/passwd: %v", err)
+					}
+
+					command.UserHome = strings.TrimSpace(string(userHome))
+					userSshDir := fmt.Sprintf("%s/.ssh", command.UserHome)
+					client, err = sftp.NewClient(command.RemoteHost.SshClient)
+					if err != nil {
+						return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error creating sftp client: %v", err)
+					}
+
+					client.MkdirAll(userSshDir)
+					_, err = client.Create(fmt.Sprintf("%s/authorized_keys", userSshDir))
+					if err != nil {
+						return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error opening file %s/authorized_keys: %v", userSshDir, err)
+					}
+					f, err = client.OpenFile(fmt.Sprintf("%s/authorized_keys", userSshDir), os.O_APPEND|os.O_CREATE|os.O_WRONLY)
+					if err != nil {
+						return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error opening file %s/authorized_keys: %v", userSshDir, err)
+					}
+					defer f.Close()
+					for _, k := range command.UserSshPubKeys {
+						buf := bytes.NewBufferString(k)
+						cmdCtxLogger.Info().Str("key", k).Msg("adding SSH key")
+						if _, err := f.ReadFrom(buf); err != nil {
+							return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error adding to authorized keys: %v", err)
+						}
+					}
+
+					commandSession, _ = command.RemoteHost.createSSHSession(opts)
+					_, err = commandSession.CombinedOutput(fmt.Sprintf("chown -R %s:%s %s", command.Username, command.Username, userHome))
+					if err != nil {
+						return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), err
+					}
+
+				}
+			}
+		}
 	}
 
 	return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), nil
@@ -622,7 +624,7 @@ func (command *Command) runScript(session *ssh.Session, cmdCtxLogger zerolog.Log
 		return collectOutput(outputBuf, command.Name, cmdCtxLogger, true), fmt.Errorf("error waiting for shell: %w", err)
 	}
 
-	return collectOutput(outputBuf, command.Name, cmdCtxLogger, command.GetOutput), nil
+	return collectOutput(outputBuf, command.Name, cmdCtxLogger, command.OutputToLog), nil
 }
 
 // runScriptFile handles the execution of script files.

pkg/backy/types.go

@@ -26,15 +26,15 @@ type (
 		ConfigFilePath     string `yaml:"config,omitempty"`
 		Host               string `yaml:"host,omitempty"`
 		HostName           string `yaml:"hostname,omitempty"`
-		KnownHostsFile     string `yaml:"knownhostsfile,omitempty"`
+		KnownHostsFile     string `yaml:"knownHostsFile,omitempty"`
 		ClientConfig       *ssh.ClientConfig
 		SSHConfigFile      *sshConfigFile
 		SshClient          *ssh.Client
 		Port               uint16 `yaml:"port,omitempty"`
 		ProxyJump          string `yaml:"proxyjump,omitempty"`
 		Password           string `yaml:"password,omitempty"`
-		PrivateKeyPath     string `yaml:"privatekeypath,omitempty"`
+		PrivateKeyPath     string `yaml:"privateKeyPath,omitempty"`
-		PrivateKeyPassword string `yaml:"privatekeypassword,omitempty"`
+		PrivateKeyPassword string `yaml:"privateKeyPassword,omitempty"`
 		useDefaultConfig   bool
 		User               string `yaml:"user,omitempty"`
 		isProxyHost        bool
@@ -51,44 +51,30 @@ type (
 	Command struct {
 		Name string `yaml:"name,omitempty"`
 
-		// command to run
 		Cmd string `yaml:"cmd"`
 
 		// See CommandType enum further down the page for acceptable values
 		Type CommandType `yaml:"type,omitempty"`
 
-		// host on which to run cmd
 		Host *string `yaml:"host,omitempty"`
 
-		// Hooks are for running commands on certain events
 		Hooks *Hooks `yaml:"hooks,omitempty"`
 
-		// hook refs are internal references of commands for each hook type
 		hookRefs map[string]map[string]*Command
 
-		// Shell specifies which shell to run the command in, if any.
 		Shell string `yaml:"shell,omitempty"`
 
 		RemoteHost *Host `yaml:"-"`
 
-		// Args is an array that holds the arguments to cmd
 		Args []string `yaml:"args,omitempty"`
 
-		/*
-			Dir specifies a directory in which to run the command.
-		*/
 		Dir *string `yaml:"dir,omitempty"`
 
-		// Env points to a file containing env variables to be used with the command
 		Env string `yaml:"env,omitempty"`
 
-		// Environment holds env variables to be used with the command
 		Environment []string `yaml:"environment,omitempty"`
 
-		// Output determines if output is requested.
+		GetOutputInList bool `yaml:"getOutputInList,omitempty"`
-		//
-		// Only for when command is in a list.
-		GetOutput bool `yaml:"getOutput,omitempty"`
 
 		ScriptEnvFile string `yaml:"scriptEnvFile"`
 
@@ -102,10 +88,8 @@ type (
 
 		PackageName string `yaml:"packageName,omitempty"`
 
-		// Version specifies the desired version for package execution
 		PackageVersion string `yaml:"packageVersion,omitempty"`
 
-		// PackageOperation specifies the action for package-related commands (e.g., "install" or "remove")
 		PackageOperation PackageOperation `yaml:"packageOperation,omitempty"`
 
 		pkgMan pkgman.PackageManager
@@ -113,42 +97,35 @@ type (
 		packageCmdSet bool
 		// END PACKAGE COMMAND FIELDS
 
-		// RemoteSource specifies a URL to fetch the command or configuration remotely
 		RemoteSource string `yaml:"remoteSource,omitempty"`
 
-		// FetchBeforeExecution determines if the remoteSource should be fetched before running
 		FetchBeforeExecution bool `yaml:"fetchBeforeExecution,omitempty"`
 
 		Fetcher remotefetcher.RemoteFetcher
 
 		// BEGIN USER COMMAND FIELDS
 
-		// Username specifies the username for user creation or related operations
 		Username string `yaml:"userName,omitempty"`
 
 		UserID string `yaml:"userID,omitempty"`
 
-		// UserGroups specifies the groups to add the user to
 		UserGroups []string `yaml:"userGroups,omitempty"`
 
-		// UserHome specifies the home directory for the user
 		UserHome string `yaml:"userHome,omitempty"`
 
-		// UserShell specifies the shell for the user
 		UserShell string `yaml:"userShell,omitempty"`
 
-		// SystemUser specifies whether the user is a system account
 		SystemUser bool `yaml:"systemUser,omitempty"`
 
-		// UserPassword specifies the password for the user (can be file: or plain text)
 		UserPassword string `yaml:"userPassword,omitempty"`
 
+		UserSshPubKeys []string `yaml:"userSshPubKeys,omitempty"`
+
 		userMan usermanager.UserManager
 
 		// OS for the command, only used when type is user
 		OS string `yaml:"OS,omitempty"`
 
-		// UserOperation specifies the action for user-related commands (e.g., "create" or "remove")
 		UserOperation string `yaml:"userOperation,omitempty"`
 
 		userCmdSet bool

pkg/backy/utils.go

@@ -181,7 +181,6 @@ func testFile(c string) error {
 			return fileOpenErr
 		}
 	}
-
 	return nil
 }
 
@@ -247,7 +246,6 @@ func (opts *ConfigOpts) loadEnv() {
 	opts.backyEnv = backyEnv
 }
 
-// expandEnvVars expands environment variables with the env used in the config
 func expandEnvVars(backyEnv map[string]string, envVars []string) {
 
 	env := func(name string) string {
@@ -259,12 +257,11 @@ func expandEnvVars(backyEnv map[string]string, envVars []string) {
 		return ""
 	}
 
-	// parse env variables using new macros
 	for indx, v := range envVars {
-		if strings.HasPrefix(v, macroStart) && strings.HasSuffix(v, macroEnd) {
+		if strings.HasPrefix(v, externDirectiveStart) && strings.HasSuffix(v, externDirectiveEnd) {
-			if strings.HasPrefix(v, envMacroStart) {
+			if strings.HasPrefix(v, envExternDirectiveStart) {
-				v = strings.TrimPrefix(v, envMacroStart)
+				v = strings.TrimPrefix(v, envExternDirectiveStart)
-				v = strings.TrimRight(v, macroEnd)
+				v = strings.TrimRight(v, externDirectiveEnd)
 				out, _ := shell.Expand(v, env)
 				envVars[indx] = out
 			}
@@ -324,7 +321,7 @@ func parsePackageVersion(output string, cmdCtxLogger zerolog.Logger, command *Co
 	// println(output)
 	if err != nil {
 		cmdCtxLogger.Error().Err(err).Str("package", command.PackageName).Msg("Error parsing package version output")
-		return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.GetOutput), err
+		return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), err
 	}
 
 	cmdCtxLogger.Info().
@@ -349,3 +346,42 @@ func parsePackageVersion(output string, cmdCtxLogger zerolog.Logger, command *Co
 	}
 	return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, false), err
 }
+
+func getExternalConfigDirectiveValue(key string, opts *ConfigOpts) string {
+	if !(strings.HasPrefix(key, externDirectiveStart) && strings.HasSuffix(key, externDirectiveEnd)) {
+		return key
+	}
+	opts.Logger.Debug().Str("expanding external key", key).Send()
+	if strings.HasPrefix(key, envExternDirectiveStart) {
+		key = strings.TrimPrefix(key, envExternDirectiveStart)
+		key = strings.TrimSuffix(key, externDirectiveEnd)
+		key = os.Getenv(key)
+	}
+	if strings.HasPrefix(key, externFileDirectiveStart) {
+		var err error
+		var keyValue []byte
+		key = strings.TrimPrefix(key, externFileDirectiveStart)
+		key = strings.TrimSuffix(key, externDirectiveEnd)
+		key, err = getFullPathWithHomeDir(key)
+		if err != nil {
+			opts.Logger.Err(err).Send()
+			return ""
+		}
+		if !path.IsAbs(key) {
+			key = path.Join(opts.ConfigDir, key)
+		}
+		keyValue, err = os.ReadFile(key)
+		if err != nil {
+			opts.Logger.Err(err).Send()
+			return ""
+		}
+		key = string(keyValue)
+	}
+	if strings.HasPrefix(key, vaultExternDirectiveStart) {
+		key = strings.TrimPrefix(key, vaultExternDirectiveStart)
+		key = strings.TrimSuffix(key, externDirectiveEnd)
+		key = GetVaultKey(key, opts, opts.Logger)
+	}
+
+	return key
+}

pkg/remotefetcher/configfetcher.go

@@ -29,7 +29,7 @@ func NewRemoteFetcher(source string, cache *Cache, options ...FetcherOption) (Re
 		option(&config)
 	}
 
-	// If FileType is empty (i.e. WithFileType was not called), yaml is the default file type
+	// WithFileType was not called. yaml is the default file type
 	if strings.TrimSpace(config.FileType) == "" {
 		config.FileType = "yaml"
 	}

pkg/remotefetcher/local.go

@@ -20,7 +20,7 @@ func (l *LocalFetcher) Fetch(source string) ([]byte, error) {
 		if l.config.IgnoreFileNotFound {
 			return nil, ErrIgnoreFileNotFound
 		}
-		return nil, nil
+		return nil, err
 	}
 	file, err := os.Open(source)
 	if err != nil {

tests/.gitignore

@@ -0,0 +1 @@
+private

tests/ErrorHook.yml

@@ -0,0 +1,17 @@
+commands:
+  echoTestFail:
+    cmd: ech
+    shell: bash
+    Args: hello world
+    hooks:
+      error:
+        - errorCmd
+
+  errorCmd:
+    name: get docker version
+    cmd: docker
+    getOutput: true
+    outputToLog: true
+    Args:
+      - "-v"
+    host: email-svr

tests/HookNotInFile.yaml

@@ -0,0 +1,14 @@
+commands:
+  echoTestFail:
+    cmd: ech
+    shell: bash
+    Args: hello world
+    hooks:
+      error:
+        - errorCm # 
+
+  errorCmd:
+    name: get docker version
+    cmd: docker
+    Args:
+      - "-v"

tests/SuccessHook.yml

@@ -0,0 +1,16 @@
+commands:
+  echoTestSuccess:
+    cmd: echo
+    shell: bash
+    Args: hello world
+    hooks:
+      success:
+        - successCmd
+
+  errorCmd:
+    name: get docker version
+    cmd: docker
+    getOutput: true
+    outputToLog: true
+    Args:
+      - "-v"

tests/backy.yaml

@@ -1,18 +0,0 @@
-commands:
-  echoTestPass:
-    cmd: echo
-    shell: bash
-    Args: hello world
-
-  runRemoteShellScriptSuccess:
-    cmd: 
-
-
-  packageCommandSuccess:
-    packageName: docker-ce
-    Args:
-      - docker-ce-cli
-    packageManager: apt
-    packageOperation: install
-
-