v0.10.0

v0.9.1
2025-03-03 23:46:26 -06:00 · 2025-03-03 23:45:28 -06:00 · 2025-03-01 01:57:51 -06:00 · 2025-03-01 01:57:33 -06:00 · 2025-03-01 01:57:05 -06:00 · 2025-02-28 17:52:28 -06:00
43 changed files with 520 additions and 202 deletions
--- a/.changes/unreleased/Added-20250220-144549.yaml
+++ b/.changes/unreleased/Added-20250220-144549.yaml
@ -1,3 +0,0 @@
-kind: Added
-body: '`list` command with subcommands `cmds` and `lists`'
-time: 2025-02-20T14:45:49.562361581-06:00
--- a/.changes/unreleased/Added-20250220-145014.yaml
+++ b/.changes/unreleased/Added-20250220-145014.yaml
@ -1,3 +0,0 @@
-kind: Added
-body: Deprecation and unsupported warnings for old config keys
-time: 2025-02-20T14:50:14.048452348-06:00
--- a/.changes/unreleased/Added-20250220-145245.yaml
+++ b/.changes/unreleased/Added-20250220-145245.yaml
@ -1,3 +0,0 @@
-kind: Added
-body: CLI flag `--cmdStdOut` to output command's stdout/stderr to stdout
-time: 2025-02-20T14:52:45.030582408-06:00
--- a/.changes/unreleased/Added-20250221-003012.yaml
+++ b/.changes/unreleased/Added-20250221-003012.yaml
@ -1,3 +0,0 @@
-kind: Added
-body: Command type `remoteScript`. See docs for more info.
-time: 2025-02-21T00:30:12.276616792-06:00
--- a/.changes/unreleased/Added-20250301-132932.yaml
+++ b/.changes/unreleased/Added-20250301-132932.yaml
@ -0,0 +1,3 @@
+kind: Added
+body: 'Hooks: improved logging when executing'
+time: 2025-03-01T13:29:32.195438013-06:00
--- a/.changes/unreleased/Added-20250303-234248.yaml
+++ b/.changes/unreleased/Added-20250303-234248.yaml
@ -0,0 +1,3 @@
+kind: Added
+body: 'User commands: adding SSH keys using config key `userSshPubKeys`'
+time: 2025-03-03T23:42:48.009294808-06:00
--- a/.changes/unreleased/Added-20250303-234505.yaml
+++ b/.changes/unreleased/Added-20250303-234505.yaml
@ -0,0 +1,3 @@
+kind: Added
+body: 'directives: added support for fetching values using directive `%{externalSource:key}%`'
+time: 2025-03-03T23:45:05.666939653-06:00
--- a/.changes/unreleased/Changed-20250223-153012.yaml
+++ b/.changes/unreleased/Changed-20250223-153012.yaml
@ -1,3 +0,0 @@
-kind: Changed
-body: change to enums for Command type
-time: 2025-02-23T15:30:12.033359922-06:00
--- a/.changes/unreleased/Changed-20250223-153305.yaml
+++ b/.changes/unreleased/Changed-20250223-153305.yaml
@ -1,3 +0,0 @@
-kind: Changed
-body: Cache now stores resources by URL hash for ease-of-lookup
-time: 2025-02-23T15:33:05.33444642-06:00
--- a/.changes/unreleased/Changed-20250228-173825.yaml
+++ b/.changes/unreleased/Changed-20250228-173825.yaml
@ -1,3 +0,0 @@
-kind: Changed
-body: Changed PackageOperation to enums
-time: 2025-02-28T17:38:25.426136588-06:00
--- a/.changes/unreleased/Changed-20250301-194321.yaml
+++ b/.changes/unreleased/Changed-20250301-194321.yaml
@ -0,0 +1,3 @@
+kind: Changed
+body: 'Commands: if dir is not specified, run in config dir'
+time: 2025-03-01T19:43:21.323077376-06:00
--- a/.changes/unreleased/Fixed-20250220-144843.yaml
+++ b/.changes/unreleased/Fixed-20250220-144843.yaml
@ -1,3 +0,0 @@
-kind: Fixed
-body: Local command's `dir` full path is now found with home directory
-time: 2025-02-20T14:48:43.475300515-06:00
--- a/.changes/unreleased/Fixed-20250301-132600.yaml
+++ b/.changes/unreleased/Fixed-20250301-132600.yaml
@ -0,0 +1,3 @@
+kind: Fixed
+body: 'LocalFetcher: return fetch error'
+time: 2025-03-01T13:26:00.330176712-06:00
--- a/.changes/unreleased/Fixed-20250301-132801.yaml
+++ b/.changes/unreleased/Fixed-20250301-132801.yaml
@ -0,0 +1,3 @@
+kind: Fixed
+body: 'Lists: load file key before attempting to load from current file'
+time: 2025-03-01T13:28:01.739467944-06:00
--- a/.changes/unreleased/Fixed-20250301-182434.yaml
+++ b/.changes/unreleased/Fixed-20250301-182434.yaml
@ -0,0 +1,3 @@
+kind: Fixed
+body: 'fix: host not in config file, but in ssh config, properly added to hosts struct'
+time: 2025-03-01T18:24:34.81395054-06:00
--- a/.changes/v0.9.0.md
+++ b/.changes/v0.9.0.md
@ -0,0 +1,12 @@
+## v0.9.0 - 2025-02-28
+### Added
+* `list` command with subcommands `cmds` and `lists`
+* Deprecation and unsupported warnings for old config keys
+* CLI flag `--cmdStdOut` to output command's stdout/stderr to stdout
+* Command type `remoteScript`. See docs for more info.
+### Changed
+* change to enums for Command type
+* Cache now stores resources by URL hash for ease-of-lookup
+* Changed PackageOperation to enums
+### Fixed
+* Local command's `dir` full path is now found with home directory
--- a/.changes/v0.9.1.md
+++ b/.changes/v0.9.1.md
@ -0,0 +1,3 @@
+## v0.9.1 - 2025-03-01
+### Changed
+* Use EnvVar AWS_PROFILE to get S3 profile
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -6,6 +6,23 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html),
 and is generated by [Changie](https://github.com/miniscruff/changie).


+## v0.9.1 - 2025-03-01
+### Changed
+* Use EnvVar AWS_PROFILE to get S3 profile
+
+## v0.9.0 - 2025-02-28
+### Added
+* `list` command with subcommands `cmds` and `lists`
+* Deprecation and unsupported warnings for old config keys
+* CLI flag `--cmdStdOut` to output command's stdout/stderr to stdout
+* Command type `remoteScript`. See docs for more info.
+### Changed
+* change to enums for Command type
+* Cache now stores resources by URL hash for ease-of-lookup
+* Changed PackageOperation to enums
+### Fixed
+* Local command's `dir` full path is now found with home directory
+
 ## v0.8.0 - 2025-02-15
 ### Changed
 * Breaking: `cmd-lists` key changed to `cmdLists`
--- a/backy.code-workspace
+++ b/backy.code-workspace
@ -8,14 +8,17 @@
 	"settings": {
 		"cSpell.words": [
 			"Autorestic",
+			"changie",
 			"Cmds",
 			"CMDSTDOUT",
+			"goreleaser",
 			"knadh",
 			"koanf",
 			"mattn",
 			"maunium",
 			"mautrix",
 			"nikoksr",
+			"rawbytes",
 			"remotefetcher",
 			"Strs"
 		]
--- a/cmd/version.go
+++ b/cmd/version.go
@ -7,7 +7,7 @@ import (
 	"github.com/spf13/cobra"
 )

-const versionStr = "0.9.0"
+const versionStr = "0.9.2"

 var (
 	versionCmd = &cobra.Command{
--- a/docs/content/config/commands/_index.md
+++ b/docs/content/config/commands/_index.md
@ -8,32 +8,7 @@ weight: 1

 ### Example Config

-```yaml
-commands:
-  stop-docker-container:
-    cmd: docker
-    Args:
-      - compose
-      - -f /some/path/to/docker-compose.yaml
-      - down
-    # if host is not defined, command will be run locally
-    # The host has to be defined in either the config file or the SSH Config files
-    host: some-host
-    hooks
-      error:
-        - some-other-command-when-failing
-      success:
-        - success-command
-      final:
-        - final-command
-  backup-docker-container-script:
-    cmd: /path/to/local/script
-    # script file is input as stdin to SSH
-    type: scriptFile # also can be script
-    environment:
-      - FOO=BAR
-      - APP=$VAR
-```
+{{% code file="/examples/example.yml" language="yaml" %}}

 Values available for this section **(case-sensitive)**:

--- a/docs/content/config/remote-resources.md
+++ b/docs/content/config/remote-resources.md
@ -12,6 +12,10 @@ For the main config file to be fetched remotely, pass the URL using `-f [url]`.

 If using S3, you should use the s3 protocol URI: `s3://bucketName/key/path`. You will also need to set the env variable `S3_ENDPOINT` to the appropriate value. The flag `--s3-endpoint` can be used to override this value or to set this value, if not already set.

+## Authentication
+
+Currently, only the AWS authentication credentials file `~/.aws/credentials` is supported. For now, the environment variable `AWS_PROFILE` is used to lookup the profile.
+
 ## Scripts

 Remote script support is currently limited to http/https endpoints.
--- a/docs/content/examples/backy.yaml
+++ b/docs/content/examples/backy.yaml
@ -0,0 +1,108 @@
+commands:
+  stop-docker-container:
+    cmd: docker
+    Args:
+      - compose
+      - -f /some/path/to/docker-compose.yaml
+      - down
+    # if host is not defined, cmd will be run locally
+    host: some-host
+    hooks:
+      final:
+        - hostname
+      error:
+        - hostname
+  backup-docker-container-script:
+    cmd: /path/to/script
+    # The host has to be defined in the config file
+    host: some-host
+    environment:
+      - FOO=BAR
+      - APP=$VAR
+  shell-cmd:
+    cmd: rsync
+    shell: bash
+    Args:
+      - -av some-host:/path/to/data ~/Docker/Backups/docker-data
+  hostname:
+    cmd: hostname
+  update-docker:
+    type: package
+    shell: zsh # best to run package commands in a shell
+    packageName: docker-ce
+    Args:
+      - docker-ce-cli
+    packageManager: apt
+    packageOperation: install
+  update-dockerApt:
+    # type: package
+    shell: zsh
+    cmd: apt
+    Args:
+      - update
+      - "&&"
+      - apt install -y docker-ce
+      - docker-ce-cli
+    packageManager: apt
+    packageOperation: install
+
+cmd-lists:
+  cmds-to-run: # this can be any name you want
+    # all commands have to be defined
+    order:
+      - stop-docker-container
+      - backup-docker-container-script
+      - shell-cmd
+      - hostname
+    notifications:
+      - matrix.matrix
+    name: backup-some-server
+    cron: "0 0 1 * * *"
+  hostname:
+    name: hostname
+    order:
+      - hostname
+    notifications:
+      - mail.prod-email
+
+hosts:
+  # any ssh_config(5) keys/values not listed here will be looked up in the config file or the default config file
+  some-host:
+    hostname: some-hostname
+    config: ~/.ssh/config
+    user: user
+    privatekeypath: /path/to/private/key
+    port: 22
+    # can also be env:VAR
+    password: file:/path/to/file
+    # only one is supported for now
+    proxyjump: some-proxy-host
+
+# optional
+logging:
+  verbose: true
+  file: ./backy.log
+  console: false
+  cmd-std-out: false
+
+
+notifications:
+  mail:
+    prod-email:
+      id: prod-email
+      type: mail
+      host: yourhost.tld
+      port: 587
+      senderAddress: email@domain.tld
+      to:
+        - admin@domain.tld
+      username: smtp-username@domain.tld
+      password: your-password-here
+  matrix:
+    matrix:
+      id: matrix
+      type: matrix
+      home-server: your-home-server.tld
+      room-id: room-id
+      access-token: your-access-token
+      user-id: your-user-id
--- a/docs/content/examples/example.yml
+++ b/docs/content/examples/example.yml
@ -0,0 +1,24 @@
+commands:
+  stop-docker-container:
+    cmd: docker
+    Args:
+      - compose
+      - -f /some/path/to/docker-compose.yaml
+      - down
+    # if host is not defined, command will be run locally
+    # The host has to be defined in either the config file or the SSH Config files
+    host: some-host
+    hooks:
+      error:
+        - some-other-command-when-failing
+      success:
+        - success-command
+      final:
+        - final-command
+  backup-docker-container-script:
+    cmd: /path/to/local/script
+    # script file is input as stdin to SSH
+    type: scriptFile # also can be script
+    environment:
+      - FOO=BAR
+      - APP=$VAR
--- a/docs/layouts/shortcodes/code.html
+++ b/docs/layouts/shortcodes/code.html
@ -0,0 +1,3 @@
+{{ $file := .Get "file" | readFile }}
+{{ $lang := .Get "language" }}
+{{ (print "```" $lang "\n" $file "\n```") }}
--- a/examples/example.yml
+++ b/examples/example.yml
@ -0,0 +1,24 @@
+commands:
+  stop-docker-container:
+    cmd: docker
+    Args:
+      - compose
+      - -f /some/path/to/docker-compose.yaml
+      - down
+    # if host is not defined, command will be run locally
+    # The host has to be defined in either the config file or the SSH Config files
+    host: some-host
+    hooks:
+      error:
+        - some-other-command-when-failing
+      success:
+        - success-command
+      final:
+        - final-command
+  backup-docker-container-script:
+    cmd: /path/to/local/script
+    # script file is input as stdin to SSH
+    type: scriptFile # also can be script
+    environment:
+      - FOO=BAR
+      - APP=$VAR
--- a/go.mod
+++ b/go.mod
@ -21,6 +21,7 @@ require (
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/nikoksr/notify v1.3.0
 	github.com/pkg/errors v0.9.1
+	github.com/pkg/sftp v1.13.7
 	github.com/rs/zerolog v1.33.0
 	github.com/sethvargo/go-password v0.3.1
 	github.com/spf13/cobra v1.8.1
@ -65,6 +66,7 @@ require (
 	github.com/klauspost/compress v1.17.11 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.9 // indirect
 	github.com/knadh/koanf/maps v0.1.1 // indirect
+	github.com/kr/fs v0.1.0 // indirect
 	github.com/mattn/go-colorable v0.1.14 // indirect
 	github.com/minio/md5-simd v1.1.2 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
--- a/go.sum
+++ b/go.sum
@ -100,6 +100,8 @@ github.com/knadh/koanf/providers/rawbytes v0.1.0 h1:dpzgu2KO6uf6oCb4aP05KDmKmAmI
 github.com/knadh/koanf/providers/rawbytes v0.1.0/go.mod h1:mMTB1/IcJ/yE++A2iEZbY1MLygX7vttU+C+S/YmPu9c=
 github.com/knadh/koanf/v2 v2.1.2 h1:I2rtLRqXRy1p01m/utEtpZSSA6dcJbgGVuE27kW2PzQ=
 github.com/knadh/koanf/v2 v2.1.2/go.mod h1:Gphfaen0q1Fc1HTgJgSTC4oRX9R2R5ErYMZJy8fLJBo=
+github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
+github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
@ -135,6 +137,8 @@ github.com/pascaldekloe/name v1.0.0/go.mod h1:Z//MfYJnH4jVpQ9wkclwu2I2MkHmXTlT9w
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/sftp v1.13.7 h1:uv+I3nNJvlKZIQGSr8JVQLNHFU9YhhNpvC14Y6KgmSM=
+github.com/pkg/sftp v1.13.7/go.mod h1:KMKI0t3T6hfA+lTR/ssZdunHo+uwq7ghoN09/FSu3DY=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
@ -179,34 +183,72 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
 github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
 github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
 github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 go.mau.fi/util v0.8.4 h1:mVKlJcXWfVo8ZW3f4vqtjGpqtZqJvX4ETekxawt2vnQ=
 go.mau.fi/util v0.8.4/go.mod h1:MOfGTs1CBuK6ERTcSL4lb5YU7/ujz09eOPVEDckuazY=
 go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
 go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
 go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
 golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus=
 golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M=
 golang.org/x/exp v0.0.0-20250207012021-f9890c6ad9f3 h1:qNgPs5exUA+G0C96DrPwNrvLSj7GT/9D+3WMWUcUg34=
 golang.org/x/exp v0.0.0-20250207012021-f9890c6ad9f3/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/mod v0.23.0 h1:Zb7khfcRGKk+kqfxFaP5tZqCnDZMjC5VtUBs87Hr6QM=
 golang.org/x/mod v0.23.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8=
 golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w=
 golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
 golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
+golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
 golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU=
 golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM=
 golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
 golang.org/x/time v0.10.0 h1:3usCWA8tQn0L8+hFJQNgzpWbd89begxN66o1Ojdn5L4=
 golang.org/x/time v0.10.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/tools v0.30.0 h1:BgcpHewrV5AUp2G9MebG4XPFI1E2W41zU1SaqVA9vJY=
 golang.org/x/tools v0.30.0/go.mod h1:c347cR/OJfw5TI+GfX7RUPNMdDRRbjvYTS0jPyvsVtY=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
--- a/pkg/backy/backy.go
+++ b/pkg/backy/backy.go
@ -144,6 +144,7 @@ func (command *Command) RunCmd(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([
 		}

 		var err error
+
 		if command.Shell != "" {
 			cmdCtxLogger.Info().Str("Command", fmt.Sprintf("Running command %s on local machine in %s", command.Name, command.Shell)).Send()

@ -182,20 +183,7 @@ func (command *Command) RunCmd(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([

 		err = localCMD.Run()

-		outScanner := bufio.NewScanner(&cmdOutBuf)
-
-		for outScanner.Scan() {
-			outMap := make(map[string]interface{})
-			outMap["cmd"] = command.Cmd
-			outMap["output"] = outScanner.Text()
-
-			if str, ok := outMap["output"].(string); ok {
-				outputArr = append(outputArr, str)
-			}
-			// if command.GetOutput {
-			cmdCtxLogger.Info().Fields(outMap).Send()
-			// }
-		}
+		outputArr = logCommandOutput(command, cmdOutBuf, cmdCtxLogger, outputArr)
 		if err != nil {
 			cmdCtxLogger.Error().Err(fmt.Errorf("error when running cmd %s: %w", command.Name, err)).Send()
 			return outputArr, err
@ -240,7 +228,7 @@ func cmdListWorker(msgTemps *msgTemplates, jobs <-chan *CmdList, results chan<-
 			}

 			// Collect output if required
-			if list.GetOutput || cmdToRun.GetOutput {
+			if list.GetOutput || cmdToRun.GetOutputInList {
 				outStructArr = append(outStructArr, outStruct{
 					CmdName:     currentCmd,
 					CmdExecuted: currentCmd,
@ -249,17 +237,14 @@ func cmdListWorker(msgTemps *msgTemplates, jobs <-chan *CmdList, results chan<-
 			}
 		}

-		// Notify success if no errors occurred
 		if !hasError && list.NotifyConfig != nil && (list.NotifyOnSuccess || list.GetOutput) {
 			notifySuccess(cmdLogger, msgTemps, list, cmdsRan, outStructArr)
 		}

-		// Execute success and final hooks for all commands
 		for _, cmd := range list.Order {
 			cmdToRun := opts.Cmds[cmd]

-			// Execute success hooks if the command succeeded
-			if !hasError || cmdsRanContains(cmd, cmdsRan) {
+			if !hasError {
 				cmdToRun.ExecuteHooks("success", opts)
 			}

@ -276,17 +261,6 @@ func cmdListWorker(msgTemps *msgTemplates, jobs <-chan *CmdList, results chan<-
 	}
 }

-// Helper to check if a command is in the list of executed commands
-func cmdsRanContains(cmd string, cmdsRan []string) bool {
-	for _, c := range cmdsRan {
-		if c == cmd {
-			return true
-		}
-	}
-	return false
-}
-
-// Helper to notify errors
 func notifyError(logger zerolog.Logger, templates *msgTemplates, list *CmdList, cmdsRan []string, outStructArr []outStruct, err error, cmd *Command) {
 	errStruct := map[string]interface{}{
 		"listName":  list.Name,
@ -355,7 +329,6 @@ func (opts *ConfigOpts) RunListConfig(cron string) {
 		result := <-results
 		opts.Logger.Debug().Msgf("Processing result for list %s, command %s", result.ListName, result.CmdName)

-		// Process final hooks for the list (already handled in worker)
 	}
 	opts.closeHostConnections()
 }
@ -367,10 +340,8 @@ func (opts *ConfigOpts) ExecuteCmds() {
 		_, runErr := cmdToRun.RunCmd(cmdLogger, opts)
 		if runErr != nil {
 			opts.Logger.Err(runErr).Send()
-
 			cmdToRun.ExecuteHooks("error", opts)
 		} else {
-
 			cmdToRun.ExecuteHooks("success", opts)
 		}

@ -378,7 +349,6 @@ func (opts *ConfigOpts) ExecuteCmds() {
 	}

 	opts.closeHostConnections()
-
 }

 func (c *ConfigOpts) closeHostConnections() {
@ -425,8 +395,9 @@ func (cmd *Command) ExecuteHooks(hookType string, opts *ConfigOpts) {
 	case "error":
 		for _, v := range cmd.Hooks.Error {
 			errCmd := opts.Cmds[v]
+			opts.Logger.Info().Msgf("Running error hook command %s", v)
 			cmdLogger := opts.Logger.With().
-				Str("backy-cmd", v).
+				Str("backy-cmd", v).Str("hookType", "error").
 				Logger()
 			errCmd.RunCmd(cmdLogger, opts)
 		}
@ -434,16 +405,18 @@ func (cmd *Command) ExecuteHooks(hookType string, opts *ConfigOpts) {
 	case "success":
 		for _, v := range cmd.Hooks.Success {
 			successCmd := opts.Cmds[v]
+			opts.Logger.Info().Msgf("Running success hook command %s", v)
 			cmdLogger := opts.Logger.With().
-				Str("backy-cmd", v).
+				Str("backy-cmd", v).Str("hookType", "success").
 				Logger()
 			successCmd.RunCmd(cmdLogger, opts)
 		}
 	case "final":
 		for _, v := range cmd.Hooks.Final {
 			finalCmd := opts.Cmds[v]
+			opts.Logger.Info().Msgf("Running final hook command %s", v)
 			cmdLogger := opts.Logger.With().
-				Str("backy-cmd", v).
+				Str("backy-cmd", v).Str("hookType", "final").
 				Logger()
 			finalCmd.RunCmd(cmdLogger, opts)
 		}
@ -481,6 +454,25 @@ func (opts *ConfigOpts) ExecCmdsSSH(cmdList []string, hostsList []string) {
 	}
 }

+func logCommandOutput(command *Command, cmdOutBuf bytes.Buffer, cmdCtxLogger zerolog.Logger, outputArr []string) []string {
+
+	outScanner := bufio.NewScanner(&cmdOutBuf)
+
+	for outScanner.Scan() {
+		outMap := make(map[string]interface{})
+		outMap["cmd"] = command.Name
+		outMap["output"] = outScanner.Text()
+
+		if str, ok := outMap["output"].(string); ok {
+			outputArr = append(outputArr, str)
+		}
+		if command.OutputToLog {
+			cmdCtxLogger.Info().Fields(outMap).Send()
+		}
+	}
+	return outputArr
+}
+
 // func executeUserCommands() []string {

 // }
--- a/pkg/backy/config.go
+++ b/pkg/backy/config.go
@ -22,10 +22,13 @@ import (
 	"github.com/rs/zerolog"
 )

-const macroStart string = "%{"
-const macroEnd string = "}%"
-const envMacroStart string = "%{env:"
-const vaultMacroStart string = "%{vault:"
+const (
+	externDirectiveStart      string = "%{"
+	externDirectiveEnd        string = "}%"
+	externFileDirectiveStart  string = "%{file:"
+	envExternDirectiveStart   string = "%{env:"
+	vaultExternDirectiveStart string = "%{vault:"
+)

 func (opts *ConfigOpts) InitConfig() {
 	var err error
@ -95,41 +98,6 @@ func (opts *ConfigOpts) InitConfig() {
 	opts.koanf = backyKoanf
 }

-func loadConfigFile(fetcher remotefetcher.RemoteFetcher, filePath string, k *koanf.Koanf, opts *ConfigOpts) {
-	data, err := fetcher.Fetch(filePath)
-	if err != nil {
-		logging.ExitWithMSG(fmt.Sprintf("Could not fetch config file %s: %v", filePath, err), 1, nil)
-	}
-
-	if err := k.Load(rawbytes.Provider(data), yaml.Parser()); err != nil {
-		logging.ExitWithMSG(fmt.Sprintf("error loading config: %v", err), 1, &opts.Logger)
-	}
-}
-
-func loadDefaultConfigFiles(fetcher remotefetcher.RemoteFetcher, configFiles []string, k *koanf.Koanf, opts *ConfigOpts) {
-	cFileFailures := 0
-	for _, c := range configFiles {
-		opts.ConfigFilePath = c
-		data, err := fetcher.Fetch(c)
-		if err != nil {
-			cFileFailures++
-			continue
-		}
-
-		if data != nil {
-			if err := k.Load(rawbytes.Provider(data), yaml.Parser()); err == nil {
-				break
-			} else {
-				logging.ExitWithMSG(fmt.Sprintf("error loading config from file %s: %v", c, err), 1, &opts.Logger)
-			}
-		}
-	}
-
-	if cFileFailures == len(configFiles) {
-		logging.ExitWithMSG("Could not find any valid local config file", 1, nil)
-	}
-}
-
 func (opts *ConfigOpts) ReadConfig() *ConfigOpts {
 	setTerminalEnv()

@ -148,7 +116,7 @@ func (opts *ConfigOpts) ReadConfig() *ConfigOpts {

 	CheckConfigValues(backyKoanf, opts.ConfigFilePath)

-	validateCommands(backyKoanf, opts)
+	validateExecCommandsFromCLI(backyKoanf, opts)

 	setLoggingOptions(backyKoanf, opts)

@ -192,6 +160,41 @@ func (opts *ConfigOpts) ReadConfig() *ConfigOpts {
 	return opts
 }

+func loadConfigFile(fetcher remotefetcher.RemoteFetcher, filePath string, k *koanf.Koanf, opts *ConfigOpts) {
+	data, err := fetcher.Fetch(filePath)
+	if err != nil {
+		logging.ExitWithMSG(generateFileFetchErrorString(filePath, "config", err), 1, nil)
+	}
+
+	if err := k.Load(rawbytes.Provider(data), yaml.Parser()); err != nil {
+		logging.ExitWithMSG(fmt.Sprintf("error loading config: %v", err), 1, &opts.Logger)
+	}
+}
+
+func loadDefaultConfigFiles(fetcher remotefetcher.RemoteFetcher, configFiles []string, k *koanf.Koanf, opts *ConfigOpts) {
+	cFileFailures := 0
+	for _, c := range configFiles {
+		opts.ConfigFilePath = c
+		data, err := fetcher.Fetch(c)
+		if err != nil {
+			cFileFailures++
+			continue
+		}
+
+		if data != nil {
+			if err := k.Load(rawbytes.Provider(data), yaml.Parser()); err == nil {
+				break
+			} else {
+				logging.ExitWithMSG(fmt.Sprintf("error loading config from file %s: %v", c, err), 1, &opts.Logger)
+			}
+		}
+	}
+
+	if cFileFailures == len(configFiles) {
+		logging.ExitWithMSG("Could not find any valid local config file", 1, nil)
+	}
+}
+
 func setTerminalEnv() {
 	if isatty.IsTerminal(os.Stdout.Fd()) || isatty.IsCygwinTerminal(os.Stdout.Fd()) {
 		os.Setenv("BACKY_TERM", "enabled")
@ -200,7 +203,7 @@ func setTerminalEnv() {
 	}
 }

-func validateCommands(k *koanf.Koanf, opts *ConfigOpts) {
+func validateExecCommandsFromCLI(k *koanf.Koanf, opts *ConfigOpts) {
 	for _, c := range opts.executeCmds {
 		if !k.Exists(getCmdFromConfig(c)) {
 			logging.ExitWithMSG(fmt.Sprintf("command %s is not in config file %s", c, opts.ConfigFilePath), 1, nil)
@ -238,7 +241,7 @@ func setupLogger(opts *ConfigOpts) zerolog.Logger {

 func unmarshalConfig(k *koanf.Koanf, key string, target interface{}, log zerolog.Logger) {
 	if err := k.UnmarshalWithConf(key, target, koanf.UnmarshalConf{Tag: "yaml"}); err != nil {
-		logging.ExitWithMSG(fmt.Sprintf("error unmarshalling key %s into struct: %v", key, err), 1, &log)
+		logging.ExitWithMSG(fmt.Sprintf("error unmarshaling key %s into struct: %v", key, err), 1, &log)
 	}
 }

@ -304,9 +307,10 @@ func loadCommandLists(opts *ConfigOpts, backyKoanf *koanf.Koanf) {
 	}

 	if backyKoanf.Exists("cmdLists") {
-		unmarshalConfig(backyKoanf, "cmdLists", &opts.CmdConfigLists, opts.Logger)
 		if backyKoanf.Exists("cmdLists.file") {
 			loadCmdListsFile(backyKoanf, listsConfig, opts)
+		} else {
+			unmarshalConfig(backyKoanf, "cmdLists", &opts.CmdConfigLists, opts.Logger)
 		}
 	}
 }
@ -366,7 +370,7 @@ func loadCmdListsFile(backyKoanf *koanf.Koanf, listsConfig *koanf.Koanf, opts *C

 	data, err := fetcher.Fetch(opts.CmdListFile)
 	if err != nil {
-		logging.ExitWithMSG(fmt.Sprintf("Could not fetch config file %s: %v", opts.CmdListFile, err), 1, nil)
+		logging.ExitWithMSG(generateFileFetchErrorString(opts.CmdListFile, "list config", err), 1, nil)
 	}

 	if err := listsConfig.Load(rawbytes.Provider(data), yaml.Parser()); err != nil {
@ -378,6 +382,10 @@ func loadCmdListsFile(backyKoanf *koanf.Koanf, listsConfig *koanf.Koanf, opts *C
 	opts.Logger.Info().Str("using lists config file", opts.CmdListFile).Send()
 }

+func generateFileFetchErrorString(file, fileType string, err error) string {
+	return fmt.Sprintf("Could not fetch %s file %s: %v", file, fileType, err)
+}
+
 func validateCommandLists(opts *ConfigOpts) {
 	var cmdNotFoundSliceErr []error
 	for cmdListName, cmdList := range opts.CmdConfigLists {
@ -455,7 +463,7 @@ func (opts *ConfigOpts) setupVault() error {

 	unmarshalErr := opts.koanf.UnmarshalWithConf("vault.keys", &opts.VaultKeys, koanf.UnmarshalConf{Tag: "yaml"})
 	if unmarshalErr != nil {
-		logging.ExitWithMSG(fmt.Sprintf("error unmarshalling vault.keys into struct: %v", unmarshalErr), 1, &opts.Logger)
+		logging.ExitWithMSG(fmt.Sprintf("error unmarshaling vault.keys into struct: %v", unmarshalErr), 1, &opts.Logger)
 	}

 	opts.vaultClient = client
@ -565,6 +573,10 @@ func processCmds(opts *ConfigOpts) error {
 					cmd.RemoteHost.HostName = host.HostName
 				}
 			} else {
+				opts.Logger.Info().Msgf("adding host %s to host list", *cmd.Host)
+				if opts.Hosts == nil {
+					opts.Hosts = make(map[string]*Host)
+				}
 				opts.Hosts[*cmd.Host] = &Host{Host: *cmd.Host}
 				cmd.RemoteHost = &Host{Host: *cmd.Host}
 			}
@ -577,10 +589,11 @@ func processCmds(opts *ConfigOpts) error {
 					return err
 				}
 				cmd.Dir = &cmdDir
+			} else {
+				cmd.Dir = &opts.ConfigDir
 			}
 		}

-		// Parse package commands
 		if cmd.Type == PackageCT {
 			if cmd.PackageManager == "" {
 				return fmt.Errorf("package manager is required for package command %s", cmd.PackageName)
@ -612,19 +625,29 @@ func processCmds(opts *ConfigOpts) error {
 				return fmt.Errorf("username is required for user command %s", cmd.Name)
 			}

-			detectOSType(cmd, opts)
-			var err error
+			err := detectOSType(cmd, opts)
+			if err != nil {
+				opts.Logger.Info().Err(err).Str("command", cmdName).Send()
+			}

 			// Validate the operation
 			switch cmd.UserOperation {
 			case "add", "remove", "modify", "checkIfExists", "delete", "password":
 				cmd.userMan, err = usermanager.NewUserManager(cmd.OS)
+				if cmd.UserOperation == "password" {
+					cmd.UserPassword = expandExternalConfigDirectives(cmd.UserPassword, opts)
+				}
 				if cmd.Host != nil {
 					host, ok := opts.Hosts[*cmd.Host]
 					if ok {
 						cmd.userMan, err = usermanager.NewUserManager(host.OS)
 					}
 				}
+				for indx, key := range cmd.UserSshPubKeys {
+					opts.Logger.Debug().Msg("adding SSH Keys")
+					key = expandExternalConfigDirectives(key, opts)
+					cmd.UserSshPubKeys[indx] = key
+				}
 				if err != nil {
 					return err
 				}
@ -656,14 +679,8 @@ func processCmds(opts *ConfigOpts) error {
 	return nil
 }

-// processHooks evaluates if hooks are valid Commands
-//
-// The cmd.hookRefs[hookType] is created with any hooks found.
-//
-// Returns an error, if any, if the hook command is not found
 func processHooks(cmd *Command, hooks []string, opts *ConfigOpts, hookType string) error {

-	// initialize hook type
 	var hookCmdFound bool
 	cmd.hookRefs = map[string]map[string]*Command{}
 	cmd.hookRefs[hookType] = map[string]*Command{}
@ -691,11 +708,14 @@ func processHooks(cmd *Command, hooks []string, opts *ConfigOpts, hookType strin

 func detectOSType(cmd *Command, opts *ConfigOpts) error {
 	if cmd.Host == nil {
-		if runtime.GOOS == "linux" { // also can be specified to FreeBSD
+		if runtime.GOOS == "linux" {
 			cmd.OS = "linux"
 			opts.Logger.Info().Msg("Unix/Linux type OS detected")
+			return nil
 		}
+		return fmt.Errorf("using an os that is not yet supported for user commands")
 	}
+
 	host, ok := opts.Hosts[*cmd.Host]
 	if ok {
 		if host.OS != "" {
--- a/pkg/backy/notification.go
+++ b/pkg/backy/notification.go
@ -71,9 +71,8 @@ func (opts *ConfigOpts) SetupNotify() {
 					opts.Logger.Info().Str("list", confName).Err(fmt.Errorf("error: configuring matrix id %s failed during setup: %w", id, mtrxErr))
 					continue
 				}
-				// append the services
 				services = append(services, mtrxConf)
-			// service is not recognized
+
 			default:
 				opts.Logger.Info().Err(fmt.Errorf("id %s not found", id)).Str("list", confName).Send()
 			}
--- a/pkg/backy/ssh.go
+++ b/pkg/backy/ssh.go
@ -17,6 +17,7 @@ import (

 	"github.com/kevinburke/ssh_config"
 	"github.com/pkg/errors"
+	"github.com/pkg/sftp"
 	"github.com/rs/zerolog"
 	"golang.org/x/crypto/ssh"
 	"golang.org/x/crypto/ssh/knownhosts"
@ -569,6 +570,57 @@ func (command *Command) RunCmdSSH(cmdCtxLogger zerolog.Logger, opts *ConfigOpts)
 		if err := commandSession.Run(ArgsStr); err != nil {
 			return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error running command: %w", err)
 		}
+
+		if command.Type == UserCT && command.UserOperation == "add" {
+			if command.UserSshPubKeys != nil {
+				var (
+					f        *sftp.File
+					err      error
+					userHome []byte
+					client   *sftp.Client
+				)
+
+				cmdCtxLogger.Info().Msg("adding SSH Keys")
+
+				commandSession, _ = command.RemoteHost.createSSHSession(opts)
+				userHome, err = commandSession.CombinedOutput(fmt.Sprintf("grep \"%s\" /etc/passwd | cut -d: -f6", command.Username))
+				if err != nil {
+					return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error finding user home from /etc/passwd: %v", err)
+				}
+
+				command.UserHome = strings.TrimSpace(string(userHome))
+				userSshDir := fmt.Sprintf("%s/.ssh", command.UserHome)
+				client, err = sftp.NewClient(command.RemoteHost.SshClient)
+				if err != nil {
+					return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error creating sftp client: %v", err)
+				}
+
+				client.MkdirAll(userSshDir)
+				_, err = client.Create(fmt.Sprintf("%s/authorized_keys", userSshDir))
+				if err != nil {
+					return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error opening file %s/authorized_keys: %v", userSshDir, err)
+				}
+				f, err = client.OpenFile(fmt.Sprintf("%s/authorized_keys", userSshDir), os.O_APPEND|os.O_CREATE|os.O_WRONLY)
+				if err != nil {
+					return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error opening file %s/authorized_keys: %v", userSshDir, err)
+				}
+				defer f.Close()
+				for _, k := range command.UserSshPubKeys {
+					buf := bytes.NewBufferString(k)
+					cmdCtxLogger.Info().Str("key", k).Msg("adding SSH key")
+					if _, err := f.ReadFrom(buf); err != nil {
+						return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error adding to authorized keys: %v", err)
+					}
+				}
+
+				commandSession, _ = command.RemoteHost.createSSHSession(opts)
+				_, err = commandSession.CombinedOutput(fmt.Sprintf("chown -R %s:%s %s", command.Username, command.Username, userHome))
+				if err != nil {
+					return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), err
+				}
+
+			}
+		}
 	}

 	return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), nil
@ -622,7 +674,7 @@ func (command *Command) runScript(session *ssh.Session, cmdCtxLogger zerolog.Log
 		return collectOutput(outputBuf, command.Name, cmdCtxLogger, true), fmt.Errorf("error waiting for shell: %w", err)
 	}

-	return collectOutput(outputBuf, command.Name, cmdCtxLogger, command.GetOutput), nil
+	return collectOutput(outputBuf, command.Name, cmdCtxLogger, command.OutputToLog), nil
 }

 // runScriptFile handles the execution of script files.
--- a/pkg/backy/types.go
+++ b/pkg/backy/types.go
@ -51,44 +51,30 @@ type (
 	Command struct {
 		Name string `yaml:"name,omitempty"`

-		// command to run
 		Cmd string `yaml:"cmd"`

 		// See CommandType enum further down the page for acceptable values
 		Type CommandType `yaml:"type,omitempty"`

-		// host on which to run cmd
 		Host *string `yaml:"host,omitempty"`

-		// Hooks are for running commands on certain events
 		Hooks *Hooks `yaml:"hooks,omitempty"`

-		// hook refs are internal references of commands for each hook type
 		hookRefs map[string]map[string]*Command

-		// Shell specifies which shell to run the command in, if any.
 		Shell string `yaml:"shell,omitempty"`

 		RemoteHost *Host `yaml:"-"`

-		// Args is an array that holds the arguments to cmd
 		Args []string `yaml:"args,omitempty"`

-		/*
-			Dir specifies a directory in which to run the command.
-		*/
 		Dir *string `yaml:"dir,omitempty"`

-		// Env points to a file containing env variables to be used with the command
 		Env string `yaml:"env,omitempty"`

-		// Environment holds env variables to be used with the command
 		Environment []string `yaml:"environment,omitempty"`

-		// Output determines if output is requested.
-		//
-		// Only for when command is in a list.
-		GetOutput bool `yaml:"getOutput,omitempty"`
+		GetOutputInList bool `yaml:"getOutputInList,omitempty"`

 		ScriptEnvFile string `yaml:"scriptEnvFile"`

@ -102,10 +88,8 @@ type (

 		PackageName string `yaml:"packageName,omitempty"`

-		// Version specifies the desired version for package execution
 		PackageVersion string `yaml:"packageVersion,omitempty"`

-		// PackageOperation specifies the action for package-related commands (e.g., "install" or "remove")
 		PackageOperation PackageOperation `yaml:"packageOperation,omitempty"`

 		pkgMan pkgman.PackageManager
@ -113,42 +97,35 @@ type (
 		packageCmdSet bool
 		// END PACKAGE COMMAND FIELDS

-		// RemoteSource specifies a URL to fetch the command or configuration remotely
 		RemoteSource string `yaml:"remoteSource,omitempty"`

-		// FetchBeforeExecution determines if the remoteSource should be fetched before running
 		FetchBeforeExecution bool `yaml:"fetchBeforeExecution,omitempty"`

 		Fetcher remotefetcher.RemoteFetcher

 		// BEGIN USER COMMAND FIELDS

-		// Username specifies the username for user creation or related operations
 		Username string `yaml:"userName,omitempty"`

 		UserID string `yaml:"userID,omitempty"`

-		// UserGroups specifies the groups to add the user to
 		UserGroups []string `yaml:"userGroups,omitempty"`

-		// UserHome specifies the home directory for the user
 		UserHome string `yaml:"userHome,omitempty"`

-		// UserShell specifies the shell for the user
 		UserShell string `yaml:"userShell,omitempty"`

-		// SystemUser specifies whether the user is a system account
 		SystemUser bool `yaml:"systemUser,omitempty"`

-		// UserPassword specifies the password for the user (can be file: or plain text)
 		UserPassword string `yaml:"userPassword,omitempty"`

+		UserSshPubKeys []string `yaml:"userSshPubKeys,omitempty"`
+
 		userMan usermanager.UserManager

 		// OS for the command, only used when type is user
 		OS string `yaml:"OS,omitempty"`

-		// UserOperation specifies the action for user-related commands (e.g., "create" or "remove")
 		UserOperation string `yaml:"userOperation,omitempty"`

 		userCmdSet bool
--- a/pkg/backy/utils.go
+++ b/pkg/backy/utils.go
@ -247,7 +247,6 @@ func (opts *ConfigOpts) loadEnv() {
 	opts.backyEnv = backyEnv
 }

-// expandEnvVars expands environment variables with the env used in the config
 func expandEnvVars(backyEnv map[string]string, envVars []string) {

 	env := func(name string) string {
@ -261,10 +260,10 @@ func expandEnvVars(backyEnv map[string]string, envVars []string) {

 	// parse env variables using new macros
 	for indx, v := range envVars {
-		if strings.HasPrefix(v, macroStart) && strings.HasSuffix(v, macroEnd) {
-			if strings.HasPrefix(v, envMacroStart) {
-				v = strings.TrimPrefix(v, envMacroStart)
-				v = strings.TrimRight(v, macroEnd)
+		if strings.HasPrefix(v, externDirectiveStart) && strings.HasSuffix(v, externDirectiveEnd) {
+			if strings.HasPrefix(v, envExternDirectiveStart) {
+				v = strings.TrimPrefix(v, envExternDirectiveStart)
+				v = strings.TrimRight(v, externDirectiveEnd)
 				out, _ := shell.Expand(v, env)
 				envVars[indx] = out
 			}
@ -324,7 +323,7 @@ func parsePackageVersion(output string, cmdCtxLogger zerolog.Logger, command *Co
 	// println(output)
 	if err != nil {
 		cmdCtxLogger.Error().Err(err).Str("package", command.PackageName).Msg("Error parsing package version output")
-		return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.GetOutput), err
+		return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), err
 	}

 	cmdCtxLogger.Info().
@ -349,3 +348,39 @@ func parsePackageVersion(output string, cmdCtxLogger zerolog.Logger, command *Co
 	}
 	return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, false), err
 }
+
+func expandExternalConfigDirectives(key string, opts *ConfigOpts) string {
+	if !(strings.HasPrefix(key, externDirectiveStart) && strings.HasSuffix(key, externDirectiveEnd)) {
+		return key
+	}
+	opts.Logger.Info().Str("expanding external key", key).Send()
+	if strings.HasPrefix(key, envExternDirectiveStart) {
+		key = strings.TrimPrefix(key, envExternDirectiveStart)
+		key = strings.TrimSuffix(key, externDirectiveEnd)
+		return os.Getenv(key)
+	}
+	if strings.HasPrefix(key, externFileDirectiveStart) {
+		var err error
+		var keyValue []byte
+		key = strings.TrimPrefix(key, externFileDirectiveStart)
+		key = strings.TrimSuffix(key, externDirectiveEnd)
+		key, err = getFullPathWithHomeDir(key)
+		if err != nil {
+			opts.Logger.Err(err).Send()
+			return ""
+		}
+		keyValue, err = os.ReadFile(key)
+		if err != nil {
+			opts.Logger.Err(err).Send()
+			return ""
+		}
+		key = string(keyValue)
+	}
+	if strings.HasPrefix(key, vaultExternDirectiveStart) {
+		key = strings.TrimPrefix(key, vaultExternDirectiveStart)
+		key = strings.TrimSuffix(key, externDirectiveEnd)
+		key = GetVaultKey(key, opts, opts.Logger)
+	}
+
+	return key
+}
--- a/pkg/remotefetcher/configfetcher.go
+++ b/pkg/remotefetcher/configfetcher.go
@ -29,7 +29,7 @@ func NewRemoteFetcher(source string, cache *Cache, options ...FetcherOption) (Re
 		option(&config)
 	}

-	// If FileType is empty (i.e. WithFileType was not called), yaml is the default file type
+	// WithFileType was not called. yaml is the default file type
 	if strings.TrimSpace(config.FileType) == "" {
 		config.FileType = "yaml"
 	}
--- a/pkg/remotefetcher/local.go
+++ b/pkg/remotefetcher/local.go
@ -20,7 +20,7 @@ func (l *LocalFetcher) Fetch(source string) ([]byte, error) {
 		if l.config.IgnoreFileNotFound {
 			return nil, ErrIgnoreFileNotFound
 		}
-		return nil, nil
+		return nil, err
 	}
 	file, err := os.Open(source)
 	if err != nil {
--- a/pkg/remotefetcher/s3.go
+++ b/pkg/remotefetcher/s3.go
@ -44,9 +44,8 @@ func NewS3Fetcher(endpoint string, options ...FetcherOption) (*S3Fetcher, error)
 	*/

 	s3Endpoint := os.Getenv("S3_ENDPOINT")
-	creds, err := getS3Credentials("default", s3Endpoint, cfg.HTTPClient)
+	creds, err := getS3Credentials(os.Getenv("AWS_PROFILE"), s3Endpoint, cfg.HTTPClient)
 	if err != nil {
-		println(err.Error())
 		return nil, err
 	}

@ -133,7 +132,7 @@ func getS3Credentials(profile, host string, httpClient *http.Client) (*credentia
 	if hdirErr != nil {
 		return nil, hdirErr
 	}
-	s3Creds := credentials.NewFileAWSCredentials(path.Join(homeDir, ".aws", "credentials"), "default")
+	s3Creds := credentials.NewFileAWSCredentials(path.Join(homeDir, ".aws", "credentials"), profile)
 	credVals, credErr := s3Creds.GetWithContext(&credentials.CredContext{Endpoint: host, Client: httpClient})
 	if credErr != nil {
 		return nil, credErr
--- a/2
+++ b/2
@ -1,7 +1,7 @@
 #!/bin/bash
 set -eou pipefail
 go generate ./...
-export CURRENT_TAG="$(go run backy.go version -V)"
+CURRENT_TAG="$(go run backy.go version -V)"
 goreleaser -f .goreleaser/github.yml check
 goreleaser -f .goreleaser/gitea.yml check
 changie batch $CURRENT_TAG
--- a/tests/.gitignore
+++ b/tests/.gitignore
@ -0,0 +1 @@
+private
--- a/tests/ErrorHook.yml
+++ b/tests/ErrorHook.yml
@ -0,0 +1,17 @@
+commands:
+  echoTestFail:
+    cmd: ech
+    shell: bash
+    Args: hello world
+    hooks:
+      error:
+        - errorCmd
+
+  errorCmd:
+    name: get docker version
+    cmd: docker
+    getOutput: true
+    outputToLog: true
+    Args:
+      - "-v"
+    host: email-svr
--- a/tests/HookNotInFile.yaml
+++ b/tests/HookNotInFile.yaml
@ -0,0 +1,14 @@
+commands:
+  echoTestFail:
+    cmd: ech
+    shell: bash
+    Args: hello world
+    hooks:
+      error:
+        - errorCm # 
+
+  errorCmd:
+    name: get docker version
+    cmd: docker
+    Args:
+      - "-v"
--- a/tests/SuccessHook.yml
+++ b/tests/SuccessHook.yml
@ -0,0 +1,16 @@
+commands:
+  echoTestSuccess:
+    cmd: echo
+    shell: bash
+    Args: hello world
+    hooks:
+      success:
+        - successCmd
+
+  errorCmd:
+    name: get docker version
+    cmd: docker
+    getOutput: true
+    outputToLog: true
+    Args:
+      - "-v"
--- a/tests/backy.yaml
+++ b/tests/backy.yaml
@ -1,18 +0,0 @@
-commands:
-  echoTestPass:
-    cmd: echo
-    shell: bash
-    Args: hello world
-
-  runRemoteShellScriptSuccess:
-    cmd: 
-
-
-  packageCommandSuccess:
-    packageName: docker-ce
-    Args:
-      - docker-ce-cli
-    packageManager: apt
-    packageOperation: install
-
-
Author	SHA1	Message	Date
Andrew Woodlee	bcba6b2086	v0.10.0	2025-03-03 23:46:26 -06:00
Andrew Woodlee	753b03861f	v0.10.0	2025-03-03 23:45:28 -06:00
Andrew Woodlee	80a45cd595	v0.9.1	2025-03-01 01:57:51 -06:00
Andrew Woodlee	551c8ad441	get S3 profile using env AWS_PROFILE	2025-03-01 01:57:33 -06:00
Andrew Woodlee	3823b1bf44	get S3 profile using env AWS_PROFILE	2025-03-01 01:57:05 -06:00
Andrew Woodlee	f777c78aad	v0.9.0	2025-02-28 17:52:28 -06:00