v0.10.2

.changes/unreleased/Added-20250301-132932.yaml

@@ -1,3 +0,0 @@
-kind: Added
-body: 'Hooks: improved logging when executing'
-time: 2025-03-01T13:29:32.195438013-06:00

.changes/unreleased/Added-20250303-234248.yaml

@@ -1,3 +0,0 @@
-kind: Added
-body: 'User commands: adding SSH keys using config key `userSshPubKeys`'
-time: 2025-03-03T23:42:48.009294808-06:00

.changes/unreleased/Added-20250303-234505.yaml

@@ -1,3 +0,0 @@
-kind: Added
-body: 'directives: added support for fetching values using directive `%{externalSource:key}%`'
-time: 2025-03-03T23:45:05.666939653-06:00

.changes/unreleased/Changed-20250301-194321.yaml

@@ -1,3 +0,0 @@
-kind: Changed
-body: 'Commands: if dir is not specified, run in config dir'
-time: 2025-03-01T19:43:21.323077376-06:00

.changes/unreleased/Changed-20250305-003415.yaml

@@ -1,3 +0,0 @@
-kind: Changed
-body: 'FileDirective: use the config directory if path is not absolute'
-time: 2025-03-05T00:34:15.689980075-06:00

.changes/unreleased/Fixed-20250301-132600.yaml

@@ -1,3 +0,0 @@
-kind: Fixed
-body: 'LocalFetcher: return fetch error'
-time: 2025-03-01T13:26:00.330176712-06:00

.changes/unreleased/Fixed-20250301-132801.yaml

@@ -1,3 +0,0 @@
-kind: Fixed
-body: 'Lists: load file key before attempting to load from current file'
-time: 2025-03-01T13:28:01.739467944-06:00

.changes/unreleased/Fixed-20250301-182434.yaml

@@ -1,3 +0,0 @@
-kind: Fixed
-body: 'fix: host not in config file, but in ssh config, properly added to hosts struct'
-time: 2025-03-01T18:24:34.81395054-06:00

.changes/unreleased/Fixed-20250304-235706.yaml

@@ -1,3 +0,0 @@
-kind: Fixed
-body: 'SSH: password authentication bugs'
-time: 2025-03-04T23:57:06.326604774-06:00

.changes/v0.10.0.md

@@ -0,0 +1,16 @@
+## v0.10.0 - 2025-03-08
+### Added
+* Hooks: improved logging when executing
+* User commands: adding SSH keys using config key `userSshPubKeys`
+* directives: added support for fetching values using directive `%{externalSource:key}%`
+### Changed
+* Commands: if dir is not specified, run in config dir
+* FileDirective: use the config directory if path is not absolute
+* Host: changes to case of some keys
+* Notifications: added external directive to sensitive keys
+### Fixed
+* LocalFetcher: return fetch error
+* Lists: load file key before attempting to load from current file
+* fix: host not in config file, but in ssh config, properly added to hosts struct
+* SSH: password authentication bugs
+* User commands: change user password works

.changes/v0.10.1.md

@@ -0,0 +1,8 @@
+## v0.10.1 - 2025-03-11
+### Added
+* UserCommands: add ssh public keys when running locally
+* UserCommands: add field CreateUserHome
+### Changed
+* UserCommands: create temp file when modifing password over SSH
+* UserCommands: change field name
+* Vault: keys are now referenced by `name`, and the actual data by `data`

.changes/v0.10.2.md

@@ -0,0 +1,6 @@
+## v0.10.2 - 2025-03-19
+### Added
+* Notifications: http service added
+* Variable support. Can be referenced with `%{var:nameOfVar}%` in select string fields.
+### Changed
+* vault: initialize vault before validating config

.woodpecker/gitea.yml

@@ -1,9 +1,7 @@
-name: goreleaser release
 steps:
   golang:
     image: golang:1.23
     commands:
-      - go mod tidy
       - go install github.com/goreleaser/goreleaser/v2@v2.7.0
       - goreleaser release -f .goreleaser/gitea.yml --release-notes=".changes/$(go run backy.go version -V).md"
     environment:

.woodpecker/go-lint.yml

@@ -5,7 +5,7 @@ steps:
       - go build
       - go test
   release:
-    image: golangci/golangci-lint:v1.53.3
+    image: golangci/golangci-lint:v1.64.7
     commands:
       - golangci-lint run -v --timeout 5m
 

CHANGELOG.md

@@ -6,6 +6,39 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html),
 and is generated by [Changie](https://github.com/miniscruff/changie).
 
 
+## v0.10.2 - 2025-03-19
+### Added
+* Notifications: http service added
+* Variable support. Can be referenced with `%{var:nameOfVar}%` in select string fields.
+### Changed
+* vault: initialize vault before validating config
+
+## v0.10.1 - 2025-03-11
+### Added
+* UserCommands: add ssh public keys when running locally
+* UserCommands: add field CreateUserHome
+### Changed
+* UserCommands: create temp file when modifing password over SSH
+* UserCommands: change field name
+* Vault: keys are now referenced by `name`, and the actual data by `data`
+
+## v0.10.0 - 2025-03-08
+### Added
+* Hooks: improved logging when executing
+* User commands: adding SSH keys using config key `userSshPubKeys`
+* directives: added support for fetching values using directive `%{externalSource:key}%`
+### Changed
+* Commands: if dir is not specified, run in config dir
+* FileDirective: use the config directory if path is not absolute
+* Host: changes to case of some keys
+* Notifications: added external directive to sensitive keys
+### Fixed
+* LocalFetcher: return fetch error
+* Lists: load file key before attempting to load from current file
+* fix: host not in config file, but in ssh config, properly added to hosts struct
+* SSH: password authentication bugs
+* User commands: change user password works
+
 ## v0.9.1 - 2025-03-01
 ### Changed
 * Use EnvVar AWS_PROFILE to get S3 profile

cmd/version.go

@@ -7,7 +7,7 @@ import (
 	"github.com/spf13/cobra"
 )
 
-const versionStr = "0.9.2"
+const versionStr = "0.10.2"
 
 var (
 	versionCmd = &cobra.Command{

docs/content/cli/exec.md

@@ -15,5 +15,5 @@ The `exec` subcommand can do some things that the configuration file can't do ye
 The commands have to be defined in the config file. The hosts need to at least be in the ssh_config(5) file.
 
 ```sh
-backy exec host [--commands=command1 -commands=command2 ... | -c command1 -c command2 ...] [--hosts=host1 --hosts=hosts2 ... | -m host1 -c host2 ...]  [flags]
+backy exec host [--commands=command1 -commands=command2 ... | -c command1 -c command2 ...] [--hosts=host1 --hosts=hosts2 ... | -m host1 -m host2 ...]  [flags]
 ```

docs/content/cli/list.md

@@ -0,0 +1,29 @@
+---
+title: List
+---
+
+
+List commands, lists, or hosts defined in config file
+
+Usage:
+```
+  backy list [command]
+```
+
+Available Commands:
+  cmds        List commands defined in config file.
+  lists       List lists defined in config file.
+
+Flags:
+```
+  -h, --help   help for list
+```
+
+Global Flags:
+```
+      --cmdStdOut            Pass to print command output to stdout
+  -f, --config string        config file to read from
+      --log-file string      log file to write to
+      --s3-endpoint string   Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable.
+  -v, --verbose              Sets verbose level
+```

docs/content/config/commands/_index.md

@@ -12,17 +12,17 @@ weight: 1
 
 Values available for this section **(case-sensitive)**:
 
-| name | notes | type | required
-| --- | --- | --- | --- |
-| `cmd` | Defines the command to execute | `string` | yes |
-| `Args` | Defines the arguments to the command | `[]string` | no |
-| `environment` | Defines environment variables for the command | `[]string` | no |
-| `type` | See documentation further down the page. Additional fields may be required. | `string` | no |
-| `getOutput` | Command(s) output is in the notification(s) | `bool` | no |
-| `host` | If not specified, the command will execute locally. | `string` | no |
-| `scriptEnvFile` | When type is `scriptFile` or `script`, this file is prepended to the input. | `string` | no |
-| `shell` | Run the command in the shell | `string` | no |
-| `hooks` | Hooks are used at the end of the individual command. Must have at least `error`, `success`, or `final`. | `map[string][]string` | no |
+| name            | notes                                                                                                   | type                  | required | External directive support |
+| ----------------| ------------------------------------------------------------------------------------------------------- | --------------------- | -------- |----------------------------|
+| `cmd`           | Defines the command to execute                                                                          | `string`              | yes      | No                         |
+| `Args`          | Defines the arguments to the command                                                                    | `[]string`            | no       | No                         |
+| `environment`   | Defines environment variables for the command                                                           | `[]string`            | no       | Partial                    |
+| `type`          | See documentation further down the page. Additional fields may be required.                             | `string`              | no       | No                         |
+| `getOutput`     | Command(s) output is in the notification(s)                                                             | `bool`                | no       | No                         |
+| `host`          | If not specified, the command will execute locally.                                                     | `string`              | no       | No                         |
+| `scriptEnvFile` | When type is `scriptFile` or `script`, this file is prepended to the input.                             | `string`              | no       | No                         |
+| `shell`         | Run the command in the shell                                                                            | `string`              | no       | No                         |
+| `hooks`         | Hooks are used at the end of the individual command. Must have at least `error`, `success`, or `final`. | `map[string][]string` | no       | No                         |
 
 #### cmd
 
@@ -95,8 +95,9 @@ The following options are available:
 The environment variables support expansion:
 
 - using escaped values `$VAR` or `${VAR}`
+- using any external directive, and if using the env directive, the variable will be read from a `.env` file
 
-For now, the variables have to be defined in an `.env` file in the same directory that the program is run from.
+<!-- For now, the variables expanded have to be defined in an `.env` file in the same directory that the program is run from. -->
 
 If using it with host specified, the SSH server has to be configured to accept those env variables.
 

docs/content/config/commands/user-commands.md

@@ -6,14 +6,18 @@ description: This is dedicated to user commands.
 
 This is dedicated to `user` commands. The command `type` field must be `user`. User is a type that allows one to perform user operations. There are several additional options available when `type` is `user`:
 
-| name | notes | type | required |
-| --- | --- | --- | --- |
-| `userName` | The name of a user to be configured. | `string` | yes |
-| `userOperation` | The type of operation to perform. | `string` | yes |
-| `userID` | The user ID to use. | `string` | yes |
-| `userGroups` | The groups the user should be added to. | `[]string` | yes |
-| `userShell` | The shell for the user. | `string` | yes |
-| `userHome` | The user's home directory. | `string` | no |
+| name            | notes                                                        | type       | required | External directive support
+| ----------------| -------------------------------------------------------------| ---------- | ---------| --------------------------|
+| `userName`      | The name of a user to be configured.                         | `string`   | yes      | no 						 |
+| `userOperation` | The type of operation to perform.                            | `string`   | yes      | no 						 |
+| `userID`        | The user ID to use.                                          | `string`   | no       | no 						 |
+| `userGroups`    | The groups the user should be added to.                      | `[]string` | no       | no 						 |
+| `systemUser`    | Create a system user.                                        | `bool`     | no       | no 						 |
+| `userCreateHome`| Create the home directory.                                   | `bool`     | no       | no 						 |
+| `userSshPubKeys`| The keys to add to the user's authorized keys.               | `[]string` | no       | yes 						 |
+| `userShell`     | The shell for the user.                                      | `string`   | no       | no 						 |
+| `userHome`      | The user's home directory.                                   | `string`   | no       | no 						 |
+| `userPassword`  | The new password value when using the `password` operation.  | `string`   | no       | yes						 |
 
 
 #### example

docs/content/config/directives.md

@@ -0,0 +1,15 @@
+---
+title: "External Directives"
+weight: 2
+description: How to set up external directives.
+---
+
+External directives are for including data that should not be in the config file. The following directives are supported:
+
+- `%{file:path/to/file}%`
+- `%{env:ENV_VAR}%`
+- `%{vault:vault-key}%`
+
+See the docs of each command if the field is supported.
+
+If the file path does not begin with a `/`, the config file's directory will be used as the starting point.

docs/content/config/hosts.md

@@ -5,19 +5,19 @@ description: >
   This page tells you how to use hosts.
 ---
 
-| Key                  | Description                                                   | Type     | Required |
-|----------------------|---------------------------------------------------------------|----------|----------|
-| `OS`                 | Operating system of the host (used for package commands)      | `string` | no       |
-| `config`             | Path to the SSH config file                                   | `string` | no       |
-| `host`               | Specifies the `Host` ssh_config(5) directive                  | `string` | yes      |
-| `hostname`           | Hostname of the host                                          | `string` | no       |
-| `knownhostsfile`     | Path to the known hosts file                                  | `string` | no       |
-| `port`               | Port number to connect to                                     | `uint16` | no       |
-| `proxyjump`          | Proxy jump hosts, comma-separated                             | `string` | no       |
-| `password`           | Password for SSH authentication                               | `string` | no       |
-| `privatekeypath`     | Path to the private key file                                  | `string` | no       |
-| `privatekeypassword` | Password for the private key file                             | `string` | no       |
-| `user`               | Username for SSH authentication                               | `string` | no       |
+| Key                  | Description                                                   | Type     | Required | External directive support |
+|----------------------|---------------------------------------------------------------|----------|----------|----------------------------|
+| `OS`                 | Operating system of the host (used for package commands)      | `string` | no       | No                         |
+| `config`             | Path to the SSH config file                                   | `string` | no       | No                         |
+| `host`               | Specifies the `Host` ssh_config(5) directive                  | `string` | yes      | No                         |
+| `hostname`           | Hostname of the host                                          | `string` | no       | No                         |
+| `knownHostsFile`     | Path to the known hosts file                                  | `string` | no       | No                         |
+| `port`               | Port number to connect to                                     | `uint16` | no       | No                         |
+| `proxyjump`          | Proxy jump hosts, comma-separated                             | `string` | no       | No                         |
+| `password`           | Password for SSH authentication                               | `string` | no       | No                         |
+| `privateKeyPath`     | Path to the private key file                                  | `string` | no       | No                         |
+| `privateKeyPassword` | Password for the private key file                             | `string` | no       | Yes                        |
+| `user`               | Username for SSH authentication                               | `string` | no       | No                         |
 
 ## exec host subcommand
 

docs/content/config/notifications.md

@@ -39,23 +39,23 @@ There must be a section with an id (eg. `mail.test-svr`) following one of these
 
 ### mail
 
-| key | description | type
-| --- | --- | ---
-| `host` | Specifies the SMTP host to connect to | `string`
-| `port` | Specifies the SMTP port | `uint16`
-| `senderaddress` | Address from which to send mail | `string`
-| `to` | Recipients to send emails to | `[]string`
-| `username` | SMTP username | `string`
-| `password` | SMTP password | `string`
+| key | description | type | External directive support |
+| --- | --- | --- | --- |
+| `host` | Specifies the SMTP host to connect to | `string` | no
+| `port` | Specifies the SMTP port | `uint16` | no
+| `senderaddress` | Address from which to send mail | `string` | no
+| `to` | Recipients to send emails to | `[]string` | no
+| `username` | SMTP username | `string` | no
+| `password` | SMTP password | `string` | yes
 
 ### matrix
 
-| key | description | type
-| --- | --- | ---
-| `home-server` | Specifies the Matrix server connect to | `string`
-| `room-id` | Specifies the room ID of the room to send messages to | `string`
-| `access-token` | Matrix access token | `string`
-| `user-id` | Matrix user ID | `string`
+| key | description | type | External directive support |
+| --- | --- | ---| ---- |
+| `home-server` | Specifies the Matrix server connect to | `string` | no
+| `room-id` | Specifies the room ID of the room to send messages to | `string` | no
+| `access-token` | Matrix access token  | `string` | yes
+| `user-id` | Matrix user ID | `string` | no
 
 To get your access token (assumes you are using [Element](https://element.io/)) :
 

docs/content/config/vault.md

@@ -6,7 +6,7 @@ description: Set up and configure vault.
 
 [Vault](https://www.vaultproject.io/) is a tool for storing secrets and other data securely.
 
-Vault config can be used by prefixing `vault:` in front of a password or ENV var.
+A Vault key can be used by prefixing `%{vault:vault.keys.name}%` in a field that supports external directives.
 
 This is the object in the config file:
 
@@ -18,10 +18,12 @@ vault:
   keys:
     - name: mongourl
       mountpath: secret
+      key: data
       path: mongo/url
-      type:  # KVv1 or KVv2
-    - name:
-      path:
-      type:
-      mountpath:
+      type: KVv2  # KVv1 or KVv2
+    - name: someKeyName
+      mountpath: secret
+      key: keyData
+      type: KVv2
+      path: some/path
 ```

docs/content/examples/backy.yaml

@@ -71,7 +71,7 @@ hosts:
     hostname: some-hostname
     config: ~/.ssh/config
     user: user
-    privatekeypath: /path/to/private/key
+    privateKeyPath: /path/to/private/key
     port: 22
     # can also be env:VAR
     password: file:/path/to/file

go.mod

@@ -2,14 +2,13 @@ module git.andrewnw.xyz/CyberShell/backy
 
 go 1.23
 
-toolchain go1.23.6
-
-replace git.andrewnw.xyz/CyberShell/backy => /home/andrew/Projects/backy
+toolchain go1.23.7
 
 require (
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.76.0
 	github.com/dmarkham/enumer v1.5.11
 	github.com/go-co-op/gocron v1.37.0
+	github.com/google/uuid v1.6.0
 	github.com/hashicorp/vault/api v1.15.0
 	github.com/joho/godotenv v1.5.1
 	github.com/kevinburke/ssh_config v1.2.0
@@ -51,7 +50,6 @@ require (
 	github.com/go-jose/go-jose/v4 v4.0.1 // indirect
 	github.com/go-viper/mapstructure/v2 v2.2.1 // indirect
 	github.com/goccy/go-json v0.10.5 // indirect
-	github.com/google/uuid v1.6.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect

pkg/backy/allowedexternaldirectives_enumer.go

@@ -0,0 +1,145 @@
+// Code generated by "enumer -linecomment -yaml -text -json -type=AllowedExternalDirectives"; DO NOT EDIT.
+
+package backy
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+)
+
+const _AllowedExternalDirectivesName = "DefaultExternalDirvaultvault-filevault-file-envfile-envfileenv"
+
+var _AllowedExternalDirectivesIndex = [...]uint8{0, 18, 23, 33, 47, 55, 59, 62}
+
+const _AllowedExternalDirectivesLowerName = "defaultexternaldirvaultvault-filevault-file-envfile-envfileenv"
+
+func (i AllowedExternalDirectives) String() string {
+	if i < 0 || i >= AllowedExternalDirectives(len(_AllowedExternalDirectivesIndex)-1) {
+		return fmt.Sprintf("AllowedExternalDirectives(%d)", i)
+	}
+	return _AllowedExternalDirectivesName[_AllowedExternalDirectivesIndex[i]:_AllowedExternalDirectivesIndex[i+1]]
+}
+
+// An "invalid array index" compiler error signifies that the constant values have changed.
+// Re-run the stringer command to generate them again.
+func _AllowedExternalDirectivesNoOp() {
+	var x [1]struct{}
+	_ = x[DefaultExternalDir-(0)]
+	_ = x[AllowedExternalDirectiveVault-(1)]
+	_ = x[AllowedExternalDirectiveVaultFile-(2)]
+	_ = x[AllowedExternalDirectiveAll-(3)]
+	_ = x[AllowedExternalDirectiveFileEnv-(4)]
+	_ = x[AllowedExternalDirectiveFile-(5)]
+	_ = x[AllowedExternalDirectiveEnv-(6)]
+}
+
+var _AllowedExternalDirectivesValues = []AllowedExternalDirectives{DefaultExternalDir, AllowedExternalDirectiveVault, AllowedExternalDirectiveVaultFile, AllowedExternalDirectiveAll, AllowedExternalDirectiveFileEnv, AllowedExternalDirectiveFile, AllowedExternalDirectiveEnv}
+
+var _AllowedExternalDirectivesNameToValueMap = map[string]AllowedExternalDirectives{
+	_AllowedExternalDirectivesName[0:18]:       DefaultExternalDir,
+	_AllowedExternalDirectivesLowerName[0:18]:  DefaultExternalDir,
+	_AllowedExternalDirectivesName[18:23]:      AllowedExternalDirectiveVault,
+	_AllowedExternalDirectivesLowerName[18:23]: AllowedExternalDirectiveVault,
+	_AllowedExternalDirectivesName[23:33]:      AllowedExternalDirectiveVaultFile,
+	_AllowedExternalDirectivesLowerName[23:33]: AllowedExternalDirectiveVaultFile,
+	_AllowedExternalDirectivesName[33:47]:      AllowedExternalDirectiveAll,
+	_AllowedExternalDirectivesLowerName[33:47]: AllowedExternalDirectiveAll,
+	_AllowedExternalDirectivesName[47:55]:      AllowedExternalDirectiveFileEnv,
+	_AllowedExternalDirectivesLowerName[47:55]: AllowedExternalDirectiveFileEnv,
+	_AllowedExternalDirectivesName[55:59]:      AllowedExternalDirectiveFile,
+	_AllowedExternalDirectivesLowerName[55:59]: AllowedExternalDirectiveFile,
+	_AllowedExternalDirectivesName[59:62]:      AllowedExternalDirectiveEnv,
+	_AllowedExternalDirectivesLowerName[59:62]: AllowedExternalDirectiveEnv,
+}
+
+var _AllowedExternalDirectivesNames = []string{
+	_AllowedExternalDirectivesName[0:18],
+	_AllowedExternalDirectivesName[18:23],
+	_AllowedExternalDirectivesName[23:33],
+	_AllowedExternalDirectivesName[33:47],
+	_AllowedExternalDirectivesName[47:55],
+	_AllowedExternalDirectivesName[55:59],
+	_AllowedExternalDirectivesName[59:62],
+}
+
+// AllowedExternalDirectivesString retrieves an enum value from the enum constants string name.
+// Throws an error if the param is not part of the enum.
+func AllowedExternalDirectivesString(s string) (AllowedExternalDirectives, error) {
+	if val, ok := _AllowedExternalDirectivesNameToValueMap[s]; ok {
+		return val, nil
+	}
+
+	if val, ok := _AllowedExternalDirectivesNameToValueMap[strings.ToLower(s)]; ok {
+		return val, nil
+	}
+	return 0, fmt.Errorf("%s does not belong to AllowedExternalDirectives values", s)
+}
+
+// AllowedExternalDirectivesValues returns all values of the enum
+func AllowedExternalDirectivesValues() []AllowedExternalDirectives {
+	return _AllowedExternalDirectivesValues
+}
+
+// AllowedExternalDirectivesStrings returns a slice of all String values of the enum
+func AllowedExternalDirectivesStrings() []string {
+	strs := make([]string, len(_AllowedExternalDirectivesNames))
+	copy(strs, _AllowedExternalDirectivesNames)
+	return strs
+}
+
+// IsAAllowedExternalDirectives returns "true" if the value is listed in the enum definition. "false" otherwise
+func (i AllowedExternalDirectives) IsAAllowedExternalDirectives() bool {
+	for _, v := range _AllowedExternalDirectivesValues {
+		if i == v {
+			return true
+		}
+	}
+	return false
+}
+
+// MarshalJSON implements the json.Marshaler interface for AllowedExternalDirectives
+func (i AllowedExternalDirectives) MarshalJSON() ([]byte, error) {
+	return json.Marshal(i.String())
+}
+
+// UnmarshalJSON implements the json.Unmarshaler interface for AllowedExternalDirectives
+func (i *AllowedExternalDirectives) UnmarshalJSON(data []byte) error {
+	var s string
+	if err := json.Unmarshal(data, &s); err != nil {
+		return fmt.Errorf("AllowedExternalDirectives should be a string, got %s", data)
+	}
+
+	var err error
+	*i, err = AllowedExternalDirectivesString(s)
+	return err
+}
+
+// MarshalText implements the encoding.TextMarshaler interface for AllowedExternalDirectives
+func (i AllowedExternalDirectives) MarshalText() ([]byte, error) {
+	return []byte(i.String()), nil
+}
+
+// UnmarshalText implements the encoding.TextUnmarshaler interface for AllowedExternalDirectives
+func (i *AllowedExternalDirectives) UnmarshalText(text []byte) error {
+	var err error
+	*i, err = AllowedExternalDirectivesString(string(text))
+	return err
+}
+
+// MarshalYAML implements a YAML Marshaler for AllowedExternalDirectives
+func (i AllowedExternalDirectives) MarshalYAML() (interface{}, error) {
+	return i.String(), nil
+}
+
+// UnmarshalYAML implements a YAML Unmarshaler for AllowedExternalDirectives
+func (i *AllowedExternalDirectives) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var s string
+	if err := unmarshal(&s); err != nil {
+		return err
+	}
+
+	var err error
+	*i, err = AllowedExternalDirectivesString(s)
+	return err
+}

pkg/backy/backy.go

@@ -11,6 +11,7 @@ import (
 	"io"
 	"os"
 	"os/exec"
+	"strings"
 	"text/template"
 
 	"embed"
@@ -95,7 +96,7 @@ func (command *Command) RunCmd(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([
 				command.Shell = "sh"
 			}
 			localCMD = exec.Command(command.Shell, command.Args...)
-			injectEnvIntoLocalCMD(envVars, localCMD, cmdCtxLogger)
+			injectEnvIntoLocalCMD(envVars, localCMD, cmdCtxLogger, opts)
 
 			cmdOutWriters = io.MultiWriter(&cmdOutBuf)
 
@@ -166,11 +167,17 @@ func (command *Command) RunCmd(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([
 			}
 		}
 
+		if command.Type == UserCT {
+			if command.UserOperation == "password" {
+				localCMD.Stdin = command.stdin
+				cmdCtxLogger.Info().Str("password", command.UserPassword).Msg("user password to be updated")
+			}
+		}
 		if command.Dir != nil {
 			localCMD.Dir = *command.Dir
 		}
 
-		injectEnvIntoLocalCMD(envVars, localCMD, cmdCtxLogger)
+		injectEnvIntoLocalCMD(envVars, localCMD, cmdCtxLogger, opts)
 
 		cmdOutWriters = io.MultiWriter(&cmdOutBuf)
 
@@ -188,6 +195,63 @@ func (command *Command) RunCmd(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([
 			cmdCtxLogger.Error().Err(fmt.Errorf("error when running cmd %s: %w", command.Name, err)).Send()
 			return outputArr, err
 		}
+
+		if command.Type == UserCT {
+
+			if command.UserOperation == "add" {
+				if command.UserSshPubKeys != nil {
+					var (
+						f        *os.File
+						err      error
+						userHome []byte
+					)
+
+					cmdCtxLogger.Info().Msg("adding SSH Keys")
+
+					localCMD := exec.Command(fmt.Sprintf("grep \"%s\" /etc/passwd | cut -d: -f6", command.Username))
+					userHome, err = localCMD.CombinedOutput()
+					if err != nil {
+						return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error finding user home from /etc/passwd: %v", err)
+					}
+
+					command.UserHome = strings.TrimSpace(string(userHome))
+					userSshDir := fmt.Sprintf("%s/.ssh", command.UserHome)
+
+					if _, err := os.Stat(userSshDir); os.IsNotExist(err) {
+						err := os.MkdirAll(userSshDir, 0700)
+						if err != nil {
+							return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error creating directory %s %v", userSshDir, err)
+						}
+					}
+
+					if _, err := os.Stat(fmt.Sprintf("%s/authorized_keys", userSshDir)); os.IsNotExist(err) {
+						_, err := os.Create(fmt.Sprintf("%s/authorized_keys", userSshDir))
+						if err != nil {
+							return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error creating file %s/authorized_keys: %v", userSshDir, err)
+						}
+					}
+
+					f, err = os.OpenFile(fmt.Sprintf("%s/authorized_keys", userSshDir), 0700, os.ModeAppend)
+					if err != nil {
+						return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error opening file %s/authorized_keys: %v", userSshDir, err)
+					}
+					defer f.Close()
+					for _, k := range command.UserSshPubKeys {
+						buf := bytes.NewBufferString(k)
+						cmdCtxLogger.Info().Str("key", k).Msg("adding SSH key")
+						if _, err := f.ReadFrom(buf); err != nil {
+							return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error adding to authorized keys: %v", err)
+						}
+					}
+					localCMD = exec.Command(fmt.Sprintf("chown -R %s:%s %s", command.Username, command.Username, userHome))
+					_, err = localCMD.CombinedOutput()
+					if err != nil {
+						return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), err
+					}
+
+				}
+			}
+		}
 	}
 	return outputArr, nil
 }
@@ -399,7 +463,7 @@ func (cmd *Command) ExecuteHooks(hookType string, opts *ConfigOpts) {
 			cmdLogger := opts.Logger.With().
 				Str("backy-cmd", v).Str("hookType", "error").
 				Logger()
-			errCmd.RunCmd(cmdLogger, opts)
+			_, _ = errCmd.RunCmd(cmdLogger, opts)
 		}
 
 	case "success":
@@ -409,7 +473,7 @@ func (cmd *Command) ExecuteHooks(hookType string, opts *ConfigOpts) {
 			cmdLogger := opts.Logger.With().
 				Str("backy-cmd", v).Str("hookType", "success").
 				Logger()
-			successCmd.RunCmd(cmdLogger, opts)
+			_, _ = successCmd.RunCmd(cmdLogger, opts)
 		}
 	case "final":
 		for _, v := range cmd.Hooks.Final {
@@ -418,7 +482,7 @@ func (cmd *Command) ExecuteHooks(hookType string, opts *ConfigOpts) {
 			cmdLogger := opts.Logger.With().
 				Str("backy-cmd", v).Str("hookType", "final").
 				Logger()
-			finalCmd.RunCmd(cmdLogger, opts)
+			_, _ = finalCmd.RunCmd(cmdLogger, opts)
 		}
 	}
 }
@@ -473,6 +537,12 @@ func logCommandOutput(command *Command, cmdOutBuf bytes.Buffer, cmdCtxLogger zer
 	return outputArr
 }
 
+func (c *Command) GetVariablesFromConf(opts *ConfigOpts) {
+	c.ScriptEnvFile = replaceVarInString(opts.Vars, c.ScriptEnvFile, opts.Logger)
+	c.Name = replaceVarInString(opts.Vars, c.Name, opts.Logger)
+	c.OutputFile = replaceVarInString(opts.Vars, c.OutputFile, opts.Logger)
+}
+
 // func executeUserCommands() []string {
 
 // }

pkg/backy/config.go

@@ -1,7 +1,6 @@
 package backy
 
 import (
-	"context"
 	"errors"
 	"fmt"
 	"net/url"
@@ -81,15 +80,16 @@ func (opts *ConfigOpts) InitConfig() {
 		logging.ExitWithMSG(fmt.Sprintf("error initializing cache: %v", err), 1, nil)
 	}
 
-	fetcher, err := remotefetcher.NewRemoteFetcher(opts.ConfigFilePath, opts.Cache)
-
 	if isRemoteURL(opts.ConfigFilePath) {
 		p, _ := getRemoteDir(opts.ConfigFilePath)
 		opts.ConfigDir = p
 	}
+
+	fetcher, err := remotefetcher.NewRemoteFetcher(opts.ConfigFilePath, opts.Cache)
 	if err != nil {
 		logging.ExitWithMSG(fmt.Sprintf("error initializing config fetcher: %v", err), 1, nil)
 	}
+
 	if opts.ConfigFilePath != "" {
 		loadConfigFile(fetcher, opts.ConfigFilePath, backyKoanf, opts)
 	} else {
@@ -103,6 +103,12 @@ func (opts *ConfigOpts) ReadConfig() *ConfigOpts {
 
 	backyKoanf := opts.koanf
 
+	if backyKoanf.Exists("variables") {
+		unmarshalConfigIntoStruct(backyKoanf, "variables", &opts.Vars, opts.Logger)
+	}
+
+	getConfigDir(opts)
+
 	opts.loadEnv()
 
 	if backyKoanf.Bool(getNestedConfig("logging", "cmd-std-out")) {
@@ -125,14 +131,23 @@ func (opts *ConfigOpts) ReadConfig() *ConfigOpts {
 
 	log.Info().Str("config file", opts.ConfigFilePath).Send()
 
-	unmarshalConfig(backyKoanf, "commands", &opts.Cmds, opts.Logger)
+	if err := opts.initVault(); err != nil {
+		log.Err(err).Send()
+	}
+
+	unmarshalConfigIntoStruct(backyKoanf, "commands", &opts.Cmds, opts.Logger)
 
 	getCommandEnvironments(opts)
 
-	unmarshalConfig(backyKoanf, "hosts", &opts.Hosts, opts.Logger)
+	unmarshalConfigIntoStruct(backyKoanf, "hosts", &opts.Hosts, opts.Logger)
 
 	resolveHostConfigs(opts)
 
+	for k, v := range opts.Vars {
+		v = getExternalConfigDirectiveValue(v, opts)
+		opts.Vars[k] = v
+	}
+
 	loadCommandLists(opts, backyKoanf)
 
 	validateCommandLists(opts)
@@ -148,15 +163,11 @@ func (opts *ConfigOpts) ReadConfig() *ConfigOpts {
 	filterExecuteLists(opts)
 
 	if backyKoanf.Exists("notifications") {
-		unmarshalConfig(backyKoanf, "notifications", &opts.NotificationConf, opts.Logger)
+		unmarshalConfigIntoStruct(backyKoanf, "notifications", &opts.NotificationConf, opts.Logger)
 	}
 
 	opts.SetupNotify()
 
-	if err := opts.setupVault(); err != nil {
-		log.Err(err).Send()
-	}
-
 	return opts
 }
 
@@ -220,6 +231,7 @@ func setLoggingOptions(k *koanf.Koanf, opts *ConfigOpts) {
 		logFile = k.String(getLoggingKeyFromConfig("file"))
 		opts.LogFilePath = logFile
 	}
+	opts.LogFilePath = logFile
 
 	zerolog.SetGlobalLevel(zerolog.InfoLevel)
 	if isLoggingVerbose {
@@ -239,7 +251,7 @@ func setupLogger(opts *ConfigOpts) zerolog.Logger {
 	return zerolog.New(writers).With().Timestamp().Logger()
 }
 
-func unmarshalConfig(k *koanf.Koanf, key string, target interface{}, log zerolog.Logger) {
+func unmarshalConfigIntoStruct(k *koanf.Koanf, key string, target interface{}, log zerolog.Logger) {
 	if err := k.UnmarshalWithConf(key, target, koanf.UnmarshalConf{Tag: "yaml"}); err != nil {
 		logging.ExitWithMSG(fmt.Sprintf("error unmarshaling key %s into struct: %v", key, err), 1, &log)
 	}
@@ -247,6 +259,9 @@ func unmarshalConfig(k *koanf.Koanf, key string, target interface{}, log zerolog
 
 func getCommandEnvironments(opts *ConfigOpts) {
 	for cmdName, cmdConf := range opts.Cmds {
+		if cmdConf.Env == "" {
+			continue
+		}
 		opts.Logger.Debug().Str("env file", cmdConf.Env).Str("cmd", cmdName).Send()
 		if err := testFile(cmdConf.Env); err != nil {
 			logging.ExitWithMSG("Could not open file"+cmdConf.Env+": "+err.Error(), 1, &opts.Logger)
@@ -278,16 +293,22 @@ func resolveProxyHosts(host *Host, opts *ConfigOpts) {
 	}
 }
 
+func getConfigDir(opts *ConfigOpts) {
+	if isRemoteURL(opts.ConfigFilePath) {
+		p, _ := getRemoteDir(opts.ConfigFilePath)
+		opts.ConfigDir = p
+	} else {
+		opts.ConfigDir = path.Dir(opts.ConfigFilePath)
+	}
+}
+
 func loadCommandLists(opts *ConfigOpts, backyKoanf *koanf.Koanf) {
 	var listConfigFiles []string
 	var u *url.URL
 	var p string
-	// if config file is remote, use the directory of the remote file
 	if isRemoteURL(opts.ConfigFilePath) {
 		p, u = getRemoteDir(opts.ConfigFilePath)
 		opts.ConfigDir = p
-		println(p)
-		// // Still use local list files if a remote config file is used, but use them last
 		listConfigFiles = []string{u.JoinPath("lists.yml").String(), u.JoinPath("lists.yaml").String()}
 	} else {
 		opts.ConfigDir = path.Dir(opts.ConfigFilePath)
@@ -310,7 +331,7 @@ func loadCommandLists(opts *ConfigOpts, backyKoanf *koanf.Koanf) {
 		if backyKoanf.Exists("cmdLists.file") {
 			loadCmdListsFile(backyKoanf, listsConfig, opts)
 		} else {
-			unmarshalConfig(backyKoanf, "cmdLists", &opts.CmdConfigLists, opts.Logger)
+			unmarshalConfigIntoStruct(backyKoanf, "cmdLists", &opts.CmdConfigLists, opts.Logger)
 		}
 	}
 }
@@ -350,7 +371,7 @@ func loadListConfigFile(filePath string, k *koanf.Koanf, opts *ConfigOpts) bool
 		return false
 	}
 
-	unmarshalConfig(k, "cmdLists", &opts.CmdConfigLists, opts.Logger)
+	unmarshalConfigIntoStruct(k, "cmdLists", &opts.CmdConfigLists, opts.Logger)
 	keyNotSupported("cmd-lists", "cmdLists", k, opts, true)
 	opts.CmdListFile = filePath
 	return true
@@ -378,7 +399,7 @@ func loadCmdListsFile(backyKoanf *koanf.Koanf, listsConfig *koanf.Koanf, opts *C
 	}
 
 	keyNotSupported("cmd-lists", "cmdLists", listsConfig, opts, true)
-	unmarshalConfig(listsConfig, "cmdLists", &opts.CmdConfigLists, opts.Logger)
+	unmarshalConfigIntoStruct(listsConfig, "cmdLists", &opts.CmdConfigLists, opts.Logger)
 	opts.Logger.Info().Str("using lists config file", opts.CmdListFile).Send()
 }
 
@@ -435,7 +456,7 @@ func getLoggingKeyFromConfig(key string) string {
 // 	return fmt.Sprintf("cmdLists.%s", list)
 // }
 
-func (opts *ConfigOpts) setupVault() error {
+func (opts *ConfigOpts) initVault() error {
 	if !opts.koanf.Bool("vault.enabled") {
 		return nil
 	}
@@ -456,7 +477,7 @@ func (opts *ConfigOpts) setupVault() error {
 		token = os.Getenv("VAULT_TOKEN")
 	}
 	if strings.TrimSpace(token) == "" {
-		return fmt.Errorf("no token found, but one was required. \n\nSet the config key vault.token or the environment variable VAULT_TOKEN")
+		return fmt.Errorf("no token found. One is required. \n\nSet the config key vault.token or the environment variable VAULT_TOKEN")
 	}
 
 	client.SetToken(token)
@@ -468,70 +489,22 @@ func (opts *ConfigOpts) setupVault() error {
 
 	opts.vaultClient = client
 
+	for _, v := range opts.VaultKeys {
+		v.Name = replaceVarInString(opts.Vars, v.Key, opts.Logger)
+		v.MountPath = replaceVarInString(opts.Vars, v.MountPath, opts.Logger)
+	}
+
 	return nil
 }
 
-func getVaultSecret(vaultClient *vault.Client, key *VaultKey) (string, error) {
-	var (
-		secret *vault.KVSecret
-		err    error
-	)
-
-	if key.ValueType == "KVv2" {
-		secret, err = vaultClient.KVv2(key.MountPath).Get(context.Background(), key.Path)
-	} else if key.ValueType == "KVv1" {
-		secret, err = vaultClient.KVv1(key.MountPath).Get(context.Background(), key.Path)
-	} else if key.ValueType != "" {
-		return "", fmt.Errorf("type %s for key %s not known. Valid types are KVv1 or KVv2", key.ValueType, key.Name)
-	} else {
-		return "", fmt.Errorf("type for key %s must be specified. Valid types are KVv1 or KVv2", key.Name)
-
-	}
-	if err != nil {
-		return "", fmt.Errorf("unable to read secret: %v", err)
-	}
-
-	value, ok := secret.Data[key.Name].(string)
-	if !ok {
-		return "", fmt.Errorf("value type assertion failed: %T %#v", secret.Data[key.Name], secret.Data[key.Name])
-	}
-
-	return value, nil
-}
-
-func parseVaultKey(keyName string, keys []*VaultKey) (*VaultKey, error) {
-
-	for _, k := range keys {
-		if k.Name == keyName {
-			return k, nil
-		}
-	}
-	return nil, fmt.Errorf("key %s not found in vault keys", keyName)
-}
-
-func GetVaultKey(str string, opts *ConfigOpts, log zerolog.Logger) string {
-	key, err := parseVaultKey(str, opts.VaultKeys)
-	if key == nil && err == nil {
-		return str
-	}
-	if err != nil && key == nil {
-		log.Err(err).Send()
-		return ""
-	}
-
-	value, secretErr := getVaultSecret(opts.vaultClient, key)
-	if secretErr != nil {
-		log.Err(secretErr).Send()
-		return value
-	}
-	return value
-}
-
 func processCmds(opts *ConfigOpts) error {
 
 	// process commands
 	for cmdName, cmd := range opts.Cmds {
-
+		for i, v := range cmd.Args {
+			v = replaceVarInString(opts.Vars, v, opts.Logger)
+			cmd.Args[i] = v
+		}
 		if cmd.Name == "" {
 			cmd.Name = cmdName
 		}
@@ -556,6 +529,10 @@ func processCmds(opts *ConfigOpts) error {
 
 		// resolve hosts
 		if cmd.Host != nil {
+			cmdHost := replaceVarInString(opts.Vars, *cmd.Host, opts.Logger)
+			if cmdHost != *cmd.Host {
+				cmd.Host = &cmdHost
+			}
 			host, hostFound := opts.Hosts[*cmd.Host]
 			if hostFound {
 				cmd.RemoteHost = host
@@ -615,7 +592,7 @@ func processCmds(opts *ConfigOpts) error {
 			if cmd.Username == "" {
 				return fmt.Errorf("username is required for user command %s", cmd.Name)
 			}
-
+			cmd.Username = replaceVarInString(opts.Vars, cmd.Username, opts.Logger)
 			err := detectOSType(cmd, opts)
 			if err != nil {
 				opts.Logger.Info().Err(err).Str("command", cmdName).Send()
@@ -625,7 +602,9 @@ func processCmds(opts *ConfigOpts) error {
 			switch cmd.UserOperation {
 			case "add", "remove", "modify", "checkIfExists", "delete", "password":
 				cmd.userMan, err = usermanager.NewUserManager(cmd.OS)
+
 				if cmd.UserOperation == "password" {
+					opts.Logger.Debug().Msg("changing password for user: " + cmd.Username)
 					cmd.UserPassword = getExternalConfigDirectiveValue(cmd.UserPassword, opts)
 				}
 				if cmd.Host != nil {
@@ -739,3 +718,24 @@ func keyNotSupported(oldKey, newKey string, koanf *koanf.Koanf, opts *ConfigOpts
 		}
 	}
 }
+
+func replaceVarInString(vars map[string]string, str string, logger zerolog.Logger) string {
+	if strings.Contains(str, "%{var:") && strings.Contains(str, "}%") {
+		logger.Debug().Msgf("replacing vars in string %s", str)
+		for k, v := range vars {
+			if strings.Contains(str, "%{var:"+k+"}%") {
+				str = strings.ReplaceAll(str, "%{var:"+k+"}%", v)
+			}
+		}
+		if strings.Contains(str, "%{var:") && strings.Contains(str, "}%") {
+			logger.Warn().Msg("could not replace all vars in string")
+		}
+	}
+	return str
+}
+
+func VariadicFunctionParameterTest(allowedKeys ...string) {
+	if contains(allowedKeys, "file") {
+		println("file param included")
+	}
+}

pkg/backy/notification.go

@@ -9,6 +9,7 @@ import (
 
 	"git.andrewnw.xyz/CyberShell/backy/pkg/logging"
 	"github.com/nikoksr/notify"
+	"github.com/nikoksr/notify/service/http"
 	"github.com/nikoksr/notify/service/mail"
 	"github.com/nikoksr/notify/service/matrix"
 	"maunium.net/go/mautrix/id"
@@ -30,6 +31,12 @@ type MailConfig struct {
 	Password      string   `yaml:"password"`
 }
 
+type HttpConfig struct {
+	URL     string              `yaml:"url"`
+	Method  string              `yaml:"method"`
+	Headers map[string][]string `yaml:"headers"`
+}
+
 // SetupNotify sets up notify instances for each command list.
 func (opts *ConfigOpts) SetupNotify() {
 
@@ -58,6 +65,8 @@ func (opts *ConfigOpts) SetupNotify() {
 					opts.Logger.Info().Err(fmt.Errorf("error: ID %s not found in mail object", confId)).Str("list", confName).Send()
 					continue
 				}
+				conf.Password = getExternalConfigDirectiveValue(conf.Password, opts)
+				opts.Logger.Debug().Str("list", confName).Str("id", confId).Msg("adding mail notification service")
 				mailConf := setupMail(conf)
 				services = append(services, mailConf)
 			case "matrix":
@@ -66,13 +75,23 @@ func (opts *ConfigOpts) SetupNotify() {
 					opts.Logger.Info().Err(fmt.Errorf("error: ID %s not found in matrix object", confId)).Str("list", confName).Send()
 					continue
 				}
+				conf.AccessToken = getExternalConfigDirectiveValue(conf.AccessToken, opts)
+				opts.Logger.Debug().Str("list", confName).Str("id", confId).Msg("adding matrix notification service")
 				mtrxConf, mtrxErr := setupMatrix(conf)
 				if mtrxErr != nil {
 					opts.Logger.Info().Str("list", confName).Err(fmt.Errorf("error: configuring matrix id %s failed during setup: %w", id, mtrxErr))
 					continue
 				}
 				services = append(services, mtrxConf)
-
+			case "http":
+				conf, ok := opts.NotificationConf.HttpConfig[confId]
+				if !ok {
+					opts.Logger.Info().Err(fmt.Errorf("error: ID %s not found in http object", confId)).Str("list", confName).Send()
+					continue
+				}
+				opts.Logger.Debug().Str("list", confName).Str("id", confId).Msg("adding http notification service")
+				httpConf := setupHttp(conf)
+				services = append(services, httpConf)
 			default:
 				opts.Logger.Info().Err(fmt.Errorf("id %s not found", id)).Str("list", confName).Send()
 			}
@@ -98,3 +117,19 @@ func setupMail(config MailConfig) *mail.Mail {
 	mailClient.BodyFormat(mail.PlainText)
 	return mailClient
 }
+
+func setupHttp(httpConf HttpConfig) *http.Service {
+
+	httpService := http.New()
+	httpService.AddReceivers(&http.Webhook{
+		URL:         httpConf.URL,
+		Header:      httpConf.Headers,
+		ContentType: "text/plain",
+		Method:      httpConf.Method,
+		BuildPayload: func(subject, message string) (payload any) {
+			return subject + "\n\n" + message
+		},
+	})
+
+	return httpService
+}

pkg/backy/ssh.go

@@ -15,6 +15,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/google/uuid"
 	"github.com/kevinburke/ssh_config"
 	"github.com/pkg/errors"
 	"github.com/pkg/sftp"
@@ -205,8 +206,12 @@ func (remoteHost *Host) GetAuthMethods(opts *ConfigOpts) error {
 
 	if remoteHost.Password != "" {
 
+		opts.Logger.Debug().Str("password", remoteHost.Password).Str("Host", remoteHost.Host).Send()
+
 		remoteHost.Password = GetPassword(remoteHost.Password, opts)
 
+		// opts.Logger.Debug().Str("actual password", remoteHost.Password).Str("Host", remoteHost.Host).Send()
+
 		remoteHost.ClientConfig.Auth = append(remoteHost.ClientConfig.Auth, ssh.Password(remoteHost.Password))
 	}
 
@@ -310,13 +315,13 @@ func (remoteHost *Host) ConnectThroughBastion(log zerolog.Logger) (*ssh.Client,
 
 // GetKnownHosts resolves the host's KnownHosts file if it is defined
 // if not defined, the default location for this file is used
-func (remotehHost *Host) GetKnownHosts() error {
+func (remoteHost *Host) GetKnownHosts() error {
 	var knownHostsFileErr error
-	if TS(remotehHost.KnownHostsFile) != "" {
-		remotehHost.KnownHostsFile, knownHostsFileErr = getFullPathWithHomeDir(remotehHost.KnownHostsFile)
+	if TS(remoteHost.KnownHostsFile) != "" {
+		remoteHost.KnownHostsFile, knownHostsFileErr = getFullPathWithHomeDir(remoteHost.KnownHostsFile)
 		return knownHostsFileErr
 	}
-	remotehHost.KnownHostsFile, knownHostsFileErr = getFullPathWithHomeDir("~/.ssh/known_hosts")
+	remoteHost.KnownHostsFile, knownHostsFileErr = getFullPathWithHomeDir("~/.ssh/known_hosts")
 	return knownHostsFileErr
 }
 
@@ -427,7 +432,6 @@ func (command *Command) RunCmdSSH(cmdCtxLogger zerolog.Logger, opts *ConfigOpts)
 			env:  command.Environment,
 		}
 	)
-	// Getting the command type must be done before concatenating the arguments
 	command = getCommandTypeAndSetCommandInfo(command)
 
 	// Prepare command arguments
@@ -503,12 +507,44 @@ func (command *Command) RunCmdSSH(cmdCtxLogger zerolog.Logger, opts *ConfigOpts)
 			ArgsStr = fmt.Sprintf("%s %s", command.Cmd, ArgsStr)
 		}
 		cmdCtxLogger.Debug().Str("cmd + args", ArgsStr).Send()
-		// Run simple command
+
+		if command.Type == UserCT && command.UserOperation == "password" {
+			// cmdCtxLogger.Debug().Msgf("adding stdin")
+
+			userNamePass := fmt.Sprintf("%s:%s", command.Username, command.UserPassword)
+			client, err := sftp.NewClient(command.RemoteHost.SshClient)
+			if err != nil {
+				return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error creating sftp client: %v", err)
+			}
+			uuidFile := uuid.New()
+			passFilePath := fmt.Sprintf("/tmp/%s", uuidFile.String())
+			passFile, passFileErr := client.Create(passFilePath)
+			if passFileErr != nil {
+				return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error creating file /tmp/%s: %v", uuidFile.String(), passFileErr)
+			}
+
+			_, err = passFile.Write([]byte(userNamePass))
+			if err != nil {
+				return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error writing to file /tmp/%s: %v", uuidFile.String(), err)
+			}
+
+			ArgsStr = fmt.Sprintf("cat %s | chpasswd", passFilePath)
+			defer passFile.Close()
+
+			rmFileFunc := func() {
+				_ = client.Remove(passFilePath)
+			}
+
+			defer rmFileFunc()
+			// commandSession.Stdin = command.stdin
+		}
 		if err := commandSession.Run(ArgsStr); err != nil {
 			return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error running command: %w", err)
 		}
 
-		if command.Type == UserCT && command.UserOperation == "add" {
+		if command.Type == UserCT {
+
+			if command.UserOperation == "add" {
 				if command.UserSshPubKeys != nil {
 					var (
 						f        *sftp.File
@@ -532,7 +568,10 @@ func (command *Command) RunCmdSSH(cmdCtxLogger zerolog.Logger, opts *ConfigOpts)
 						return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error creating sftp client: %v", err)
 					}
 
-				client.MkdirAll(userSshDir)
+					err = client.MkdirAll(userSshDir)
+					if err != nil {
+						return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error creating directory %s: %v", userSshDir, err)
+					}
 					_, err = client.Create(fmt.Sprintf("%s/authorized_keys", userSshDir))
 					if err != nil {
 						return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error opening file %s/authorized_keys: %v", userSshDir, err)
@@ -559,6 +598,7 @@ func (command *Command) RunCmdSSH(cmdCtxLogger zerolog.Logger, opts *ConfigOpts)
 				}
 			}
 		}
+	}
 
 	return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), nil
 }

pkg/backy/types.go

@@ -26,15 +26,15 @@ type (
 		ConfigFilePath     string `yaml:"config,omitempty"`
 		Host               string `yaml:"host,omitempty"`
 		HostName           string `yaml:"hostname,omitempty"`
-		KnownHostsFile     string `yaml:"knownhostsfile,omitempty"`
+		KnownHostsFile     string `yaml:"knownHostsFile,omitempty"`
 		ClientConfig       *ssh.ClientConfig
 		SSHConfigFile      *sshConfigFile
 		SshClient          *ssh.Client
 		Port               uint16 `yaml:"port,omitempty"`
 		ProxyJump          string `yaml:"proxyjump,omitempty"`
 		Password           string `yaml:"password,omitempty"`
-		PrivateKeyPath     string `yaml:"privatekeypath,omitempty"`
-		PrivateKeyPassword string `yaml:"privatekeypassword,omitempty"`
+		PrivateKeyPath     string `yaml:"privateKeyPath,omitempty"`
+		PrivateKeyPassword string `yaml:"privateKeyPassword,omitempty"`
 		useDefaultConfig   bool
 		User               string `yaml:"user,omitempty"`
 		isProxyHost        bool
@@ -115,7 +115,9 @@ type (
 
 		UserShell string `yaml:"userShell,omitempty"`
 
-		SystemUser bool `yaml:"systemUser,omitempty"`
+		UserCreateHome bool `yaml:"userCreateHome,omitempty"`
+
+		UserIsSystem bool `yaml:"userIsSystem,omitempty"`
 
 		UserPassword string `yaml:"userPassword,omitempty"`
 
@@ -203,6 +205,8 @@ type (
 
 		List ListConfig
 
+		Vars map[string]string `yaml:"variables"`
+
 		VaultKeys []*VaultKey `yaml:"keys"`
 
 		koanf *koanf.Koanf
@@ -221,6 +225,7 @@ type (
 
 	VaultKey struct {
 		Name      string `yaml:"name"`
+		Key       string `yaml:"key"`
 		Path      string `yaml:"path"`
 		ValueType string `yaml:"type"`
 		MountPath string `yaml:"mountpath"`
@@ -236,6 +241,7 @@ type (
 	Notifications struct {
 		MailConfig   map[string]MailConfig   `yaml:"mail,omitempty"`
 		MatrixConfig map[string]MatrixStruct `yaml:"matrix,omitempty"`
+		HttpConfig   map[string]HttpConfig   `yaml:"http,omitempty"`
 	}
 
 	CmdOutput struct {
@@ -274,6 +280,7 @@ type (
 	// use ints so we can use enums
 	CommandType               int
 	PackageOperation          int
+	AllowedExternalDirectives int
 )
 
 //go:generate go run github.com/dmarkham/enumer -linecomment -yaml -text -json -type=CommandType
@@ -296,3 +303,14 @@ const (
 	PackOpCheckVersion                         // checkVersion
 	PackOpIsInstalled                          // isInstalled
 )
+
+//go:generate go run github.com/dmarkham/enumer -linecomment -yaml -text -json -type=AllowedExternalDirectives
+const (
+	DefaultExternalDir                AllowedExternalDirectives = iota
+	AllowedExternalDirectiveVault                               // vault
+	AllowedExternalDirectiveVaultFile                           // vault-file
+	AllowedExternalDirectiveAll                                 // vault-file-env
+	AllowedExternalDirectiveFileEnv                             // file-env
+	AllowedExternalDirectiveFile                                // file
+	AllowedExternalDirectiveEnv                                 // env
+)

pkg/backy/utils.go

@@ -6,6 +6,7 @@ package backy
 
 import (
 	"bytes"
+	"context"
 	"errors"
 	"fmt"
 	"os"
@@ -16,6 +17,7 @@ import (
 
 	"git.andrewnw.xyz/CyberShell/backy/pkg/logging"
 	"git.andrewnw.xyz/CyberShell/backy/pkg/remotefetcher"
+	vault "github.com/hashicorp/vault/api"
 	"github.com/joho/godotenv"
 	"github.com/knadh/koanf/v2"
 	"github.com/rs/zerolog"
@@ -108,7 +110,11 @@ func injectEnvIntoSSH(envVarsToInject environmentVars, process *ssh.Session, opt
 			goto errEnvFile
 		}
 		for key, val := range envMap {
-			process.Setenv(key, GetVaultKey(val, opts, log))
+			err = process.Setenv(key, GetVaultKey(val, opts, log))
+			if err != nil {
+				log.Error().Err(err).Send()
+
+			}
 		}
 	}
 
@@ -119,12 +125,16 @@ errEnvFile:
 		if strings.Contains(envVal, "=") {
 			envVarArr := strings.Split(envVal, "=")
 
-			process.Setenv(envVarArr[0], GetVaultKey(envVarArr[1], opts, log))
+			err := process.Setenv(envVarArr[0], getExternalConfigDirectiveValue(envVarArr[1], opts))
+			if err != nil {
+				log.Error().Err(err).Send()
+
+			}
 		}
 	}
 }
 
-func injectEnvIntoLocalCMD(envVarsToInject environmentVars, process *exec.Cmd, log zerolog.Logger) {
+func injectEnvIntoLocalCMD(envVarsToInject environmentVars, process *exec.Cmd, log zerolog.Logger, opts *ConfigOpts) {
 	if envVarsToInject.file != "" {
 		envPath, _ := getFullPathWithHomeDir(envVarsToInject.file)
 
@@ -148,7 +158,8 @@ errEnvFile:
 
 	for _, envVal := range envVarsToInject.env {
 		if strings.Contains(envVal, "=") {
-			process.Env = append(process.Env, envVal)
+			envVarArr := strings.Split(envVal, "=")
+			process.Env = append(process.Env, fmt.Sprintf("%s=%s", envVarArr[0], getExternalConfigDirectiveValue(envVarArr[1], opts)))
 		}
 	}
 	process.Env = append(process.Env, os.Environ()...)
@@ -181,7 +192,6 @@ func testFile(c string) error {
 			return fileOpenErr
 		}
 	}
-
 	return nil
 }
 
@@ -250,7 +260,6 @@ func (opts *ConfigOpts) loadEnv() {
 func expandEnvVars(backyEnv map[string]string, envVars []string) {
 
 	env := func(name string) string {
-		name = strings.ToUpper(name)
 		envVar, found := backyEnv[name]
 		if found {
 			return envVar
@@ -259,14 +268,14 @@ func expandEnvVars(backyEnv map[string]string, envVars []string) {
 	}
 
 	for indx, v := range envVars {
-		if strings.HasPrefix(v, externDirectiveStart) && strings.HasSuffix(v, externDirectiveEnd) {
-			if strings.HasPrefix(v, envExternDirectiveStart) {
+
+		if strings.HasPrefix(v, envExternDirectiveStart) && strings.HasSuffix(v, externDirectiveEnd) {
 			v = strings.TrimPrefix(v, envExternDirectiveStart)
 			v = strings.TrimRight(v, externDirectiveEnd)
 			out, _ := shell.Expand(v, env)
 			envVars[indx] = out
 		}
-		}
+
 	}
 }
 
@@ -294,7 +303,8 @@ func getCommandTypeAndSetCommandInfo(command *Command) *Command {
 				command.Username,
 				command.UserHome,
 				command.UserShell,
-				command.SystemUser,
+				command.UserIsSystem,
+				command.UserCreateHome,
 				command.UserGroups,
 				command.Args)
 		case "modify":
@@ -352,7 +362,8 @@ func getExternalConfigDirectiveValue(key string, opts *ConfigOpts) string {
 	if !(strings.HasPrefix(key, externDirectiveStart) && strings.HasSuffix(key, externDirectiveEnd)) {
 		return key
 	}
-	opts.Logger.Info().Str("expanding external key", key).Send()
+	key = replaceVarInString(opts.Vars, key, opts.Logger)
+	opts.Logger.Debug().Str("expanding external key", key).Send()
 	if strings.HasPrefix(key, envExternDirectiveStart) {
 		key = strings.TrimPrefix(key, envExternDirectiveStart)
 		key = strings.TrimSuffix(key, externDirectiveEnd)
@@ -386,3 +397,57 @@ func getExternalConfigDirectiveValue(key string, opts *ConfigOpts) string {
 
 	return key
 }
+
+func getVaultSecret(vaultClient *vault.Client, key *VaultKey) (string, error) {
+	var (
+		secret *vault.KVSecret
+		err    error
+	)
+
+	if key.ValueType == "KVv2" {
+		secret, err = vaultClient.KVv2(key.MountPath).Get(context.Background(), key.Path)
+	} else if key.ValueType == "KVv1" {
+		secret, err = vaultClient.KVv1(key.MountPath).Get(context.Background(), key.Path)
+	} else if key.ValueType != "" {
+		return "", fmt.Errorf("type %s for key %s not known. Valid types are KVv1 or KVv2", key.ValueType, key.Name)
+	} else {
+		return "", fmt.Errorf("type for key %s must be specified. Valid types are KVv1 or KVv2", key.Name)
+	}
+	if err != nil {
+		return "", fmt.Errorf("unable to read secret: %v", err)
+	}
+
+	value, ok := secret.Data[key.Key].(string)
+	if !ok {
+		return "", fmt.Errorf("value type assertion failed for vault key %s: %T %#v", key.Name, secret.Data[key.Name], secret.Data[key.Name])
+	}
+
+	return value, nil
+}
+
+func getVaultKeyData(keyName string, keys []*VaultKey) (*VaultKey, error) {
+	for _, k := range keys {
+		if k.Name == keyName {
+			return k, nil
+		}
+	}
+	return nil, fmt.Errorf("key %s not found in vault keys", keyName)
+}
+
+func GetVaultKey(str string, opts *ConfigOpts, log zerolog.Logger) string {
+	key, err := getVaultKeyData(str, opts.VaultKeys)
+	if key == nil && err == nil {
+		return str
+	}
+	if err != nil && key == nil {
+		log.Err(err).Send()
+		return ""
+	}
+
+	value, secretErr := getVaultSecret(opts.vaultClient, key)
+	if secretErr != nil {
+		log.Err(secretErr).Send()
+		return value
+	}
+	return value
+}

pkg/remotefetcher/cache.go

@@ -116,7 +116,7 @@ func (c *Cache) Set(source, hash string, data []byte, dataType string) (CacheDat
 	path := filepath.Join(c.dir, fmt.Sprintf("%s-%s", fileName, sourceHash))
 
 	if _, err := os.Stat(path); os.IsNotExist(err) {
-		os.MkdirAll(c.dir, 0700)
+		_ = os.MkdirAll(c.dir, 0700)
 	}
 
 	err := os.WriteFile(path, data, 0644)
@@ -171,7 +171,7 @@ func (cf *CachedFetcher) Hash(data []byte) string {
 func LoadMetadataFromFile(filePath string) ([]*CacheData, error) {
 	if _, err := os.Stat(filePath); os.IsNotExist(err) {
 		// Create the file if it does not exist
-		os.MkdirAll(path.Dir(filePath), 0700)
+		_ = os.MkdirAll(path.Dir(filePath), 0700)
 		emptyData := []byte("[]")
 		err := os.WriteFile(filePath, emptyData, 0644)
 		if err != nil {

pkg/usermanager/linux/linux.go

@@ -15,7 +15,7 @@ func (l LinuxUserManager) NewLinuxManager() *LinuxUserManager {
 }
 
 // AddUser adds a new user to the system.
-func (l LinuxUserManager) AddUser(username, homeDir, shell string, isSystem bool, groups, args []string) (string, []string) {
+func (l LinuxUserManager) AddUser(username, homeDir, shell string, isSystem, createHome bool, groups, args []string) (string, []string) {
 	baseArgs := []string{}
 
 	if isSystem {
@@ -38,6 +38,10 @@ func (l LinuxUserManager) AddUser(username, homeDir, shell string, isSystem bool
 		baseArgs = append(baseArgs, args...)
 	}
 
+	if createHome {
+		baseArgs = append(baseArgs, "-m")
+	}
+
 	args = append(baseArgs, username)
 
 	cmd := "useradd"

pkg/usermanager/userman.go

@@ -10,7 +10,7 @@ import (
 // UserManager defines the interface for user management operations.
 // All functions but one return a string for the command and any args.
 type UserManager interface {
-	AddUser(username, homeDir, shell string, isSystem bool, groups, args []string) (string, []string)
+	AddUser(username, homeDir, shell string, createHome, isSystem bool, groups, args []string) (string, []string)
 	RemoveUser(username string) (string, []string)
 	ModifyUser(username, homeDir, shell string, groups []string) (string, []string)
 	// Modify password uses chpasswd for Linux systems to build the command to change the password

release

@@ -1,5 +1,6 @@
 #!/bin/bash
 set -eou pipefail
+go mod tidy
 go generate ./...
 CURRENT_TAG="$(go run backy.go version -V)"
 goreleaser -f .goreleaser/github.yml check

tests/VaultTest.yml

@@ -0,0 +1,27 @@
+commands:
+  vaultEnvVar:
+    cmd: echo
+    shell: /bin/zsh
+    Args:
+      - ${VAULT_VAR}
+    environment:
+      "VAULT_VAR=%{vault:vaultTestSecret}%"
+
+logging:
+  verbose: true
+
+vault:
+  token: root
+  address: http://127.0.0.1:8200
+  enabled: true
+  keys:
+    - name: vaultTestSecret
+      key: data
+      mountpath: secret
+      path: test/var
+      type: KVv2 # KVv1 or KVv2
+
+cmdLists:
+  addUsers:
+    order:
+      - vaultEnvVar