Vault: keys are now referenced by name, and the actual data by data

.changes/unreleased/Changed-20250311-212638.yaml

@@ -0,0 +1,3 @@
+kind: Changed
+body: 'Vault: keys are now referenced by `name`, and the actual data by `data`'
+time: 2025-03-11T21:26:38.085271797-05:00

docs/content/config/commands/_index.md

@@ -16,7 +16,7 @@ Values available for this section **(case-sensitive)**:
 | ----------------| ------------------------------------------------------------------------------------------------------- | --------------------- | -------- |----------------------------|
 | `cmd`           | Defines the command to execute                                                                          | `string`              | yes      | No                         |
 | `Args`          | Defines the arguments to the command                                                                    | `[]string`            | no       | No                         |
-| `environment`   | Defines environment variables for the command                                                           | `[]string`            | no       | No                         |
+| `environment`   | Defines environment variables for the command                                                           | `[]string`            | no       | Partial                    |
 | `type`          | See documentation further down the page. Additional fields may be required.                             | `string`              | no       | No                         |
 | `getOutput`     | Command(s) output is in the notification(s)                                                             | `bool`                | no       | No                         |
 | `host`          | If not specified, the command will execute locally.                                                     | `string`              | no       | No                         |
@@ -95,6 +95,7 @@ The following options are available:
 The environment variables support expansion:
 
 - using escaped values `$VAR` or `${VAR}`
+- using the directive`%{env:VAR}%`
 
 For now, the variables have to be defined in an `.env` file in the same directory that the program is run from.
 

docs/content/config/vault.md

@@ -6,7 +6,7 @@ description: Set up and configure vault.
 
 [Vault](https://www.vaultproject.io/) is a tool for storing secrets and other data securely.
 
-Vault config can be used by prefixing `vault:` in front of a password or ENV var.
+A Vault key can be used by prefixing `%{vault:vault.keys.name}%` in a field that supports external directives.
 
 This is the object in the config file:
 
@@ -18,10 +18,12 @@ vault:
   keys:
     - name: mongourl
       mountpath: secret
+      key: data
       path: mongo/url
       type:  # KVv1 or KVv2
-    - name:
+    - name: someKeyName
-      path:
+      mountpath: secret
-      type:
+      key: keyData
-      mountpath:
+      type: KVv2
+      path: some/path
 ```

pkg/backy/backy.go

@@ -96,7 +96,7 @@ func (command *Command) RunCmd(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([
 				command.Shell = "sh"
 			}
 			localCMD = exec.Command(command.Shell, command.Args...)
-			injectEnvIntoLocalCMD(envVars, localCMD, cmdCtxLogger)
+			injectEnvIntoLocalCMD(envVars, localCMD, cmdCtxLogger, opts)
 
 			cmdOutWriters = io.MultiWriter(&cmdOutBuf)
 
@@ -177,7 +177,7 @@ func (command *Command) RunCmd(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([
 			localCMD.Dir = *command.Dir
 		}
 
-		injectEnvIntoLocalCMD(envVars, localCMD, cmdCtxLogger)
+		injectEnvIntoLocalCMD(envVars, localCMD, cmdCtxLogger, opts)
 
 		cmdOutWriters = io.MultiWriter(&cmdOutBuf)
 

pkg/backy/config.go

@@ -1,7 +1,6 @@
 package backy
 
 import (
-	"context"
 	"errors"
 	"fmt"
 	"net/url"
@@ -475,62 +474,6 @@ func (opts *ConfigOpts) setupVault() error {
 	return nil
 }
 
-func getVaultSecret(vaultClient *vault.Client, key *VaultKey) (string, error) {
-	var (
-		secret *vault.KVSecret
-		err    error
-	)
-
-	if key.ValueType == "KVv2" {
-		secret, err = vaultClient.KVv2(key.MountPath).Get(context.Background(), key.Path)
-	} else if key.ValueType == "KVv1" {
-		secret, err = vaultClient.KVv1(key.MountPath).Get(context.Background(), key.Path)
-	} else if key.ValueType != "" {
-		return "", fmt.Errorf("type %s for key %s not known. Valid types are KVv1 or KVv2", key.ValueType, key.Name)
-	} else {
-		return "", fmt.Errorf("type for key %s must be specified. Valid types are KVv1 or KVv2", key.Name)
-
-	}
-	if err != nil {
-		return "", fmt.Errorf("unable to read secret: %v", err)
-	}
-
-	value, ok := secret.Data[key.Name].(string)
-	if !ok {
-		return "", fmt.Errorf("value type assertion failed: %T %#v", secret.Data[key.Name], secret.Data[key.Name])
-	}
-
-	return value, nil
-}
-
-func parseVaultKey(keyName string, keys []*VaultKey) (*VaultKey, error) {
-
-	for _, k := range keys {
-		if k.Name == keyName {
-			return k, nil
-		}
-	}
-	return nil, fmt.Errorf("key %s not found in vault keys", keyName)
-}
-
-func GetVaultKey(str string, opts *ConfigOpts, log zerolog.Logger) string {
-	key, err := parseVaultKey(str, opts.VaultKeys)
-	if key == nil && err == nil {
-		return str
-	}
-	if err != nil && key == nil {
-		log.Err(err).Send()
-		return ""
-	}
-
-	value, secretErr := getVaultSecret(opts.vaultClient, key)
-	if secretErr != nil {
-		log.Err(secretErr).Send()
-		return value
-	}
-	return value
-}
-
 func processCmds(opts *ConfigOpts) error {
 
 	// process commands

pkg/backy/types.go

@@ -223,6 +223,7 @@ type (
 
 	VaultKey struct {
 		Name      string `yaml:"name"`
+		Key       string `yaml:"key"`
 		Path      string `yaml:"path"`
 		ValueType string `yaml:"type"`
 		MountPath string `yaml:"mountpath"`

pkg/backy/utils.go

@@ -6,6 +6,7 @@ package backy
 
 import (
 	"bytes"
+	"context"
 	"errors"
 	"fmt"
 	"os"
@@ -16,6 +17,7 @@ import (
 
 	"git.andrewnw.xyz/CyberShell/backy/pkg/logging"
 	"git.andrewnw.xyz/CyberShell/backy/pkg/remotefetcher"
+	vault "github.com/hashicorp/vault/api"
 	"github.com/joho/godotenv"
 	"github.com/knadh/koanf/v2"
 	"github.com/rs/zerolog"
@@ -119,12 +121,12 @@ errEnvFile:
 		if strings.Contains(envVal, "=") {
 			envVarArr := strings.Split(envVal, "=")
 
-			process.Setenv(envVarArr[0], GetVaultKey(envVarArr[1], opts, log))
+			process.Setenv(envVarArr[0], getExternalConfigDirectiveValue(envVarArr[1], opts))
 		}
 	}
 }
 
-func injectEnvIntoLocalCMD(envVarsToInject environmentVars, process *exec.Cmd, log zerolog.Logger) {
+func injectEnvIntoLocalCMD(envVarsToInject environmentVars, process *exec.Cmd, log zerolog.Logger, opts *ConfigOpts) {
 	if envVarsToInject.file != "" {
 		envPath, _ := getFullPathWithHomeDir(envVarsToInject.file)
 
@@ -148,7 +150,8 @@ errEnvFile:
 
 	for _, envVal := range envVarsToInject.env {
 		if strings.Contains(envVal, "=") {
-			process.Env = append(process.Env, envVal)
+			envVarArr := strings.Split(envVal, "=")
+			process.Env = append(process.Env, fmt.Sprintf("%s=%s", envVarArr[0], getExternalConfigDirectiveValue(envVarArr[1], opts)))
 		}
 	}
 	process.Env = append(process.Env, os.Environ()...)
@@ -249,7 +252,6 @@ func (opts *ConfigOpts) loadEnv() {
 func expandEnvVars(backyEnv map[string]string, envVars []string) {
 
 	env := func(name string) string {
-		name = strings.ToUpper(name)
 		envVar, found := backyEnv[name]
 		if found {
 			return envVar
@@ -258,14 +260,14 @@ func expandEnvVars(backyEnv map[string]string, envVars []string) {
 	}
 
 	for indx, v := range envVars {
-		if strings.HasPrefix(v, externDirectiveStart) && strings.HasSuffix(v, externDirectiveEnd) {
+
-			if strings.HasPrefix(v, envExternDirectiveStart) {
+		if strings.HasPrefix(v, envExternDirectiveStart) && strings.HasSuffix(v, externDirectiveEnd) {
-				v = strings.TrimPrefix(v, envExternDirectiveStart)
+			v = strings.TrimPrefix(v, envExternDirectiveStart)
-				v = strings.TrimRight(v, externDirectiveEnd)
+			v = strings.TrimRight(v, externDirectiveEnd)
-				out, _ := shell.Expand(v, env)
+			out, _ := shell.Expand(v, env)
-				envVars[indx] = out
+			envVars[indx] = out
-			}
 		}
+
 	}
 }
 
@@ -383,6 +385,62 @@ func getExternalConfigDirectiveValue(key string, opts *ConfigOpts) string {
 		key = strings.TrimSuffix(key, externDirectiveEnd)
 		key = GetVaultKey(key, opts, opts.Logger)
 	}
+	println(key)
 
 	return key
 }
+
+func getVaultSecret(vaultClient *vault.Client, key *VaultKey) (string, error) {
+	var (
+		secret *vault.KVSecret
+		err    error
+	)
+
+	if key.ValueType == "KVv2" {
+		secret, err = vaultClient.KVv2(key.MountPath).Get(context.Background(), key.Path)
+	} else if key.ValueType == "KVv1" {
+		secret, err = vaultClient.KVv1(key.MountPath).Get(context.Background(), key.Path)
+	} else if key.ValueType != "" {
+		return "", fmt.Errorf("type %s for key %s not known. Valid types are KVv1 or KVv2", key.ValueType, key.Name)
+	} else {
+		return "", fmt.Errorf("type for key %s must be specified. Valid types are KVv1 or KVv2", key.Name)
+	}
+	if err != nil {
+		return "", fmt.Errorf("unable to read secret: %v", err)
+	}
+
+	value, ok := secret.Data[key.Key].(string)
+	println(value)
+	if !ok {
+		return "", fmt.Errorf("value type assertion failed for vault key %s: %T %#v", key.Name, secret.Data[key.Name], secret.Data[key.Name])
+	}
+
+	return value, nil
+}
+
+func getVaultKeyData(keyName string, keys []*VaultKey) (*VaultKey, error) {
+	for _, k := range keys {
+		if k.Name == keyName {
+			return k, nil
+		}
+	}
+	return nil, fmt.Errorf("key %s not found in vault keys", keyName)
+}
+
+func GetVaultKey(str string, opts *ConfigOpts, log zerolog.Logger) string {
+	key, err := getVaultKeyData(str, opts.VaultKeys)
+	if key == nil && err == nil {
+		return str
+	}
+	if err != nil && key == nil {
+		log.Err(err).Send()
+		return ""
+	}
+
+	value, secretErr := getVaultSecret(opts.vaultClient, key)
+	if secretErr != nil {
+		log.Err(secretErr).Send()
+		return value
+	}
+	return value
+}

tests/VaultTest.yml

@@ -0,0 +1,27 @@
+commands:
+  vaultEnvVar:
+    cmd: echo
+    shell: /bin/zsh
+    Args:
+      - ${VAULT_VAR}
+    environment:
+      "VAULT_VAR=%{vault:vaultTestSecret}%"
+
+logging:
+  verbose: true
+
+vault:
+  token: root
+  address: http://127.0.0.1:8200
+  enabled: true
+  keys:
+    - name: vaultTestSecret
+      key: data
+      mountpath: secret
+      path: test/var
+      type: KVv2 # KVv1 or KVv2
+
+cmdLists:
+  addUsers:
+    order:
+      - vaultEnvVar