CyberShell/backy
1
1
Fork 0
Code Issues 10 Pull Requests Packages Projects 1 Releases 20 Wiki Activity

added getting ENV vars from Vault

Browse Source
This commit is contained in:
Andrew
2023-05-12 00:42:14 -05:00
parent f7676e73ba
commit 5e7c52997c
10 changed files with 387 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

139
pkg/backy/backy.go
View File

@ -11,6 +11,7 @@ import (
"io"
"os"
"os/exec"
"strings"
"text/template"
"embed"
@ -29,7 +30,7 @@ var Sprintf = fmt.Sprintf
// The environment of local commands will be the machine's environment plus any extra
// variables specified in the Env file or Environment.
// Dir can also be specified for local commands.
func (command *Command) RunCmd(log *zerolog.Logger, backyConf *BackyConfigFile) ([]string, error) {
func (command *Command) RunCmd(log *zerolog.Logger, backyConf *BackyConfigFile, opts *BackyConfigOpts) ([]string, error) {
var (
outputArr []string
@ -48,7 +49,13 @@ func (command *Command) RunCmd(log *zerolog.Logger, backyConf *BackyConfigFile)
}
if command.Host != nil {
log.Info().Str("Command", fmt.Sprintf("Running command %s %s on host %s", command.Cmd, ArgsStr, *command.Host)).Send()
command.Type = strings.TrimSpace(command.Type)
if command.Type != "" {
log.Info().Str("Command", fmt.Sprintf("Running script %s on host %s", command.Cmd, *command.Host)).Send()
} else {
log.Info().Str("Command", fmt.Sprintf("Running command %s %s on host %s", command.Cmd, ArgsStr, *command.Host)).Send()
}
if command.RemoteHost.SshClient == nil {
err := command.RemoteHost.ConnectToSSHHost(log, backyConf)
@ -58,12 +65,11 @@ func (command *Command) RunCmd(log *zerolog.Logger, backyConf *BackyConfigFile)
}
commandSession, err := command.RemoteHost.SshClient.NewSession()
if err != nil {
log.Err(fmt.Errorf("new ssh session: %w", err)).Send()
return nil, err
}
defer commandSession.Close()
injectEnvIntoSSH(envVars, commandSession, log)
injectEnvIntoSSH(envVars, commandSession, opts)
cmd := command.Cmd
for _, a := range command.Args {
cmd += " " + a
@ -76,6 +82,94 @@ func (command *Command) RunCmd(log *zerolog.Logger, backyConf *BackyConfigFile)
commandSession.Stdout = cmdOutWriters
commandSession.Stderr = cmdOutWriters
if command.Type != "" {
if command.Type == "script" {
script := bytes.NewBufferString(cmd + "\n")
commandSession.Stdin = script
if err := commandSession.Shell(); err != nil {
return nil, err
}
if err := commandSession.Wait(); err != nil {
outScanner := bufio.NewScanner(&cmdOutBuf)
for outScanner.Scan() {
outMap := make(map[string]interface{})
outMap["cmd"] = cmd
outMap["output"] = outScanner.Text()
if str, ok := outMap["output"].(string); ok {
outputArr = append(outputArr, str)
}
log.Info().Fields(outMap).Send()
}
return outputArr, err
}
outScanner := bufio.NewScanner(&cmdOutBuf)
for outScanner.Scan() {
outMap := make(map[string]interface{})
outMap["cmd"] = cmd
outMap["output"] = outScanner.Text()
if str, ok := outMap["output"].(string); ok {
outputArr = append(outputArr, str)
}
log.Info().Fields(outMap).Send()
}
return outputArr, nil
}
if command.Type == "scriptFile" {
var buffer bytes.Buffer
var dirErr error
command.Cmd, dirErr = resolveDir(command.Cmd)
if dirErr != nil {
return nil, dirErr
}
file, err := os.Open(command.Cmd)
if err != nil {
return nil, err
}
defer file.Close()
_, err = io.Copy(&buffer, file)
if err != nil {
return nil, err
}
script := &buffer
commandSession.Stdin = script
if err := commandSession.Shell(); err != nil {
return nil, err
}
if err := commandSession.Wait(); err != nil {
outScanner := bufio.NewScanner(&cmdOutBuf)
for outScanner.Scan() {
outMap := make(map[string]interface{})
outMap["cmd"] = cmd
outMap["output"] = outScanner.Text()
if str, ok := outMap["output"].(string); ok {
outputArr = append(outputArr, str)
}
log.Info().Fields(outMap).Send()
}
return outputArr, err
}
outScanner := bufio.NewScanner(&cmdOutBuf)
for outScanner.Scan() {
outMap := make(map[string]interface{})
outMap["cmd"] = cmd
outMap["output"] = outScanner.Text()
if str, ok := outMap["output"].(string); ok {
outputArr = append(outputArr, str)
}
log.Info().Fields(outMap).Send()
}
return outputArr, nil
}
return nil, fmt.Errorf("command type not recognized")
}
err = commandSession.Run(cmd)
outScanner := bufio.NewScanner(&cmdOutBuf)
for outScanner.Scan() {
@ -93,10 +187,6 @@ func (command *Command) RunCmd(log *zerolog.Logger, backyConf *BackyConfigFile)
return outputArr, err
}
} else {
cmdExists := command.checkCmdExists()
if !cmdExists {
log.Info().Str(command.Cmd, "not found").Send()
}
var err error
if command.Shell != "" {
@ -177,10 +267,9 @@ func (command *Command) RunCmd(log *zerolog.Logger, backyConf *BackyConfigFile)
return outputArr, nil
}
func cmdListWorker(msgTemps *msgTemplates, jobs <-chan *CmdList, config *BackyConfigFile, results chan<- string) {
func cmdListWorker(msgTemps *msgTemplates, jobs <-chan *CmdList, config *BackyConfigFile, results chan<- string, opts *BackyConfigOpts) {
for list := range jobs {
var currentCmd string
fieldsMap := make(map[string]interface{})
fieldsMap["list"] = list.Name
@ -188,13 +277,14 @@ func cmdListWorker(msgTemps *msgTemplates, jobs <-chan *CmdList, config *BackyCo
var count int
var cmdsRan []string
var outStructArr []outStruct
for _, cmd := range list.Order {
currentCmd = config.Cmds[cmd].Cmd
currentCmd := config.Cmds[cmd].Cmd
fieldsMap["cmd"] = config.Cmds[cmd].Cmd
cmdLog.Fields(fieldsMap).Send()
cmdToRun := config.Cmds[cmd]
cmdLog.Fields(fieldsMap).Send()
cmdLogger := config.Logger.With().
Str("backy-cmd", cmd).Str("Host", "local machine").
@ -206,7 +296,17 @@ func cmdListWorker(msgTemps *msgTemplates, jobs <-chan *CmdList, config *BackyCo
Logger()
}
outputArr, runOutErr := cmdToRun.RunCmd(&cmdLogger, config)
outputArr, runOutErr := cmdToRun.RunCmd(&cmdLogger, config, opts)
if cmdToRun.Output {
outputStruct := outStruct{
CmdName: cmd,
CmdExecuted: currentCmd,
Output: outputArr,
}
outStructArr = append(outStructArr, outputStruct)
}
count++
if runOutErr != nil {
var errMsg bytes.Buffer
@ -215,10 +315,14 @@ func cmdListWorker(msgTemps *msgTemplates, jobs <-chan *CmdList, config *BackyCo
errStruct["listName"] = list.Name
errStruct["Command"] = currentCmd
errStruct["Cmd"] = cmd
errStruct["Args"] = config.Cmds[cmd].Args
errStruct["Err"] = runOutErr
errStruct["CmdsRan"] = cmdsRan
errStruct["Output"] = outputArr
errStruct["CmdOutput"] = outStructArr
tmpErr := msgTemps.err.Execute(&errMsg, errStruct)
if tmpErr != nil {
@ -243,9 +347,12 @@ func cmdListWorker(msgTemps *msgTemplates, jobs <-chan *CmdList, config *BackyCo
if list.NotifyConfig != nil {
successStruct := make(map[string]interface{})
successStruct["listName"] = list.Name
successStruct["CmdsRan"] = cmdsRan
successStruct["CmdOutput"] = outStructArr
tmpErr := msgTemps.success.Execute(&successMsg, successStruct)
if tmpErr != nil {
@ -271,7 +378,7 @@ func cmdListWorker(msgTemps *msgTemplates, jobs <-chan *CmdList, config *BackyCo
}
// RunBackyConfig runs a command list from the BackyConfigFile.
func (config *BackyConfigFile) RunBackyConfig(cron string) {
func (config *BackyConfigFile) RunBackyConfig(cron string, opts *BackyConfigOpts) {
mTemps := &msgTemplates{
err: template.Must(template.New("error.txt").ParseFS(templates, "templates/error.txt")),
success: template.Must(template.New("success.txt").ParseFS(templates, "templates/success.txt")),
@ -283,7 +390,7 @@ func (config *BackyConfigFile) RunBackyConfig(cron string) {
// This starts up 3 workers, initially blocked
// because there are no jobs yet.
for w := 1; w <= configListsLen; w++ {
go cmdListWorker(mTemps, listChan, config, results)
go cmdListWorker(mTemps, listChan, config, results, opts)
}
// Here we send 5 `jobs` and then `close` that
@ -316,7 +423,7 @@ func (config *BackyConfigFile) ExecuteCmds(opts *BackyConfigOpts) {
cmdLogger := config.Logger.With().
Str("backy-cmd", cmd).
Logger()
_, runErr := cmdToRun.RunCmd(&cmdLogger, config)
_, runErr := cmdToRun.RunCmd(&cmdLogger, config, opts)
if runErr != nil {
config.Logger.Err(runErr).Send()
}

153
pkg/backy/config.go
View File

@ -1,6 +1,7 @@
package backy
import (
"context"
"errors"
"fmt"
"os"
@ -8,12 +9,39 @@ import (
"strings"
"git.andrewnw.xyz/CyberShell/backy/pkg/logging"
"github.com/joho/godotenv"
vault "github.com/hashicorp/vault/api"
"github.com/mattn/go-isatty"
"github.com/rs/zerolog"
"github.com/spf13/viper"
)
func (opts *BackyConfigOpts) InitConfig() {
if opts.viper != nil {
return
}
backyViper := viper.New()
if strings.TrimSpace(opts.ConfigFilePath) != "" {
err := testFile(opts.ConfigFilePath)
if err != nil {
logging.ExitWithMSG(fmt.Sprintf("Could not open config file %s: %v", opts.ConfigFilePath, err), 1, nil)
}
backyViper.SetConfigFile(opts.ConfigFilePath)
} else {
backyViper.SetConfigName("backy.yml") // name of config file (with extension)
backyViper.SetConfigName("backy.yaml") // name of config file (with extension)
backyViper.SetConfigType("yaml") // REQUIRED if the config file does not have the extension in the name
backyViper.AddConfigPath(".") // optionally look for config in the working directory
backyViper.AddConfigPath("$HOME/.config/backy") // call multiple times to add many search paths
}
err := backyViper.ReadInConfig() // Find and read the config file
if err != nil { // Handle errors reading the config file
msg := fmt.Sprintf("fatal error reading config file %s: %v", backyViper.ConfigFileUsed(), err)
logging.ExitWithMSG(msg, 1, nil)
}
opts.viper = backyViper
}
// ReadConfig validates and reads the config file.
func ReadConfig(opts *BackyConfigOpts) *BackyConfigFile {
@ -28,10 +56,10 @@ func ReadConfig(opts *BackyConfigOpts) *BackyConfigFile {
backyConfigFile := NewConfig()
backyViper := opts.viper
opts.loadEnv()
envFileInConfigDir := fmt.Sprintf("%s/.env", path.Dir(backyViper.ConfigFileUsed()))
// envFileInConfigDir := fmt.Sprintf("%s/.env", path.Dir(backyViper.ConfigFileUsed()))
// load the .env file in config file directory
_ = godotenv.Load(envFileInConfigDir)
// _ = godotenv.Load(envFileInConfigDir)
if backyViper.GetBool(getNestedConfig("logging", "cmd-std-out")) {
os.Setenv("BACKY_STDOUT", "enabled")
@ -239,6 +267,11 @@ func ReadConfig(opts *BackyConfigOpts) *BackyConfigFile {
}
backyConfigFile.SetupNotify()
opts.ConfigFile = backyConfigFile
if err := opts.setupVault(); err != nil {
log.Err(err).Send()
}
opts.ConfigFile = backyConfigFile
return backyConfigFile
}
@ -261,29 +294,101 @@ func getCmdListFromConfig(list string) string {
return fmt.Sprintf("cmd-configs.%s", list)
}
func (opts *BackyConfigOpts) InitConfig() {
if opts.viper != nil {
return
func (opts *BackyConfigOpts) setupVault() error {
if !opts.viper.GetBool("vault.enabled") {
return nil
}
backyViper := viper.New()
config := vault.DefaultConfig()
if strings.TrimSpace(opts.ConfigFilePath) != "" {
err := testFile(opts.ConfigFilePath)
if err != nil {
logging.ExitWithMSG(fmt.Sprintf("Could not open config file %s: %v", opts.ConfigFilePath, err), 1, nil)
}
backyViper.SetConfigFile(opts.ConfigFilePath)
} else {
backyViper.SetConfigName("backy.yml") // name of config file (with extension)
backyViper.SetConfigName("backy.yaml") // name of config file (with extension)
backyViper.SetConfigType("yaml") // REQUIRED if the config file does not have the extension in the name
backyViper.AddConfigPath(".") // optionally look for config in the working directory
backyViper.AddConfigPath("$HOME/.config/backy") // call multiple times to add many search paths
config.Address = opts.viper.GetString("vault.address")
if strings.TrimSpace(config.Address) == "" {
config.Address = os.Getenv("VAULT_ADDR")
}
err := backyViper.ReadInConfig() // Find and read the config file
if err != nil { // Handle errors reading the config file
msg := fmt.Sprintf("fatal error reading config file %s: %v", backyViper.ConfigFileUsed(), err)
logging.ExitWithMSG(msg, 1, nil)
client, err := vault.NewClient(config)
if err != nil {
return err
}
opts.viper = backyViper
token := opts.viper.GetString("vault.token")
if strings.TrimSpace(token) == "" {
token = os.Getenv("VAULT_TOKEN")
}
client.SetToken(token)
cmdListCfg := opts.viper.Sub("viper.keys")
unmarshalErr := cmdListCfg.Unmarshal(&opts.VaultKeys)
if unmarshalErr != nil {
panic(fmt.Errorf("error unmarshalling viper.keys into struct: %w", unmarshalErr))
}
opts.vaultClient = client
return nil
}
func getVaultSecret(vaultClient *vault.Client, key *VaultKey) (string, error) {
var (
secret *vault.KVSecret
err error
)
if key.ValueType == "KVv2" {
secret, err = vaultClient.KVv2(key.MountPath).Get(context.Background(), key.Path)
} else if key.ValueType == "KVv1" {
secret, err = vaultClient.KVv1(key.MountPath).Get(context.Background(), key.Path)
} else if key.ValueType != "" {
return "", fmt.Errorf("type %s for key %s not known. Valid types are KVv1 or KVv2", key.ValueType, key.Name)
} else {
return "", fmt.Errorf("type for key %s must be specified. Valid types are KVv1 or KVv2", key.Name)
}
if err != nil {
return "", fmt.Errorf("unable to read secret: %v", err)
}
value, ok := secret.Data[key.Name].(string)
if !ok {
return "", fmt.Errorf("value type assertion failed: %T %#v", secret.Data[key.Name], secret.Data[key.Name])
}
return value, nil
}
func isVaultKey(str string) (string, bool) {
str = strings.TrimSpace(str)
return strings.TrimPrefix(str, "vault:"), strings.HasPrefix(str, "vault:")
}
func parseVaultKey(str string, keys []*VaultKey) (*VaultKey, error) {
keyName, isKey := isVaultKey(str)
if !isKey {
return nil, nil
}
for _, k := range keys {
if k.Name == keyName {
return k, nil
}
}
return nil, fmt.Errorf("key %s not found in vault keys", keyName)
}
func GetVaultKey(str string, opts *BackyConfigOpts) string {
key, err := parseVaultKey(str, opts.VaultKeys)
if key == nil && err == nil {
return str
}
if err != nil && key == nil {
opts.ConfigFile.Logger.Err(err).Send()
return ""
}
value, secretErr := getVaultSecret(opts.vaultClient, key)
if secretErr != nil {
opts.ConfigFile.Logger.Err(secretErr).Send()
return value
}
return value
}

12
pkg/backy/templates/error.txt
View File

@ -1,4 +1,8 @@
Command list {{.listName }} failed on running {{.Command}}.
Command list {{.listName }} failed.
The command run was {{.Cmd}}.
The command executed was {{.Command}} {{ if .Args }} {{- range .Args}} {{.}} {{end}} {{end}}
{{ if .Err }} The error was {{ .Err }}{{ end }}
@ -9,4 +13,10 @@ The following commands ran:
{{- range .CmdsRan}}
- {{. -}}
{{end}}
{{ end }}
{{ if .CmdOutput }}{{- range .CmdOutput }}Commad output for {{ .CmdName }}:
{{- range .Output}}
{{ . }}
{{ end }}{{ end }}
{{ end }}

9
pkg/backy/templates/success.txt
View File

@ -1,7 +1,12 @@
Command list {{ .listName }} was completed successfully.
Command list {{ .listName }} completed successfully.
The following commands ran:
{{- range .CmdsRan}}
- {{. -}}
{{end}}
{{ if .CmdOutput }}{{- range .CmdOutput }}Commad output for {{ .CmdName }}:
{{- range .Output}}
{{ . }}
{{ end }}{{ end }}
{{ end }}

33
pkg/backy/types.go
View File

@ -4,6 +4,7 @@ import (
"bytes"
"text/template"
vaultapi "github.com/hashicorp/vault/api"
"github.com/kevinburke/ssh_config"
"github.com/nikoksr/notify"
"github.com/rs/zerolog"
@ -66,6 +67,10 @@ type (
// command to run
Cmd string `yaml:"cmd"`
// Possible values: script, scriptFile
// If blank, it is regualar command.
Type string `yaml:"type"`
// host on which to run cmd
Host *string `yaml:"host,omitempty"`
@ -91,6 +96,10 @@ type (
// Environment holds env variables to be used with the command
Environment []string `yaml:"environment,omitempty"`
// Output determines if output is requested.
// Only works if command is in a list.
Output bool `yaml:"output,omitempty"`
}
BackyOptionFunc func(*BackyConfigOpts)
@ -146,9 +155,33 @@ type (
// Holds env vars from .env file
backyEnv map[string]string
vaultClient *vaultapi.Client
VaultKeys []*VaultKey `yaml:"keys"`
viper *viper.Viper
}
outStruct struct {
CmdName string
CmdExecuted string
Output []string
}
VaultKey struct {
Name string `yaml:"name"`
Path string `yaml:"path"`
ValueType string `yaml:"type"`
MountPath string `yaml:"mountpath"`
}
VaultConfig struct {
Token string `yaml:"token"`
Address string `yaml:"address"`
Enabled string `yaml:"enabled"`
Keys []*VaultKey `yaml:"keys"`
}
NotificationsConfig struct {
Config *viper.Viper
Enabled bool

28
pkg/backy/utils.go
View File

@ -21,25 +21,25 @@ import (
"mvdan.cc/sh/v3/shell"
)
func injectEnvIntoSSH(envVarsToInject environmentVars, process *ssh.Session, log *zerolog.Logger) {
func injectEnvIntoSSH(envVarsToInject environmentVars, process *ssh.Session, opts *BackyConfigOpts) {
if envVarsToInject.file != "" {
envPath, envPathErr := resolveDir(envVarsToInject.file)
if envPathErr != nil {
log.Fatal().Str("envFile", envPath).Err(envPathErr).Send()
opts.ConfigFile.Logger.Fatal().Str("envFile", envPath).Err(envPathErr).Send()
}
file, err := os.Open(envPath)
if err != nil {
log.Fatal().Str("envFile", envPath).Err(err).Send()
opts.ConfigFile.Logger.Fatal().Str("envFile", envPath).Err(err).Send()
}
defer file.Close()
envMap, err := godotenv.Parse(file)
if err != nil {
log.Error().Str("envFile", envPath).Err(err).Send()
opts.ConfigFile.Logger.Error().Str("envFile", envPath).Err(err).Send()
goto errEnvFile
}
for key, val := range envMap {
process.Setenv(key, val)
process.Setenv(key, GetVaultKey(val, opts))
}
}
@ -49,7 +49,8 @@ errEnvFile:
// don't append env Vars for Backy
if strings.Contains(envVal, "=") {
envVarArr := strings.Split(envVal, "=")
process.Setenv(envVarArr[0], envVarArr[1])
process.Setenv(envVarArr[0], GetVaultKey(envVarArr[1], opts))
}
}
}
@ -58,10 +59,11 @@ func injectEnvIntoLocalCMD(envVarsToInject environmentVars, process *exec.Cmd, l
if envVarsToInject.file != "" {
envPath, _ := resolveDir(envVarsToInject.file)
file, _ := os.Open(envPath)
// if err != nil {
// log.Fatal().Str("envFile", envPath).Err(err).Send()
// }
file, fileErr := os.Open(envPath)
if fileErr != nil {
log.Error().Str("envFile", envPath).Err(fileErr).Send()
goto errEnvFile
}
defer file.Close()
envMap, err := godotenv.Parse(file)
if err != nil {
@ -69,6 +71,7 @@ func injectEnvIntoLocalCMD(envVarsToInject environmentVars, process *exec.Cmd, l
goto errEnvFile
}
for key, val := range envMap {
process.Env = append(process.Env, fmt.Sprintf("%s=%s", key, val))
}
@ -83,11 +86,6 @@ errEnvFile:
process.Env = append(process.Env, os.Environ()...)
}
func (cmd *Command) checkCmdExists() bool {
_, err := exec.LookPath(cmd.Cmd)
return err == nil
}
func contains(s []string, e string) bool {
for _, a := range s {
if a == e {