Compare commits

...

2 Commits

Author SHA1 Message Date
51f9e9a776 Merge branch 'develop' 2023-02-11 23:52:00 -06:00
37c20aaafa added new features
- expanding `environment` vars in cmd section
- support for connecting to one proxy/bastion host
- better notification text layout
- better error message on private key failing to open
2023-02-11 23:50:19 -06:00
15 changed files with 413 additions and 234 deletions

View File

@ -8,10 +8,15 @@ builds:
- env: - env:
- CGO_ENABLED=0 - CGO_ENABLED=0
goos: goos:
- freebsd
- linux - linux
goarch:
- "386"
- amd64
- arm64
archives: archives:
- format: tar.gz - format: binary
# this name template makes the OS and Arch compatible with the results of uname. # this name template makes the OS and Arch compatible with the results of uname.
name_template: >- name_template: >-
{{ .ProjectName }} {{ .ProjectName }}

View File

@ -14,11 +14,17 @@ You can also download binaries [here](https://git.andrewnw.xyz/CyberShell/backy/
## Features ## Features
- Define lists of commands and run them - Allows easy configuration of executable commands
- Execute commands over SSH - Allows for commands to be run on many hosts over SSH
- More to come. - Commands can be grouped in list to run in specific order
- Notifications on completion and failure
- Run in cron mode
- For any command, especially backup commands
To run a config: To run a config:
@ -29,84 +35,11 @@ Or to use a specific file:
If you leave the config path blank, the following paths will be searched in order: If you leave the config path blank, the following paths will be searched in order:
- `./backy.yml`
- `./backy.yaml` - `./backy.yaml`
- `~/.config/backy.yml`
- `~/.config/backy.yaml` - `~/.config/backy.yaml`
Create a file at `~/.config/backy.yaml`: Create a file at `~/.config/backy.yml`.
```yaml See the config file in the examples directory to configure it.
commands:
stop-docker-container:
cmd: docker
Args:
- compose
- -f /some/path/to/docker-compose.yaml
- down
# if host is not defined, cmd will be run locally
host: some-host
backup-docker-container-script:
cmd: /path/to/script
# The host has to be defined in the config file
host: some-host
shell-cmd:
cmd: rsync
shell: bash
Args:
- -av some-host:/path/to/data ~/Docker/Backups/docker-data
hostname:
cmd: hostname
cmd-configs:
cmds-to-run: # this can be any name you want
# all commands have to be defined
order:
- stop-docker-container
- backup-docker-container-script
- shell-cmd
- hostname
notifications:
- matrix
name: backup-some-server
hostname:
name: hostname
order:
- hostname
notifications:
- prod-email
hosts:
some-host:
hostname: some-hostname
config: ~/.ssh/config
user: user
privatekeypath: /path/to/private/key
port: 22
password:
logging:
verbose: true
file: /path/to/logs/commands.log
console: false
cmd-std-out: false
notifications:
prod-email:
id: prod-email
type: mail
host: yourhost.tld:port
senderAddress: email@domain.tld
to:
- admin@domain.tld
username: smtp-username@domain.tld
password: your-password-here
matrix:
id: matrix
type: matrix
home-server: your-home-server.tld
room-id: room-id
access-token: your-access-token
user-id: your-user-id
```

View File

@ -25,7 +25,7 @@ var cmdLists []string
func init() { func init() {
backupCmd.Flags().StringSliceVarP(&cmdLists, "lists", "l", nil, "Accepts a comma-separated names of command lists to execute.") backupCmd.Flags().StringSliceVarP(&cmdLists, "lists", "l", nil, "Accepts comma-separated names of command lists to execute.")
} }

31
cmd/config.go Normal file
View File

@ -0,0 +1,31 @@
package cmd
// import (
// "git.andrewnw.xyz/CyberShell/backy/pkg/backy"
// "github.com/spf13/cobra"
// )
// var (
// configCmd = &cobra.Command{
// Use: "config list ...",
// Short: "Runs commands defined in config file.",
// Long: `Cron executes commands at the time defined in config file.`,
// Run: config,
// }
// cmds []string
// lists []string
// )
// func config(cmd *cobra.Command, args []string) {
// opts := backy.NewOpts(cfgFile, backy.UseCron())
// opts.InitConfig()
// }
// func init() {
// configCmd.PersistentFlags().StringArrayVarP(&cmds, "cmds", "c", nil, "Accepts comma-seperated list of commands to list")
// }

View File

@ -11,6 +11,9 @@ commands:
cmd: /path/to/script cmd: /path/to/script
# The host has to be defined in the config file # The host has to be defined in the config file
host: some-host host: some-host
environment:
- FOO=BAR
- APP=$VAR
shell-cmd: shell-cmd:
cmd: rsync cmd: rsync
shell: bash shell: bash
@ -38,15 +41,19 @@ cmd-configs:
- prod-email - prod-email
hosts: hosts:
# any ssh_config(5) keys/values not listed here will be looked up in the config file or the default config file
some-host: some-host:
hostname: some-hostname hostname: some-hostname
config: ~/.ssh/config config: ~/.ssh/config
user: user user: user
privatekeypath: /path/to/private/key privatekeypath: /path/to/private/key
port: 22 port: 22
password: # can also be env:VAR
password: file:/path/to/file
# only one is supported for now
proxyjump: some-proxy-host
# optional
logging: logging:
verbose: true verbose: true
file: /path/to/logs/commands.log file: /path/to/logs/commands.log
@ -58,7 +65,8 @@ notifications:
prod-email: prod-email:
id: prod-email id: prod-email
type: mail type: mail
host: yourhost.tld:port host: yourhost.tld
port: 587
senderAddress: email@domain.tld senderAddress: email@domain.tld
to: to:
- admin@domain.tld - admin@domain.tld

View File

@ -11,11 +11,17 @@ import (
"io" "io"
"os" "os"
"os/exec" "os/exec"
"text/template"
"embed"
"github.com/rs/zerolog" "github.com/rs/zerolog"
) )
var requiredKeys = []string{"commands", "cmd-configs", "logging"} //go:embed templates/*.txt
var templates embed.FS
var requiredKeys = []string{"commands", "cmd-configs"}
var Sprintf = fmt.Sprintf var Sprintf = fmt.Sprintf
@ -23,7 +29,7 @@ var Sprintf = fmt.Sprintf
// The environment of local commands will be the machine's environment plus any extra // The environment of local commands will be the machine's environment plus any extra
// variables specified in the Env file or Environment. // variables specified in the Env file or Environment.
// Dir can also be specified for local commands. // Dir can also be specified for local commands.
func (command *Command) RunCmd(log *zerolog.Logger) error { func (command *Command) RunCmd(log *zerolog.Logger, hosts map[string]*Host) error {
var ( var (
ArgsStr string ArgsStr string
@ -44,12 +50,12 @@ func (command *Command) RunCmd(log *zerolog.Logger) error {
if command.Host != nil { if command.Host != nil {
log.Info().Str("Command", fmt.Sprintf("Running command: %s %s on host %s", command.Cmd, ArgsStr, *command.Host)).Send() log.Info().Str("Command", fmt.Sprintf("Running command: %s %s on host %s", command.Cmd, ArgsStr, *command.Host)).Send()
sshc, err := command.RemoteHost.ConnectToSSHHost(log) err := command.RemoteHost.ConnectToSSHHost(log, hosts)
if err != nil { if err != nil {
return err return err
} }
defer sshc.Close() defer command.RemoteHost.SshClient.Close()
commandSession, err := sshc.NewSession() commandSession, err := command.RemoteHost.SshClient.NewSession()
if err != nil { if err != nil {
log.Err(fmt.Errorf("new ssh session: %w", err)).Send() log.Err(fmt.Errorf("new ssh session: %w", err)).Send()
return err return err
@ -158,7 +164,7 @@ func cmdListWorker(id int, jobs <-chan *CmdList, config *BackyConfigFile, result
fieldsMap["list"] = list.Name fieldsMap["list"] = list.Name
cmdLog := config.Logger.Info() cmdLog := config.Logger.Info()
var count int var count int
var Msg string var cmdsRan []string
for _, cmd := range list.Order { for _, cmd := range list.Order {
currentCmd = config.Cmds[cmd].Cmd currentCmd = config.Cmds[cmd].Cmd
fieldsMap["cmd"] = config.Cmds[cmd].Cmd fieldsMap["cmd"] = config.Cmds[cmd].Cmd
@ -167,12 +173,22 @@ func cmdListWorker(id int, jobs <-chan *CmdList, config *BackyConfigFile, result
cmdLogger := config.Logger.With(). cmdLogger := config.Logger.With().
Str("backy-cmd", cmd). Str("backy-cmd", cmd).
Logger() Logger()
runOutErr := cmdToRun.RunCmd(&cmdLogger) runOutErr := cmdToRun.RunCmd(&cmdLogger, config.Hosts)
count++ count++
if runOutErr != nil { if runOutErr != nil {
var errMsg bytes.Buffer
if list.NotifyConfig != nil { if list.NotifyConfig != nil {
notifySendErr := list.NotifyConfig.Send(context.Background(), fmt.Sprintf("List %s failed on command %s ", list.Name, cmd), errStruct := make(map[string]interface{})
fmt.Sprintf("List %s failed on command %s running command %s. \n Error: %v", list.Name, cmd, currentCmd, runOutErr)) errStruct["listName"] = list.Name
errStruct["Command"] = currentCmd
errStruct["Err"] = runOutErr
errStruct["CmdsRan"] = cmdsRan
t := template.Must(template.New("error.txt").ParseFS(templates, "templates/error.txt"))
tmpErr := t.Execute(&errMsg, errStruct)
if tmpErr != nil {
config.Logger.Err(tmpErr).Send()
}
notifySendErr := list.NotifyConfig.Send(context.Background(), fmt.Sprintf("List %s failed on command %s ", list.Name, cmd), errMsg.String())
if notifySendErr != nil { if notifySendErr != nil {
config.Logger.Err(notifySendErr).Send() config.Logger.Err(notifySendErr).Send()
} }
@ -182,22 +198,32 @@ func cmdListWorker(id int, jobs <-chan *CmdList, config *BackyConfigFile, result
} else { } else {
if count == len(list.Order) { if count == len(list.Order) {
Msg += fmt.Sprintf("%s ", cmd) cmdsRan = append(cmdsRan, cmd)
var successMsg bytes.Buffer
if list.NotifyConfig != nil { if list.NotifyConfig != nil {
err := list.NotifyConfig.Send(context.Background(), fmt.Sprintf("List %s succeded", list.Name), successStruct := make(map[string]interface{})
fmt.Sprintf("Command list %s was completed successfully. The following commands ran:\n %s", list.Name, Msg)) successStruct["listName"] = list.Name
successStruct["CmdsRan"] = cmdsRan
t := template.Must(template.New("success.txt").ParseFS(templates, "templates/success.txt"))
tmpErr := t.Execute(&successMsg, successStruct)
if tmpErr != nil {
config.Logger.Err(tmpErr).Send()
break
}
err := list.NotifyConfig.Send(context.Background(), fmt.Sprintf("List %s succeded", list.Name), successMsg.String())
if err != nil { if err != nil {
config.Logger.Err(err).Send() config.Logger.Err(err).Send()
} }
} }
} else { } else {
Msg += fmt.Sprintf("%s, ", cmd) cmdsRan = append(cmdsRan, cmd)
} }
} }
} }
results <- "done" results <- "done"
} }
} }
// RunBackyConfig runs a command list from the BackyConfigFile. // RunBackyConfig runs a command list from the BackyConfigFile.
@ -208,7 +234,7 @@ func (config *BackyConfigFile) RunBackyConfig(cron string) {
// This starts up 3 workers, initially blocked // This starts up 3 workers, initially blocked
// because there are no jobs yet. // because there are no jobs yet.
for w := 1; w <= 3; w++ { for w := 1; w <= configListsLen; w++ {
go cmdListWorker(w, listChan, config, results) go cmdListWorker(w, listChan, config, results)
} }
@ -216,7 +242,10 @@ func (config *BackyConfigFile) RunBackyConfig(cron string) {
// Here we send 5 `jobs` and then `close` that // Here we send 5 `jobs` and then `close` that
// channel to indicate that's all the work we have. // channel to indicate that's all the work we have.
// configChan <- config.Cmds // configChan <- config.Cmds
for _, cmdConfig := range config.CmdConfigLists { for listName, cmdConfig := range config.CmdConfigLists {
if cmdConfig.Name == "" {
cmdConfig.Name = listName
}
if cron != "" { if cron != "" {
if cron == cmdConfig.Cron { if cron == cmdConfig.Cron {
listChan <- cmdConfig listChan <- cmdConfig
@ -235,6 +264,9 @@ func (config *BackyConfigFile) RunBackyConfig(cron string) {
func (config *BackyConfigFile) ExecuteCmds() { func (config *BackyConfigFile) ExecuteCmds() {
for _, cmd := range config.Cmds { for _, cmd := range config.Cmds {
cmd.RunCmd(&config.Logger) runErr := cmd.RunCmd(&config.Logger, config.Hosts)
if runErr != nil {
config.Logger.Err(runErr).Send()
}
} }
} }

View File

@ -12,7 +12,6 @@ import (
"github.com/mattn/go-isatty" "github.com/mattn/go-isatty"
"github.com/rs/zerolog" "github.com/rs/zerolog"
"github.com/spf13/viper" "github.com/spf13/viper"
"mvdan.cc/sh/v3/shell"
) )
// ReadConfig validates and reads the config file. // ReadConfig validates and reads the config file.
@ -28,13 +27,12 @@ func ReadConfig(opts *BackyConfigOpts) *BackyConfigFile {
backyConfigFile := NewConfig() backyConfigFile := NewConfig()
backyViper := opts.viper backyViper := opts.viper
// loadEnv(backyViper) opts.loadEnv()
envFileInConfigDir := fmt.Sprintf("%s/.env", path.Dir(backyViper.ConfigFileUsed())) envFileInConfigDir := fmt.Sprintf("%s/.env", path.Dir(backyViper.ConfigFileUsed()))
envFileErr := godotenv.Load() // load the .env file in config file directory
if envFileErr != nil { _ = godotenv.Load(envFileInConfigDir)
_ = godotenv.Load(envFileInConfigDir)
}
if backyViper.GetBool(getNestedConfig("logging", "cmd-std-out")) { if backyViper.GetBool(getNestedConfig("logging", "cmd-std-out")) {
os.Setenv("BACKY_STDOUT", "enabled") os.Setenv("BACKY_STDOUT", "enabled")
} }
@ -52,24 +50,28 @@ func ReadConfig(opts *BackyConfigOpts) *BackyConfigFile {
} }
} }
var backyLoggingOpts *viper.Viper var (
isBackyLoggingOptsSet := backyViper.IsSet("logging") // backyLoggingOpts *viper.Viper
if isBackyLoggingOptsSet { verbose bool
backyLoggingOpts = backyViper.Sub("logging") logFile string
} )
verbose := backyLoggingOpts.GetBool("verbose")
logFile := backyLoggingOpts.GetString("file") verbose = backyViper.GetBool(getLoggingKeyFromConfig("verbose"))
logFile = fmt.Sprintf("%s/backy.log", path.Dir(backyViper.ConfigFileUsed()))
if backyViper.IsSet(getLoggingKeyFromConfig("file")) {
logFile = backyViper.GetString(getLoggingKeyFromConfig("file"))
}
zerolog.SetGlobalLevel(zerolog.InfoLevel) zerolog.SetGlobalLevel(zerolog.InfoLevel)
if verbose { if verbose {
zerolog.SetGlobalLevel(zerolog.DebugLevel) zerolog.SetGlobalLevel(zerolog.DebugLevel)
globalLvl := zerolog.GlobalLevel() globalLvl := zerolog.GlobalLevel()
os.Setenv("BACKY_LOGLEVEL", Sprintf("%x", globalLvl)) os.Setenv("BACKY_LOGLEVEL", Sprintf("%v", globalLvl))
} }
consoleLoggingEnabled := backyLoggingOpts.GetBool("console") consoleLoggingEnabled := backyViper.GetBool(getLoggingKeyFromConfig("console"))
// Other qualifiers can go here as well // Other qualifiers can go here as well
if consoleLoggingEnabled { if consoleLoggingEnabled {
@ -78,12 +80,13 @@ func ReadConfig(opts *BackyConfigOpts) *BackyConfigFile {
os.Setenv("BACKY_CONSOLE_LOGGING", "") os.Setenv("BACKY_CONSOLE_LOGGING", "")
} }
writers := logging.SetLoggingWriters(backyLoggingOpts, logFile) writers := logging.SetLoggingWriters(logFile)
log := zerolog.New(writers).With().Timestamp().Logger() log := zerolog.New(writers).With().Timestamp().Logger()
backyConfigFile.Logger = log backyConfigFile.Logger = log
log.Info().Str("config file", backyViper.ConfigFileUsed()).Send()
commandsMap := backyViper.GetStringMapString("commands") commandsMap := backyViper.GetStringMapString("commands")
commandsMapViper := backyViper.Sub("commands") commandsMapViper := backyViper.Sub("commands")
unmarshalErr := commandsMapViper.Unmarshal(&backyConfigFile.Cmds) unmarshalErr := commandsMapViper.Unmarshal(&backyConfigFile.Cmds)
@ -100,6 +103,8 @@ func ReadConfig(opts *BackyConfigOpts) *BackyConfigFile {
os.Exit(1) os.Exit(1)
} }
expandEnvVars(opts.backyEnv, cmdConf.Environment)
host := cmdConf.Host host := cmdConf.Host
if host != nil { if host != nil {
if backyViper.IsSet(getNestedConfig("hosts", *host)) { if backyViper.IsSet(getNestedConfig("hosts", *host)) {
@ -114,12 +119,37 @@ func ReadConfig(opts *BackyConfigOpts) *BackyConfigFile {
if unmarshalErr != nil { if unmarshalErr != nil {
panic(fmt.Errorf("error unmarshalling hosts struct: %w", unmarshalErr)) panic(fmt.Errorf("error unmarshalling hosts struct: %w", unmarshalErr))
} }
for _, v := range backyConfigFile.Hosts { for _, host := range backyConfigFile.Hosts {
if host.ProxyJump != "" {
if v.JumpHost != "" { proxyHosts := strings.Split(host.ProxyJump, ",")
proxyHost, defined := backyConfigFile.Hosts[v.JumpHost] if len(proxyHosts) > 1 {
if defined { for hostNum, h := range proxyHosts {
v.ProxyHost = proxyHost if hostNum > 1 {
proxyHost, defined := backyConfigFile.Hosts[h]
if defined {
host.ProxyHost = append(host.ProxyHost, proxyHost)
} else {
newProxy := &Host{Host: h}
host.ProxyHost = append(host.ProxyHost, newProxy)
}
} else {
proxyHost, defined := backyConfigFile.Hosts[h]
if defined {
host.ProxyHost = append(host.ProxyHost, proxyHost)
} else {
newHost := &Host{Host: h}
host.ProxyHost = append(host.ProxyHost, newHost)
}
}
}
} else {
proxyHost, defined := backyConfigFile.Hosts[proxyHosts[0]]
if defined {
host.ProxyHost = append(host.ProxyHost, proxyHost)
} else {
newProxy := &Host{Host: proxyHosts[0]}
host.ProxyHost = append(host.ProxyHost, newProxy)
}
} }
} }
} }
@ -157,10 +187,7 @@ func ReadConfig(opts *BackyConfigOpts) *BackyConfigFile {
cmdNotFoundErrorLog.Errs("commands not found", cmdNotFoundSliceErr).Send() cmdNotFoundErrorLog.Errs("commands not found", cmdNotFoundSliceErr).Send()
} }
if opts.useCron && len(backyConfigFile.CmdConfigLists) > 0 { if opts.useCron && (len(backyConfigFile.CmdConfigLists) == 0) {
log.Info().Msg("Starting cron mode...")
} else if opts.useCron && (len(backyConfigFile.CmdConfigLists) == 0) {
logging.ExitWithMSG("No cron fields detected in any command lists", 1, nil) logging.ExitWithMSG("No cron fields detected in any command lists", 1, nil)
} }
@ -217,6 +244,14 @@ func getNestedConfig(nestedConfig, key string) string {
func getCmdFromConfig(key string) string { func getCmdFromConfig(key string) string {
return fmt.Sprintf("commands.%s", key) return fmt.Sprintf("commands.%s", key)
} }
func getLoggingKeyFromConfig(key string) string {
if key == "" {
return "logging"
}
return fmt.Sprintf("logging.%s", key)
}
func getCmdListFromConfig(list string) string { func getCmdListFromConfig(list string) string {
return fmt.Sprintf("cmd-configs.%s", list) return fmt.Sprintf("cmd-configs.%s", list)
} }
@ -228,8 +263,13 @@ func (opts *BackyConfigOpts) InitConfig() {
backyViper := viper.New() backyViper := viper.New()
if strings.TrimSpace(opts.ConfigFilePath) != "" { if strings.TrimSpace(opts.ConfigFilePath) != "" {
err := testFile(opts.ConfigFilePath)
if err != nil {
logging.ExitWithMSG(fmt.Sprintf("Could not open config file %s: %v", opts.ConfigFilePath, err), 1, nil)
}
backyViper.SetConfigFile(opts.ConfigFilePath) backyViper.SetConfigFile(opts.ConfigFilePath)
} else { } else {
backyViper.SetConfigName("backy.yml") // name of config file (with extension)
backyViper.SetConfigName("backy.yaml") // name of config file (with extension) backyViper.SetConfigName("backy.yaml") // name of config file (with extension)
backyViper.SetConfigType("yaml") // REQUIRED if the config file does not have the extension in the name backyViper.SetConfigType("yaml") // REQUIRED if the config file does not have the extension in the name
backyViper.AddConfigPath(".") // optionally look for config in the working directory backyViper.AddConfigPath(".") // optionally look for config in the working directory
@ -237,40 +277,8 @@ func (opts *BackyConfigOpts) InitConfig() {
} }
err := backyViper.ReadInConfig() // Find and read the config file err := backyViper.ReadInConfig() // Find and read the config file
if err != nil { // Handle errors reading the config file if err != nil { // Handle errors reading the config file
panic(fmt.Errorf("fatal error reading config file %s: %w", backyViper.ConfigFileUsed(), err)) msg := fmt.Sprintf("fatal error reading config file %s: %v", backyViper.ConfigFileUsed(), err)
logging.ExitWithMSG(msg, 1, nil)
} }
opts.viper = backyViper opts.viper = backyViper
} }
func loadEnv(backyViper *viper.Viper) {
envFileInConfigDir := fmt.Sprintf("%s/.env", path.Dir(backyViper.ConfigFileUsed()))
var backyEnv map[string]string
backyEnv, envFileErr := godotenv.Read()
// envFile, envFileErr := os.Open(".env")
if envFileErr != nil {
backyEnv, _ = godotenv.Read(envFileInConfigDir)
}
envFileErr = godotenv.Load()
if envFileErr != nil {
_ = godotenv.Load(envFileInConfigDir)
}
env := func(name string) string {
name = strings.ToUpper(name)
envVar, found := backyEnv[name]
if found {
return envVar
}
return ""
}
envVars := []string{"APP=${BACKY_APP}"}
for indx, v := range envVars {
if strings.Contains(v, "$") || (strings.Contains(v, "${") && strings.Contains(v, "}")) {
out, _ := shell.Expand(v, env)
envVars[indx] = out
// println(out)
}
}
}

View File

@ -14,15 +14,21 @@ import (
func (conf *BackyConfigFile) Cron() { func (conf *BackyConfigFile) Cron() {
s := gocron.NewScheduler(time.Local) s := gocron.NewScheduler(time.Local)
s.TagsUnique() s.TagsUnique()
for _, config := range conf.CmdConfigLists { for listName, config := range conf.CmdConfigLists {
if strings.TrimSpace(config.Cron) != "" { if config.Name == "" {
_, err := s.CronWithSeconds(config.Cron).Tag(config.Name).Do(func(cron string) { config.Name = listName
}
cron := strings.TrimSpace(config.Cron)
if cron != "" {
conf.Logger.Info().Str("Scheduling cron list", config.Name).Str("Time", cron).Send()
_, err := s.CronWithSeconds(cron).Tag(config.Name).Do(func(cron string) {
conf.RunBackyConfig(cron) conf.RunBackyConfig(cron)
}, config.Cron) }, cron)
if err != nil { if err != nil {
panic(err) panic(err)
} }
} }
} }
conf.Logger.Info().Msg("Starting cron mode...")
s.StartBlocking() s.StartBlocking()
} }

View File

@ -87,5 +87,6 @@ func setupMail(config mailConfig) *mail.Mail {
mailClient := mail.New(config.senderaddress, config.host+":"+config.port) mailClient := mail.New(config.senderaddress, config.host+":"+config.port)
mailClient.AuthenticateSMTP("", config.username, config.password, config.host) mailClient.AuthenticateSMTP("", config.username, config.password, config.host)
mailClient.AddReceivers(config.to...) mailClient.AddReceivers(config.to...)
mailClient.BodyFormat(mail.PlainText)
return mailClient return mailClient
} }

View File

@ -7,9 +7,9 @@ package backy
import ( import (
"bufio" "bufio"
"fmt" "fmt"
"log"
"os" "os"
"os/user" "os/user"
"strconv"
"strings" "strings"
"time" "time"
@ -20,83 +20,98 @@ import (
"golang.org/x/crypto/ssh/knownhosts" "golang.org/x/crypto/ssh/knownhosts"
) )
var ErrPrivateKeyFileFailedToOpen = errors.New("Private key file failed to open.") var ErrPrivateKeyFileFailedToOpen = errors.New("Failed to open private key file. If encrypted, make sure the password is specified.")
var TS = strings.TrimSpace var TS = strings.TrimSpace
// ConnectToSSHHost connects to a host by looking up the config values in the directory ~/.ssh/config // ConnectToSSHHost connects to a host by looking up the config values in the directory ~/.ssh/config
// It uses any set values and looks up an unset values in the config files // It uses any set values and looks up an unset values in the config files
// It returns an ssh.Client used to run commands against. // It returns an ssh.Client used to run commands against.
func (remoteConfig *Host) ConnectToSSHHost(log *zerolog.Logger) (*ssh.Client, error) { // If configFile is empty, any required configuration is looked up in the default config files
// If any value is not found, defaults are used
func (remoteConfig *Host) ConnectToSSHHost(log *zerolog.Logger, hosts map[string]*Host) error {
var sshClient *ssh.Client // var sshClient *ssh.Client
var connectErr error var connectErr error
// TODO: add JumpHost config check // TODO: add JumpHost config check
// if !remoteConfig.UseConfigFiles {
// log.Info().Msg("Not using config files")
// }
if TS(remoteConfig.ConfigFilePath) == "" { if TS(remoteConfig.ConfigFilePath) == "" {
remoteConfig.useDefaultConfig = true remoteConfig.useDefaultConfig = true
} }
if remoteConfig.ProxyHost != nil {
for _, proxyHost := range remoteConfig.ProxyHost {
log.Info().Msgf("Proxy Host %s", proxyHost.Host)
err := proxyHost.GetProxyJumpConfig(hosts)
if err != nil {
return err
}
}
}
khPath, khPathErr := GetKnownHosts(remoteConfig.KnownHostsFile) khPath, khPathErr := GetKnownHosts(remoteConfig.KnownHostsFile)
if khPathErr != nil { if khPathErr != nil {
return nil, khPathErr return khPathErr
} }
if remoteConfig.ClientConfig == nil { if remoteConfig.ClientConfig == nil {
remoteConfig.ClientConfig = &ssh.ClientConfig{} remoteConfig.ClientConfig = &ssh.ClientConfig{}
} }
var sshConfigFile *os.File var configFile *os.File
var sshConfigFileOpenErr error var sshConfigFileOpenErr error
if !remoteConfig.useDefaultConfig { if !remoteConfig.useDefaultConfig {
configFile, sshConfigFileOpenErr = os.Open(remoteConfig.ConfigFilePath)
sshConfigFile, sshConfigFileOpenErr = os.Open(remoteConfig.ConfigFilePath)
if sshConfigFileOpenErr != nil { if sshConfigFileOpenErr != nil {
return nil, sshConfigFileOpenErr return sshConfigFileOpenErr
} }
} else { } else {
defaultConfig, _ := resolveDir("~/.ssh/config") defaultConfig, _ := resolveDir("~/.ssh/config")
sshConfigFile, sshConfigFileOpenErr = os.Open(defaultConfig) configFile, sshConfigFileOpenErr = os.Open(defaultConfig)
if sshConfigFileOpenErr != nil { if sshConfigFileOpenErr != nil {
return nil, sshConfigFileOpenErr return sshConfigFileOpenErr
} }
} }
remoteConfig.SSHConfigFile = &sshConfigFile{}
remoteConfig.SSHConfigFile.DefaultUserSettings = ssh_config.DefaultUserSettings remoteConfig.SSHConfigFile.DefaultUserSettings = ssh_config.DefaultUserSettings
var decodeErr error
cfg, decodeErr := ssh_config.Decode(sshConfigFile) remoteConfig.SSHConfigFile.SshConfigFile, decodeErr = ssh_config.Decode(configFile)
if decodeErr != nil { if decodeErr != nil {
return nil, decodeErr return decodeErr
} }
remoteConfig.SSHConfigFile.SshConfigFile = cfg remoteConfig.ClientConfig.Timeout = time.Second * 30
remoteConfig.GetPrivateKeyFromConfig() remoteConfig.GetPrivateKeyFileFromConfig()
remoteConfig.GetHostNameWithPort() remoteConfig.GetPort()
remoteConfig.GetHostName()
remoteConfig.CombineHostNameWithPort()
remoteConfig.GetSshUserFromConfig() remoteConfig.GetSshUserFromConfig()
log.Info().Msgf("Port: %v", remoteConfig.Port)
if remoteConfig.HostName == "" { if remoteConfig.HostName == "" {
return nil, errors.New("No hostname found or specified") return errors.New("No hostname found or specified")
} }
err := remoteConfig.GetAuthMethods() err := remoteConfig.GetAuthMethods()
if err != nil { if err != nil {
return nil, err return err
} }
// TODO: Add value/option to config for host key and add bool to check for host key
hostKeyCallback, err := knownhosts.New(khPath) hostKeyCallback, err := knownhosts.New(khPath)
if err != nil { if err != nil {
return nil, errors.Wrap(err, "could not create hostkeycallback function") return errors.Wrap(err, "could not create hostkeycallback function")
} }
remoteConfig.ClientConfig.HostKeyCallback = hostKeyCallback remoteConfig.ClientConfig.HostKeyCallback = hostKeyCallback
log.Info().Str("user", remoteConfig.ClientConfig.User).Send() log.Info().Str("user", remoteConfig.ClientConfig.User).Send()
log.Info().Msgf("Connecting to host %s", remoteConfig.HostName) remoteConfig.SshClient, connectErr = remoteConfig.ConnectThroughBastion(log)
remoteConfig.ClientConfig.Timeout = time.Second * 30
sshClient, connectErr = ssh.Dial("tcp", remoteConfig.HostName, remoteConfig.ClientConfig)
if connectErr != nil { if connectErr != nil {
return nil, connectErr return connectErr
} }
return sshClient, nil if remoteConfig.SshClient != nil {
return nil
}
log.Info().Msgf("Connecting to host %s", remoteConfig.HostName)
remoteConfig.SshClient, connectErr = ssh.Dial("tcp", remoteConfig.HostName, remoteConfig.ClientConfig)
if connectErr != nil {
return connectErr
}
return nil
} }
func (remoteHost *Host) GetSshUserFromConfig() { func (remoteHost *Host) GetSshUserFromConfig() {
@ -155,9 +170,9 @@ func (remoteHost *Host) GetAuthMethods() error {
// GetPrivateKeyFromConfig checks to see if the privateKeyPath is empty. // GetPrivateKeyFromConfig checks to see if the privateKeyPath is empty.
// If not, it keeps the value. // If not, it keeps the value.
// If empty, the key is looked for in the specified config file. // If empty, the key is looked for in the specified config file.
// If that path is empty, the default config file is searched // If that path is empty, the default config file is searched.
// If not found in the default file, the privateKeyPath is set to ~/.ssh/id_rsa // If not found in the default file, the privateKeyPath is set to ~/.ssh/id_rsa
func (remoteHost *Host) GetPrivateKeyFromConfig() { func (remoteHost *Host) GetPrivateKeyFileFromConfig() {
var identityFile string var identityFile string
if remoteHost.PrivateKeyPath == "" { if remoteHost.PrivateKeyPath == "" {
identityFile, _ = remoteHost.SSHConfigFile.SshConfigFile.Get(remoteHost.Host, "IdentityFile") identityFile, _ = remoteHost.SSHConfigFile.SshConfigFile.Get(remoteHost.Host, "IdentityFile")
@ -175,18 +190,11 @@ func (remoteHost *Host) GetPrivateKeyFromConfig() {
remoteHost.PrivateKeyPath, _ = resolveDir(identityFile) remoteHost.PrivateKeyPath, _ = resolveDir(identityFile)
} }
// GetHostNameWithPort checks if the port from the config file is 0 // GetPort checks if the port from the config file is 0
// If it is the port is searched in the SSH config file(s) // If it is the port is searched in the SSH config file(s)
func (remoteHost *Host) GetHostNameWithPort() { func (remoteHost *Host) GetPort() {
port := fmt.Sprintf("%v", remoteHost.Port) port := fmt.Sprintf("%v", remoteHost.Port)
// port specifed?
if remoteHost.HostName == "" {
remoteHost.HostName, _ = remoteHost.SSHConfigFile.SshConfigFile.Get(remoteHost.Host, "HostName")
if remoteHost.HostName == "" {
remoteHost.HostName = remoteHost.SSHConfigFile.DefaultUserSettings.Get(remoteHost.Host, "HostName")
}
}
// no port specifed
if port == "0" { if port == "0" {
port, _ = remoteHost.SSHConfigFile.SshConfigFile.Get(remoteHost.Host, "Port") port, _ = remoteHost.SSHConfigFile.SshConfigFile.Get(remoteHost.Host, "Port")
if port == "" { if port == "" {
@ -195,16 +203,34 @@ func (remoteHost *Host) GetHostNameWithPort() {
port = "22" port = "22"
} }
} }
println(port)
} }
if !strings.HasSuffix(remoteHost.HostName, ":"+port) { portNum, _ := strconv.ParseUint(port, 10, 32)
remoteHost.HostName = remoteHost.HostName + ":" + port remoteHost.Port = uint16(portNum)
}
func (remoteHost *Host) CombineHostNameWithPort() {
remoteHost.HostName = fmt.Sprintf("%s:%v", remoteHost.HostName, remoteHost.Port)
}
func (remoteHost *Host) GetHostName() {
if remoteHost.HostName == "" {
remoteHost.HostName, _ = remoteHost.SSHConfigFile.SshConfigFile.Get(remoteHost.Host, "HostName")
if remoteHost.HostName == "" {
remoteHost.HostName = remoteHost.SSHConfigFile.DefaultUserSettings.Get(remoteHost.Host, "HostName")
}
} }
} }
func (remoteHost *Host) ConnectThroughBastion() (*ssh.Client, error) { func (remoteHost *Host) ConnectThroughBastion(log *zerolog.Logger) (*ssh.Client, error) {
if remoteHost.ProxyHost == nil {
return nil, nil
}
log.Info().Msgf("Connecting to proxy host %s", remoteHost.ProxyHost[0].HostName)
// connect to the bastion host // connect to the bastion host
bClient, err := ssh.Dial("tcp", remoteHost.ProxyHost.HostName, remoteHost.ProxyHost.ClientConfig) bClient, err := ssh.Dial("tcp", remoteHost.ProxyHost[0].HostName, remoteHost.ProxyHost[0].ClientConfig)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -214,10 +240,10 @@ func (remoteHost *Host) ConnectThroughBastion() (*ssh.Client, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
log.Info().Msgf("Connecting to host %s", remoteHost.HostName)
ncc, chans, reqs, err := ssh.NewClientConn(conn, remoteHost.HostName, remoteHost.ClientConfig) ncc, chans, reqs, err := ssh.NewClientConn(conn, remoteHost.HostName, remoteHost.ClientConfig)
if err != nil { if err != nil {
log.Fatal(err) return nil, err
} }
sClient := ssh.NewClient(ncc, chans, reqs) sClient := ssh.NewClient(ncc, chans, reqs)
@ -258,14 +284,14 @@ func GetPrivateKeyPassword(key string) (string, error) {
return prKeyPassword, nil return prKeyPassword, nil
} }
func GetPassword(key string) (string, error) { func GetPassword(pass string) (string, error) {
key = strings.TrimSpace(key) pass = strings.TrimSpace(pass)
if key == "" { if pass == "" {
return "", nil return "", nil
} }
var password string var password string
if strings.HasPrefix(key, "file:") { if strings.HasPrefix(pass, "file:") {
passFilePath := strings.TrimPrefix(key, "file:") passFilePath := strings.TrimPrefix(pass, "file:")
passFilePath, _ = resolveDir(passFilePath) passFilePath, _ = resolveDir(passFilePath)
keyFile, keyFileErr := os.Open(passFilePath) keyFile, keyFileErr := os.Open(passFilePath)
if keyFileErr != nil { if keyFileErr != nil {
@ -275,14 +301,94 @@ func GetPassword(key string) (string, error) {
for passwordScanner.Scan() { for passwordScanner.Scan() {
password = passwordScanner.Text() password = passwordScanner.Text()
} }
} else if strings.HasPrefix(key, "env:") { } else if strings.HasPrefix(pass, "env:") {
passEnv := strings.TrimPrefix(key, "env:") passEnv := strings.TrimPrefix(pass, "env:")
passEnv = strings.TrimPrefix(passEnv, "${") passEnv = strings.TrimPrefix(passEnv, "${")
passEnv = strings.TrimSuffix(passEnv, "}") passEnv = strings.TrimSuffix(passEnv, "}")
passEnv = strings.TrimPrefix(passEnv, "$") passEnv = strings.TrimPrefix(passEnv, "$")
password = os.Getenv(passEnv) password = os.Getenv(passEnv)
} else { } else {
password = key password = pass
} }
return password, nil return password, nil
} }
func (remoteConfig *Host) GetProxyJumpFromConfig(hosts map[string]*Host) error {
proxyJump, _ := remoteConfig.SSHConfigFile.SshConfigFile.Get(remoteConfig.Host, "ProxyJump")
if proxyJump == "" {
proxyJump = remoteConfig.SSHConfigFile.DefaultUserSettings.Get(remoteConfig.Host, "ProxyJump")
}
if remoteConfig.ProxyJump == "" && proxyJump != "" {
remoteConfig.ProxyJump = proxyJump
}
proxyJumpHosts := strings.Split(remoteConfig.ProxyJump, ",")
if remoteConfig.ProxyHost == nil && len(proxyJumpHosts) == 1 {
remoteConfig.ProxyJump = proxyJump
proxyHost, proxyHostFound := hosts[proxyJump]
if proxyHostFound {
remoteConfig.ProxyHost = append(remoteConfig.ProxyHost, proxyHost)
} else {
newProxy := &Host{Host: proxyJump}
remoteConfig.ProxyHost = append(remoteConfig.ProxyHost, newProxy)
}
}
return nil
}
func (remoteConfig *Host) GetProxyJumpConfig(hosts map[string]*Host) error {
if TS(remoteConfig.ConfigFilePath) == "" {
remoteConfig.useDefaultConfig = true
}
// log.Info().Msgf("Proxy Host %s", remoteConfig.ProxyHost[0].Host)
khPath, khPathErr := GetKnownHosts(remoteConfig.KnownHostsFile)
if khPathErr != nil {
return khPathErr
}
if remoteConfig.ClientConfig == nil {
remoteConfig.ClientConfig = &ssh.ClientConfig{}
}
var configFile *os.File
var sshConfigFileOpenErr error
if !remoteConfig.useDefaultConfig {
configFile, sshConfigFileOpenErr = os.Open(remoteConfig.ConfigFilePath)
if sshConfigFileOpenErr != nil {
return sshConfigFileOpenErr
}
} else {
defaultConfig, _ := resolveDir("~/.ssh/config")
configFile, sshConfigFileOpenErr = os.Open(defaultConfig)
if sshConfigFileOpenErr != nil {
return sshConfigFileOpenErr
}
}
remoteConfig.SSHConfigFile = &sshConfigFile{}
remoteConfig.SSHConfigFile.DefaultUserSettings = ssh_config.DefaultUserSettings
var decodeErr error
remoteConfig.SSHConfigFile.SshConfigFile, decodeErr = ssh_config.Decode(configFile)
if decodeErr != nil {
return decodeErr
}
remoteConfig.GetPrivateKeyFileFromConfig()
remoteConfig.GetPort()
remoteConfig.GetHostName()
remoteConfig.CombineHostNameWithPort()
remoteConfig.GetSshUserFromConfig()
if remoteConfig.HostName == "" {
return errors.New("No hostname found or specified")
}
err := remoteConfig.GetAuthMethods()
if err != nil {
return err
}
// TODO: Add value/option to config for host key and add bool to check for host key
hostKeyCallback, err := knownhosts.New(khPath)
if err != nil {
return errors.Wrap(err, "could not create hostkeycallback function")
}
remoteConfig.ClientConfig.HostKeyCallback = hostKeyCallback
return nil
}

View File

@ -0,0 +1,8 @@
Command list {{.listName }} failed on running {{.Command}}.
The error was {{ .Err }}
The following commands ran:
{{- range .CmdsRan}}
- {{. -}}
{{end}}

View File

@ -0,0 +1,7 @@
Command list {{ .listName }} was completed successfully.
The following commands ran:
{{- range .CmdsRan}}
- {{. -}}
{{end}}

View File

@ -41,17 +41,18 @@ type Host struct {
HostName string `yaml:"hostname,omitempty"` HostName string `yaml:"hostname,omitempty"`
KnownHostsFile string `yaml:"knownhostsfile,omitempty"` KnownHostsFile string `yaml:"knownhostsfile,omitempty"`
ClientConfig *ssh.ClientConfig ClientConfig *ssh.ClientConfig
SSHConfigFile sshConfigFile SSHConfigFile *sshConfigFile
SshClient *ssh.Client
Port uint16 `yaml:"port,omitempty"` Port uint16 `yaml:"port,omitempty"`
JumpHost string `yaml:"jumphost,omitempty"` ProxyJump string `yaml:"proxyjump,omitempty"`
Password string `yaml:"password,omitempty"` Password string `yaml:"password,omitempty"`
PrivateKeyPath string `yaml:"privatekeypath,omitempty"` PrivateKeyPath string `yaml:"privatekeypath,omitempty"`
PrivateKeyPassword string `yaml:"privatekeypassword,omitempty"` PrivateKeyPassword string `yaml:"privatekeypassword,omitempty"`
UseConfigFiles bool `yaml:"use_config_files,omitempty"` UseConfigFiles bool `yaml:"use_config_files,omitempty"`
useDefaultConfig bool useDefaultConfig bool
User string `yaml:"user,omitempty"` User string `yaml:"user,omitempty"`
// ProxyHost holds the configuration for a JumpHost host // ProxyHost holds the configuration for a ProxyJump host
ProxyHost *Host ProxyHost []*Host
} }
type sshConfigFile struct { type sshConfigFile struct {
@ -144,6 +145,9 @@ type BackyConfigOpts struct {
// Holds commands to execute for the exec command // Holds commands to execute for the exec command
executeLists []string executeLists []string
// Holds env vars from .env file
backyEnv map[string]string
viper *viper.Viper viper *viper.Viper
} }

View File

@ -9,6 +9,7 @@ import (
"fmt" "fmt"
"os" "os"
"os/exec" "os/exec"
"path"
"path/filepath" "path/filepath"
"strings" "strings"
@ -17,6 +18,7 @@ import (
"github.com/rs/zerolog" "github.com/rs/zerolog"
"github.com/spf13/viper" "github.com/spf13/viper"
"golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh"
"mvdan.cc/sh/v3/shell"
) )
func injectEnvIntoSSH(envVarsToInject environmentVars, process *ssh.Session, log *zerolog.Logger) { func injectEnvIntoSSH(envVarsToInject environmentVars, process *ssh.Session, log *zerolog.Logger) {
@ -207,3 +209,33 @@ func resolveDir(path string) (string, error) {
} }
return path, nil return path, nil
} }
func (opts *BackyConfigOpts) loadEnv() {
envFileInConfigDir := fmt.Sprintf("%s/.env", path.Dir(opts.viper.ConfigFileUsed()))
var backyEnv map[string]string
backyEnv, envFileErr := godotenv.Read(envFileInConfigDir)
if envFileErr != nil {
return
}
opts.backyEnv = backyEnv
}
func expandEnvVars(backyEnv map[string]string, envVars []string) {
env := func(name string) string {
name = strings.ToUpper(name)
envVar, found := backyEnv[name]
if found {
return envVar
}
return ""
}
for indx, v := range envVars {
if strings.Contains(v, "$") || (strings.Contains(v, "${") && strings.Contains(v, "}")) {
out, _ := shell.Expand(v, env)
envVars[indx] = out
}
}
}

View File

@ -7,7 +7,6 @@ import (
"time" "time"
"github.com/rs/zerolog" "github.com/rs/zerolog"
"github.com/spf13/viper"
"gopkg.in/natefinch/lumberjack.v2" "gopkg.in/natefinch/lumberjack.v2"
) )
@ -25,7 +24,7 @@ func ExitWithMSG(msg string, code int, log *zerolog.Logger) {
os.Exit(code) os.Exit(code)
} }
func SetLoggingWriters(v *viper.Viper, logFile string) (writers zerolog.LevelWriter) { func SetLoggingWriters(logFile string) (writers zerolog.LevelWriter) {
console := zerolog.ConsoleWriter{} console := zerolog.ConsoleWriter{}
if IsConsoleLoggingEnabled() { if IsConsoleLoggingEnabled() {
@ -55,12 +54,7 @@ func SetLoggingWriters(v *viper.Viper, logFile string) (writers zerolog.LevelWri
MaxAge: 28, //days MaxAge: 28, //days
Compress: true, // disabled by default Compress: true, // disabled by default
} }
if strings.TrimSpace(logFile) != "" { fileLogger.Filename = logFile
fileLogger.Filename = logFile
} else {
fileLogger.Filename = "./backy.log"
}
// UNIX Time is faster and smaller than most timestamps // UNIX Time is faster and smaller than most timestamps
zerolog.TimeFieldFormat = zerolog.TimeFormatUnix zerolog.TimeFieldFormat = zerolog.TimeFormatUnix
// zerolog.TimeFieldFormat = time.RFC1123 // zerolog.TimeFieldFormat = time.RFC1123
@ -75,3 +69,7 @@ func SetLoggingWriters(v *viper.Viper, logFile string) (writers zerolog.LevelWri
func IsConsoleLoggingEnabled() bool { func IsConsoleLoggingEnabled() bool {
return os.Getenv("BACKY_CONSOLE_LOGGING") == "enabled" return os.Getenv("BACKY_CONSOLE_LOGGING") == "enabled"
} }
// func IsTerminal() bool {
// return os.Getenv("BACKY_TERM") == "enabled"
// }