CyberShell/backy
1
1
Fork 0
Code Issues 10 Pull Requests Packages Projects 1 Releases 20 Wiki Activity

v0.3.0

Browse Source 
* Getting environment variables and passwords from Vault (not tested
yet)
* Vault configuration to config (not tested yet)
* Ability to run scripts from file on local machine on the remote host
* Ability to get ouput in the notification of a list for individual
commands or all commands
* Make SSH connections close after all commands have been run; reuse
previous connections if needed
This commit is contained in:
Andrew Woodlee
2023-07-01 21:46:54 -05:00
parent 5e7c52997c
commit 4b382bddd9
831 changed files with 83782 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
pkg/backy/backy.go
View File

@ -30,7 +30,7 @@ var Sprintf = fmt.Sprintf
// The environment of local commands will be the machine's environment plus any extra
// variables specified in the Env file or Environment.
// Dir can also be specified for local commands.
func (command *Command) RunCmd(log *zerolog.Logger, backyConf *BackyConfigFile, opts *BackyConfigOpts) ([]string, error) {
func (command *Command) RunCmd(log zerolog.Logger, backyConf *ConfigFile, opts *ConfigOpts) ([]string, error) {
var (
outputArr []string
@ -58,7 +58,7 @@ func (command *Command) RunCmd(log *zerolog.Logger, backyConf *BackyConfigFile,
}
if command.RemoteHost.SshClient == nil {
err := command.RemoteHost.ConnectToSSHHost(log, backyConf)
err := command.RemoteHost.ConnectToSSHHost(opts, backyConf)
if err != nil {
return nil, err
}
@ -69,7 +69,7 @@ func (command *Command) RunCmd(log *zerolog.Logger, backyConf *BackyConfigFile,
}
defer commandSession.Close()
injectEnvIntoSSH(envVars, commandSession, opts)
injectEnvIntoSSH(envVars, commandSession, opts, log)
cmd := command.Cmd
for _, a := range command.Args {
cmd += " " + a
@ -267,7 +267,7 @@ func (command *Command) RunCmd(log *zerolog.Logger, backyConf *BackyConfigFile,
return outputArr, nil
}
func cmdListWorker(msgTemps *msgTemplates, jobs <-chan *CmdList, config *BackyConfigFile, results chan<- string, opts *BackyConfigOpts) {
func cmdListWorker(msgTemps *msgTemplates, jobs <-chan *CmdList, config *ConfigFile, results chan<- string, opts *ConfigOpts) {
for list := range jobs {
fieldsMap := make(map[string]interface{})
@ -296,16 +296,19 @@ func cmdListWorker(msgTemps *msgTemplates, jobs <-chan *CmdList, config *BackyCo
Logger()
}
outputArr, runOutErr := cmdToRun.RunCmd(&cmdLogger, config, opts)
if cmdToRun.Output {
outputStruct := outStruct{
CmdName: cmd,
CmdExecuted: currentCmd,
Output: outputArr,
outputArr, runOutErr := cmdToRun.RunCmd(cmdLogger, config, opts)
if list.NotifyConfig != nil {
if cmdToRun.GetOutput || list.GetOutput {
outputStruct := outStruct{
CmdName: cmd,
CmdExecuted: currentCmd,
Output: outputArr,
}
outStructArr = append(outStructArr, outputStruct)
}
outStructArr = append(outStructArr, outputStruct)
}
count++
if runOutErr != nil {
@ -377,8 +380,8 @@ func cmdListWorker(msgTemps *msgTemplates, jobs <-chan *CmdList, config *BackyCo
}
// RunBackyConfig runs a command list from the BackyConfigFile.
func (config *BackyConfigFile) RunBackyConfig(cron string, opts *BackyConfigOpts) {
// RunListConfig runs a command list from the ConfigFile.
func (config *ConfigFile) RunListConfig(cron string, opts *ConfigOpts) {
mTemps := &msgTemplates{
err: template.Must(template.New("error.txt").ParseFS(templates, "templates/error.txt")),
success: template.Must(template.New("success.txt").ParseFS(templates, "templates/success.txt")),
@ -417,13 +420,13 @@ func (config *BackyConfigFile) RunBackyConfig(cron string, opts *BackyConfigOpts
config.closeHostConnections()
}
func (config *BackyConfigFile) ExecuteCmds(opts *BackyConfigOpts) {
func (config *ConfigFile) ExecuteCmds(opts *ConfigOpts) {
for _, cmd := range opts.executeCmds {
cmdToRun := config.Cmds[cmd]
cmdLogger := config.Logger.With().
Str("backy-cmd", cmd).
Logger()
_, runErr := cmdToRun.RunCmd(&cmdLogger, config, opts)
_, runErr := cmdToRun.RunCmd(cmdLogger, config, opts)
if runErr != nil {
config.Logger.Err(runErr).Send()
}
@ -433,7 +436,7 @@ func (config *BackyConfigFile) ExecuteCmds(opts *BackyConfigOpts) {
}
func (c *BackyConfigFile) closeHostConnections() {
func (c *ConfigFile) closeHostConnections() {
for _, host := range c.Hosts {
if host.isProxyHost {
continue

16
pkg/backy/config.go
View File

@ -15,7 +15,7 @@ import (
"github.com/spf13/viper"
)
func (opts *BackyConfigOpts) InitConfig() {
func (opts *ConfigOpts) InitConfig() {
if opts.viper != nil {
return
}
@ -43,7 +43,7 @@ func (opts *BackyConfigOpts) InitConfig() {
}
// ReadConfig validates and reads the config file.
func ReadConfig(opts *BackyConfigOpts) *BackyConfigFile {
func ReadConfig(opts *ConfigOpts) *ConfigFile {
if isatty.IsTerminal(os.Stdout.Fd()) {
os.Setenv("BACKY_TERM", "enabled")
@ -271,7 +271,6 @@ func ReadConfig(opts *BackyConfigOpts) *BackyConfigFile {
log.Err(err).Send()
}
opts.ConfigFile = backyConfigFile
return backyConfigFile
}
@ -294,7 +293,7 @@ func getCmdListFromConfig(list string) string {
return fmt.Sprintf("cmd-configs.%s", list)
}
func (opts *BackyConfigOpts) setupVault() error {
func (opts *ConfigOpts) setupVault() error {
if !opts.viper.GetBool("vault.enabled") {
return nil
}
@ -314,6 +313,9 @@ func (opts *BackyConfigOpts) setupVault() error {
if strings.TrimSpace(token) == "" {
token = os.Getenv("VAULT_TOKEN")
}
if strings.TrimSpace(token) == "" {
return fmt.Errorf("no token found, but one was required. \n\nSet the config key vault.token or the environment variable VAULT_TOKEN")
}
client.SetToken(token)
@ -375,19 +377,19 @@ func parseVaultKey(str string, keys []*VaultKey) (*VaultKey, error) {
return nil, fmt.Errorf("key %s not found in vault keys", keyName)
}
func GetVaultKey(str string, opts *BackyConfigOpts) string {
func GetVaultKey(str string, opts *ConfigOpts, log zerolog.Logger) string {
key, err := parseVaultKey(str, opts.VaultKeys)
if key == nil && err == nil {
return str
}
if err != nil && key == nil {
opts.ConfigFile.Logger.Err(err).Send()
log.Err(err).Send()
return ""
}
value, secretErr := getVaultSecret(opts.vaultClient, key)
if secretErr != nil {
opts.ConfigFile.Logger.Err(secretErr).Send()
log.Err(secretErr).Send()
return value
}
return value

11
pkg/backy/cron.go
View File

@ -11,24 +11,25 @@ import (
"github.com/go-co-op/gocron"
)
func (conf *BackyConfigFile) Cron() {
func (opts *ConfigOpts) Cron() {
s := gocron.NewScheduler(time.Local)
s.TagsUnique()
for listName, config := range conf.CmdConfigLists {
cmdLists := opts.ConfigFile.CmdConfigLists
for listName, config := range cmdLists {
if config.Name == "" {
config.Name = listName
}
cron := strings.TrimSpace(config.Cron)
if cron != "" {
conf.Logger.Info().Str("Scheduling cron list", config.Name).Str("Time", cron).Send()
opts.ConfigFile.Logger.Info().Str("Scheduling cron list", config.Name).Str("Time", cron).Send()
_, err := s.CronWithSeconds(cron).Tag(config.Name).Do(func(cron string) {
conf.RunBackyConfig(cron)
opts.ConfigFile.RunListConfig(cron, opts)
}, cron)
if err != nil {
panic(err)
}
}
}
conf.Logger.Info().Msg("Starting cron mode...")
opts.ConfigFile.Logger.Info().Msg("Starting cron mode...")
s.StartBlocking()
}

4
pkg/backy/mongo.go
View File

@ -16,7 +16,7 @@ import (
const mongoConfigKey = "global.mongo"
func (opts *BackyConfigOpts) InitMongo() {
func (opts *ConfigOpts) InitMongo() {
if !opts.viper.GetBool(getMongoConfigKey("enabled")) {
return
@ -68,7 +68,7 @@ func (opts *BackyConfigOpts) InitMongo() {
opts.DB = backyDB
}
func getMongoConfigFromEnv(opts *BackyConfigOpts) error {
func getMongoConfigFromEnv(opts *ConfigOpts) error {
mongoEnvFile, err := os.Open(opts.viper.GetString("global.mongo.env"))
if err != nil {
return err

4
pkg/backy/notification.go
View File

@ -28,13 +28,13 @@ type mailConfig struct {
port string
}
func SetupCommandsNotifiers(backyConfig BackyConfigFile, ids ...string) {
func SetupCommandsNotifiers(backyConfig ConfigFile, ids ...string) {
}
// SetupNotify sets up notify instances for each command list.
func (backyConfig *BackyConfigFile) SetupNotify() {
func (backyConfig *ConfigFile) SetupNotify() {
for _, cmdConfig := range backyConfig.CmdConfigLists {
var services []notify.Notifier

37
pkg/backy/ssh.go
View File

@ -20,7 +20,7 @@ import (
"golang.org/x/crypto/ssh/knownhosts"
)
var PrivateKeyExtraInfoErr = errors.New("Private key may be encrypted. \nIf encrypted, make sure the password is specified correctly in the correct section: \n privatekeypassword: password (not recommended) \n privatekeypassword: env:PR_KEY_PASS \n privatekeypassword: file:/path/to/password-file. \n ")
var PrivateKeyExtraInfoErr = errors.New("Private key may be encrypted. \nIf encrypted, make sure the password is specified correctly in the correct section: \n privatekeypassword: env:PR_KEY_PASS \n privatekeypassword: file:/path/to/password-file \n privatekeypassword: password (not recommended). \n ")
var TS = strings.TrimSpace
// ConnectToSSHHost connects to a host by looking up the config values in the directory ~/.ssh/config
@ -28,13 +28,11 @@ var TS = strings.TrimSpace
// It returns an ssh.Client used to run commands against.
// If configFile is empty, any required configuration is looked up in the default config files
// If any value is not found, defaults are used
func (remoteConfig *Host) ConnectToSSHHost(log *zerolog.Logger, config *BackyConfigFile) error {
func (remoteConfig *Host) ConnectToSSHHost(opts *ConfigOpts, config *ConfigFile) error {
// var sshClient *ssh.Client
var connectErr error
// TODO: add JumpHost config check
if TS(remoteConfig.ConfigFilePath) == "" {
remoteConfig.useDefaultConfig = true
}
@ -74,8 +72,8 @@ func (remoteConfig *Host) ConnectToSSHHost(log *zerolog.Logger, config *BackyCon
}
if remoteConfig.ProxyHost != nil {
for _, proxyHost := range remoteConfig.ProxyHost {
err := proxyHost.GetProxyJumpConfig(config.Hosts)
log.Info().Msgf("Proxy host: %s", proxyHost.Host)
err := proxyHost.GetProxyJumpConfig(config.Hosts, opts)
opts.ConfigFile.Logger.Info().Msgf("Proxy host: %s", proxyHost.Host)
if err != nil {
return err
}
@ -93,7 +91,7 @@ func (remoteConfig *Host) ConnectToSSHHost(log *zerolog.Logger, config *BackyCon
return errors.Errorf("No hostname found or specified for host %s", remoteConfig.Host)
}
err = remoteConfig.GetAuthMethods()
err = remoteConfig.GetAuthMethods(opts)
if err != nil {
return err
}
@ -103,9 +101,9 @@ func (remoteConfig *Host) ConnectToSSHHost(log *zerolog.Logger, config *BackyCon
return errors.Wrap(err, "could not create hostkeycallback function")
}
remoteConfig.ClientConfig.HostKeyCallback = hostKeyCallback
log.Info().Str("user", remoteConfig.ClientConfig.User).Send()
opts.ConfigFile.Logger.Info().Str("user", remoteConfig.ClientConfig.User).Send()
remoteConfig.SshClient, connectErr = remoteConfig.ConnectThroughBastion(log)
remoteConfig.SshClient, connectErr = remoteConfig.ConnectThroughBastion(opts.ConfigFile.Logger)
if connectErr != nil {
return connectErr
}
@ -114,7 +112,7 @@ func (remoteConfig *Host) ConnectToSSHHost(log *zerolog.Logger, config *BackyCon
return nil
}
log.Info().Msgf("Connecting to host %s", remoteConfig.HostName)
opts.ConfigFile.Logger.Info().Msgf("Connecting to host %s", remoteConfig.HostName)
remoteConfig.SshClient, connectErr = ssh.Dial("tcp", remoteConfig.HostName, remoteConfig.ClientConfig)
if connectErr != nil {
return connectErr
@ -136,7 +134,7 @@ func (remoteHost *Host) GetSshUserFromConfig() {
}
remoteHost.ClientConfig.User = remoteHost.User
}
func (remoteHost *Host) GetAuthMethods() error {
func (remoteHost *Host) GetAuthMethods(opts *ConfigOpts) error {
var signer ssh.Signer
var err error
var privateKey []byte
@ -148,7 +146,7 @@ func (remoteHost *Host) GetAuthMethods() error {
if err != nil {
return err
}
remoteHost.PrivateKeyPassword, err = GetPrivateKeyPassword(remoteHost.PrivateKeyPassword)
remoteHost.PrivateKeyPassword, err = GetPrivateKeyPassword(remoteHost.PrivateKeyPassword, opts, opts.ConfigFile.Logger)
if err != nil {
return err
}
@ -167,7 +165,7 @@ func (remoteHost *Host) GetAuthMethods() error {
}
}
if remoteHost.Password == "" {
remoteHost.Password, err = GetPassword(remoteHost.Password)
remoteHost.Password, err = GetPassword(remoteHost.Password, opts, opts.ConfigFile.Logger)
if err != nil {
return err
}
@ -235,7 +233,7 @@ func (remoteHost *Host) GetHostName() {
}
}
func (remoteHost *Host) ConnectThroughBastion(log *zerolog.Logger) (*ssh.Client, error) {
func (remoteHost *Host) ConnectThroughBastion(log zerolog.Logger) (*ssh.Client, error) {
if remoteHost.ProxyHost == nil {
return nil, nil
}
@ -273,7 +271,7 @@ func GetKnownHosts(khPath string) (string, error) {
return resolveDir("~/.ssh/known_hosts")
}
func GetPrivateKeyPassword(key string) (string, error) {
func GetPrivateKeyPassword(key string, opts *ConfigOpts, log zerolog.Logger) (string, error) {
var prKeyPassword string
if strings.HasPrefix(key, "file:") {
privKeyPassFilePath := strings.TrimPrefix(key, "file:")
@ -295,10 +293,11 @@ func GetPrivateKeyPassword(key string) (string, error) {
} else {
prKeyPassword = key
}
prKeyPassword = GetVaultKey(prKeyPassword, opts, opts.ConfigFile.Logger)
return prKeyPassword, nil
}
func GetPassword(pass string) (string, error) {
func GetPassword(pass string, opts *ConfigOpts, log zerolog.Logger) (string, error) {
pass = strings.TrimSpace(pass)
if pass == "" {
return "", nil
@ -324,6 +323,8 @@ func GetPassword(pass string) (string, error) {
} else {
password = pass
}
password = GetVaultKey(password, opts, opts.ConfigFile.Logger)
return password, nil
}
@ -352,7 +353,7 @@ func (remoteConfig *Host) GetProxyJumpFromConfig(hosts map[string]*Host) error {
return nil
}
func (remoteConfig *Host) GetProxyJumpConfig(hosts map[string]*Host) error {
func (remoteConfig *Host) GetProxyJumpConfig(hosts map[string]*Host, opts *ConfigOpts) error {
if TS(remoteConfig.ConfigFilePath) == "" {
remoteConfig.useDefaultConfig = true
}
@ -396,7 +397,7 @@ func (remoteConfig *Host) GetProxyJumpConfig(hosts map[string]*Host) error {
if remoteConfig.HostName == "" {
return errors.Errorf("No hostname found or specified for host %s", remoteConfig.Host)
}
err := remoteConfig.GetAuthMethods()
err := remoteConfig.GetAuthMethods(opts)
if err != nil {
return err
}

13
pkg/backy/types.go
View File

@ -99,22 +99,23 @@ type (
// Output determines if output is requested.
// Only works if command is in a list.
Output bool `yaml:"output,omitempty"`
GetOutput bool `yaml:"getOutput,omitempty"`
}
BackyOptionFunc func(*BackyConfigOpts)
BackyOptionFunc func(*ConfigOpts)
CmdList struct {
Name string `yaml:"name,omitempty"`
Cron string `yaml:"cron,omitempty"`
Order []string `yaml:"order,omitempty"`
Notifications []string `yaml:"notifications,omitempty"`
GetOutput bool `yaml:"getOutput,omitempty"`
NotifyConfig *notify.Notify
// NotificationsConfig map[string]*NotificationsConfig
// NotifyConfig map[string]*notify.Notify
}
BackyConfigFile struct {
ConfigFile struct {
// Cmds holds the commands for a list.
// Key is the name of the command,
@ -134,11 +135,11 @@ type (
Logger zerolog.Logger
}
BackyConfigOpts struct {
ConfigOpts struct {
// Global log level
BackyLogLvl *string
// Holds config file
ConfigFile *BackyConfigFile
ConfigFile *ConfigFile
// Holds config file
ConfigFilePath string
@ -149,7 +150,7 @@ type (
useCron bool
// Holds commands to execute for the exec command
executeCmds []string
// Holds commands to execute for the exec command
// Holds lists to execute for the backup command
executeLists []string
// Holds env vars from .env file

43
pkg/backy/utils.go
View File

@ -21,25 +21,25 @@ import (
"mvdan.cc/sh/v3/shell"
)
func injectEnvIntoSSH(envVarsToInject environmentVars, process *ssh.Session, opts *BackyConfigOpts) {
func injectEnvIntoSSH(envVarsToInject environmentVars, process *ssh.Session, opts *ConfigOpts, log zerolog.Logger) {
if envVarsToInject.file != "" {
envPath, envPathErr := resolveDir(envVarsToInject.file)
if envPathErr != nil {
opts.ConfigFile.Logger.Fatal().Str("envFile", envPath).Err(envPathErr).Send()
log.Fatal().Str("envFile", envPath).Err(envPathErr).Send()
}
file, err := os.Open(envPath)
if err != nil {
opts.ConfigFile.Logger.Fatal().Str("envFile", envPath).Err(err).Send()
log.Fatal().Str("envFile", envPath).Err(err).Send()
}
defer file.Close()
envMap, err := godotenv.Parse(file)
if err != nil {
opts.ConfigFile.Logger.Error().Str("envFile", envPath).Err(err).Send()
log.Error().Str("envFile", envPath).Err(err).Send()
goto errEnvFile
}
for key, val := range envMap {
process.Setenv(key, GetVaultKey(val, opts))
process.Setenv(key, GetVaultKey(val, opts, log))
}
}
@ -50,12 +50,12 @@ errEnvFile:
if strings.Contains(envVal, "=") {
envVarArr := strings.Split(envVal, "=")
process.Setenv(envVarArr[0], GetVaultKey(envVarArr[1], opts))
process.Setenv(envVarArr[0], GetVaultKey(envVarArr[1], opts, log))
}
}
}
func injectEnvIntoLocalCMD(envVarsToInject environmentVars, process *exec.Cmd, log *zerolog.Logger) {
func injectEnvIntoLocalCMD(envVarsToInject environmentVars, process *exec.Cmd, log zerolog.Logger) {
if envVarsToInject.file != "" {
envPath, _ := resolveDir(envVarsToInject.file)
@ -71,7 +71,6 @@ func injectEnvIntoLocalCMD(envVarsToInject environmentVars, process *exec.Cmd, l
goto errEnvFile
}
for key, val := range envMap {
process.Env = append(process.Env, fmt.Sprintf("%s=%s", key, val))
}
@ -117,43 +116,43 @@ func testFile(c string) error {
return nil
}
func (c *BackyConfigOpts) LogLvl(level string) BackyOptionFunc {
func (c *ConfigOpts) LogLvl(level string) BackyOptionFunc {
return func(bco *BackyConfigOpts) {
return func(bco *ConfigOpts) {
c.BackyLogLvl = &level
}
}
// AddCommands adds commands to BackyConfigOpts
// AddCommands adds commands to ConfigOpts
func AddCommands(commands []string) BackyOptionFunc {
return func(bco *BackyConfigOpts) {
return func(bco *ConfigOpts) {
bco.executeCmds = append(bco.executeCmds, commands...)
}
}
// AddCommandLists adds lists to BackyConfigOpts
// AddCommandLists adds lists to ConfigOpts
func AddCommandLists(lists []string) BackyOptionFunc {
return func(bco *BackyConfigOpts) {
return func(bco *ConfigOpts) {
bco.executeLists = append(bco.executeLists, lists...)
}
}
// UseCron enables the execution of command lists at specified times
func UseCron() BackyOptionFunc {
return func(bco *BackyConfigOpts) {
return func(bco *ConfigOpts) {
bco.useCron = true
}
}
// UseCron enables the execution of command lists at specified times
func (c *BackyConfigOpts) AddViper(v *viper.Viper) BackyOptionFunc {
return func(bco *BackyConfigOpts) {
func (c *ConfigOpts) AddViper(v *viper.Viper) BackyOptionFunc {
return func(bco *ConfigOpts) {
c.viper = v
}
}
func NewOpts(configFilePath string, opts ...BackyOptionFunc) *BackyConfigOpts {
b := &BackyConfigOpts{}
func NewOpts(configFilePath string, opts ...BackyOptionFunc) *ConfigOpts {
b := &ConfigOpts{}
b.ConfigFilePath = configFilePath
for _, opt := range opts {
if opt != nil {
@ -166,8 +165,8 @@ func NewOpts(configFilePath string, opts ...BackyOptionFunc) *BackyConfigOpts {
/*
NewConfig initializes new config that holds information from the config file
*/
func NewConfig() *BackyConfigFile {
return &BackyConfigFile{
func NewConfig() *ConfigFile {
return &ConfigFile{
Cmds: make(map[string]*Command),
CmdConfigLists: make(map[string]*CmdList),
Hosts: make(map[string]*Host),
@ -203,7 +202,7 @@ func resolveDir(path string) (string, error) {
return path, nil
}
func (opts *BackyConfigOpts) loadEnv() {
func (opts *ConfigOpts) loadEnv() {
envFileInConfigDir := fmt.Sprintf("%s/.env", path.Dir(opts.viper.ConfigFileUsed()))
var backyEnv map[string]string
backyEnv, envFileErr := godotenv.Read(envFileInConfigDir)