@ -1,136 +1,121 @@
// backy.go
// Copyright (C) Andrew Woodlee 2023
// License: Apache-2.0
package backy
import (
"bufio"
"bytes"
"errors"
"fmt"
"io"
"os"
"os/exec"
"os/user"
"path/filepath"
"strings"
"time"
"git.andrewnw.xyz/CyberShell/backy/pkg/logging"
"github.com/rs/zerolog"
"github.com/spf13/viper"
"golang.org/x/crypto/ssh"
"gopkg.in/natefinch/lumberjack.v2"
)
// Host defines a host to which to connect
// If not provided, the values will be looked up in the default ssh config files
type Host struct {
ConfigFilePath string
UseConfigFile bool
Empty bool
Host string
HostName string
Port uint16
PrivateKeyPath string
PrivateKeyPassword string
User string
}
type Command struct {
Remote bool ` yaml:"remote,omitempty" `
// command to run
Cmd string ` yaml:"cmd" `
// host on which to run cmd
Host * string ` yaml:"host,omitempty" `
var requiredKeys = [ ] string { "commands" , "cmd-configs" }
/ *
Shell specifies which shell to run the command in , if any .
Not applicable when host is defined .
* /
Shell string ` yaml:"shell,omitempty" `
var Sprintf = fmt . Sprintf
RemoteHost Host ` yaml:"-" `
// cmdArgs is an array that holds the arguments to cmd
CmdArgs [ ] string ` yaml:"cmdArgs,omitempty" `
type BackyOptionFunc func ( * BackyConfigOpts )
/ *
Dir specifies a directory in which to run the command .
Ignored if Host is set .
* /
Dir * string ` yaml:"dir,omitempty" `
func ( c * BackyConfigOpts ) LogLvl ( level string ) BackyOptionFunc {
return func ( bco * BackyConfigOpts ) {
c . BackyLogLvl = & level
}
}
type BackyGlobalOpts struct {
func ( c * BackyConfigOpts ) GetConfig ( ) {
c . ConfigFile = ReadAndParseConfigFile ( c . ConfigFilePath )
}
type BackyConfigFile struct {
/ *
Cmds holds the commands for a list .
Key is the name of the command ,
* /
Cmds map [ string ] Command ` yaml:"commands" `
/ *
CmdLists holds the lists of commands to be run in order .
Key is the command list name .
* /
CmdLists map [ string ] [ ] string ` yaml:"cmd-lists" `
/ *
Hosts holds the Host config .
key is the host .
* /
Hosts map [ string ] Host ` yaml:"hosts" `
Logger zerolog . Logger
func NewOpts ( configFilePath string , opts ... BackyOptionFunc ) * BackyConfigOpts {
b := & BackyConfigOpts { }
b . ConfigFilePath = configFilePath
for _ , opt := range opts {
opt ( b )
}
return b
}
// BackupConfig is a configuration struct that is used to define backups
type BackupConfig struct {
Name string
BackupType string
ConfigPath string
/ *
NewConfig initializes new config that holds information from the config file
* /
func NewConfig ( ) * BackyConfigFile {
return & BackyConfigFile {
Cmds : make ( map [ string ] Command ) ,
CmdConfigLists : make ( map [ string ] * CmdConfig ) ,
Hosts : make ( map [ string ] Host ) ,
Notifications : make ( map [ string ] * NotificationsConfig ) ,
}
}
Cmd Command
type environmentVars struct {
file string
env [ ] string
}
/ *
* Runs a backup configuration
* /
func ( command Command ) RunCmd ( log * zerolog . Logger ) {
func ( command * Command ) RunCmd ( log * zerolog . Logger ) {
var envVars = environmentVars {
file : command . Env ,
env : command . Environment ,
}
envVars . env = append ( envVars . env , os . Environ ( ) ... )
var cmdArgsStr string
for _ , v := range command . CmdArgs {
cmdArgsStr += fmt . Sprintf ( " %s" , v )
}
fmt . Printf ( "\n\nRunning command: " + command . Cmd + " " + cmdArgsStr + " on host " + * command . Host + "...\n\n" )
var hostStr string
if command . Host != nil {
hostStr = * command . Host
}
log . Info ( ) . Str ( "Command" , fmt . Sprintf ( "Running command: %s %s on host %s" , command . Cmd , cmdArgsStr , hostStr ) ) . Send ( )
if command . Host != nil {
command . RemoteHost . Host = * command . Host
command . RemoteHost . Port = 22
sshc , err := command . RemoteHost . ConnectToSSHHost ( log )
if err != nil {
panic ( fmt . Errorf ( "ssh dial: %w" , err ) )
log . Err ( fmt . Errorf ( "ssh dial: %w" , err ) ) . Send ( )
}
defer sshc . Close ( )
, err := sshc . NewSession ( )
commandSe ssion , err := sshc . NewSession ( )
if err != nil {
panic ( fmt . Errorf ( "new ssh session: %w" , err ) )
log . Err ( fmt . Errorf ( "new ssh session: %w" , err ) ) . Send ( )
}
defer . Close ( )
defer commandSe ssion . Close ( )
injectEnvIntoSSH ( envVars , commandSession , log )
cmd := command . Cmd
for _ , a := range command . CmdArgs {
cmd += " " + a
}
var stdoutBuf , stderrBuf bytes . Buffer
. Stdout = io . MultiWriter ( os . Stdout , & stdoutBuf )
. Stderr = io . MultiWriter ( os . Stderr , & stderrBuf )
err = . Run ( cmd )
commandSe ssion . Stdout = io . MultiWriter ( os . Stdout , & stdoutBuf )
commandSe ssion . Stderr = io . MultiWriter ( os . Stderr , & stderrBuf )
err = commandSe ssion . Run ( cmd )
log . Info ( ) . Bytes ( fmt . Sprintf ( "%s stdout" , command . Cmd ) , stdoutBuf . Bytes ( ) ) . Send ( )
log . Info ( ) . Bytes ( fmt . Sprintf ( "%s stderr" , command . Cmd ) , stderrBuf . Bytes ( ) ) . Send ( )
if err != nil {
panic ( fmt . Errorf ( "error when running cmd " + cmd + "\n Error: " + err . Error ( ) ) )
log . Error ( ) . Err ( fmt . Errorf ( "error when running cmd: %s: %w" , command . Cmd , err ) ) . Send ( )
}
} else {
// shell := "/bin/bash"
@ -145,12 +130,13 @@ func (command Command) RunCmd(log *zerolog.Logger) {
var stdoutBuf , stderrBuf bytes . Buffer
localCMD . Stdout = io . MultiWriter ( os . Stdout , & stdoutBuf )
localCMD . Stderr = io . MultiWriter ( os . Stderr , & stderrBuf )
injectEnvIntoLocalCMD ( envVars , localCMD , log )
err = localCMD . Run ( )
log . Info ( ) . Bytes ( fmt . Sprintf ( "%s stdout" , command . Cmd ) , stdoutBuf . Bytes ( ) ) . Send ( )
log . Info ( ) . Bytes ( fmt . Sprintf ( "%s stderr" , command . Cmd ) , stderrBuf . Bytes ( ) ) . Send ( )
if err != nil {
panic ( fmt . Errorf ( "error when running cmd: %s: %w" , command . Cmd , err ) )
log . Error ( ) . Err ( fmt . Errorf ( "error when running cmd: %s: %w" , command . Cmd , err ) ) . Send ( )
}
return
}
@ -161,70 +147,58 @@ func (command Command) RunCmd(log *zerolog.Logger) {
var stdoutBuf , stderrBuf bytes . Buffer
localCMD . Stdout = io . MultiWriter ( os . Stdout , & stdoutBuf )
localCMD . Stderr = io . MultiWriter ( os . Stderr , & stderrBuf )
injectEnvIntoLocalCMD ( envVars , localCMD , log )
err = localCMD . Run ( )
log . Info ( ) . Bytes ( fmt . Sprintf ( "%s stdout" , command . Cmd ) , stdoutBuf . Bytes ( ) ) . Send ( )
log . Info ( ) . Bytes ( fmt . Sprintf ( "%s stderr" , command . Cmd ) , stderrBuf . Bytes ( ) ) . Send ( )
if err != nil {
panic ( fmt . Errorf ( "error when running cmd: %s: %w" , command . Cmd , err ) )
log . Error ( ) . Err ( fmt . Errorf ( "error when running cmd: %s: %w" , command . Cmd , err ) ) . Send ( )
}
}
}
func ( config * BackyConfigFile ) RunBackyConfig ( ) {
for _ , list := range config . CmdLists {
for _ , cmd := range list {
func cmdListWorker ( id int , jobs <- chan * CmdConfig , config * BackyConfigFile , results chan <- string ) {
for j := range jobs {
// fmt.Println("worker", id, "started job", j)
for _ , cmd := range j . Order {
cmdToRun := config . Cmds [ cmd ]
cmdToRun . RunCmd ( & config . Logger )
}
// fmt.Println("worker", id, "finished job", j)
results <- "done"
}
}
type BackyConfigOpts struct {
// Holds config file
ConfigFile * BackyConfigFile
// Holds config file
ConfigFilePath string
// Global log level
BackyLogLvl * string
}
// RunBackyConfig runs a command list from the BackyConfigFile.
func ( config * BackyConfigFile ) RunBackyConfig ( ) {
configListsLen := len ( config . CmdConfigLists )
jobs := make ( chan * CmdConfig , configListsLen )
results := make ( chan string )
// configChan := make(chan map[string]Command)
type BackyOptionFunc func ( * BackyConfigOpts )
// This starts up 3 workers, initially blocked
// because there are no jobs yet.
for w := 1 ; w <= 3 ; w ++ {
go cmdListWorker ( w , jobs , config , results )
func ( c * BackyConfigOpts ) LogLvl ( level string ) BackyOptionFunc {
return func ( bco * BackyConfigOpts ) {
c . BackyLogLvl = & level
}
}
func ( c * BackyConfigOpts ) GetConfig ( ) {
c . ConfigFile = ReadAndParseConfigFile ( c . ConfigFilePath )
}
func New ( ) BackupConfig {
return BackupConfig { }
}
func NewOpts ( configFilePath string , opts ... BackyOptionFunc ) * BackyConfigOpts {
b := & BackyConfigOpts { }
b . ConfigFilePath = configFilePath
for _ , opt := range opts {
opt ( b )
// Here we send 5 `jobs` and then `close` that
// channel to indicate that's all the work we have.
// configChan <- config.Cmds
for _ , cmdConfig := range config . CmdConfigLists {
jobs <- cmdConfig
// fmt.Println("sent job", config.Order)
}
return b
}
close ( jobs )
/ *
* NewConfig initializes new config that holds information
* from the config file
* /
func NewConfig ( ) * BackyConfigFile {
return & BackyConfigFile {
Cmds : make ( map [ string ] Command ) ,
CmdLists : make ( map [ string ] [ ] string ) ,
Hosts : make ( map [ string ] Host ) ,
for a := 1 ; a <= configListsLen ; a ++ {
<- results
}
}
// ReadAndParseConfigFile validates and reads the config file.
func ReadAndParseConfigFile ( configFile string ) * BackyConfigFile {
backyConfigFile := NewConfig ( )
@ -234,22 +208,28 @@ func ReadAndParseConfigFile(configFile string) *BackyConfigFile {
if configFile != "" {
backyViper . SetConfigFile ( configFile )
} else {
backyViper . SetConfigName ( "backy ") // name of config file (with out extension)
backyViper . SetConfigName ( "backy .yaml ") // name of config file (with extension)
backyViper . SetConfigType ( "yaml" ) // REQUIRED if the config file does not have the extension in the name
backyViper . AddConfigPath ( "." ) // optionally look for config in the working directory
backyViper . AddConfigPath ( "$HOME/.config/backy" ) // call multiple times to add many search paths
}
err := backyViper . ReadInConfig ( ) // Find and read the config file
if err != nil { // Handle errors reading the config file
panic ( fmt . Errorf ( "fatal error finding config file: %w" , err ) )
panic ( fmt . Errorf ( "fatal error reading config file %s: %w", backyViper . ConfigFileUsed ( ) , err ) )
}
backyLoggingOpts := backyViper . Sub ( "logging" )
CheckForConfigValues ( backyViper )
var backyLoggingOpts * viper . Viper
backyLoggingOptsSet := backyViper . IsSet ( "logging" )
if backyLoggingOptsSet {
backyLoggingOpts = backyViper . Sub ( "logging" )
}
verbose := backyLoggingOpts . GetBool ( "verbose" )
logFile := backyLoggingOpts . GetString ( "file" )
if verbose {
zerolog . Level . String ( zerolog . DebugLevel )
zerolog . SetGlobalLevel ( zerolog . Error )
}
output := zerolog . ConsoleWriter { Out : os . Stdout , TimeFormat : time . RFC1123 }
output . FormatLevel = func ( i interface { } ) string {
@ -271,7 +251,7 @@ func ReadAndParseConfigFile(configFile string) *BackyConfigFile {
MaxAge : 28 , //days
Compress : true , // disabled by default
}
if strings . Trim ( logFile , " " ) != "" {
if strings . Trim Space ( logFile ) != "" {
fileLogger . Filename = logFile
} else {
fileLogger . Filename = "./backy.log"
@ -290,24 +270,8 @@ func ReadAndParseConfigFile(configFile string) *BackyConfigFile {
unmarshalErr := commandsMapViper . Unmarshal ( & backyConfigFile . Cmds )
if unmarshalErr != nil {
panic ( fmt . Errorf ( "error unmarshalling cmds struct: %w" , unmarshalErr ) )
} else {
for cmdName , cmdConf := range backyConfigFile . Cmds {
fmt . Printf ( "\nCommand Name: %s\n" , cmdName )
fmt . Printf ( "Shell: %v\n" , cmdConf . Shell )
fmt . Printf ( "Command: %s\n" , cmdConf . Cmd )
if len ( cmdConf . CmdArgs ) > 0 {
fmt . Println ( "\nCmd Args:" )
for _ , args := range cmdConf . CmdArgs {
fmt . Printf ( "%s\n" , args )
}
}
if cmdConf . Host != nil {
fmt . Printf ( "Host: %s\n" , * backyConfigFile . Cmds [ cmdName ] . Host )
}
}
os . Exit ( 0 )
}
var cmdNames [ ] string
for k := range commandsMap {
cmdNames = append ( cmdNames , k )
@ -336,24 +300,84 @@ func ReadAndParseConfigFile(configFile string) *BackyConfigFile {
}
cmdListCfg := backyViper . GetStringMapStringSlice ( "cmd-lists" )
cmdListCfg := backyViper . Sub ( "cmd-configs" )
unmarshalErr = cmdListCfg . Unmarshal ( & backyConfigFile . CmdConfigLists )
if unmarshalErr != nil {
panic ( fmt . Errorf ( "error unmarshalling cmd list struct: %w" , unmarshalErr ) )
}
var cmdNotFoundSliceErr [ ] error
for cmdListName , cmdList := range cmdListCfg {
for _ , cmdInList := range cmdList {
for cmdListName , cmdList := range backyConfigFile . CmdConfigLists {
for _ , cmdInList := range cmdList . Order {
// log.Info().Msgf("CmdList %s Cmd %s", cmdListName, cmdInList)
_ , cmdNameFound := backyConfigFile . Cmds [ cmdInList ]
if ! backyViper. IsSet ( getNestedConfig ( "commands" , cmdInList ) ) && ! cmdNameFound {
cmdNotFoundStr := fmt . Sprintf ( "command definition %s is not in config file\n ", cmdInList )
if ! {
cmdNotFoundStr := fmt . Sprintf ( "command %s is not defined in config file", cmdInList )
cmdNotFoundErr := errors . New ( cmdNotFoundStr )
cmdNotFoundSliceErr = append ( cmdNotFoundSliceErr , cmdNotFoundErr )
} else {
backyConfigFile . CmdLists [ cmdListName ] = append ( backyConfigFile . CmdLists [ cmdListName ] , cmdInList )
log . Info ( ) . Str ( cmdInList , "found in " + cmdListName ) . Send ( )
// backyConfigFile.CmdLists[cmdListName] = append(backyConfigFile.CmdLists[cmdListName], cmdInList)
}
}
}
for _ , err := range cmdNotFoundSliceErr {
if err != nil {
fmt . Println ( err . Error ( ) )
if len ( cmdNotFoundSliceErr ) > 0 {
var cmdNotFoundErrorLog = log . Fatal ( )
for _ , err := range cmdNotFoundSliceErr {
if err != nil {
cmdNotFoundErrorLog . Err ( err )
}
}
cmdNotFoundErrorLog . Send ( )
}
// var notificationSlice []string
for name , cmdCfg := range backyConfigFile . CmdConfigLists {
for _ , notificationID := range cmdCfg . Notifications {
// if !contains(notificationSlice, notificationID) {
cmdCfg . NotificationsConfig = make ( map [ string ] * NotificationsConfig )
notifConfig := backyViper . Sub ( getNestedConfig ( "notifications" , notificationID ) )
config := & NotificationsConfig {
Config : notifConfig ,
Enabled : true ,
}
cmdCfg . NotificationsConfig [ notificationID ] = config
// First we get a "copy" of the entry
if entry , ok := cmdCfg . NotificationsConfig [ notificationID ] ; ok {
// Then we modify the copy
entry . Config = notifConfig
entry . Enabled = true
// Then we reassign the copy
cmdCfg . NotificationsConfig [ notificationID ] = entry
}
backyConfigFile . CmdConfigLists [ name ] . NotificationsConfig [ notificationID ] = config
}
// }
}
var notificationsMap = make ( map [ string ] interface { } )
if backyViper . IsSet ( "notifications" ) {
notificationsMap = backyViper . GetStringMap ( "notifications" )
for id := range notificationsMap {
notifConfig := backyViper . Sub ( getNestedConfig ( "notifications" , id ) )
config := & NotificationsConfig {
Config : notifConfig ,
Enabled : true ,
}
backyConfigFile . Notifications [ id ] = config
}
// for _, notif := range backyConfigFile.Notifications {
// fmt.Printf("Type: %s\n", notif.Config.GetString("type"))
// notificationID := notif.Config.GetString("id")
// if !contains(notificationSlice, notificationID) {
// config := backyConfigFile.Notifications[notificationID]
// config.Enabled = false
// backyConfigFile.Notifications[notificationID] = config
// }
// }
}
return backyConfigFile
@ -363,6 +387,101 @@ func getNestedConfig(nestedConfig, key string) string {
return fmt . Sprintf ( "%s.%s" , nestedConfig , key )
}
func getNestedSSHConfig ( key string ) string {
return fmt . Sprintf ( "hosts.%s.config" , key )
func resolveDir ( path string ) ( string , error ) {
usr , err := user . Current ( )
if err != nil {
return path , err
}
dir := usr . HomeDir
if path == "~" {
// In case of "~", which won't be caught by the "else if"
path = dir
} else if strings . HasPrefix ( path , "~/" ) {
// Use strings.HasPrefix so we don't match paths like
// "/something/~/something/"
path = filepath . Join ( dir , path [ 2 : ] )
}
return path , nil
}
func injectEnvIntoSSH ( envVarsToInject environmentVars , process * ssh . Session , log * zerolog . Logger ) {
if envVarsToInject . file != "" {
envPath , envPathErr := resolveDir ( envVarsToInject . file )
if envPathErr != nil {
log . Error ( ) . Err ( envPathErr ) . Send ( )
}
file , err := os . Open ( envPath )
if err != nil {
log . Err ( err ) . Send ( )
}
defer file . Close ( )
scanner := bufio . NewScanner ( file )
for scanner . Scan ( ) {
envVar := scanner . Text ( )
envVarArr := strings . Split ( envVar , "=" )
process . Setenv ( envVarArr [ 0 ] , envVarArr [ 1 ] )
}
if err := scanner . Err ( ) ; err != nil {
log . Err ( err ) . Send ( )
}
}
if len ( envVarsToInject . env ) > 0 {
for _ , envVal := range envVarsToInject . env {
if strings . Contains ( envVal , "=" ) {
envVarArr := strings . Split ( envVal , "=" )
process . Setenv ( strings . ToUpper ( envVarArr [ 0 ] ) , envVarArr [ 1 ] )
}
}
}
}
func injectEnvIntoLocalCMD ( envVarsToInject environmentVars , process * exec . Cmd , log * zerolog . Logger ) {
if envVarsToInject . file != "" {
envPath , envPathErr := resolveDir ( envVarsToInject . file )
if envPathErr != nil {
log . Error ( ) . Err ( envPathErr ) . Send ( )
}
file , err := os . Open ( envPath )
if err != nil {
log . Err ( err ) . Send ( )
}
defer file . Close ( )
scanner := bufio . NewScanner ( file )
for scanner . Scan ( ) {
envVar := scanner . Text ( )
process . Env = append ( process . Env , envVar )
}
if err := scanner . Err ( ) ; err != nil {
log . Err ( err ) . Send ( )
}
}
if len ( envVarsToInject . env ) > 0 {
for _ , envVal := range envVarsToInject . env {
if strings . Contains ( envVal , "=" ) {
process . Env = append ( process . Env , envVal )
}
}
}
}
func contains ( s [ ] string , e string ) bool {
for _ , a := range s {
if a == e {
return true
}
}
return false
}
func CheckForConfigValues ( config * viper . Viper ) {
for _ , key := range requiredKeys {
isKeySet := config . IsSet ( key )
if ! isKeySet {
logging . ExitWithMSG ( Sprintf ( "Config key %s is not defined in %s" , key , config . ConfigFileUsed ( ) ) , 1 , nil )
}
}
}
