You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
534 lines
14 KiB
534 lines
14 KiB
// backy.go
|
|
// Copyright (C) Andrew Woodlee 2023
|
|
// License: Apache-2.0
|
|
package backy
|
|
|
|
import (
|
|
"bufio"
|
|
"bytes"
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"os/exec"
|
|
"strings"
|
|
"text/template"
|
|
|
|
"embed"
|
|
|
|
"github.com/rs/zerolog"
|
|
"github.com/rs/zerolog/log"
|
|
)
|
|
|
|
//go:embed templates/*.txt
|
|
var templates embed.FS
|
|
|
|
var requiredKeys = []string{"commands"}
|
|
|
|
var Sprintf = fmt.Sprintf
|
|
|
|
// RunCmd runs a Command.
|
|
// The environment of local commands will be the machine's environment plus any extra
|
|
// variables specified in the Env file or Environment.
|
|
// Dir can also be specified for local commands.
|
|
func (command *Command) RunCmd(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([]string, error) {
|
|
|
|
var (
|
|
outputArr []string
|
|
ArgsStr string
|
|
cmdOutBuf bytes.Buffer
|
|
cmdOutWriters io.Writer
|
|
|
|
envVars = environmentVars{
|
|
file: command.Env,
|
|
env: command.Environment,
|
|
}
|
|
)
|
|
|
|
for _, v := range command.Args {
|
|
ArgsStr += fmt.Sprintf(" %s", v)
|
|
}
|
|
|
|
if command.Host != nil {
|
|
command.Type = strings.TrimSpace(command.Type)
|
|
|
|
if command.Type != "" {
|
|
cmdCtxLogger.Info().Str("Command", fmt.Sprintf("Running script %s on host %s", command.Cmd, *command.Host)).Send()
|
|
} else {
|
|
cmdCtxLogger.Info().Str("Command", fmt.Sprintf("Running command %s %s on host %s", command.Cmd, ArgsStr, *command.Host)).Send()
|
|
}
|
|
|
|
if command.RemoteHost.SshClient == nil {
|
|
err := command.RemoteHost.ConnectToSSHHost(opts)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
commandSession, err := command.RemoteHost.SshClient.NewSession()
|
|
|
|
// Retry connecting to host; if that fails, error. If it does not fail, try to create new session
|
|
if err != nil {
|
|
connErr := command.RemoteHost.ConnectToSSHHost(opts)
|
|
if connErr != nil {
|
|
return nil, fmt.Errorf("error creating session: %v, and error creating new connection to host: %v", err, connErr)
|
|
}
|
|
commandSession, err = command.RemoteHost.SshClient.NewSession()
|
|
|
|
if err != nil {
|
|
return nil, fmt.Errorf("error creating session: %v", err)
|
|
}
|
|
}
|
|
|
|
defer commandSession.Close()
|
|
|
|
injectEnvIntoSSH(envVars, commandSession, opts, cmdCtxLogger)
|
|
cmd := command.Cmd
|
|
for _, a := range command.Args {
|
|
cmd += " " + a
|
|
}
|
|
cmdOutWriters = io.MultiWriter(&cmdOutBuf)
|
|
|
|
if IsCmdStdOutEnabled() {
|
|
cmdOutWriters = io.MultiWriter(os.Stdout, &cmdOutBuf)
|
|
}
|
|
|
|
commandSession.Stdout = cmdOutWriters
|
|
commandSession.Stderr = cmdOutWriters
|
|
if command.Type != "" {
|
|
|
|
// did the program panic while writing to the buffer?
|
|
defer func() {
|
|
if err := recover(); err != nil {
|
|
cmdCtxLogger.Info().Msg(fmt.Sprintf("panic occured writing to buffer: %x", err))
|
|
}
|
|
}()
|
|
|
|
if command.Type == "script" {
|
|
script := bytes.NewBufferString(cmd + "\n")
|
|
|
|
commandSession.Stdin = script
|
|
if err := commandSession.Shell(); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if err := commandSession.Wait(); err != nil {
|
|
outScanner := bufio.NewScanner(&cmdOutBuf)
|
|
for outScanner.Scan() {
|
|
outMap := make(map[string]interface{})
|
|
outMap["cmd"] = cmd
|
|
outMap["output"] = outScanner.Text()
|
|
if str, ok := outMap["output"].(string); ok {
|
|
outputArr = append(outputArr, str)
|
|
}
|
|
cmdCtxLogger.Info().Fields(outMap).Send()
|
|
}
|
|
return outputArr, err
|
|
}
|
|
|
|
outScanner := bufio.NewScanner(&cmdOutBuf)
|
|
for outScanner.Scan() {
|
|
outMap := make(map[string]interface{})
|
|
outMap["cmd"] = cmd
|
|
outMap["output"] = outScanner.Text()
|
|
if str, ok := outMap["output"].(string); ok {
|
|
outputArr = append(outputArr, str)
|
|
}
|
|
log.Info().Fields(outMap).Send()
|
|
}
|
|
return outputArr, nil
|
|
}
|
|
if command.Type == "scriptFile" {
|
|
var (
|
|
buffer bytes.Buffer
|
|
scriptEnvFileBuffer bytes.Buffer
|
|
scriptFileBuffer bytes.Buffer
|
|
dirErr error
|
|
scriptEnvFilePresent bool
|
|
)
|
|
|
|
if command.ScriptEnvFile != "" {
|
|
command.ScriptEnvFile, dirErr = resolveDir(command.ScriptEnvFile)
|
|
if dirErr != nil {
|
|
return nil, dirErr
|
|
}
|
|
file, err := os.Open(command.ScriptEnvFile)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer file.Close()
|
|
_, err = io.Copy(&scriptEnvFileBuffer, file)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
scriptEnvFilePresent = true
|
|
}
|
|
|
|
command.Cmd, dirErr = resolveDir(command.Cmd)
|
|
if dirErr != nil {
|
|
return nil, dirErr
|
|
}
|
|
file, err := os.Open(command.Cmd)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
_, err = io.Copy(&scriptFileBuffer, file)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
defer file.Close()
|
|
|
|
if scriptEnvFilePresent {
|
|
_, err := buffer.WriteString(scriptEnvFileBuffer.String())
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
// write newline
|
|
buffer.WriteByte(0x0A)
|
|
|
|
_, err = buffer.WriteString(scriptFileBuffer.String())
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
} else {
|
|
_, err = io.Copy(&buffer, file)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
|
|
script := &buffer
|
|
|
|
commandSession.Stdin = script
|
|
if err := commandSession.Shell(); err != nil {
|
|
return nil, err
|
|
}
|
|
if err := commandSession.Wait(); err != nil {
|
|
outScanner := bufio.NewScanner(&cmdOutBuf)
|
|
for outScanner.Scan() {
|
|
outMap := make(map[string]interface{})
|
|
outMap["cmd"] = cmd
|
|
outMap["output"] = outScanner.Text()
|
|
if str, ok := outMap["output"].(string); ok {
|
|
outputArr = append(outputArr, str)
|
|
}
|
|
cmdCtxLogger.Info().Fields(outMap).Send()
|
|
}
|
|
return outputArr, err
|
|
}
|
|
|
|
outScanner := bufio.NewScanner(&cmdOutBuf)
|
|
for outScanner.Scan() {
|
|
outMap := make(map[string]interface{})
|
|
outMap["cmd"] = cmd
|
|
outMap["output"] = outScanner.Text()
|
|
if str, ok := outMap["output"].(string); ok {
|
|
outputArr = append(outputArr, str)
|
|
}
|
|
cmdCtxLogger.Info().Fields(outMap).Send()
|
|
}
|
|
return outputArr, nil
|
|
}
|
|
return nil, fmt.Errorf("command type not recognized")
|
|
}
|
|
|
|
err = commandSession.Run(cmd)
|
|
outScanner := bufio.NewScanner(&cmdOutBuf)
|
|
for outScanner.Scan() {
|
|
outMap := make(map[string]interface{})
|
|
outMap["cmd"] = cmd
|
|
outMap["output"] = outScanner.Text()
|
|
if str, ok := outMap["output"].(string); ok {
|
|
outputArr = append(outputArr, str)
|
|
}
|
|
cmdCtxLogger.Info().Fields(outMap).Send()
|
|
}
|
|
|
|
if err != nil {
|
|
cmdCtxLogger.Error().Err(fmt.Errorf("error when running cmd: %s: %w", command.Cmd, err)).Send()
|
|
return outputArr, err
|
|
}
|
|
} else {
|
|
|
|
var err error
|
|
if command.Shell != "" {
|
|
cmdCtxLogger.Info().Str("Command", fmt.Sprintf("Running command %s %s on local machine in %s", command.Cmd, ArgsStr, command.Shell)).Send()
|
|
|
|
ArgsStr = fmt.Sprintf("%s %s", command.Cmd, ArgsStr)
|
|
|
|
localCMD := exec.Command(command.Shell, "-c", ArgsStr)
|
|
|
|
if command.Dir != nil {
|
|
localCMD.Dir = *command.Dir
|
|
}
|
|
injectEnvIntoLocalCMD(envVars, localCMD, cmdCtxLogger)
|
|
|
|
cmdOutWriters = io.MultiWriter(&cmdOutBuf)
|
|
|
|
if IsCmdStdOutEnabled() {
|
|
cmdOutWriters = io.MultiWriter(os.Stdout, &cmdOutBuf)
|
|
}
|
|
|
|
localCMD.Stdout = cmdOutWriters
|
|
localCMD.Stderr = cmdOutWriters
|
|
|
|
err = localCMD.Run()
|
|
|
|
outScanner := bufio.NewScanner(&cmdOutBuf)
|
|
|
|
for outScanner.Scan() {
|
|
outMap := make(map[string]interface{})
|
|
outMap["cmd"] = command.Cmd
|
|
outMap["output"] = outScanner.Text()
|
|
if str, ok := outMap["output"].(string); ok {
|
|
outputArr = append(outputArr, str)
|
|
}
|
|
log.Info().Fields(outMap).Send()
|
|
}
|
|
|
|
if err != nil {
|
|
cmdCtxLogger.Error().Err(fmt.Errorf("error when running cmd %s: %w", command.Cmd, err)).Send()
|
|
return outputArr, err
|
|
}
|
|
return outputArr, nil
|
|
}
|
|
cmdCtxLogger.Info().Str("Command", fmt.Sprintf("Running command %s %s on local machine", command.Cmd, ArgsStr)).Send()
|
|
|
|
localCMD := exec.Command(command.Cmd, command.Args...)
|
|
if command.Dir != nil {
|
|
localCMD.Dir = *command.Dir
|
|
}
|
|
// fmt.Printf("%v\n", envVars.env)
|
|
|
|
injectEnvIntoLocalCMD(envVars, localCMD, cmdCtxLogger)
|
|
cmdOutWriters = io.MultiWriter(&cmdOutBuf)
|
|
// fmt.Printf("%v\n", localCMD.Environ())
|
|
|
|
if IsCmdStdOutEnabled() {
|
|
cmdOutWriters = io.MultiWriter(os.Stdout, &cmdOutBuf)
|
|
}
|
|
localCMD.Stdout = cmdOutWriters
|
|
localCMD.Stderr = cmdOutWriters
|
|
err = localCMD.Run()
|
|
outScanner := bufio.NewScanner(&cmdOutBuf)
|
|
for outScanner.Scan() {
|
|
outMap := make(map[string]interface{})
|
|
outMap["cmd"] = command.Cmd
|
|
outMap["output"] = outScanner.Text()
|
|
|
|
if str, ok := outMap["output"].(string); ok {
|
|
outputArr = append(outputArr, str)
|
|
}
|
|
cmdCtxLogger.Info().Fields(outMap).Send()
|
|
}
|
|
if err != nil {
|
|
cmdCtxLogger.Error().Err(fmt.Errorf("error when running cmd %s: %w", command.Cmd, err)).Send()
|
|
return outputArr, err
|
|
}
|
|
}
|
|
return outputArr, nil
|
|
}
|
|
|
|
func cmdListWorker(msgTemps *msgTemplates, jobs <-chan *CmdList, results chan<- string, opts *ConfigOpts) {
|
|
|
|
for list := range jobs {
|
|
fieldsMap := make(map[string]interface{})
|
|
fieldsMap["list"] = list.Name
|
|
|
|
cmdLog := opts.Logger.Info()
|
|
|
|
var count int
|
|
var cmdsRan []string
|
|
var outStructArr []outStruct
|
|
|
|
for _, cmd := range list.Order {
|
|
currentCmd := opts.Cmds[cmd].Cmd
|
|
|
|
fieldsMap["cmd"] = opts.Cmds[cmd].Cmd
|
|
cmdToRun := opts.Cmds[cmd]
|
|
cmdLog.Fields(fieldsMap).Send()
|
|
|
|
cmdLogger := opts.Logger.With().
|
|
Str("backy-cmd", cmd).Str("Host", "local machine").
|
|
Logger()
|
|
|
|
if cmdToRun.Host != nil {
|
|
cmdLogger = opts.Logger.With().
|
|
Str("backy-cmd", cmd).Str("Host", *cmdToRun.Host).
|
|
Logger()
|
|
}
|
|
|
|
outputArr, runOutErr := cmdToRun.RunCmd(cmdLogger, opts)
|
|
if list.NotifyConfig != nil {
|
|
|
|
if cmdToRun.GetOutput || list.GetOutput {
|
|
outputStruct := outStruct{
|
|
CmdName: cmd,
|
|
CmdExecuted: currentCmd,
|
|
Output: outputArr,
|
|
}
|
|
|
|
outStructArr = append(outStructArr, outputStruct)
|
|
|
|
}
|
|
}
|
|
count++
|
|
if runOutErr != nil {
|
|
var errMsg bytes.Buffer
|
|
if list.NotifyConfig != nil {
|
|
errStruct := make(map[string]interface{})
|
|
|
|
errStruct["listName"] = list.Name
|
|
errStruct["Command"] = currentCmd
|
|
errStruct["Cmd"] = cmd
|
|
errStruct["Args"] = opts.Cmds[cmd].Args
|
|
errStruct["Err"] = runOutErr
|
|
errStruct["CmdsRan"] = cmdsRan
|
|
errStruct["Output"] = outputArr
|
|
|
|
errStruct["CmdOutput"] = outStructArr
|
|
|
|
tmpErr := msgTemps.err.Execute(&errMsg, errStruct)
|
|
|
|
if tmpErr != nil {
|
|
opts.Logger.Err(tmpErr).Send()
|
|
}
|
|
|
|
notifySendErr := list.NotifyConfig.Send(context.Background(), fmt.Sprintf("List %s failed", list.Name), errMsg.String())
|
|
|
|
if notifySendErr != nil {
|
|
opts.Logger.Err(notifySendErr).Send()
|
|
}
|
|
}
|
|
|
|
opts.Logger.Err(runOutErr).Send()
|
|
|
|
break
|
|
} else {
|
|
|
|
if count == len(list.Order) {
|
|
cmdsRan = append(cmdsRan, cmd)
|
|
var successMsg bytes.Buffer
|
|
|
|
if list.NotifyConfig != nil {
|
|
successStruct := make(map[string]interface{})
|
|
|
|
successStruct["listName"] = list.Name
|
|
successStruct["CmdsRan"] = cmdsRan
|
|
|
|
successStruct["CmdOutput"] = outStructArr
|
|
|
|
tmpErr := msgTemps.success.Execute(&successMsg, successStruct)
|
|
|
|
if tmpErr != nil {
|
|
opts.Logger.Err(tmpErr).Send()
|
|
break
|
|
}
|
|
|
|
err := list.NotifyConfig.Send(context.Background(), fmt.Sprintf("List %s succeded", list.Name), successMsg.String())
|
|
|
|
if err != nil {
|
|
opts.Logger.Err(err).Send()
|
|
}
|
|
}
|
|
} else {
|
|
cmdsRan = append(cmdsRan, cmd)
|
|
}
|
|
}
|
|
}
|
|
|
|
results <- "done"
|
|
}
|
|
|
|
}
|
|
|
|
// RunListConfig runs a command list from the ConfigFile.
|
|
func (opts *ConfigOpts) RunListConfig(cron string) {
|
|
mTemps := &msgTemplates{
|
|
err: template.Must(template.New("error.txt").ParseFS(templates, "templates/error.txt")),
|
|
success: template.Must(template.New("success.txt").ParseFS(templates, "templates/success.txt")),
|
|
}
|
|
configListsLen := len(opts.CmdConfigLists)
|
|
listChan := make(chan *CmdList, configListsLen)
|
|
results := make(chan string)
|
|
|
|
// This starts up list workers, initially blocked
|
|
// because there are no jobs yet.
|
|
for w := 1; w <= configListsLen; w++ {
|
|
go cmdListWorker(mTemps, listChan, results, opts)
|
|
}
|
|
|
|
for listName, cmdConfig := range opts.CmdConfigLists {
|
|
if cmdConfig.Name == "" {
|
|
cmdConfig.Name = listName
|
|
}
|
|
if cron != "" {
|
|
if cron == cmdConfig.Cron {
|
|
listChan <- cmdConfig
|
|
}
|
|
} else {
|
|
listChan <- cmdConfig
|
|
}
|
|
}
|
|
close(listChan)
|
|
|
|
for a := 1; a <= configListsLen; a++ {
|
|
<-results
|
|
}
|
|
|
|
opts.closeHostConnections()
|
|
}
|
|
|
|
func (config *ConfigOpts) ExecuteCmds(opts *ConfigOpts) {
|
|
for _, cmd := range opts.executeCmds {
|
|
cmdToRun := opts.Cmds[cmd]
|
|
cmdLogger := opts.Logger.With().
|
|
Str("backy-cmd", cmd).
|
|
Logger()
|
|
_, runErr := cmdToRun.RunCmd(cmdLogger, opts)
|
|
if runErr != nil {
|
|
opts.Logger.Err(runErr).Send()
|
|
}
|
|
}
|
|
|
|
opts.closeHostConnections()
|
|
|
|
}
|
|
|
|
func (c *ConfigOpts) closeHostConnections() {
|
|
for _, host := range c.Hosts {
|
|
c.Logger.Info().Str("server", host.HostName)
|
|
if host.isProxyHost {
|
|
continue
|
|
}
|
|
if host.SshClient != nil {
|
|
if _, err := host.SshClient.NewSession(); err == nil {
|
|
c.Logger.Info().Msgf("Closing host connection %s", host.HostName)
|
|
host.SshClient.Close()
|
|
host.SshClient = nil
|
|
}
|
|
}
|
|
for _, proxyHost := range host.ProxyHost {
|
|
if proxyHost.isProxyHost {
|
|
continue
|
|
}
|
|
if proxyHost.SshClient != nil {
|
|
if _, err := host.SshClient.NewSession(); err == nil {
|
|
c.Logger.Info().Msgf("Closing connection to proxy host %s", host.HostName)
|
|
host.SshClient.Close()
|
|
host.SshClient = nil
|
|
}
|
|
}
|
|
}
|
|
}
|
|
for _, host := range c.Hosts {
|
|
if host.SshClient != nil {
|
|
if _, err := host.SshClient.NewSession(); err == nil {
|
|
c.Logger.Info().Msgf("Closing proxy host connection %s", host.HostName)
|
|
host.SshClient.Close()
|
|
host.SshClient = nil
|
|
}
|
|
}
|
|
}
|
|
}
|