2 Commits

Author SHA1 Message Date
b2d89352a3 v0.11.4
All checks were successful
ci/woodpecker/push/publish-docs Pipeline was successful
ci/woodpecker/tag/gitea Pipeline was successful
ci/woodpecker/tag/publish-docs Pipeline was successful
ci/woodpecker/release/publish-docs Pipeline was successful
2026-02-01 07:21:20 -06:00
765ef2ee36 v0.11.3
All checks were successful
ci/woodpecker/push/publish-docs Pipeline was successful
ci/woodpecker/tag/gitea Pipeline was successful
ci/woodpecker/tag/publish-docs Pipeline was successful
ci/woodpecker/release/publish-docs Pipeline was successful
2026-01-31 01:06:18 -06:00
29 changed files with 887 additions and 69 deletions

7
.changes/v0.11.3.md Normal file
View File

@@ -0,0 +1,7 @@
## v0.11.3 - 2026-01-31
### Added
* Command: saveShellHistory for scriptFile commands over SSH
* Starting on Variables and Templates
### Changed
* File output for commands now adds hostname to beginning of filename
* Testing: docker testing infra

3
.changes/v0.11.4.md Normal file
View File

@@ -0,0 +1,3 @@
## v0.11.4 - 2026-02-01
### Changed
* Command.[name].output.file: now appends correctly to the beginning of file in an absolute path

View File

@@ -34,6 +34,8 @@ snapshot:
version_template: "{{ incpatch .Version }}-next"
changelog:
disable: false
release:
prerelease: auto
gitea_urls:
api: https://git.andrewnw.xyz/api/v1

View File

@@ -31,6 +31,8 @@ archives:
formats: [zip]
checksum:
name_template: 'checksums.txt'
release:
prerelease: auto
snapshot:
version_template: "{{ incpatch .Version }}-next"
changelog:

View File

@@ -6,6 +6,18 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html),
and is generated by [Changie](https://github.com/miniscruff/changie).
## v0.11.4 - 2026-02-01
### Changed
* Command.[name].output.file: now appends correctly to the beginning of file in an absolute path
## v0.11.3 - 2026-01-31
### Added
* Command: saveShellHistory for scriptFile commands over SSH
* Starting on Variables and Templates
### Changed
* File output for commands now adds hostname to beginning of filename
* Testing: docker testing infra
## v0.11.2 - 2025-12-27
### Added
* Upgraded GoCron; web ui viewer for viewing cron jobs

View File

@@ -7,7 +7,7 @@ import (
"github.com/spf13/cobra"
)
const versionStr = "0.11.2"
const versionStr = "0.11.4"
var (
versionCmd = &cobra.Command{

View File

@@ -14,10 +14,12 @@ import (
"strings"
"sync"
"text/template"
"time"
"embed"
"github.com/rs/zerolog"
"gopkg.in/natefinch/lumberjack.v2"
)
//go:embed templates/*.txt
@@ -65,10 +67,10 @@ func (e *PackageCommandExecutor) Run(cmd *Command, opts *ConfigOpts, logger zero
// Execute the package version command
execCmd := exec.Command(cmd.Cmd, cmd.Args...)
cmdOutWriters = io.MultiWriter(&cmdOutBuf)
if IsCmdStdOutEnabled() {
cmdOutWriters = io.MultiWriter(os.Stdout, &cmdOutBuf)
var err error
cmdOutWriters, _, err = makeCmdOutWriters(&cmdOutBuf, "")
if err != nil {
return nil, err
}
execCmd.Stdout = cmdOutWriters
execCmd.Stderr = cmdOutWriters
@@ -145,6 +147,48 @@ func (e *LocalCommandExecutor) Run(cmd *Command, opts *ConfigOpts, logger zerolo
return outputArr, nil
}
// makeCmdOutWriters constructs an io.Writer that writes to the provided buffer,
// optionally also to stdout and/or a file. If a file path is provided the
// caller is responsible for closing the returned *os.File when non-nil.
func makeCmdOutWriters(buf *bytes.Buffer, outputFile string) (io.Writer, *os.File, error) {
writers := io.MultiWriter(buf)
if IsCmdStdOutEnabled() {
console := zerolog.ConsoleWriter{Out: os.Stdout, TimeFormat: time.RFC1123}
console.FormatLevel = func(i interface{}) string {
return strings.ToUpper(fmt.Sprintf("| %-6s|", i))
}
console.FormatMessage = func(i any) string {
if i == nil {
return ""
}
return fmt.Sprintf("MSG: %s", i)
}
console.FormatFieldName = func(i interface{}) string {
return fmt.Sprintf("%s: ", i)
}
console.FormatFieldValue = func(i interface{}) string {
return fmt.Sprintf("%s", i)
// return strings.ToUpper(fmt.Sprintf("%s", i))
}
writers = io.MultiWriter(console, writers)
}
if outputFile != "" {
fileLogger := &lumberjack.Logger{
MaxSize: 50, // megabytes
MaxBackups: 3,
MaxAge: 28, //days
Compress: false, // disabled by default
}
fileLogger.Filename = outputFile
writers = io.MultiWriter(fileLogger, writers)
}
return writers, nil, nil
}
// ensureRemoteHost ensures localCmd.RemoteHost is set for the given host.
// It prefers opts.Hosts lookup and falls back to a minimal Host entry so remote execution can proceed.
func (opts *ConfigOpts) ensureRemoteHost(localCmd *Command, host string) {
@@ -250,6 +294,7 @@ func (command *Command) RunCmd(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([
}
if command.Type == UserCommandType {
if command.UserOperation == "password" {
cmdCtxLogger.Info().Str("password", command.UserPassword).Msg("user password to be updated")
}
@@ -283,23 +328,13 @@ func (command *Command) RunCmd(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([
localCMD = exec.Command(command.Shell, command.Args...)
injectEnvIntoLocalCMD(envVars, localCMD, cmdCtxLogger, opts)
cmdOutWriters = io.MultiWriter(&cmdOutBuf)
if IsCmdStdOutEnabled() {
cmdOutWriters = io.MultiWriter(os.Stdout, &cmdOutBuf)
var outFile *os.File
cmdOutWriters, outFile, err := makeCmdOutWriters(&cmdOutBuf, command.Output.File)
if err != nil {
return nil, err
}
if command.Output.File != "" {
file, err := os.Create(command.Output.File)
if err != nil {
return nil, fmt.Errorf("error creating output file: %w", err)
}
defer file.Close()
cmdOutWriters = io.MultiWriter(file, &cmdOutBuf)
if IsCmdStdOutEnabled() {
cmdOutWriters = io.MultiWriter(os.Stdout, file, &cmdOutBuf)
}
if outFile != nil {
defer outFile.Close()
}
localCMD.Stdin = bytes.NewReader(script)
@@ -370,10 +405,9 @@ func (command *Command) RunCmd(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([
injectEnvIntoLocalCMD(envVars, localCMD, cmdCtxLogger, opts)
cmdOutWriters = io.MultiWriter(&cmdOutBuf)
if IsCmdStdOutEnabled() {
cmdOutWriters = io.MultiWriter(os.Stdout, &cmdOutBuf)
cmdOutWriters, _, err = makeCmdOutWriters(&cmdOutBuf, "")
if err != nil {
return outputArr, err
}
localCMD.Stdout = cmdOutWriters
@@ -958,6 +992,19 @@ func (cmd *Command) GenerateLogger(opts *ConfigOpts) zerolog.Logger {
return cmdLogger
}
func (cmd *Command) GenerateLoggerForCmd(logger zerolog.Logger) zerolog.Logger {
cmdLogger := logger.With().
Str("Backy-cmd", cmd.Name).Str("Host", "local machine").
Logger()
if !IsHostLocal(cmd.Host) {
cmdLogger = logger.With().
Str("Backy-cmd", cmd.Name).Str("Host", cmd.Host).
Logger()
}
return cmdLogger
}
func (opts *ConfigOpts) ExecCmdsOnHosts(cmdList []string, hostsList []string) {
// Iterate over hosts and exec commands
for _, h := range hostsList {

View File

@@ -8,11 +8,11 @@ import (
"strings"
)
const _CommandTypeName = "scriptscriptFileremoteScriptpackageuser"
const _CommandTypeName = "scriptscriptFileremoteScriptpackageuserfile"
var _CommandTypeIndex = [...]uint8{0, 0, 6, 16, 28, 35, 39}
var _CommandTypeIndex = [...]uint8{0, 0, 6, 16, 28, 35, 39, 43}
const _CommandTypeLowerName = "scriptscriptfileremotescriptpackageuser"
const _CommandTypeLowerName = "scriptscriptfileremotescriptpackageuserfile"
func (i CommandType) String() string {
if i < 0 || i >= CommandType(len(_CommandTypeIndex)-1) {
@@ -31,9 +31,10 @@ func _CommandTypeNoOp() {
_ = x[RemoteScriptCommandType-(3)]
_ = x[PackageCommandType-(4)]
_ = x[UserCommandType-(5)]
_ = x[FileCommandType-(6)]
}
var _CommandTypeValues = []CommandType{DefaultCommandType, ScriptCommandType, ScriptFileCommandType, RemoteScriptCommandType, PackageCommandType, UserCommandType}
var _CommandTypeValues = []CommandType{DefaultCommandType, ScriptCommandType, ScriptFileCommandType, RemoteScriptCommandType, PackageCommandType, UserCommandType, FileCommandType}
var _CommandTypeNameToValueMap = map[string]CommandType{
_CommandTypeName[0:0]: DefaultCommandType,
@@ -48,6 +49,8 @@ var _CommandTypeNameToValueMap = map[string]CommandType{
_CommandTypeLowerName[28:35]: PackageCommandType,
_CommandTypeName[35:39]: UserCommandType,
_CommandTypeLowerName[35:39]: UserCommandType,
_CommandTypeName[39:43]: FileCommandType,
_CommandTypeLowerName[39:43]: FileCommandType,
}
var _CommandTypeNames = []string{
@@ -57,6 +60,7 @@ var _CommandTypeNames = []string{
_CommandTypeName[16:28],
_CommandTypeName[28:35],
_CommandTypeName[35:39],
_CommandTypeName[39:43],
}
// CommandTypeString retrieves an enum value from the enum constants string name.

View File

@@ -549,7 +549,6 @@ func processCmds(opts *ConfigOpts) error {
}
if !IsHostLocal(cmd.Host) {
cmdHost := replaceVarInString(opts.Vars, cmd.Host, opts.Logger)
if cmdHost != cmd.Host {
cmd.Host = cmdHost

193
pkg/backy/file.go Normal file
View File

@@ -0,0 +1,193 @@
package backy
import (
"fmt"
"io/fs"
"os"
"github.com/pkg/sftp"
"golang.org/x/crypto/ssh"
)
type LocalFileCommandExecutor struct{}
func (f *LocalFileCommandExecutor) Execute(cmd *Command) error {
localExecutor := LocalCommandExecutor{}
switch cmd.FileOperation {
case "copy":
return localExecutor.copyFile(cmd.Source, cmd.Destination, cmd.Permissions)
default:
return fmt.Errorf("unsupported file operation: %s", cmd.FileOperation)
}
}
func (f *LocalFileCommandExecutor) ReadLocalFile(path string) ([]byte, error) {
data, err := os.ReadFile(path)
if err != nil {
return nil, err
}
return data, nil
}
func (f *LocalFileCommandExecutor) WriteLocalFile(path string, data []byte, Perms fs.FileMode) error {
err := os.WriteFile(path, data, Perms)
if err != nil {
return err
}
return nil
}
func (f *LocalCommandExecutor) copyFile(source, destination string, Perms fs.FileMode) error {
input, err := os.ReadFile(source)
if err != nil {
return err
}
err = os.WriteFile(destination, input, Perms)
if err != nil {
return err
}
return nil
}
type RemoteFileCommandExecutor struct{}
func (r *RemoteFileCommandExecutor) Execute(cmd *Command) error {
remoteExecutor := RemoteFileCommandExecutor{}
sourceTypeLocal := false
if cmd.SourceType == "local" {
sourceTypeLocal = true
}
switch cmd.FileOperation {
case "copy":
return remoteExecutor.copyFile(cmd.Source, cmd.Destination, cmd.Permissions, cmd.RemoteHost.SshClient, sourceTypeLocal)
default:
return fmt.Errorf("unsupported file operation: %s", cmd.FileOperation)
}
}
func (r *RemoteFileCommandExecutor) copyFile(source, destination string, Perms fs.FileMode, sshClient *ssh.Client, sourceTypeLocal bool) error {
if sourceTypeLocal {
input, err := os.ReadFile(source)
if err != nil {
return err
}
sftpClient, sftpErr := sftp.NewClient(sshClient)
if sftpErr != nil {
return sftpErr
}
defer sftpClient.Close()
destFile, err := sftpClient.Create(destination)
if err != nil {
return err
}
defer destFile.Close()
_, err = destFile.Write(input)
if err != nil {
return err
}
err = destFile.Chmod(Perms)
if err != nil {
return err
}
return nil
}
client, err := sftp.NewClient(sshClient)
if err != nil {
return err
}
srcFile, err := client.Open(source)
if err != nil {
return err
}
defer srcFile.Close()
destFile, err := client.Create(destination)
if err != nil {
return err
}
defer destFile.Close()
_, err = srcFile.WriteTo(destFile)
if err != nil {
return err
}
err = destFile.Chmod(Perms)
if err != nil {
return err
}
return nil
}
func (r *RemoteFileCommandExecutor) ReadRemoteFile(path string, sshClient *ssh.Client) ([]byte, error) {
sftpClient, sftpErr := sftp.NewClient(sshClient)
if sftpErr != nil {
return nil, sftpErr
}
defer sftpClient.Close()
file, err := sftpClient.Open(path)
if err != nil {
return nil, err
}
defer file.Close()
var fileData []byte
_, err = file.Read(fileData)
if err != nil {
return nil, err
}
return fileData, nil
}
func (r *RemoteFileCommandExecutor) WriteRemoteFile(path string, data []byte, Perms fs.FileMode, sshClient *ssh.Client) error {
sftpClient, sftpErr := sftp.NewClient(sshClient)
if sftpErr != nil {
return sftpErr
}
defer sftpClient.Close()
file, err := sftpClient.Create(path)
if err != nil {
return err
}
defer file.Close()
_, err = file.Write(data)
if err != nil {
return err
}
err = file.Chmod(Perms)
if err != nil {
return err
}
return nil
}
type FileCommandExecutor interface {
Execute(cmd *Command) error
}
func NewFileCommandExecutor(isRemote bool) FileCommandExecutor {
if isRemote {
return &RemoteFileCommandExecutor{}
}
return &LocalFileCommandExecutor{}
}

78
pkg/backy/file_test.go Normal file
View File

@@ -0,0 +1,78 @@
package backy
import (
"os"
"testing"
)
func TestCopyFileLocal(t *testing.T) {
FileCommand := Command{}
FileCommand.Type = FileCommandType
FileCommand.FileOperation = "copy"
FileCommand.Permissions = 0644
FileCommand.Source = "/home/andrew/Projects/backy/tests/data/fileops/source.txt"
FileCommand.Destination = "/home/andrew/Projects/backy/tests/data/fileops/destination.txt"
var FileCommandExecutor = LocalFileCommandExecutor{}
err := FileCommandExecutor.Execute(&FileCommand)
if err != nil {
t.Errorf("Error executing file command: %v", err)
}
srcBytes, srcErr := os.ReadFile(FileCommand.Source)
if srcErr != nil {
t.Errorf("Error reading source file: %v", srcErr)
}
destBytes, destErr := os.ReadFile(FileCommand.Destination)
if destErr != nil {
t.Errorf("Error reading destination file: %v", destErr)
}
if string(srcBytes) != string(destBytes) {
t.Errorf("Source and destination files do not match")
}
// Additional checks can be added here to verify the file was copied correctly
}
func TestCopyFileRemote(t *testing.T) {
opts := NewConfigOptions("")
opts.Hosts = map[string]*Host{}
RemoteFileCommand := Command{}
RemoteFileCommand.Type = FileCommandType
RemoteFileCommand.FileOperation = "copy"
RemoteFileCommand.Permissions = 0644
RemoteFileCommand.Destination = "/home/backy/destination.txt"
RemoteFileCommand.Source = "/home/andrew/Projects/backy/tests/data/fileops/source.txt"
RemoteFileCommand.Host = "localhost"
RemoteFileCommand.RemoteHost = &Host{
HostName: "localhost",
User: "backy",
Password: "backy",
Port: 2222,
PrivateKeyPath: "/home/andrew/Projects/backy/tests/docker/backytest",
KnownHostsFile: "/home/andrew/Projects/backy/tests/docker/known_hosts",
}
sshErr := RemoteFileCommand.RemoteHost.ConnectToHost(opts)
if sshErr != nil {
t.Errorf("Error connecting to remote host: %v", sshErr)
}
var RemoteFileCommandExecutor = RemoteFileCommandExecutor{}
err := RemoteFileCommandExecutor.Execute(&RemoteFileCommand)
if err != nil {
t.Errorf("Error executing remote file command: %v", err)
}
srcBytes, srcErr := os.ReadFile(RemoteFileCommand.Source)
if srcErr != nil {
t.Errorf("Error reading source file: %v", srcErr)
}
destBytes, destErr := RemoteFileCommandExecutor.ReadRemoteFile(RemoteFileCommand.Destination, RemoteFileCommand.RemoteHost.SshClient)
if destErr != nil {
t.Errorf("Error reading destination file: %v", destErr)
}
if string(srcBytes) != string(destBytes) {
t.Errorf("Source and destination files do not match")
}
// Additional checks can be added here to verify the file was copied correctly
}

View File

@@ -0,0 +1,141 @@
// Code generated by "enumer -linecomment -yaml -text -json -type=FileCommandOperation"; DO NOT EDIT.
package backy
import (
"encoding/json"
"fmt"
"strings"
)
const _FileCommandOperationName = "copymovedeletechownchmod"
var _FileCommandOperationIndex = [...]uint8{0, 0, 4, 8, 14, 19, 24}
const _FileCommandOperationLowerName = "copymovedeletechownchmod"
func (i FileCommandOperation) String() string {
if i < 0 || i >= FileCommandOperation(len(_FileCommandOperationIndex)-1) {
return fmt.Sprintf("FileCommandOperation(%d)", i)
}
return _FileCommandOperationName[_FileCommandOperationIndex[i]:_FileCommandOperationIndex[i+1]]
}
// An "invalid array index" compiler error signifies that the constant values have changed.
// Re-run the stringer command to generate them again.
func _FileCommandOperationNoOp() {
var x [1]struct{}
_ = x[DefaultFCO-(0)]
_ = x[FileCommandOperationCopy-(1)]
_ = x[FileCommandOperationMove-(2)]
_ = x[FileCommandOperationDelete-(3)]
_ = x[FileCommandOperationChown-(4)]
_ = x[FileCommandOperationChmod-(5)]
}
var _FileCommandOperationValues = []FileCommandOperation{DefaultFCO, FileCommandOperationCopy, FileCommandOperationMove, FileCommandOperationDelete, FileCommandOperationChown, FileCommandOperationChmod}
var _FileCommandOperationNameToValueMap = map[string]FileCommandOperation{
_FileCommandOperationName[0:0]: DefaultFCO,
_FileCommandOperationLowerName[0:0]: DefaultFCO,
_FileCommandOperationName[0:4]: FileCommandOperationCopy,
_FileCommandOperationLowerName[0:4]: FileCommandOperationCopy,
_FileCommandOperationName[4:8]: FileCommandOperationMove,
_FileCommandOperationLowerName[4:8]: FileCommandOperationMove,
_FileCommandOperationName[8:14]: FileCommandOperationDelete,
_FileCommandOperationLowerName[8:14]: FileCommandOperationDelete,
_FileCommandOperationName[14:19]: FileCommandOperationChown,
_FileCommandOperationLowerName[14:19]: FileCommandOperationChown,
_FileCommandOperationName[19:24]: FileCommandOperationChmod,
_FileCommandOperationLowerName[19:24]: FileCommandOperationChmod,
}
var _FileCommandOperationNames = []string{
_FileCommandOperationName[0:0],
_FileCommandOperationName[0:4],
_FileCommandOperationName[4:8],
_FileCommandOperationName[8:14],
_FileCommandOperationName[14:19],
_FileCommandOperationName[19:24],
}
// FileCommandOperationString retrieves an enum value from the enum constants string name.
// Throws an error if the param is not part of the enum.
func FileCommandOperationString(s string) (FileCommandOperation, error) {
if val, ok := _FileCommandOperationNameToValueMap[s]; ok {
return val, nil
}
if val, ok := _FileCommandOperationNameToValueMap[strings.ToLower(s)]; ok {
return val, nil
}
return 0, fmt.Errorf("%s does not belong to FileCommandOperation values", s)
}
// FileCommandOperationValues returns all values of the enum
func FileCommandOperationValues() []FileCommandOperation {
return _FileCommandOperationValues
}
// FileCommandOperationStrings returns a slice of all String values of the enum
func FileCommandOperationStrings() []string {
strs := make([]string, len(_FileCommandOperationNames))
copy(strs, _FileCommandOperationNames)
return strs
}
// IsAFileCommandOperation returns "true" if the value is listed in the enum definition. "false" otherwise
func (i FileCommandOperation) IsAFileCommandOperation() bool {
for _, v := range _FileCommandOperationValues {
if i == v {
return true
}
}
return false
}
// MarshalJSON implements the json.Marshaler interface for FileCommandOperation
func (i FileCommandOperation) MarshalJSON() ([]byte, error) {
return json.Marshal(i.String())
}
// UnmarshalJSON implements the json.Unmarshaler interface for FileCommandOperation
func (i *FileCommandOperation) UnmarshalJSON(data []byte) error {
var s string
if err := json.Unmarshal(data, &s); err != nil {
return fmt.Errorf("FileCommandOperation should be a string, got %s", data)
}
var err error
*i, err = FileCommandOperationString(s)
return err
}
// MarshalText implements the encoding.TextMarshaler interface for FileCommandOperation
func (i FileCommandOperation) MarshalText() ([]byte, error) {
return []byte(i.String()), nil
}
// UnmarshalText implements the encoding.TextUnmarshaler interface for FileCommandOperation
func (i *FileCommandOperation) UnmarshalText(text []byte) error {
var err error
*i, err = FileCommandOperationString(string(text))
return err
}
// MarshalYAML implements a YAML Marshaler for FileCommandOperation
func (i FileCommandOperation) MarshalYAML() (interface{}, error) {
return i.String(), nil
}
// UnmarshalYAML implements a YAML Unmarshaler for FileCommandOperation
func (i *FileCommandOperation) UnmarshalYAML(unmarshal func(interface{}) error) error {
var s string
if err := unmarshal(&s); err != nil {
return err
}
var err error
*i, err = FileCommandOperationString(s)
return err
}

View File

@@ -5,12 +5,12 @@
package backy
import (
"bufio"
"bytes"
"fmt"
"io"
"os"
"os/user"
"path/filepath"
"strconv"
"strings"
"time"
@@ -472,10 +472,31 @@ func (command *Command) RunCmdOnHost(cmdCtxLogger zerolog.Logger, opts *ConfigOp
defer commandSession.Close()
// Set output writers
cmdOutWriters = io.MultiWriter(&cmdOutBuf)
if IsCmdStdOutEnabled() {
cmdOutWriters = io.MultiWriter(os.Stdout, &cmdOutBuf)
var file *os.File
if !IsHostLocal(command.Host) && command.Output.File != "" {
if filepath.IsAbs(command.Output.File) {
fileName := filepath.Base(command.Output.File)
fileName = fmt.Sprintf("%s_%s", command.RemoteHost.Host, fileName)
command.Output.File = filepath.Join(filepath.Dir(command.Output.File), fileName)
} else {
command.Output.File = fmt.Sprintf("%s_%s", command.RemoteHost.Host, command.Output.File)
}
}
cmdOutWriters, file, err = makeCmdOutWriters(&cmdOutBuf, command.Output.File)
if err != nil {
return nil, fmt.Errorf("error creating command output writers: %w", err)
}
defer func() {
if file != nil {
file.Close()
}
}()
// cmdOutWriters = logging.SetLoggingWriterForCommand(&cmdOutBuf, command.Output.File, IsCmdStdOutEnabled())
cmdCtxLogger = zerolog.New(cmdOutWriters).With().Timestamp().Logger()
cmdCtxLogger = command.GenerateLoggerForCmd(cmdCtxLogger)
// cmdCtxLogger.Info().Msgf("Executing %s", command.Cmd)
commandSession.Stdout = cmdOutWriters
commandSession.Stderr = cmdOutWriters
@@ -496,7 +517,9 @@ func (command *Command) RunCmdOnHost(cmdCtxLogger zerolog.Logger, opts *ConfigOp
case RemoteScriptCommandType:
return command.runRemoteScript(commandSession, cmdCtxLogger, &cmdOutBuf)
case ScriptFileCommandType:
return command.runScriptFile(commandSession, cmdCtxLogger, &cmdOutBuf)
commandSession.Stdout = nil
commandSession.Stderr = nil
return command.runScriptFile(commandSession, cmdCtxLogger, opts.Logger, &cmdOutBuf)
case PackageCommandType:
var remoteHostPackageExecutor RemoteHostPackageExecutor
return remoteHostPackageExecutor.RunCmdOnHost(command, commandSession, cmdCtxLogger, cmdOutBuf)
@@ -662,22 +685,56 @@ func (command *Command) runScript(session *ssh.Session, cmdCtxLogger zerolog.Log
}
// runScriptFile handles the execution of script files.
func (command *Command) runScriptFile(session *ssh.Session, cmdCtxLogger zerolog.Logger, outputBuf *bytes.Buffer) ([]string, error) {
func (command *Command) runScriptFile(session *ssh.Session, cmdCtxLogger, globalLogger zerolog.Logger, outputBuf *bytes.Buffer) ([]string, error) {
script, err := command.prepareScriptFileBuffer()
if err != nil {
return nil, err
}
session.Stdin = script
// session.Stdin = script
modes := ssh.TerminalModes{
ssh.ECHO: 0,
ssh.ECHOCTL: 0,
ssh.TTY_OP_ISPEED: 14400,
ssh.TTY_OP_OSPEED: 14400,
}
session.RequestPty("xterm", 80, 40, modes)
stdin, _ := session.StdinPipe()
stdout, stdOutErr := session.StdoutPipe()
if stdOutErr != nil {
return nil, fmt.Errorf("error getting stdout pipe: %w", stdOutErr)
}
if err := session.Shell(); err != nil {
return nil, fmt.Errorf("error starting shell: %w", err)
}
if err := session.Wait(); err != nil {
return collectOutput(outputBuf, command.Name, cmdCtxLogger, true), fmt.Errorf("error waiting for shell: %w", err)
var LogOutputToFile bool
if command.Output.File != "" || command.Output.ToLog {
if command.Output.File != "" {
globalLogger.Info().Str("file", command.Output.File).Msg("Writing script output to file")
}
LogOutputToFile = true
}
return collectOutput(outputBuf, command.Name, cmdCtxLogger, command.Output.ToLog), nil
stdin.Write(script.Bytes())
stdOutput, stdoOutReadErr := io.ReadAll(stdout)
if err := session.Wait(); err != nil {
stdOutBuff := bytes.NewBuffer(stdOutput)
// outputBuf.Write(stdOutBuff.Bytes())
// Read output
return collectOutput(stdOutBuff, command.Name, cmdCtxLogger, LogOutputToFile), fmt.Errorf("error waiting for shell: %w", err)
}
// Read output
if stdoOutReadErr != nil {
return collectOutput(outputBuf, command.Name, cmdCtxLogger, LogOutputToFile), fmt.Errorf("error reading stdout after shell error: %w", stdoOutReadErr)
}
stdOutBuff := bytes.NewBuffer(stdOutput)
return collectOutput(stdOutBuff, command.Name, cmdCtxLogger, LogOutputToFile), nil
}
// prepareScriptBuffer prepares a buffer for inline scripts.
@@ -685,7 +742,7 @@ func (command *Command) prepareScriptBuffer() (*bytes.Buffer, error) {
var buffer bytes.Buffer
for _, envVar := range command.Environment {
buffer.WriteString(fmt.Sprintf("export %s", envVar))
fmt.Fprintf(&buffer, "export %s", envVar)
buffer.WriteByte('\n')
}
@@ -710,8 +767,12 @@ func (command *Command) prepareScriptBuffer() (*bytes.Buffer, error) {
func (command *Command) prepareScriptFileBuffer() (*bytes.Buffer, error) {
var buffer bytes.Buffer
if !command.SaveShellHistory {
buffer.WriteString("unset HISTFILE\nexport HISTSIZE=0\nexport SAVEHIST=0\n")
}
for _, envVar := range command.Environment {
buffer.WriteString(fmt.Sprintf("export %s", envVar))
fmt.Fprintf(&buffer, "export %s", envVar)
buffer.WriteByte('\n')
}
@@ -774,20 +835,6 @@ func readFileToBuffer(filePath string) (*bytes.Buffer, error) {
return &buffer, nil
}
// collectOutput collects output from a buffer and logs it.
func collectOutput(buf *bytes.Buffer, commandName string, logger zerolog.Logger, wantOutput bool) []string {
var outputArr []string
scanner := bufio.NewScanner(buf)
for scanner.Scan() {
line := scanner.Text()
outputArr = append(outputArr, line)
if wantOutput {
logger.Info().Str("cmd", commandName).Str("output", line).Send()
}
}
return outputArr
}
// createSSHSession attempts to create a new SSH session and retries on failure.
func (h *Host) createSSHSession(opts *ConfigOpts) (*ssh.Session, error) {
session, err := h.SshClient.NewSession()

71
pkg/backy/template.go Normal file
View File

@@ -0,0 +1,71 @@
package backy
import (
"bytes"
"os"
"path/filepath"
"text/template"
"gopkg.in/yaml.v3"
)
func LoadVarsYAML(path string) (map[string]interface{}, error) {
b, err := os.ReadFile(path)
if err != nil {
return nil, err
}
var m map[string]interface{}
if err := yaml.Unmarshal(b, &m); err != nil {
return nil, err
}
return m, nil
}
func RenderTemplateFile(templatePath string, vars map[string]interface{}) ([]byte, error) {
tmplText, err := os.ReadFile(templatePath)
if err != nil {
return nil, err
}
funcs := template.FuncMap{
"env": func(k, d string) string {
if v := os.Getenv(k); v != "" {
return v
}
return d
},
"default": func(def interface{}, v interface{}) interface{} {
if v == nil {
return def
}
if s, ok := v.(string); ok && s == "" {
return def
}
return v
},
"toYaml": func(v interface{}) string {
b, _ := yaml.Marshal(v)
return string(b)
},
}
t := template.New(filepath.Base(templatePath)).Funcs(funcs)
t, err = t.Parse(string(tmplText))
if err != nil {
return nil, err
}
var buf bytes.Buffer
if err := t.Execute(&buf, vars); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
func WriteRenderedFile(templatePath string, vars map[string]interface{}, dest string, perm os.FileMode) error {
out, err := RenderTemplateFile(templatePath, vars)
if err != nil {
return err
}
if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil {
return err
}
return os.WriteFile(dest, out, perm)
}

View File

@@ -2,6 +2,7 @@ package backy
import (
"bytes"
"io/fs"
"text/template"
"strings"
@@ -79,6 +80,8 @@ type (
ScriptEnvFile string `yaml:"scriptEnvFile"`
SaveShellHistory bool `yaml:"saveShellHistory,omitempty"`
Output struct {
File string `yaml:"file,omitempty"`
ToLog bool `yaml:"toLog,omitempty"`
@@ -138,7 +141,20 @@ type (
// stdin only for userOperation = password (for now)
stdin *strings.Reader
// END USER STRUCommandType FIELDS
// END USER CommandType FIELDS
// BEGIN FILE COMMAND FIELDS
FileOperation string `yaml:"fileOperation,omitempty"`
Source string `yaml:"source,omitempty"`
DestinationType string `yaml:"destinationType,omitempty"`
SourceType string `yaml:"sourceType,omitempty"`
Destination string `yaml:"destination,omitempty"`
Permissions fs.FileMode `yaml:"permissions,omitempty"`
Owner string `yaml:"owner,omitempty"`
Group string `yaml:"group,omitempty"`
// END FILE COMMAND FIELDS
}
RemoteSource struct {
@@ -312,6 +328,7 @@ type (
CommandType int
PackageOperation int
AllowedExternalDirectives int
FileCommandOperation int
)
//go:generate go run github.com/dmarkham/enumer -linecomment -yaml -text -json -type=CommandType
@@ -322,6 +339,7 @@ const (
RemoteScriptCommandType // remoteScript
PackageCommandType // package
UserCommandType // user
FileCommandType // file
)
//go:generate go run github.com/dmarkham/enumer -linecomment -yaml -text -json -type=PackageOperation
@@ -335,6 +353,16 @@ const (
PackageOperationIsInstalled // isInstalled
)
//go:generate go run github.com/dmarkham/enumer -linecomment -yaml -text -json -type=FileCommandOperation
const (
DefaultFCO FileCommandOperation = iota //
FileCommandOperationCopy // copy
FileCommandOperationMove // move
FileCommandOperationDelete // delete
FileCommandOperationChown // chown
FileCommandOperationChmod // chmod
)
//go:generate go run github.com/dmarkham/enumer -linecomment -yaml -text -json -type=AllowedExternalDirectives
const (
DefaultExternalDir AllowedExternalDirectives = iota

View File

@@ -5,6 +5,7 @@
package backy
import (
"bufio"
"bytes"
"context"
"errors"
@@ -173,7 +174,7 @@ errEnvFile:
}
func prependEnvVarsToCommand(envVars environmentVars, opts *ConfigOpts, command string, args []string, cmdCtxLogger zerolog.Logger) string {
var envPrefix string
var envPrefix strings.Builder
if envVars.file != "" {
envPath, envPathErr := getFullPathWithHomeDir(envVars.file)
if envPathErr != nil {
@@ -190,15 +191,15 @@ func prependEnvVarsToCommand(envVars environmentVars, opts *ConfigOpts, command
log.Fatal().Str("envFile", envPath).Err(err).Send()
}
for key, val := range envMap {
envPrefix += fmt.Sprintf("%s=%s ", key, getExternalConfigDirectiveValue(val, opts, AllowedExternalDirectiveVaultEnv))
fmt.Fprintf(&envPrefix, "%s=%s ", key, getExternalConfigDirectiveValue(val, opts, AllowedExternalDirectiveVaultEnv))
}
}
for _, value := range envVars.env {
envVarArr := strings.Split(value, "=")
envPrefix += fmt.Sprintf("%s=%s ", envVarArr[0], getExternalConfigDirectiveValue(envVarArr[1], opts, AllowedExternalDirectiveVault))
envPrefix += "\n"
fmt.Fprintf(&envPrefix, "%s=%s ", envVarArr[0], getExternalConfigDirectiveValue(envVarArr[1], opts, AllowedExternalDirectiveVault))
envPrefix.WriteString("\n")
}
return envPrefix + command + " " + strings.Join(args, " ")
return envPrefix.String() + command + " " + strings.Join(args, " ")
}
func contains(s []string, e string) bool {
@@ -220,6 +221,47 @@ func CheckConfigValues(config *koanf.Koanf, file string) {
}
}
// collectOutput collects output from a buffer and logs it.
func collectOutput(buf *bytes.Buffer, commandName string, logger zerolog.Logger, wantOutput bool) []string {
var outputArr []string
scanner := bufio.NewScanner(buf)
for scanner.Scan() {
line := scanner.Text()
clean := sanitizeString(line)
outputArr = append(outputArr, clean)
if wantOutput {
logger.Info().Str("cmd", commandName).Str("output", clean).Send()
}
}
return outputArr
}
// sanitizeString removes ANSI escape sequences and non-printable control characters
// while preserving tabs. This helps remove color codes and other terminal control
// characters from remote command output.
func sanitizeString(s string) string {
// Remove common ANSI CSI sequences like "\x1b[31m" etc.
var ansiCSI = regexp.MustCompile(`\x1b\[[0-9;?]*[ -/]*[@-~]`)
s = ansiCSI.ReplaceAllString(s, "")
// Remove OSC sequences started by ESC ] and terminated by BEL or ESC\
var osc = regexp.MustCompile(`(?s)"].*?(?:|\\)`)
s = osc.ReplaceAllString(s, "")
// Sometimes the ESC has been stripped earlier and we are left with sequences like "]2;title]1;"
// Remove leftover bracketed sequences like "]<digits>;<text>"
var leftoverOSC = regexp.MustCompile(`\][0-9]+[^\]]*`)
s = leftoverOSC.ReplaceAllString(s, "")
var b strings.Builder
for _, r := range s {
if r == '\t' || (r >= 0x20 && r != 0x7f) {
b.WriteRune(r)
}
}
return b.String()
}
func testFile(c string) error {
if strings.TrimSpace(c) != "" {
file, fileOpenErr := os.Open(c)

View File

@@ -1,6 +1,7 @@
package logging
import (
"bytes"
"fmt"
"os"
"strings"
@@ -66,6 +67,48 @@ func SetLoggingWriters(logFile string) (writers zerolog.LevelWriter) {
return
}
func SetLoggingWriterForCommand(buf *bytes.Buffer, logFile string, logToConsole bool) (writers zerolog.LevelWriter) {
console := zerolog.ConsoleWriter{}
if logToConsole {
console = zerolog.ConsoleWriter{Out: os.Stdout, TimeFormat: time.RFC1123}
console.FormatLevel = func(i interface{}) string {
return strings.ToUpper(fmt.Sprintf("| %-6s|", i))
}
console.FormatMessage = func(i any) string {
if i == nil {
return ""
}
return fmt.Sprintf("MSG: %s", i)
}
console.FormatFieldName = func(i interface{}) string {
return fmt.Sprintf("%s: ", i)
}
console.FormatFieldValue = func(i interface{}) string {
return fmt.Sprintf("%s", i)
// return strings.ToUpper(fmt.Sprintf("%s", i))
}
}
fileLogger := &lumberjack.Logger{
MaxSize: 50, // megabytes
MaxBackups: 3,
MaxAge: 28, //days
Compress: true, // disabled by default
}
fileLogger.Filename = logFile
// UNIX Time is faster and smaller than most timestamps
zerolog.TimeFieldFormat = zerolog.TimeFormatUnix
// zerolog.TimeFieldFormat = time.RFC1123
writers = zerolog.MultiLevelWriter(fileLogger)
if logToConsole {
writers = zerolog.MultiLevelWriter(console, fileLogger)
}
return
}
func IsConsoleLoggingEnabled() bool {
return os.Getenv("BACKY_CONSOLE_LOGGING") == "enabled"
}

12
tests/FileOps.yml Normal file
View File

@@ -0,0 +1,12 @@
commands:
copyFile:
type: file
fileOperation: copy
source: /home/andrew/Projects/backy/tests/data/fileops/source.txt
destination: /home/andrew/Projects/backy/tests/data/fileops/destination.txt
copyRemoteFile:
type: file
fileOperation: copy
sourceType: rempte
source: ssh://backy@localhost:2222/home/backy/remote_source.txt
destination: /home/andrew/Projects/backy/tests/data/fileops/remote_destination.txt

View File

@@ -10,7 +10,8 @@ commands:
successCmd:
name: get docker version
cmd: docker
getOutput: true
outputToLog: true
output:
file: docker_version_success.txt
toLog: true
Args:
- "-v"

View File

@@ -0,0 +1 @@
This is some test data.

View File

@@ -0,0 +1 @@
This is some test data.

39
tests/docker/README.md Normal file
View File

@@ -0,0 +1,39 @@
SSH test container
==================
This folder contains a simple Docker-based SSH server used for integration tests.
Quick start
-----------
Start the container (builds image if needed):
```bash
./start.sh
```
Stop the container:
```bash
./stop.sh
```
Access
------
- SSH endpoint: `localhost:2222`
- Test user: `backy` with password `backy` (password auth enabled)
- Root user: `root` with password `test`
- Public key `backytest.pub` is installed for both `backy` and `root`
Running tests
-------------
1. Start the container (`./start.sh`).
2. From the repo root, run your tests (example):
```bash
GO_TEST_SSH_ADDR=localhost:2222 go test ./... -v
```
If your tests rely on an SSH private key, use `tests/docker/backytest` as the private key and restrict access appropriately.

View File

@@ -1,4 +1,11 @@
cd ~/Projects/backy/tests/docker
docker container rm -f ssh_server_container
docker build -t ssh_server_image .
docker run -d -p 2222:22 --name ssh_server_container ssh_server_image
#!/usr/bin/env bash
set -euo pipefail
# Build and run the test SSH container from the tests/docker directory
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
docker container rm -f ssh_server_container 2>/dev/null || true
docker build -t ssh_server_image "$SCRIPT_DIR"
docker run -d -p 2222:22 --name ssh_server_container ssh_server_image
sleep 5
ssh-keyscan -p 2222 localhost > $SCRIPT_DIR/known_hosts

8
tests/docker/compose.yml Normal file
View File

@@ -0,0 +1,8 @@
services:
ssh_server:
build: .
image: backy_ssh_server:latest
container_name: backy_ssh_server
ports:
- "2222:22"
restart: "no"

8
tests/docker/known_hosts Normal file
View File

@@ -0,0 +1,8 @@
# localhost:2222 SSH-2.0-OpenSSH_9.6p1 Ubuntu-3ubuntu13.14
# localhost:2222 SSH-2.0-OpenSSH_9.6p1 Ubuntu-3ubuntu13.14
[localhost]:2222 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDATufWA1HRnNayIQLjSpA2+P9N6h0WF+jP+abMaINlZkiHFnFVDAoqD5/onVXymskrgQaKEYmBOs+Kv0t+Acvdor2IcvYgFueSm+jkslpSK/uuf1mx0gVJO77S2BIjqyWtUzVv96Iy4Gjt2RsrnalgYNYmi3OyPkG0IUA+3Im+2gztSECCy+nW3R/vaoPLwr4kImpLlrijcSHc4mHOY6BurrcWKNuGrsvTAOKgUZqlya6uDd+yD7fUfsmL1MqBKwZqfP3JAdp/Dd+laNNGcvEM4WhzYFSPfhqblewD0rjbto9MSOSXLyQz5RPmdITj/m5M4lj2ECmcI2gzraDMoj8ZkuJAss50oX6fmVUZestN5jlz7Y7XKEvXuH8qfLHKwaOUTZlcGbfAMz6uSrh8DNT6KzRG4j5nZ9Z5pTn1huz/p6jnJUGuHt2Ez3EK+isM+sHS6TntXavIkebaq7ErcBCO8A1fZFZlhlHoI9o9W62tMY7gbtlGodW8dKxK89+1a88=
# localhost:2222 SSH-2.0-OpenSSH_9.6p1 Ubuntu-3ubuntu13.14
[localhost]:2222 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBK9fEYfiGGgu0Eh7X2JT4jR4+utcfpm6Ee+Cer1x/XbMHzCPZg6YmYy6OaCSms/0VJ/QWxD+0HlsO7sqO5oeO60=
# localhost:2222 SSH-2.0-OpenSSH_9.6p1 Ubuntu-3ubuntu13.14
[localhost]:2222 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKT5+Cbi/ynOAPzwv0IaOVBtGFYtW33LIvNUuBKYqqyJ
# localhost:2222 SSH-2.0-OpenSSH_9.6p1 Ubuntu-3ubuntu13.14

7
tests/docker/start.sh Executable file
View File

@@ -0,0 +1,7 @@
#!/usr/bin/env bash
set -euo pipefail
DIR="$(cd "$(dirname "$0")" && pwd)"
echo "Starting SSH test container (building if needed)..."
docker compose -f "$DIR/compose.yml" up -d --build
echo "Container started on localhost:2222"

7
tests/docker/stop.sh Executable file
View File

@@ -0,0 +1,7 @@
#!/usr/bin/env bash
set -euo pipefail
DIR="$(cd "$(dirname "$0")" && pwd)"
echo "Stopping and removing SSH test container..."
docker compose -f "$DIR/compose.yml" down --remove-orphans
echo "Stopped."

5
tests/example.tmpl Normal file
View File

@@ -0,0 +1,5 @@
{{ .greeting }}, {{ .name }}!
port: {{ .port | default "8080" }}
envHOME: {{ env "HOME" "" }}
debugYaml:
{{ toYaml . }}

3
tests/vars.yaml Normal file
View File

@@ -0,0 +1,3 @@
name: Alice
greeting: Hello
port: 9090