From 753b03861f40260d9a0d271bfd4de74ae11460b0 Mon Sep 17 00:00:00 2001 From: Andrew Woodlee Date: Mon, 3 Mar 2025 23:45:28 -0600 Subject: [PATCH] v0.10.0 --- backy.code-workspace | 1 + cmd/version.go | 2 +- docs/content/config/commands/_index.md | 27 +---- go.mod | 2 + go.sum | 42 ++++++++ pkg/backy/backy.go | 66 ++++++------- pkg/backy/config.go | 130 ++++++++++++++----------- pkg/backy/notification.go | 3 +- pkg/backy/ssh.go | 54 +++++++++- pkg/backy/types.go | 29 +----- pkg/backy/utils.go | 47 +++++++-- pkg/remotefetcher/configfetcher.go | 2 +- pkg/remotefetcher/local.go | 2 +- tests/backy.yaml | 18 ---- 14 files changed, 251 insertions(+), 174 deletions(-) delete mode 100644 tests/backy.yaml diff --git a/backy.code-workspace b/backy.code-workspace index d99e318..c3fb412 100644 --- a/backy.code-workspace +++ b/backy.code-workspace @@ -18,6 +18,7 @@ "maunium", "mautrix", "nikoksr", + "rawbytes", "remotefetcher", "Strs" ] diff --git a/cmd/version.go b/cmd/version.go index b511a2e..143a1f5 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -7,7 +7,7 @@ import ( "github.com/spf13/cobra" ) -const versionStr = "0.9.1" +const versionStr = "0.9.2" var ( versionCmd = &cobra.Command{ diff --git a/docs/content/config/commands/_index.md b/docs/content/config/commands/_index.md index d462525..7658ee0 100644 --- a/docs/content/config/commands/_index.md +++ b/docs/content/config/commands/_index.md @@ -8,32 +8,7 @@ weight: 1 ### Example Config -```yaml -commands: - stop-docker-container: - cmd: docker - Args: - - compose - - -f /some/path/to/docker-compose.yaml - - down - # if host is not defined, command will be run locally - # The host has to be defined in either the config file or the SSH Config files - host: some-host - hooks - error: - - some-other-command-when-failing - success: - - success-command - final: - - final-command - backup-docker-container-script: - cmd: /path/to/local/script - # script file is input as stdin to SSH - type: scriptFile # also can be script - environment: - - FOO=BAR - - APP=$VAR -``` +{{% code file="/examples/example.yml" language="yaml" %}} Values available for this section **(case-sensitive)**: diff --git a/go.mod b/go.mod index 72bbb71..e098406 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,7 @@ require ( github.com/mitchellh/go-homedir v1.1.0 github.com/nikoksr/notify v1.3.0 github.com/pkg/errors v0.9.1 + github.com/pkg/sftp v1.13.7 github.com/rs/zerolog v1.33.0 github.com/sethvargo/go-password v0.3.1 github.com/spf13/cobra v1.8.1 @@ -65,6 +66,7 @@ require ( github.com/klauspost/compress v1.17.11 // indirect github.com/klauspost/cpuid/v2 v2.2.9 // indirect github.com/knadh/koanf/maps v0.1.1 // indirect + github.com/kr/fs v0.1.0 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/minio/md5-simd v1.1.2 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect diff --git a/go.sum b/go.sum index 6ce2098..d4c4adb 100644 --- a/go.sum +++ b/go.sum @@ -100,6 +100,8 @@ github.com/knadh/koanf/providers/rawbytes v0.1.0 h1:dpzgu2KO6uf6oCb4aP05KDmKmAmI github.com/knadh/koanf/providers/rawbytes v0.1.0/go.mod h1:mMTB1/IcJ/yE++A2iEZbY1MLygX7vttU+C+S/YmPu9c= github.com/knadh/koanf/v2 v2.1.2 h1:I2rtLRqXRy1p01m/utEtpZSSA6dcJbgGVuE27kW2PzQ= github.com/knadh/koanf/v2 v2.1.2/go.mod h1:Gphfaen0q1Fc1HTgJgSTC4oRX9R2R5ErYMZJy8fLJBo= +github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= @@ -135,6 +137,8 @@ github.com/pascaldekloe/name v1.0.0/go.mod h1:Z//MfYJnH4jVpQ9wkclwu2I2MkHmXTlT9w github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.13.7 h1:uv+I3nNJvlKZIQGSr8JVQLNHFU9YhhNpvC14Y6KgmSM= +github.com/pkg/sftp v1.13.7/go.mod h1:KMKI0t3T6hfA+lTR/ssZdunHo+uwq7ghoN09/FSu3DY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= @@ -179,34 +183,72 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.mau.fi/util v0.8.4 h1:mVKlJcXWfVo8ZW3f4vqtjGpqtZqJvX4ETekxawt2vnQ= go.mau.fi/util v0.8.4/go.mod h1:MOfGTs1CBuK6ERTcSL4lb5YU7/ujz09eOPVEDckuazY= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus= golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M= golang.org/x/exp v0.0.0-20250207012021-f9890c6ad9f3 h1:qNgPs5exUA+G0C96DrPwNrvLSj7GT/9D+3WMWUcUg34= golang.org/x/exp v0.0.0-20250207012021-f9890c6ad9f3/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.23.0 h1:Zb7khfcRGKk+kqfxFaP5tZqCnDZMjC5VtUBs87Hr6QM= golang.org/x/mod v0.23.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8= golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU= golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= golang.org/x/time v0.10.0 h1:3usCWA8tQn0L8+hFJQNgzpWbd89begxN66o1Ojdn5L4= golang.org/x/time v0.10.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.30.0 h1:BgcpHewrV5AUp2G9MebG4XPFI1E2W41zU1SaqVA9vJY= golang.org/x/tools v0.30.0/go.mod h1:c347cR/OJfw5TI+GfX7RUPNMdDRRbjvYTS0jPyvsVtY= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= diff --git a/pkg/backy/backy.go b/pkg/backy/backy.go index 3fbdf09..5c45431 100644 --- a/pkg/backy/backy.go +++ b/pkg/backy/backy.go @@ -144,6 +144,7 @@ func (command *Command) RunCmd(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([ } var err error + if command.Shell != "" { cmdCtxLogger.Info().Str("Command", fmt.Sprintf("Running command %s on local machine in %s", command.Name, command.Shell)).Send() @@ -182,20 +183,7 @@ func (command *Command) RunCmd(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([ err = localCMD.Run() - outScanner := bufio.NewScanner(&cmdOutBuf) - - for outScanner.Scan() { - outMap := make(map[string]interface{}) - outMap["cmd"] = command.Cmd - outMap["output"] = outScanner.Text() - - if str, ok := outMap["output"].(string); ok { - outputArr = append(outputArr, str) - } - // if command.GetOutput { - cmdCtxLogger.Info().Fields(outMap).Send() - // } - } + outputArr = logCommandOutput(command, cmdOutBuf, cmdCtxLogger, outputArr) if err != nil { cmdCtxLogger.Error().Err(fmt.Errorf("error when running cmd %s: %w", command.Name, err)).Send() return outputArr, err @@ -240,7 +228,7 @@ func cmdListWorker(msgTemps *msgTemplates, jobs <-chan *CmdList, results chan<- } // Collect output if required - if list.GetOutput || cmdToRun.GetOutput { + if list.GetOutput || cmdToRun.GetOutputInList { outStructArr = append(outStructArr, outStruct{ CmdName: currentCmd, CmdExecuted: currentCmd, @@ -249,17 +237,14 @@ func cmdListWorker(msgTemps *msgTemplates, jobs <-chan *CmdList, results chan<- } } - // Notify success if no errors occurred if !hasError && list.NotifyConfig != nil && (list.NotifyOnSuccess || list.GetOutput) { notifySuccess(cmdLogger, msgTemps, list, cmdsRan, outStructArr) } - // Execute success and final hooks for all commands for _, cmd := range list.Order { cmdToRun := opts.Cmds[cmd] - // Execute success hooks if the command succeeded - if !hasError || cmdsRanContains(cmd, cmdsRan) { + if !hasError { cmdToRun.ExecuteHooks("success", opts) } @@ -276,17 +261,6 @@ func cmdListWorker(msgTemps *msgTemplates, jobs <-chan *CmdList, results chan<- } } -// Helper to check if a command is in the list of executed commands -func cmdsRanContains(cmd string, cmdsRan []string) bool { - for _, c := range cmdsRan { - if c == cmd { - return true - } - } - return false -} - -// Helper to notify errors func notifyError(logger zerolog.Logger, templates *msgTemplates, list *CmdList, cmdsRan []string, outStructArr []outStruct, err error, cmd *Command) { errStruct := map[string]interface{}{ "listName": list.Name, @@ -355,7 +329,6 @@ func (opts *ConfigOpts) RunListConfig(cron string) { result := <-results opts.Logger.Debug().Msgf("Processing result for list %s, command %s", result.ListName, result.CmdName) - // Process final hooks for the list (already handled in worker) } opts.closeHostConnections() } @@ -367,10 +340,8 @@ func (opts *ConfigOpts) ExecuteCmds() { _, runErr := cmdToRun.RunCmd(cmdLogger, opts) if runErr != nil { opts.Logger.Err(runErr).Send() - cmdToRun.ExecuteHooks("error", opts) } else { - cmdToRun.ExecuteHooks("success", opts) } @@ -378,7 +349,6 @@ func (opts *ConfigOpts) ExecuteCmds() { } opts.closeHostConnections() - } func (c *ConfigOpts) closeHostConnections() { @@ -425,8 +395,9 @@ func (cmd *Command) ExecuteHooks(hookType string, opts *ConfigOpts) { case "error": for _, v := range cmd.Hooks.Error { errCmd := opts.Cmds[v] + opts.Logger.Info().Msgf("Running error hook command %s", v) cmdLogger := opts.Logger.With(). - Str("backy-cmd", v). + Str("backy-cmd", v).Str("hookType", "error"). Logger() errCmd.RunCmd(cmdLogger, opts) } @@ -434,16 +405,18 @@ func (cmd *Command) ExecuteHooks(hookType string, opts *ConfigOpts) { case "success": for _, v := range cmd.Hooks.Success { successCmd := opts.Cmds[v] + opts.Logger.Info().Msgf("Running success hook command %s", v) cmdLogger := opts.Logger.With(). - Str("backy-cmd", v). + Str("backy-cmd", v).Str("hookType", "success"). Logger() successCmd.RunCmd(cmdLogger, opts) } case "final": for _, v := range cmd.Hooks.Final { finalCmd := opts.Cmds[v] + opts.Logger.Info().Msgf("Running final hook command %s", v) cmdLogger := opts.Logger.With(). - Str("backy-cmd", v). + Str("backy-cmd", v).Str("hookType", "final"). Logger() finalCmd.RunCmd(cmdLogger, opts) } @@ -481,6 +454,25 @@ func (opts *ConfigOpts) ExecCmdsSSH(cmdList []string, hostsList []string) { } } +func logCommandOutput(command *Command, cmdOutBuf bytes.Buffer, cmdCtxLogger zerolog.Logger, outputArr []string) []string { + + outScanner := bufio.NewScanner(&cmdOutBuf) + + for outScanner.Scan() { + outMap := make(map[string]interface{}) + outMap["cmd"] = command.Name + outMap["output"] = outScanner.Text() + + if str, ok := outMap["output"].(string); ok { + outputArr = append(outputArr, str) + } + if command.OutputToLog { + cmdCtxLogger.Info().Fields(outMap).Send() + } + } + return outputArr +} + // func executeUserCommands() []string { // } diff --git a/pkg/backy/config.go b/pkg/backy/config.go index 2902788..26ad032 100644 --- a/pkg/backy/config.go +++ b/pkg/backy/config.go @@ -22,10 +22,13 @@ import ( "github.com/rs/zerolog" ) -const macroStart string = "%{" -const macroEnd string = "}%" -const envMacroStart string = "%{env:" -const vaultMacroStart string = "%{vault:" +const ( + externDirectiveStart string = "%{" + externDirectiveEnd string = "}%" + externFileDirectiveStart string = "%{file:" + envExternDirectiveStart string = "%{env:" + vaultExternDirectiveStart string = "%{vault:" +) func (opts *ConfigOpts) InitConfig() { var err error @@ -95,41 +98,6 @@ func (opts *ConfigOpts) InitConfig() { opts.koanf = backyKoanf } -func loadConfigFile(fetcher remotefetcher.RemoteFetcher, filePath string, k *koanf.Koanf, opts *ConfigOpts) { - data, err := fetcher.Fetch(filePath) - if err != nil { - logging.ExitWithMSG(fmt.Sprintf("Could not fetch config file %s: %v", filePath, err), 1, nil) - } - - if err := k.Load(rawbytes.Provider(data), yaml.Parser()); err != nil { - logging.ExitWithMSG(fmt.Sprintf("error loading config: %v", err), 1, &opts.Logger) - } -} - -func loadDefaultConfigFiles(fetcher remotefetcher.RemoteFetcher, configFiles []string, k *koanf.Koanf, opts *ConfigOpts) { - cFileFailures := 0 - for _, c := range configFiles { - opts.ConfigFilePath = c - data, err := fetcher.Fetch(c) - if err != nil { - cFileFailures++ - continue - } - - if data != nil { - if err := k.Load(rawbytes.Provider(data), yaml.Parser()); err == nil { - break - } else { - logging.ExitWithMSG(fmt.Sprintf("error loading config from file %s: %v", c, err), 1, &opts.Logger) - } - } - } - - if cFileFailures == len(configFiles) { - logging.ExitWithMSG("Could not find any valid local config file", 1, nil) - } -} - func (opts *ConfigOpts) ReadConfig() *ConfigOpts { setTerminalEnv() @@ -148,7 +116,7 @@ func (opts *ConfigOpts) ReadConfig() *ConfigOpts { CheckConfigValues(backyKoanf, opts.ConfigFilePath) - validateCommands(backyKoanf, opts) + validateExecCommandsFromCLI(backyKoanf, opts) setLoggingOptions(backyKoanf, opts) @@ -192,6 +160,41 @@ func (opts *ConfigOpts) ReadConfig() *ConfigOpts { return opts } +func loadConfigFile(fetcher remotefetcher.RemoteFetcher, filePath string, k *koanf.Koanf, opts *ConfigOpts) { + data, err := fetcher.Fetch(filePath) + if err != nil { + logging.ExitWithMSG(generateFileFetchErrorString(filePath, "config", err), 1, nil) + } + + if err := k.Load(rawbytes.Provider(data), yaml.Parser()); err != nil { + logging.ExitWithMSG(fmt.Sprintf("error loading config: %v", err), 1, &opts.Logger) + } +} + +func loadDefaultConfigFiles(fetcher remotefetcher.RemoteFetcher, configFiles []string, k *koanf.Koanf, opts *ConfigOpts) { + cFileFailures := 0 + for _, c := range configFiles { + opts.ConfigFilePath = c + data, err := fetcher.Fetch(c) + if err != nil { + cFileFailures++ + continue + } + + if data != nil { + if err := k.Load(rawbytes.Provider(data), yaml.Parser()); err == nil { + break + } else { + logging.ExitWithMSG(fmt.Sprintf("error loading config from file %s: %v", c, err), 1, &opts.Logger) + } + } + } + + if cFileFailures == len(configFiles) { + logging.ExitWithMSG("Could not find any valid local config file", 1, nil) + } +} + func setTerminalEnv() { if isatty.IsTerminal(os.Stdout.Fd()) || isatty.IsCygwinTerminal(os.Stdout.Fd()) { os.Setenv("BACKY_TERM", "enabled") @@ -200,7 +203,7 @@ func setTerminalEnv() { } } -func validateCommands(k *koanf.Koanf, opts *ConfigOpts) { +func validateExecCommandsFromCLI(k *koanf.Koanf, opts *ConfigOpts) { for _, c := range opts.executeCmds { if !k.Exists(getCmdFromConfig(c)) { logging.ExitWithMSG(fmt.Sprintf("command %s is not in config file %s", c, opts.ConfigFilePath), 1, nil) @@ -238,7 +241,7 @@ func setupLogger(opts *ConfigOpts) zerolog.Logger { func unmarshalConfig(k *koanf.Koanf, key string, target interface{}, log zerolog.Logger) { if err := k.UnmarshalWithConf(key, target, koanf.UnmarshalConf{Tag: "yaml"}); err != nil { - logging.ExitWithMSG(fmt.Sprintf("error unmarshalling key %s into struct: %v", key, err), 1, &log) + logging.ExitWithMSG(fmt.Sprintf("error unmarshaling key %s into struct: %v", key, err), 1, &log) } } @@ -304,9 +307,10 @@ func loadCommandLists(opts *ConfigOpts, backyKoanf *koanf.Koanf) { } if backyKoanf.Exists("cmdLists") { - unmarshalConfig(backyKoanf, "cmdLists", &opts.CmdConfigLists, opts.Logger) if backyKoanf.Exists("cmdLists.file") { loadCmdListsFile(backyKoanf, listsConfig, opts) + } else { + unmarshalConfig(backyKoanf, "cmdLists", &opts.CmdConfigLists, opts.Logger) } } } @@ -366,7 +370,7 @@ func loadCmdListsFile(backyKoanf *koanf.Koanf, listsConfig *koanf.Koanf, opts *C data, err := fetcher.Fetch(opts.CmdListFile) if err != nil { - logging.ExitWithMSG(fmt.Sprintf("Could not fetch config file %s: %v", opts.CmdListFile, err), 1, nil) + logging.ExitWithMSG(generateFileFetchErrorString(opts.CmdListFile, "list config", err), 1, nil) } if err := listsConfig.Load(rawbytes.Provider(data), yaml.Parser()); err != nil { @@ -378,6 +382,10 @@ func loadCmdListsFile(backyKoanf *koanf.Koanf, listsConfig *koanf.Koanf, opts *C opts.Logger.Info().Str("using lists config file", opts.CmdListFile).Send() } +func generateFileFetchErrorString(file, fileType string, err error) string { + return fmt.Sprintf("Could not fetch %s file %s: %v", file, fileType, err) +} + func validateCommandLists(opts *ConfigOpts) { var cmdNotFoundSliceErr []error for cmdListName, cmdList := range opts.CmdConfigLists { @@ -455,7 +463,7 @@ func (opts *ConfigOpts) setupVault() error { unmarshalErr := opts.koanf.UnmarshalWithConf("vault.keys", &opts.VaultKeys, koanf.UnmarshalConf{Tag: "yaml"}) if unmarshalErr != nil { - logging.ExitWithMSG(fmt.Sprintf("error unmarshalling vault.keys into struct: %v", unmarshalErr), 1, &opts.Logger) + logging.ExitWithMSG(fmt.Sprintf("error unmarshaling vault.keys into struct: %v", unmarshalErr), 1, &opts.Logger) } opts.vaultClient = client @@ -565,6 +573,10 @@ func processCmds(opts *ConfigOpts) error { cmd.RemoteHost.HostName = host.HostName } } else { + opts.Logger.Info().Msgf("adding host %s to host list", *cmd.Host) + if opts.Hosts == nil { + opts.Hosts = make(map[string]*Host) + } opts.Hosts[*cmd.Host] = &Host{Host: *cmd.Host} cmd.RemoteHost = &Host{Host: *cmd.Host} } @@ -577,10 +589,11 @@ func processCmds(opts *ConfigOpts) error { return err } cmd.Dir = &cmdDir + } else { + cmd.Dir = &opts.ConfigDir } } - // Parse package commands if cmd.Type == PackageCT { if cmd.PackageManager == "" { return fmt.Errorf("package manager is required for package command %s", cmd.PackageName) @@ -612,19 +625,29 @@ func processCmds(opts *ConfigOpts) error { return fmt.Errorf("username is required for user command %s", cmd.Name) } - detectOSType(cmd, opts) - var err error + err := detectOSType(cmd, opts) + if err != nil { + opts.Logger.Info().Err(err).Str("command", cmdName).Send() + } // Validate the operation switch cmd.UserOperation { case "add", "remove", "modify", "checkIfExists", "delete", "password": cmd.userMan, err = usermanager.NewUserManager(cmd.OS) + if cmd.UserOperation == "password" { + cmd.UserPassword = expandExternalConfigDirectives(cmd.UserPassword, opts) + } if cmd.Host != nil { host, ok := opts.Hosts[*cmd.Host] if ok { cmd.userMan, err = usermanager.NewUserManager(host.OS) } } + for indx, key := range cmd.UserSshPubKeys { + opts.Logger.Debug().Msg("adding SSH Keys") + key = expandExternalConfigDirectives(key, opts) + cmd.UserSshPubKeys[indx] = key + } if err != nil { return err } @@ -656,14 +679,8 @@ func processCmds(opts *ConfigOpts) error { return nil } -// processHooks evaluates if hooks are valid Commands -// -// The cmd.hookRefs[hookType] is created with any hooks found. -// -// Returns an error, if any, if the hook command is not found func processHooks(cmd *Command, hooks []string, opts *ConfigOpts, hookType string) error { - // initialize hook type var hookCmdFound bool cmd.hookRefs = map[string]map[string]*Command{} cmd.hookRefs[hookType] = map[string]*Command{} @@ -691,11 +708,14 @@ func processHooks(cmd *Command, hooks []string, opts *ConfigOpts, hookType strin func detectOSType(cmd *Command, opts *ConfigOpts) error { if cmd.Host == nil { - if runtime.GOOS == "linux" { // also can be specified to FreeBSD + if runtime.GOOS == "linux" { cmd.OS = "linux" opts.Logger.Info().Msg("Unix/Linux type OS detected") + return nil } + return fmt.Errorf("using an os that is not yet supported for user commands") } + host, ok := opts.Hosts[*cmd.Host] if ok { if host.OS != "" { diff --git a/pkg/backy/notification.go b/pkg/backy/notification.go index effa34c..21cf509 100644 --- a/pkg/backy/notification.go +++ b/pkg/backy/notification.go @@ -71,9 +71,8 @@ func (opts *ConfigOpts) SetupNotify() { opts.Logger.Info().Str("list", confName).Err(fmt.Errorf("error: configuring matrix id %s failed during setup: %w", id, mtrxErr)) continue } - // append the services services = append(services, mtrxConf) - // service is not recognized + default: opts.Logger.Info().Err(fmt.Errorf("id %s not found", id)).Str("list", confName).Send() } diff --git a/pkg/backy/ssh.go b/pkg/backy/ssh.go index 1472530..8a64fbe 100644 --- a/pkg/backy/ssh.go +++ b/pkg/backy/ssh.go @@ -17,6 +17,7 @@ import ( "github.com/kevinburke/ssh_config" "github.com/pkg/errors" + "github.com/pkg/sftp" "github.com/rs/zerolog" "golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh/knownhosts" @@ -569,6 +570,57 @@ func (command *Command) RunCmdSSH(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) if err := commandSession.Run(ArgsStr); err != nil { return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error running command: %w", err) } + + if command.Type == UserCT && command.UserOperation == "add" { + if command.UserSshPubKeys != nil { + var ( + f *sftp.File + err error + userHome []byte + client *sftp.Client + ) + + cmdCtxLogger.Info().Msg("adding SSH Keys") + + commandSession, _ = command.RemoteHost.createSSHSession(opts) + userHome, err = commandSession.CombinedOutput(fmt.Sprintf("grep \"%s\" /etc/passwd | cut -d: -f6", command.Username)) + if err != nil { + return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error finding user home from /etc/passwd: %v", err) + } + + command.UserHome = strings.TrimSpace(string(userHome)) + userSshDir := fmt.Sprintf("%s/.ssh", command.UserHome) + client, err = sftp.NewClient(command.RemoteHost.SshClient) + if err != nil { + return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error creating sftp client: %v", err) + } + + client.MkdirAll(userSshDir) + _, err = client.Create(fmt.Sprintf("%s/authorized_keys", userSshDir)) + if err != nil { + return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error opening file %s/authorized_keys: %v", userSshDir, err) + } + f, err = client.OpenFile(fmt.Sprintf("%s/authorized_keys", userSshDir), os.O_APPEND|os.O_CREATE|os.O_WRONLY) + if err != nil { + return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error opening file %s/authorized_keys: %v", userSshDir, err) + } + defer f.Close() + for _, k := range command.UserSshPubKeys { + buf := bytes.NewBufferString(k) + cmdCtxLogger.Info().Str("key", k).Msg("adding SSH key") + if _, err := f.ReadFrom(buf); err != nil { + return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error adding to authorized keys: %v", err) + } + } + + commandSession, _ = command.RemoteHost.createSSHSession(opts) + _, err = commandSession.CombinedOutput(fmt.Sprintf("chown -R %s:%s %s", command.Username, command.Username, userHome)) + if err != nil { + return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), err + } + + } + } } return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), nil @@ -622,7 +674,7 @@ func (command *Command) runScript(session *ssh.Session, cmdCtxLogger zerolog.Log return collectOutput(outputBuf, command.Name, cmdCtxLogger, true), fmt.Errorf("error waiting for shell: %w", err) } - return collectOutput(outputBuf, command.Name, cmdCtxLogger, command.GetOutput), nil + return collectOutput(outputBuf, command.Name, cmdCtxLogger, command.OutputToLog), nil } // runScriptFile handles the execution of script files. diff --git a/pkg/backy/types.go b/pkg/backy/types.go index 8e0a442..6289074 100644 --- a/pkg/backy/types.go +++ b/pkg/backy/types.go @@ -51,44 +51,30 @@ type ( Command struct { Name string `yaml:"name,omitempty"` - // command to run Cmd string `yaml:"cmd"` // See CommandType enum further down the page for acceptable values Type CommandType `yaml:"type,omitempty"` - // host on which to run cmd Host *string `yaml:"host,omitempty"` - // Hooks are for running commands on certain events Hooks *Hooks `yaml:"hooks,omitempty"` - // hook refs are internal references of commands for each hook type hookRefs map[string]map[string]*Command - // Shell specifies which shell to run the command in, if any. Shell string `yaml:"shell,omitempty"` RemoteHost *Host `yaml:"-"` - // Args is an array that holds the arguments to cmd Args []string `yaml:"args,omitempty"` - /* - Dir specifies a directory in which to run the command. - */ Dir *string `yaml:"dir,omitempty"` - // Env points to a file containing env variables to be used with the command Env string `yaml:"env,omitempty"` - // Environment holds env variables to be used with the command Environment []string `yaml:"environment,omitempty"` - // Output determines if output is requested. - // - // Only for when command is in a list. - GetOutput bool `yaml:"getOutput,omitempty"` + GetOutputInList bool `yaml:"getOutputInList,omitempty"` ScriptEnvFile string `yaml:"scriptEnvFile"` @@ -102,10 +88,8 @@ type ( PackageName string `yaml:"packageName,omitempty"` - // Version specifies the desired version for package execution PackageVersion string `yaml:"packageVersion,omitempty"` - // PackageOperation specifies the action for package-related commands (e.g., "install" or "remove") PackageOperation PackageOperation `yaml:"packageOperation,omitempty"` pkgMan pkgman.PackageManager @@ -113,42 +97,35 @@ type ( packageCmdSet bool // END PACKAGE COMMAND FIELDS - // RemoteSource specifies a URL to fetch the command or configuration remotely RemoteSource string `yaml:"remoteSource,omitempty"` - // FetchBeforeExecution determines if the remoteSource should be fetched before running FetchBeforeExecution bool `yaml:"fetchBeforeExecution,omitempty"` Fetcher remotefetcher.RemoteFetcher // BEGIN USER COMMAND FIELDS - // Username specifies the username for user creation or related operations Username string `yaml:"userName,omitempty"` UserID string `yaml:"userID,omitempty"` - // UserGroups specifies the groups to add the user to UserGroups []string `yaml:"userGroups,omitempty"` - // UserHome specifies the home directory for the user UserHome string `yaml:"userHome,omitempty"` - // UserShell specifies the shell for the user UserShell string `yaml:"userShell,omitempty"` - // SystemUser specifies whether the user is a system account SystemUser bool `yaml:"systemUser,omitempty"` - // UserPassword specifies the password for the user (can be file: or plain text) UserPassword string `yaml:"userPassword,omitempty"` + UserSshPubKeys []string `yaml:"userSshPubKeys,omitempty"` + userMan usermanager.UserManager // OS for the command, only used when type is user OS string `yaml:"OS,omitempty"` - // UserOperation specifies the action for user-related commands (e.g., "create" or "remove") UserOperation string `yaml:"userOperation,omitempty"` userCmdSet bool diff --git a/pkg/backy/utils.go b/pkg/backy/utils.go index 78cfcf5..30ecbd9 100644 --- a/pkg/backy/utils.go +++ b/pkg/backy/utils.go @@ -247,7 +247,6 @@ func (opts *ConfigOpts) loadEnv() { opts.backyEnv = backyEnv } -// expandEnvVars expands environment variables with the env used in the config func expandEnvVars(backyEnv map[string]string, envVars []string) { env := func(name string) string { @@ -261,10 +260,10 @@ func expandEnvVars(backyEnv map[string]string, envVars []string) { // parse env variables using new macros for indx, v := range envVars { - if strings.HasPrefix(v, macroStart) && strings.HasSuffix(v, macroEnd) { - if strings.HasPrefix(v, envMacroStart) { - v = strings.TrimPrefix(v, envMacroStart) - v = strings.TrimRight(v, macroEnd) + if strings.HasPrefix(v, externDirectiveStart) && strings.HasSuffix(v, externDirectiveEnd) { + if strings.HasPrefix(v, envExternDirectiveStart) { + v = strings.TrimPrefix(v, envExternDirectiveStart) + v = strings.TrimRight(v, externDirectiveEnd) out, _ := shell.Expand(v, env) envVars[indx] = out } @@ -324,7 +323,7 @@ func parsePackageVersion(output string, cmdCtxLogger zerolog.Logger, command *Co // println(output) if err != nil { cmdCtxLogger.Error().Err(err).Str("package", command.PackageName).Msg("Error parsing package version output") - return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.GetOutput), err + return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), err } cmdCtxLogger.Info(). @@ -349,3 +348,39 @@ func parsePackageVersion(output string, cmdCtxLogger zerolog.Logger, command *Co } return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, false), err } + +func expandExternalConfigDirectives(key string, opts *ConfigOpts) string { + if !(strings.HasPrefix(key, externDirectiveStart) && strings.HasSuffix(key, externDirectiveEnd)) { + return key + } + opts.Logger.Info().Str("expanding external key", key).Send() + if strings.HasPrefix(key, envExternDirectiveStart) { + key = strings.TrimPrefix(key, envExternDirectiveStart) + key = strings.TrimSuffix(key, externDirectiveEnd) + return os.Getenv(key) + } + if strings.HasPrefix(key, externFileDirectiveStart) { + var err error + var keyValue []byte + key = strings.TrimPrefix(key, externFileDirectiveStart) + key = strings.TrimSuffix(key, externDirectiveEnd) + key, err = getFullPathWithHomeDir(key) + if err != nil { + opts.Logger.Err(err).Send() + return "" + } + keyValue, err = os.ReadFile(key) + if err != nil { + opts.Logger.Err(err).Send() + return "" + } + key = string(keyValue) + } + if strings.HasPrefix(key, vaultExternDirectiveStart) { + key = strings.TrimPrefix(key, vaultExternDirectiveStart) + key = strings.TrimSuffix(key, externDirectiveEnd) + key = GetVaultKey(key, opts, opts.Logger) + } + + return key +} diff --git a/pkg/remotefetcher/configfetcher.go b/pkg/remotefetcher/configfetcher.go index 43d8407..1819bf9 100644 --- a/pkg/remotefetcher/configfetcher.go +++ b/pkg/remotefetcher/configfetcher.go @@ -29,7 +29,7 @@ func NewRemoteFetcher(source string, cache *Cache, options ...FetcherOption) (Re option(&config) } - // If FileType is empty (i.e. WithFileType was not called), yaml is the default file type + // WithFileType was not called. yaml is the default file type if strings.TrimSpace(config.FileType) == "" { config.FileType = "yaml" } diff --git a/pkg/remotefetcher/local.go b/pkg/remotefetcher/local.go index b131597..89f7825 100644 --- a/pkg/remotefetcher/local.go +++ b/pkg/remotefetcher/local.go @@ -20,7 +20,7 @@ func (l *LocalFetcher) Fetch(source string) ([]byte, error) { if l.config.IgnoreFileNotFound { return nil, ErrIgnoreFileNotFound } - return nil, nil + return nil, err } file, err := os.Open(source) if err != nil { diff --git a/tests/backy.yaml b/tests/backy.yaml deleted file mode 100644 index e15da07..0000000 --- a/tests/backy.yaml +++ /dev/null @@ -1,18 +0,0 @@ -commands: - echoTestPass: - cmd: echo - shell: bash - Args: hello world - - runRemoteShellScriptSuccess: - cmd: - - - packageCommandSuccess: - packageName: docker-ce - Args: - - docker-ce-cli - packageManager: apt - packageOperation: install - - \ No newline at end of file