Compare commits

...

9 Commits

Author SHA1 Message Date
1ad50ebcf8 v0.8.1 WIP
Some checks failed
ci/woodpecker/push/go-lint Pipeline failed
2025-02-20 14:53:44 -06:00
c483a1056f v0.8.0
Some checks failed
ci/woodpecker/push/publish-docs Pipeline was successful
ci/woodpecker/tag/gitea Pipeline was successful
ci/woodpecker/tag/publish-docs Pipeline was successful
ci/woodpecker/push/go-lint Pipeline failed
ci/woodpecker/release/publish-docs Pipeline was successful
2025-02-15 23:00:10 -06:00
3b9f569310 CI config changed
Some checks failed
ci/woodpecker/push/publish-docs Pipeline failed
2025-02-15 22:54:11 -06:00
843be7968b CI config changed 2025-02-15 22:50:44 -06:00
d477d850ac CI config changed 2025-02-15 22:46:49 -06:00
8eb3229af7 v0.8.0 2025-02-15 22:29:19 -06:00
d89a208bbd v0.8.0 2025-02-15 22:28:01 -06:00
0d28d6afcf breaking changes to keys 2025-02-15 22:27:11 -06:00
7c42a9a7cd v0.7.8
Some checks failed
ci/woodpecker/push/publish-docs Pipeline was successful
ci/woodpecker/tag/publish-docs Pipeline was successful
ci/woodpecker/tag/gitea Pipeline failed
2025-02-14 14:46:10 -06:00
22 changed files with 221 additions and 61 deletions

4
.changes/v0.7.8.md Normal file
View File

@ -0,0 +1,4 @@
## v0.7.8 - 2025-02-14
### Fixed
* Github CI config
* v0.7.1: Incorrect local config file loading logic caused files to not be detected

6
.changes/v0.8.0.md Normal file
View File

@ -0,0 +1,6 @@
## v0.8.0 - 2025-02-15
### Changed
* Breaking: `cmd-lists` key changed to `cmdLists`
* Properly load list config
* Config file loading properly errors
* CI Configs

View File

@ -15,11 +15,11 @@ jobs:
goreleaser:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/checkout@v4
with:
fetch-depth: 0
- run: git fetch --force --tags
- uses: actions/setup-go@v4
- uses: actions/setup-go@v5
with:
go-version: '1.23'
cache: true
@ -34,7 +34,7 @@ jobs:
with:
distribution: goreleaser
version: 2.7.0
run: goreleaser release --release-notes=".changes/$(go run backy.go version -V).md" -f .goreleaser/github.yml --clean
args: release --release-notes=".changes/${{ env.GIT_TAG_NAME }}.md" -f .goreleaser/github.yml --clean
env:
GITHUB_TOKEN: ${{ secrets.GORELEASER_TOKEN }}
# GIT_TAG_NAME: ${{ steps.tagName.outputs.tag }}
GIT_TAG_NAME: ${{ steps.tagName.outputs.tag }}

6
.gitignore vendored
View File

@ -4,3 +4,9 @@ dist/
*.log
*.sh
*.yaml
*.yml
+.changie.yaml
+.changes/

View File

@ -6,7 +6,10 @@ steps:
- go mod tidy
- go install github.com/goreleaser/goreleaser/v2@v2.7.0
- goreleaser release -f .goreleaser/gitea.yml --release-notes=".changes/$(go run backy.go version -V).md"
secrets: [ gitea_token ]
environment:
GITEA_TOKEN:
from_secret: gitea_token
when:
event: tag
# release:

View File

@ -23,7 +23,14 @@ steps:
- echo "$SSH_DEPLOY_KEY" | tr -d '\r' | DISPLAY=":0.0" SSH_ASKPASS=~/.ssh/.print_ssh_password setsid ssh-add -
- rsync -atv --delete --progress public/ backy@backy.cybershell.xyz:docs
- rsync -atv --delete --progress vangen/ backy@backy.cybershell.xyz:vangen-go
secrets: [ ssh_host_key, ssh_deploy_key, ssh_passphrase ]
environment:
SSH_HOST_KEY:
from_secret: ssh_host_key
SSH_DEPLOY_KEY:
from_secret: ssh_deploy_key
SSH_PASSPHRASE:
from_secret: ssh_passphrase
when:
- branch: master
- path: 'docs/**'

View File

@ -6,6 +6,18 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html),
and is generated by [Changie](https://github.com/miniscruff/changie).
## v0.8.0 - 2025-02-15
### Changed
* Breaking: `cmd-lists` key changed to `cmdLists`
* Properly load list config
* Config file loading properly errors
* CI Configs
## v0.7.8 - 2025-02-14
### Fixed
* Github CI config
* v0.7.1: Incorrect local config file loading logic caused files to not be detected
## v0.7.7 - 2025-02-14
### Fixed
* v0.7.1: Incorrect local config file loading logic caused files to not be detected

View File

@ -8,13 +8,14 @@
"settings": {
"cSpell.words": [
"Cmds",
"remotefetcher",
"CMDSTDOUT",
"knadh",
"koanf",
"mattn",
"maunium",
"mautrix",
"nikoksr",
"remotefetcher",
"Strs"
]
}

View File

@ -30,7 +30,7 @@ func init() {
}
func Backup(cmd *cobra.Command, args []string) {
backyConfOpts := backy.NewOpts(cfgFile, backy.AddCommandLists(cmdLists))
backyConfOpts := backy.NewOpts(cfgFile, backy.AddCommandLists(cmdLists), backy.SetLogFile(logFile), backy.SetCmdStdOut(cmdStdOut))
backyConfOpts.InitConfig()
backyConfOpts.ReadConfig()

View File

@ -18,7 +18,7 @@ var (
func cron(cmd *cobra.Command, args []string) {
parseS3Config()
opts := backy.NewOpts(cfgFile, backy.EnableCron())
opts := backy.NewOpts(cfgFile, backy.EnableCron(), backy.SetLogFile(logFile), backy.SetCmdStdOut(cmdStdOut))
opts.InitConfig()
opts.ReadConfig()

View File

@ -32,7 +32,7 @@ func execute(cmd *cobra.Command, args []string) {
logging.ExitWithMSG("Please provide a command to run. Pass --help to see options.", 1, nil)
}
opts := backy.NewOpts(cfgFile, backy.AddCommands(args), backy.SetLogFile(logFile))
opts := backy.NewOpts(cfgFile, backy.AddCommands(args), backy.SetLogFile(logFile), backy.SetCmdStdOut(cmdStdOut))
opts.InitConfig()
opts.ReadConfig()
opts.ExecuteCmds()

View File

@ -35,7 +35,7 @@ func init() {
// 2. stdin (on command line) (TODO)
func Host(cmd *cobra.Command, args []string) {
backyConfOpts := backy.NewOpts(cfgFile, backy.SetLogFile(logFile))
backyConfOpts := backy.NewOpts(cfgFile, backy.SetLogFile(logFile), backy.SetCmdStdOut(cmdStdOut))
backyConfOpts.InitConfig()
backyConfOpts.ReadConfig()

View File

@ -6,16 +6,29 @@ package cmd
import (
"git.andrewnw.xyz/CyberShell/backy/pkg/backy"
"git.andrewnw.xyz/CyberShell/backy/pkg/logging"
"github.com/spf13/cobra"
)
var (
listCmd = &cobra.Command{
Use: "list [--list=list1,list2,... | -l list1, list2,...] [ -cmd cmd1 cmd2 cmd3...]",
Use: "list [command]",
Short: "Lists commands, lists, or hosts defined in config file.",
Long: "Backup lists commands or groups defined in config file.\nUse the --lists or -l flag to list the specified lists. If not flag is not given, all lists will be executed.",
Run: List,
}
listCmds = &cobra.Command{
Use: "cmds [cmd1 cmd2 cmd3...]",
Short: "Lists commands, lists, or hosts defined in config file.",
Long: "Backup lists commands or groups defined in config file.\nUse the --lists or -l flag to list the specified lists. If not flag is not given, all lists will be executed.",
Run: ListCmds,
}
listCmdLists = &cobra.Command{
Use: "lists [list1 list2 ...]",
Short: "Lists commands, lists, or hosts defined in config file.",
Long: "Backup lists commands or groups defined in config file.\nUse the --lists or -l flag to list the specified lists. If not flag is not given, all lists will be executed.",
Run: ListCmdLists,
}
)
@ -23,27 +36,51 @@ var listsToList []string
var cmdsToList []string
func init() {
listCmd.Flags().StringSliceVarP(&listsToList, "lists", "l", nil, "Accepts comma-separated names of command lists to list.")
listCmd.Flags().StringSliceVarP(&cmdsToList, "cmds", "c", nil, "Accepts comma-separated names of commands to list.")
listCmd.AddCommand(listCmds, listCmdLists)
}
func List(cmd *cobra.Command, args []string) {
func ListCmds(cmd *cobra.Command, args []string) {
// setup based on whats passed in:
// - cmds
// - lists
// - if none, list all commands
if cmdLists != nil {
if len(args) > 0 {
cmdsToList = args
} else {
logging.ExitWithMSG("Error: list cmds subcommand needs commands to list", 1, nil)
}
parseS3Config()
opts := backy.NewOpts(cfgFile)
opts := backy.NewOpts(cfgFile, backy.SetLogFile(logFile))
opts.InitConfig()
opts.ReadConfig()
opts.ListCommand("rm-sn-db")
for _, v := range cmdsToList {
opts.ListCommand(v)
}
}
func ListCmdLists(cmd *cobra.Command, args []string) {
parseS3Config()
if len(args) > 0 {
listsToList = args
} else {
logging.ExitWithMSG("Error: lists subcommand needs lists", 1, nil)
}
opts := backy.NewOpts(cfgFile, backy.SetLogFile(logFile))
opts.InitConfig()
opts.ReadConfig()
for _, v := range listsToList {
opts.ListCommandList(v)
}
}

View File

@ -15,6 +15,7 @@ var (
// Used for flags.
cfgFile string
verbose bool
cmdStdOut bool
logFile string
s3Endpoint string
@ -35,6 +36,7 @@ func Execute() {
func init() {
rootCmd.PersistentFlags().StringVar(&logFile, "log-file", "", "log file to write to")
rootCmd.PersistentFlags().BoolVar(&cmdStdOut, "cmdStdOut", false, "Pass to print command output to stdout")
rootCmd.PersistentFlags().StringVarP(&cfgFile, "config", "f", "", "config file to read from")
rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "Sets verbose level")

View File

@ -7,7 +7,7 @@ import (
"github.com/spf13/cobra"
)
const versionStr = "0.7.7"
const versionStr = "0.8.0"
var (
versionCmd = &cobra.Command{

View File

@ -2,7 +2,7 @@
title: "Command Lists"
weight: 2
description: >
This page tells you how to get use command lists.
This page tells you how to use command lists.
---
Command lists are for executing commands in sequence and getting notifications from them.
@ -11,8 +11,8 @@ The top-level object key can be anything you want but not the same as another.
Lists can go in a separate file. Command lists should be in a separate file if:
1. key 'cmd-lists.file' is specified
2. lists.yml or lists.yaml is found in the same directory as the backy config file
1. key 'cmdLists.file' is specified
2. lists.yml or lists.yaml is found in the same directory as the backy config file (this includes remote config files as of v0.7.0)
{{% notice info %}}
The lists file is also checked in remote resources.
@ -70,14 +70,14 @@ Name is optional. If name is not defined, name will be the object's map key.
Backy also has a cron mode, so one can run `backy cron` and start a process that schedules jobs to run at times defined in the configuration file.
Adding `cron: 0 0 1 * * *` to a `cmd-lists` object will schedule the list at 1 in the morning. See [https://crontab.guru/](https://crontab.guru/) for reference.
Adding `cron: 0 0 1 * * *` to a `cmdLists` object will schedule the list at 1 in the morning. See [https://crontab.guru/](https://crontab.guru/) for reference.
{{% notice tip %}}
Note: Backy uses the second field of cron, so add anything except `*` to the beginning of a regular cron expression.
{{% /notice %}}
```yaml {lineNos="true" wrap="true" title="yaml"}
cmd-lists:
cmdLists:
  docker-container-backup: # this can be any name you want
    # all commands have to be defined
    order:

View File

@ -48,7 +48,7 @@ commands:
To execute groups of commands in sequence, use a list configuration.
```yaml
cmd-lists:
cmdLists:
cmds-to-run: # this can be any name you want
# all commands have to be defined in the commands section
order:
@ -97,7 +97,7 @@ hosts:
The notifications object can have two forms.
For more, [see the notification object documentation](/config/notifications). The top-level map key is id that has to be referenced by the `cmd-lists` key `notifications`.
For more, [see the notification object documentation](/config/notifications). The top-level map key is id that has to be referenced by the `cmdLists` key `notifications`.
```yaml
notifications:

View File

@ -90,7 +90,6 @@ func (opts *ConfigOpts) InitConfig() {
} else {
loadDefaultConfigFiles(fetcher, configFiles, backyKoanf, opts)
}
opts.koanf = backyKoanf
}
@ -117,7 +116,9 @@ func loadDefaultConfigFiles(fetcher remotefetcher.RemoteFetcher, configFiles []s
if data != nil {
if err := k.Load(rawbytes.Provider(data), yaml.Parser()); err == nil {
continue
break
} else {
logging.ExitWithMSG(fmt.Sprintf("error loading config from file %s: %v", c, err), 1, &opts.Logger)
}
}
}
@ -135,7 +136,12 @@ func (opts *ConfigOpts) ReadConfig() *ConfigOpts {
opts.loadEnv()
if backyKoanf.Bool(getNestedConfig("logging", "cmd-std-out")) {
os.Setenv("BACKY_STDOUT", "enabled")
os.Setenv("BACKY_CMDSTDOUT", "enabled")
}
// override the default value of cmd-std-out if flag is set
if opts.CmdStdOut {
os.Setenv("BACKY_CMDSTDOUT", "enabled")
}
CheckConfigValues(backyKoanf, opts.ConfigFilePath)
@ -230,7 +236,7 @@ func setupLogger(opts *ConfigOpts) zerolog.Logger {
func unmarshalConfig(k *koanf.Koanf, key string, target interface{}, log zerolog.Logger) {
if err := k.UnmarshalWithConf(key, target, koanf.UnmarshalConf{Tag: "yaml"}); err != nil {
logging.ExitWithMSG(fmt.Sprintf("error unmarshalling %s struct: %v", key, err), 1, &log)
logging.ExitWithMSG(fmt.Sprintf("error unmarshalling key %s into struct: %v", key, err), 1, &log)
}
}
@ -274,10 +280,12 @@ func loadCommandLists(opts *ConfigOpts, backyKoanf *koanf.Koanf) {
// if config file is remote, use the directory of the remote file
if isRemoteURL(opts.ConfigFilePath) {
_, u = getRemoteDir(opts.ConfigFilePath)
// // Still use local list files if a remote config file is used, but use them last
listConfigFiles = []string{u.JoinPath("lists.yml").String(), u.JoinPath("lists.yaml").String()}
} else {
backyConfigFileDir = path.Dir(opts.ConfigFilePath)
listConfigFiles = []string{
// "./lists.yml", "./lists.yaml",
path.Join(backyConfigFileDir, "lists.yml"),
path.Join(backyConfigFileDir, "lists.yaml"),
}
@ -291,9 +299,9 @@ func loadCommandLists(opts *ConfigOpts, backyKoanf *koanf.Koanf) {
}
}
if backyKoanf.Exists("cmd-lists") {
unmarshalConfig(backyKoanf, "cmd-lists", &opts.CmdConfigLists, opts.Logger)
if backyKoanf.Exists("cmd-lists.file") {
if backyKoanf.Exists("cmdLists") {
unmarshalConfig(backyKoanf, "cmdLists", &opts.CmdConfigLists, opts.Logger)
if backyKoanf.Exists("cmdLists.file") {
loadCmdListsFile(backyKoanf, listsConfig, opts)
}
}
@ -319,6 +327,7 @@ func loadListConfigFile(filePath string, k *koanf.Koanf, opts *ConfigOpts) bool
if err != nil {
// if file not found, ignore
if errors.Is(err, remotefetcher.ErrIgnoreFileNotFound) {
println("File not found", filePath)
return true
}
@ -334,12 +343,14 @@ func loadListConfigFile(filePath string, k *koanf.Koanf, opts *ConfigOpts) bool
return false
}
unmarshalConfig(k, "cmdLists", &opts.CmdConfigLists, opts.Logger)
keyNotSupported("cmd-lists", "cmdLists", k, opts, true)
opts.CmdListFile = filePath
return true
}
func loadCmdListsFile(backyKoanf *koanf.Koanf, listsConfig *koanf.Koanf, opts *ConfigOpts) {
opts.CmdListFile = strings.TrimSpace(backyKoanf.String("cmd-lists.file"))
opts.CmdListFile = strings.TrimSpace(backyKoanf.String("cmdLists.file"))
if !path.IsAbs(opts.CmdListFile) {
opts.CmdListFile = path.Join(path.Dir(opts.ConfigFilePath), opts.CmdListFile)
}
@ -359,14 +370,17 @@ func loadCmdListsFile(backyKoanf *koanf.Koanf, listsConfig *koanf.Koanf, opts *C
logging.ExitWithMSG(fmt.Sprintf("error loading config: %v", err), 1, &opts.Logger)
}
unmarshalConfig(listsConfig, "cmd-lists", &opts.CmdConfigLists, opts.Logger)
keyNotSupported("cmd-lists", "cmdLists", listsConfig, opts, true)
unmarshalConfig(listsConfig, "cmdLists", &opts.CmdConfigLists, opts.Logger)
opts.Logger.Info().Str("using lists config file", opts.CmdListFile).Send()
}
func validateCommandLists(opts *ConfigOpts) {
var cmdNotFoundSliceErr []error
for cmdListName, cmdList := range opts.CmdConfigLists {
// if cron is enabled and cron is not set, delete the list
if opts.cronEnabled && strings.TrimSpace(cmdList.Cron) == "" {
opts.Logger.Debug().Str("cron", "enabled").Str("list", cmdListName).Msg("cron not set, deleting list")
delete(opts.CmdConfigLists, cmdListName)
continue
}
@ -406,9 +420,9 @@ func getLoggingKeyFromConfig(key string) string {
return fmt.Sprintf("logging.%s", key)
}
func getCmdListFromConfig(list string) string {
return fmt.Sprintf("cmd-lists.%s", list)
}
// func getCmdListFromConfig(list string) string {
// return fmt.Sprintf("cmdLists.%s", list)
// }
func (opts *ConfigOpts) setupVault() error {
if !opts.koanf.Bool("vault.enabled") {
@ -551,6 +565,16 @@ func processCmds(opts *ConfigOpts) error {
opts.Hosts[*cmd.Host] = &Host{Host: *cmd.Host}
cmd.RemoteHost = &Host{Host: *cmd.Host}
}
} else {
if cmd.Dir != nil {
cmdDir, err := resolveDir(*cmd.Dir)
if err != nil {
return err
}
cmd.Dir = &cmdDir
}
}
// Parse package commands
@ -677,3 +701,14 @@ func detectOSType(cmd *Command, opts *ConfigOpts) error {
}
return nil
}
func keyNotSupported(oldKey, newKey string, koanf *koanf.Koanf, opts *ConfigOpts, deprecated bool) {
if koanf.Exists(oldKey) {
if deprecated {
opts.Logger.Warn().Str("key", oldKey).Msg("key is deprecated. Use " + newKey + " instead.")
} else {
opts.Logger.Fatal().Err(fmt.Errorf("key %s found; it has changed to %s", oldKey, newKey)).Send()
}
}
}

View File

@ -17,10 +17,7 @@ func (opts *ConfigOpts) Cron() {
s := gocron.NewScheduler(time.Local)
s.TagsUnique()
cmdLists := opts.CmdConfigLists
for listName, config := range cmdLists {
if config.Name == "" {
config.Name = listName
}
for _, config := range cmdLists {
cron := strings.TrimSpace(config.Cron)
if cron != "" {

View File

@ -23,8 +23,7 @@ func (opts *ConfigOpts) ListCommand(cmd string) {
var cmdFound bool = false
var cmdInfo *Command
// check commands in file against cmd
for _, cmdInFile := range opts.executeCmds {
print(cmdInFile)
for cmdInFile := range opts.Cmds {
cmdFound = false
if cmd == cmdInFile {
@ -37,28 +36,34 @@ func (opts *ConfigOpts) ListCommand(cmd string) {
// print the command's information
if cmdFound {
print("Command: ")
println("Command: ")
print(cmdInfo.Cmd)
if len(cmdInfo.Args) >= 0 {
for _, v := range cmdInfo.Args {
print(" ") // print space between command and args
print(v) // print command arg
}
for _, v := range cmdInfo.Args {
print(" ") // print space between command and args
print(v) // print command arg
}
// is is remote or local
// is it remote or local
if cmdInfo.Host != nil {
print("Host: ", cmdInfo.Host)
println()
print("Host: ", *cmdInfo.Host)
println()
} else {
println()
print("Host: Runs on Local Machine\n\n")
}
if cmdInfo.Dir != nil {
println()
print("Directory: ", *cmdInfo.Dir)
println()
}
} else {
fmt.Printf("Command %s not found. Check spelling.\n", cmd)
@ -66,3 +71,38 @@ func (opts *ConfigOpts) ListCommand(cmd string) {
}
}
func (opts *ConfigOpts) ListCommandList(list string) {
// bool for commands not found
// gets set to false if a command is not found
// set to true if the command is found
var listFound bool = false
var listInfo *CmdList
// check commands in file against cmd
for listInFile, l := range opts.CmdConfigLists {
listFound = false
if list == listInFile {
listFound = true
listInfo = l
break
}
}
// print the command's information
if listFound {
println("List: ", list)
println()
for _, v := range listInfo.Order {
println()
opts.ListCommand(v)
}
} else {
fmt.Printf("List %s not found. Check spelling.\n", list)
}
}

View File

@ -77,7 +77,6 @@ type (
/*
Dir specifies a directory in which to run the command.
Ignored if Host is set.
*/
Dir *string `yaml:"dir,omitempty"`
@ -187,7 +186,7 @@ type (
// CmdConfigLists holds the lists of commands to be run in order.
// Key is the command list name.
CmdConfigLists map[string]*CmdList `yaml:"cmd-lists"`
CmdConfigLists map[string]*CmdList `yaml:"cmdLists"`
// Hosts holds the Host config.
// key is the host.
@ -198,6 +197,8 @@ type (
// Global log level
BackyLogLvl *string
CmdStdOut bool
// Holds config file
ConfigFilePath string

View File

@ -64,6 +64,13 @@ func SetLogFile(logFile string) BackyOptionFunc {
}
}
// SetCmdStdOut forces the command output to stdout
func SetCmdStdOut(setStdOut bool) BackyOptionFunc {
return func(bco *ConfigOpts) {
bco.CmdStdOut = setStdOut
}
}
// EnableCron enables the execution of command lists at specified times
func EnableCron() BackyOptionFunc {
return func(bco *ConfigOpts) {
@ -182,10 +189,12 @@ func IsTerminalActive() bool {
}
func IsCmdStdOutEnabled() bool {
return os.Getenv("BACKY_STDOUT") == "enabled"
return os.Getenv("BACKY_CMDSTDOUT") == "enabled"
}
func resolveDir(path string) (string, error) {
path = strings.TrimSpace(path)
if path == "~" {
homeDir, err := os.UserHomeDir()
if err != nil {