v0.8.1 WIP

.changes/v0.8.0.md

@@ -0,0 +1,6 @@
+## v0.8.0 - 2025-02-15
+### Changed
+* Breaking: `cmd-lists` key changed to `cmdLists`
+* Properly load list config
+* Config file loading properly errors
+* CI Configs

.gitignore

@@ -4,3 +4,9 @@ dist/
 
 *.log
 *.sh
+*.yaml
+*.yml
+
+
++.changie.yaml
++.changes/

.woodpecker/gitea.yml

@@ -6,7 +6,10 @@ steps:
       - go mod tidy
       - go install github.com/goreleaser/goreleaser/v2@v2.7.0
       - goreleaser release -f .goreleaser/gitea.yml --release-notes=".changes/$(go run backy.go version -V).md"
-    secrets: [ gitea_token ]
+    environment:
+      GITEA_TOKEN:
+        from_secret: gitea_token
+        
     when:
       event: tag
   # release:

.woodpecker/publish-docs.yml

@@ -23,7 +23,14 @@ steps:
       - echo "$SSH_DEPLOY_KEY" | tr -d '\r' | DISPLAY=":0.0" SSH_ASKPASS=~/.ssh/.print_ssh_password setsid ssh-add -
       - rsync -atv --delete --progress public/ backy@backy.cybershell.xyz:docs
       - rsync -atv --delete --progress vangen/ backy@backy.cybershell.xyz:vangen-go
-    secrets: [ ssh_host_key, ssh_deploy_key, ssh_passphrase ]
+    environment:
+      SSH_HOST_KEY:
+        from_secret: ssh_host_key
+      SSH_DEPLOY_KEY:
+        from_secret: ssh_deploy_key
+      SSH_PASSPHRASE:
+        from_secret: ssh_passphrase
 
 when:
   - branch: master
+  - path: 'docs/**'

CHANGELOG.md

@@ -6,6 +6,13 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html),
 and is generated by [Changie](https://github.com/miniscruff/changie).
 
 
+## v0.8.0 - 2025-02-15
+### Changed
+* Breaking: `cmd-lists` key changed to `cmdLists`
+* Properly load list config
+* Config file loading properly errors
+* CI Configs
+
 ## v0.7.8 - 2025-02-14
 ### Fixed
 * Github CI config

backy.code-workspace

@@ -8,13 +8,14 @@
 	"settings": {
 		"cSpell.words": [
 			"Cmds",
-			"remotefetcher",
+			"CMDSTDOUT",
 			"knadh",
 			"koanf",
 			"mattn",
 			"maunium",
 			"mautrix",
 			"nikoksr",
+			"remotefetcher",
 			"Strs"
 		]
 	}

cmd/backup.go

@@ -30,7 +30,7 @@ func init() {
 }
 
 func Backup(cmd *cobra.Command, args []string) {
-	backyConfOpts := backy.NewOpts(cfgFile, backy.AddCommandLists(cmdLists))
+	backyConfOpts := backy.NewOpts(cfgFile, backy.AddCommandLists(cmdLists), backy.SetLogFile(logFile), backy.SetCmdStdOut(cmdStdOut))
 	backyConfOpts.InitConfig()
 	backyConfOpts.ReadConfig()
 

cmd/cron.go

@@ -18,7 +18,7 @@ var (
 func cron(cmd *cobra.Command, args []string) {
 	parseS3Config()
 
-	opts := backy.NewOpts(cfgFile, backy.EnableCron())
+	opts := backy.NewOpts(cfgFile, backy.EnableCron(), backy.SetLogFile(logFile), backy.SetCmdStdOut(cmdStdOut))
 	opts.InitConfig()
 	opts.ReadConfig()
 

cmd/exec.go

@@ -32,7 +32,7 @@ func execute(cmd *cobra.Command, args []string) {
 		logging.ExitWithMSG("Please provide a command to run. Pass --help to see options.", 1, nil)
 	}
 
-	opts := backy.NewOpts(cfgFile, backy.AddCommands(args), backy.SetLogFile(logFile))
+	opts := backy.NewOpts(cfgFile, backy.AddCommands(args), backy.SetLogFile(logFile), backy.SetCmdStdOut(cmdStdOut))
 	opts.InitConfig()
 	opts.ReadConfig()
 	opts.ExecuteCmds()

cmd/host.go

@@ -35,7 +35,7 @@ func init() {
 //    2. stdin (on command line) (TODO)
 
 func Host(cmd *cobra.Command, args []string) {
-	backyConfOpts := backy.NewOpts(cfgFile, backy.SetLogFile(logFile))
+	backyConfOpts := backy.NewOpts(cfgFile, backy.SetLogFile(logFile), backy.SetCmdStdOut(cmdStdOut))
 	backyConfOpts.InitConfig()
 
 	backyConfOpts.ReadConfig()

cmd/list.go

@@ -6,16 +6,29 @@ package cmd
 
 import (
 	"git.andrewnw.xyz/CyberShell/backy/pkg/backy"
+	"git.andrewnw.xyz/CyberShell/backy/pkg/logging"
 
 	"github.com/spf13/cobra"
 )
 
 var (
 	listCmd = &cobra.Command{
-		Use:   "list [--list=list1,list2,... | -l list1, list2,...] [ -cmd cmd1 cmd2 cmd3...]",
+		Use:   "list [command]",
 		Short: "Lists commands, lists, or hosts defined in config file.",
 		Long:  "Backup lists commands or groups defined in config file.\nUse the --lists or -l flag to list the specified lists. If not flag is not given, all lists will be executed.",
-		Run:   List,
+	}
+
+	listCmds = &cobra.Command{
+		Use:   "cmds [cmd1 cmd2 cmd3...]",
+		Short: "Lists commands, lists, or hosts defined in config file.",
+		Long:  "Backup lists commands or groups defined in config file.\nUse the --lists or -l flag to list the specified lists. If not flag is not given, all lists will be executed.",
+		Run:   ListCmds,
+	}
+	listCmdLists = &cobra.Command{
+		Use:   "lists [list1 list2 ...]",
+		Short: "Lists commands, lists, or hosts defined in config file.",
+		Long:  "Backup lists commands or groups defined in config file.\nUse the --lists or -l flag to list the specified lists. If not flag is not given, all lists will be executed.",
+		Run:   ListCmdLists,
 	}
 )
 
@@ -23,27 +36,51 @@ var listsToList []string
 var cmdsToList []string
 
 func init() {
-
-	listCmd.Flags().StringSliceVarP(&listsToList, "lists", "l", nil, "Accepts comma-separated names of command lists to list.")
-	listCmd.Flags().StringSliceVarP(&cmdsToList, "cmds", "c", nil, "Accepts comma-separated names of commands to list.")
+	listCmd.AddCommand(listCmds, listCmdLists)
 
 }
 
-func List(cmd *cobra.Command, args []string) {
+func ListCmds(cmd *cobra.Command, args []string) {
 
 	// setup based on whats passed in:
 	//   - cmds
 	//   - lists
 	//   - if none, list all commands
-	if cmdLists != nil {
-
+	if len(args) > 0 {
+		cmdsToList = args
+	} else {
+		logging.ExitWithMSG("Error: list cmds subcommand needs commands to list", 1, nil)
 	}
+
 	parseS3Config()
 
-	opts := backy.NewOpts(cfgFile)
+	opts := backy.NewOpts(cfgFile, backy.SetLogFile(logFile))
 
 	opts.InitConfig()
 	opts.ReadConfig()
 
-	opts.ListCommand("rm-sn-db")
+	for _, v := range cmdsToList {
+		opts.ListCommand(v)
+	}
+}
+
+func ListCmdLists(cmd *cobra.Command, args []string) {
+
+	parseS3Config()
+
+	if len(args) > 0 {
+		listsToList = args
+	} else {
+		logging.ExitWithMSG("Error: lists subcommand needs lists", 1, nil)
+	}
+
+	opts := backy.NewOpts(cfgFile, backy.SetLogFile(logFile))
+
+	opts.InitConfig()
+	opts.ReadConfig()
+
+	for _, v := range listsToList {
+		opts.ListCommandList(v)
+	}
+
 }

cmd/root.go

@@ -15,6 +15,7 @@ var (
 	// Used for flags.
 	cfgFile    string
 	verbose    bool
+	cmdStdOut  bool
 	logFile    string
 	s3Endpoint string
 
@@ -35,6 +36,7 @@ func Execute() {
 
 func init() {
 	rootCmd.PersistentFlags().StringVar(&logFile, "log-file", "", "log file to write to")
+	rootCmd.PersistentFlags().BoolVar(&cmdStdOut, "cmdStdOut", false, "Pass to print command output to stdout")
 
 	rootCmd.PersistentFlags().StringVarP(&cfgFile, "config", "f", "", "config file to read from")
 	rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "Sets verbose level")

cmd/version.go

@@ -7,7 +7,7 @@ import (
 	"github.com/spf13/cobra"
 )
 
-const versionStr = "0.7.8"
+const versionStr = "0.8.0"
 
 var (
 	versionCmd = &cobra.Command{

docs/content/config/command-lists.md

@@ -2,7 +2,7 @@
 title: "Command Lists"
 weight: 2
 description: >
-  This page tells you how to get use command lists.
+  This page tells you how to use command lists.
 ---
 
 Command lists are for executing commands in sequence and getting notifications from them.
@@ -11,8 +11,8 @@ The top-level object key can be anything you want but not the same as another.
 
 Lists can go in a separate file. Command lists should be in a separate file if:
 
-1. key 'cmd-lists.file' is specified
-2. lists.yml or lists.yaml is found in the same directory as the backy config file
+1. key 'cmdLists.file' is specified
+2. lists.yml or lists.yaml is found in the same directory as the backy config file (this includes remote config files as of v0.7.0)
 
 {{% notice info %}}
 The lists file is also checked in remote resources. 
@@ -70,14 +70,14 @@ Name is optional. If name is not defined, name will be the object's map key.
 
 Backy also has a cron mode, so one can run `backy cron` and start a process that schedules jobs to run at times defined in the configuration file.
 
-Adding `cron: 0 0 1 * * *` to a `cmd-lists` object will schedule the list at 1 in the morning. See [https://crontab.guru/](https://crontab.guru/) for reference.
+Adding `cron: 0 0 1 * * *` to a `cmdLists` object will schedule the list at 1 in the morning. See [https://crontab.guru/](https://crontab.guru/) for reference.
 
 {{% notice tip %}}
 Note: Backy uses the second field of cron, so add anything except `*` to the beginning of a regular cron expression.
 {{% /notice %}}
 
 ```yaml {lineNos="true" wrap="true" title="yaml"}
-cmd-lists:
+cmdLists:
   docker-container-backup: # this can be any name you want
     # all commands have to be defined
     order:

docs/content/getting-started/config.md

@@ -48,7 +48,7 @@ commands:
 To execute groups of commands in sequence, use a list configuration.
 
 ```yaml
-cmd-lists:
+cmdLists:
   cmds-to-run: # this can be any name you want
     # all commands have to be defined in the commands section
     order:
@@ -97,7 +97,7 @@ hosts:
 
 The notifications object can have two forms.
 
-For more, [see the notification object documentation](/config/notifications). The top-level map key is id that has to be referenced by the `cmd-lists` key `notifications`.
+For more, [see the notification object documentation](/config/notifications). The top-level map key is id that has to be referenced by the `cmdLists` key `notifications`.
 
 ```yaml
 notifications:

pkg/backy/config.go

@@ -90,7 +90,6 @@ func (opts *ConfigOpts) InitConfig() {
 	} else {
 		loadDefaultConfigFiles(fetcher, configFiles, backyKoanf, opts)
 	}
-
 	opts.koanf = backyKoanf
 }
 
@@ -117,7 +116,9 @@ func loadDefaultConfigFiles(fetcher remotefetcher.RemoteFetcher, configFiles []s
 
 		if data != nil {
 			if err := k.Load(rawbytes.Provider(data), yaml.Parser()); err == nil {
-				continue
+				break
+			} else {
+				logging.ExitWithMSG(fmt.Sprintf("error loading config from file %s: %v", c, err), 1, &opts.Logger)
 			}
 		}
 	}
@@ -135,7 +136,12 @@ func (opts *ConfigOpts) ReadConfig() *ConfigOpts {
 	opts.loadEnv()
 
 	if backyKoanf.Bool(getNestedConfig("logging", "cmd-std-out")) {
-		os.Setenv("BACKY_STDOUT", "enabled")
+		os.Setenv("BACKY_CMDSTDOUT", "enabled")
+	}
+
+	// override the default value of cmd-std-out if flag is set
+	if opts.CmdStdOut {
+		os.Setenv("BACKY_CMDSTDOUT", "enabled")
 	}
 
 	CheckConfigValues(backyKoanf, opts.ConfigFilePath)
@@ -230,7 +236,7 @@ func setupLogger(opts *ConfigOpts) zerolog.Logger {
 
 func unmarshalConfig(k *koanf.Koanf, key string, target interface{}, log zerolog.Logger) {
 	if err := k.UnmarshalWithConf(key, target, koanf.UnmarshalConf{Tag: "yaml"}); err != nil {
-		logging.ExitWithMSG(fmt.Sprintf("error unmarshalling %s struct: %v", key, err), 1, &log)
+		logging.ExitWithMSG(fmt.Sprintf("error unmarshalling key %s into struct: %v", key, err), 1, &log)
 	}
 }
 
@@ -274,10 +280,12 @@ func loadCommandLists(opts *ConfigOpts, backyKoanf *koanf.Koanf) {
 	// if config file is remote, use the directory of the remote file
 	if isRemoteURL(opts.ConfigFilePath) {
 		_, u = getRemoteDir(opts.ConfigFilePath)
+		// // Still use local list files if a remote config file is used, but use them last
 		listConfigFiles = []string{u.JoinPath("lists.yml").String(), u.JoinPath("lists.yaml").String()}
 	} else {
 		backyConfigFileDir = path.Dir(opts.ConfigFilePath)
 		listConfigFiles = []string{
+			// "./lists.yml", "./lists.yaml",
 			path.Join(backyConfigFileDir, "lists.yml"),
 			path.Join(backyConfigFileDir, "lists.yaml"),
 		}
@@ -291,9 +299,9 @@ func loadCommandLists(opts *ConfigOpts, backyKoanf *koanf.Koanf) {
 		}
 	}
 
-	if backyKoanf.Exists("cmd-lists") {
-		unmarshalConfig(backyKoanf, "cmd-lists", &opts.CmdConfigLists, opts.Logger)
-		if backyKoanf.Exists("cmd-lists.file") {
+	if backyKoanf.Exists("cmdLists") {
+		unmarshalConfig(backyKoanf, "cmdLists", &opts.CmdConfigLists, opts.Logger)
+		if backyKoanf.Exists("cmdLists.file") {
 			loadCmdListsFile(backyKoanf, listsConfig, opts)
 		}
 	}
@@ -319,6 +327,7 @@ func loadListConfigFile(filePath string, k *koanf.Koanf, opts *ConfigOpts) bool
 	if err != nil {
 		// if file not found, ignore
 		if errors.Is(err, remotefetcher.ErrIgnoreFileNotFound) {
+			println("File not found", filePath)
 			return true
 		}
 
@@ -334,12 +343,14 @@ func loadListConfigFile(filePath string, k *koanf.Koanf, opts *ConfigOpts) bool
 		return false
 	}
 
+	unmarshalConfig(k, "cmdLists", &opts.CmdConfigLists, opts.Logger)
+	keyNotSupported("cmd-lists", "cmdLists", k, opts, true)
 	opts.CmdListFile = filePath
 	return true
 }
 
 func loadCmdListsFile(backyKoanf *koanf.Koanf, listsConfig *koanf.Koanf, opts *ConfigOpts) {
-	opts.CmdListFile = strings.TrimSpace(backyKoanf.String("cmd-lists.file"))
+	opts.CmdListFile = strings.TrimSpace(backyKoanf.String("cmdLists.file"))
 	if !path.IsAbs(opts.CmdListFile) {
 		opts.CmdListFile = path.Join(path.Dir(opts.ConfigFilePath), opts.CmdListFile)
 	}
@@ -359,14 +370,17 @@ func loadCmdListsFile(backyKoanf *koanf.Koanf, listsConfig *koanf.Koanf, opts *C
 		logging.ExitWithMSG(fmt.Sprintf("error loading config: %v", err), 1, &opts.Logger)
 	}
 
-	unmarshalConfig(listsConfig, "cmd-lists", &opts.CmdConfigLists, opts.Logger)
+	keyNotSupported("cmd-lists", "cmdLists", listsConfig, opts, true)
+	unmarshalConfig(listsConfig, "cmdLists", &opts.CmdConfigLists, opts.Logger)
 	opts.Logger.Info().Str("using lists config file", opts.CmdListFile).Send()
 }
 
 func validateCommandLists(opts *ConfigOpts) {
 	var cmdNotFoundSliceErr []error
 	for cmdListName, cmdList := range opts.CmdConfigLists {
+		// if cron is enabled and cron is not set, delete the list
 		if opts.cronEnabled && strings.TrimSpace(cmdList.Cron) == "" {
+			opts.Logger.Debug().Str("cron", "enabled").Str("list", cmdListName).Msg("cron not set, deleting list")
 			delete(opts.CmdConfigLists, cmdListName)
 			continue
 		}
@@ -406,9 +420,9 @@ func getLoggingKeyFromConfig(key string) string {
 	return fmt.Sprintf("logging.%s", key)
 }
 
-func getCmdListFromConfig(list string) string {
-	return fmt.Sprintf("cmd-lists.%s", list)
-}
+// func getCmdListFromConfig(list string) string {
+// 	return fmt.Sprintf("cmdLists.%s", list)
+// }
 
 func (opts *ConfigOpts) setupVault() error {
 	if !opts.koanf.Bool("vault.enabled") {
@@ -551,6 +565,16 @@ func processCmds(opts *ConfigOpts) error {
 				opts.Hosts[*cmd.Host] = &Host{Host: *cmd.Host}
 				cmd.RemoteHost = &Host{Host: *cmd.Host}
 			}
+		} else {
+
+			if cmd.Dir != nil {
+
+				cmdDir, err := resolveDir(*cmd.Dir)
+				if err != nil {
+					return err
+				}
+				cmd.Dir = &cmdDir
+			}
 		}
 
 		// Parse package commands
@@ -677,3 +701,14 @@ func detectOSType(cmd *Command, opts *ConfigOpts) error {
 	}
 	return nil
 }
+
+func keyNotSupported(oldKey, newKey string, koanf *koanf.Koanf, opts *ConfigOpts, deprecated bool) {
+
+	if koanf.Exists(oldKey) {
+		if deprecated {
+			opts.Logger.Warn().Str("key", oldKey).Msg("key is deprecated. Use " + newKey + " instead.")
+		} else {
+			opts.Logger.Fatal().Err(fmt.Errorf("key %s found; it has changed to %s", oldKey, newKey)).Send()
+		}
+	}
+}

pkg/backy/cron.go

@@ -17,10 +17,7 @@ func (opts *ConfigOpts) Cron() {
 	s := gocron.NewScheduler(time.Local)
 	s.TagsUnique()
 	cmdLists := opts.CmdConfigLists
-	for listName, config := range cmdLists {
-		if config.Name == "" {
-			config.Name = listName
-		}
+	for _, config := range cmdLists {
 
 		cron := strings.TrimSpace(config.Cron)
 		if cron != "" {

pkg/backy/list.go

@@ -23,8 +23,7 @@ func (opts *ConfigOpts) ListCommand(cmd string) {
 	var cmdFound bool = false
 	var cmdInfo *Command
 	// check commands in file against cmd
-	for _, cmdInFile := range opts.executeCmds {
-		print(cmdInFile)
+	for cmdInFile := range opts.Cmds {
 		cmdFound = false
 
 		if cmd == cmdInFile {
@@ -37,28 +36,34 @@ func (opts *ConfigOpts) ListCommand(cmd string) {
 	// print the command's information
 	if cmdFound {
 
-		print("Command: ")
+		println("Command: ")
 
 		print(cmdInfo.Cmd)
-		if len(cmdInfo.Args) >= 0 {
 
 		for _, v := range cmdInfo.Args {
 			print(" ") // print space between command and args
 			print(v)   // print command arg
 		}
-		}
 
-		// is is remote or local
+		// is it remote or local
 		if cmdInfo.Host != nil {
-
-			print("Host: ", cmdInfo.Host)
+			println()
+			print("Host: ", *cmdInfo.Host)
+			println()
 
 		} else {
 
+			println()
 			print("Host: Runs on Local Machine\n\n")
 
 		}
 
+		if cmdInfo.Dir != nil {
+			println()
+			print("Directory: ", *cmdInfo.Dir)
+			println()
+		}
+
 	} else {
 
 		fmt.Printf("Command %s not found. Check spelling.\n", cmd)
@@ -66,3 +71,38 @@ func (opts *ConfigOpts) ListCommand(cmd string) {
 	}
 
 }
+
+func (opts *ConfigOpts) ListCommandList(list string) {
+	// bool for commands not found
+	// gets set to false if a command is not found
+	// set to true if the command is found
+	var listFound bool = false
+	var listInfo *CmdList
+	// check commands in file against cmd
+	for listInFile, l := range opts.CmdConfigLists {
+		listFound = false
+
+		if list == listInFile {
+			listFound = true
+			listInfo = l
+			break
+		}
+	}
+
+	// print the command's information
+	if listFound {
+
+		println("List: ", list)
+		println()
+
+		for _, v := range listInfo.Order {
+			println()
+			opts.ListCommand(v)
+		}
+
+	} else {
+
+		fmt.Printf("List %s not found. Check spelling.\n", list)
+
+	}
+}

pkg/backy/types.go

@@ -77,7 +77,6 @@ type (
 
 		/*
 			Dir specifies a directory in which to run the command.
-			Ignored if Host is set.
 		*/
 		Dir *string `yaml:"dir,omitempty"`
 
@@ -187,7 +186,7 @@ type (
 
 		// CmdConfigLists holds the lists of commands to be run in order.
 		// Key is the command list name.
-		CmdConfigLists map[string]*CmdList `yaml:"cmd-lists"`
+		CmdConfigLists map[string]*CmdList `yaml:"cmdLists"`
 
 		// Hosts holds the Host config.
 		// key is the host.
@@ -198,6 +197,8 @@ type (
 		// Global log level
 		BackyLogLvl *string
 
+		CmdStdOut bool
+
 		// Holds config file
 		ConfigFilePath string
 

pkg/backy/utils.go

@@ -64,6 +64,13 @@ func SetLogFile(logFile string) BackyOptionFunc {
 	}
 }
 
+// SetCmdStdOut forces the command output to stdout
+func SetCmdStdOut(setStdOut bool) BackyOptionFunc {
+	return func(bco *ConfigOpts) {
+		bco.CmdStdOut = setStdOut
+	}
+}
+
 // EnableCron enables the execution of command lists at specified times
 func EnableCron() BackyOptionFunc {
 	return func(bco *ConfigOpts) {
@@ -182,10 +189,12 @@ func IsTerminalActive() bool {
 }
 
 func IsCmdStdOutEnabled() bool {
-	return os.Getenv("BACKY_STDOUT") == "enabled"
+	return os.Getenv("BACKY_CMDSTDOUT") == "enabled"
 }
 
 func resolveDir(path string) (string, error) {
+	path = strings.TrimSpace(path)
+
 	if path == "~" {
 		homeDir, err := os.UserHomeDir()
 		if err != nil {