v0.9.0

.changes/unreleased/.gitkeep

@@ -0,0 +1 @@
+*.yaml

.changes/v0.9.0.md

@@ -0,0 +1,12 @@
+## v0.9.0 - 2025-02-28
+### Added
+* `list` command with subcommands `cmds` and `lists`
+* Deprecation and unsupported warnings for old config keys
+* CLI flag `--cmdStdOut` to output command's stdout/stderr to stdout
+* Command type `remoteScript`. See docs for more info.
+### Changed
+* change to enums for Command type
+* Cache now stores resources by URL hash for ease-of-lookup
+* Changed PackageOperation to enums
+### Fixed
+* Local command's `dir` full path is now found with home directory

.gitignore

@@ -1,12 +1,10 @@
+!.changie.yaml
+!.changes/**
 
 dist/
 .codegpt
 
 *.log
 *.sh
-*.yaml
-*.yml
-
-
-+.changie.yaml
-+.changes/
+/*.yaml
+/*.yml

CHANGELOG.md

@@ -6,6 +6,19 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html),
 and is generated by [Changie](https://github.com/miniscruff/changie).
 
 
+## v0.9.0 - 2025-02-28
+### Added
+* `list` command with subcommands `cmds` and `lists`
+* Deprecation and unsupported warnings for old config keys
+* CLI flag `--cmdStdOut` to output command's stdout/stderr to stdout
+* Command type `remoteScript`. See docs for more info.
+### Changed
+* change to enums for Command type
+* Cache now stores resources by URL hash for ease-of-lookup
+* Changed PackageOperation to enums
+### Fixed
+* Local command's `dir` full path is now found with home directory
+
 ## v0.8.0 - 2025-02-15
 ### Changed
 * Breaking: `cmd-lists` key changed to `cmdLists`

backy.code-workspace

@@ -7,14 +7,18 @@
 	],
 	"settings": {
 		"cSpell.words": [
+			"Autorestic",
+			"changie",
 			"Cmds",
-			"remotefetcher",
+			"CMDSTDOUT",
+			"goreleaser",
 			"knadh",
 			"koanf",
 			"mattn",
 			"maunium",
 			"mautrix",
 			"nikoksr",
+			"remotefetcher",
 			"Strs"
 		]
 	}

cmd/backup.go

@@ -30,7 +30,7 @@ func init() {
 }
 
 func Backup(cmd *cobra.Command, args []string) {
-	backyConfOpts := backy.NewOpts(cfgFile, backy.AddCommandLists(cmdLists))
+	backyConfOpts := backy.NewOpts(cfgFile, backy.AddCommandLists(cmdLists), backy.SetLogFile(logFile), backy.SetCmdStdOut(cmdStdOut))
 	backyConfOpts.InitConfig()
 	backyConfOpts.ReadConfig()
 

cmd/cron.go

@@ -18,7 +18,11 @@ var (
 func cron(cmd *cobra.Command, args []string) {
 	parseS3Config()
 
-	opts := backy.NewOpts(cfgFile, backy.EnableCron())
+	opts := backy.NewOpts(cfgFile,
+		backy.EnableCron(),
+		backy.SetLogFile(logFile),
+		backy.SetCmdStdOut(cmdStdOut))
+
 	opts.InitConfig()
 	opts.ReadConfig()
 

cmd/exec.go

@@ -32,7 +32,7 @@ func execute(cmd *cobra.Command, args []string) {
 		logging.ExitWithMSG("Please provide a command to run. Pass --help to see options.", 1, nil)
 	}
 
-	opts := backy.NewOpts(cfgFile, backy.AddCommands(args), backy.SetLogFile(logFile))
+	opts := backy.NewOpts(cfgFile, backy.AddCommands(args), backy.SetLogFile(logFile), backy.SetCmdStdOut(cmdStdOut))
 	opts.InitConfig()
 	opts.ReadConfig()
 	opts.ExecuteCmds()

cmd/host.go

@@ -35,7 +35,7 @@ func init() {
 //    2. stdin (on command line) (TODO)
 
 func Host(cmd *cobra.Command, args []string) {
-	backyConfOpts := backy.NewOpts(cfgFile, backy.SetLogFile(logFile))
+	backyConfOpts := backy.NewOpts(cfgFile, backy.SetLogFile(logFile), backy.SetCmdStdOut(cmdStdOut))
 	backyConfOpts.InitConfig()
 
 	backyConfOpts.ReadConfig()

cmd/list.go

@@ -6,16 +6,29 @@ package cmd
 
 import (
 	"git.andrewnw.xyz/CyberShell/backy/pkg/backy"
+	"git.andrewnw.xyz/CyberShell/backy/pkg/logging"
 
 	"github.com/spf13/cobra"
 )
 
 var (
 	listCmd = &cobra.Command{
-		Use:   "list [--list=list1,list2,... | -l list1, list2,...] [ -cmd cmd1 cmd2 cmd3...]",
-		Short: "Lists commands, lists, or hosts defined in config file.",
-		Long:  "Backup lists commands or groups defined in config file.\nUse the --lists or -l flag to list the specified lists. If not flag is not given, all lists will be executed.",
-		Run:   List,
+		Use:   "list [command]",
+		Short: "List commands, lists, or hosts defined in config file.",
+		Long:  "List commands, lists, or hosts defined in config file",
+	}
+
+	listCmds = &cobra.Command{
+		Use:   "cmds [cmd1 cmd2 cmd3...]",
+		Short: "List commands defined in config file.",
+		Long:  "List commands defined in config file",
+		Run:   ListCmds,
+	}
+	listCmdLists = &cobra.Command{
+		Use:   "lists [list1 list2 ...]",
+		Short: "List lists defined in config file.",
+		Long:  "List lists defined in config file",
+		Run:   ListCmdLists,
 	}
 )
 
@@ -23,27 +36,51 @@ var listsToList []string
 var cmdsToList []string
 
 func init() {
-
-	listCmd.Flags().StringSliceVarP(&listsToList, "lists", "l", nil, "Accepts comma-separated names of command lists to list.")
-	listCmd.Flags().StringSliceVarP(&cmdsToList, "cmds", "c", nil, "Accepts comma-separated names of commands to list.")
+	listCmd.AddCommand(listCmds, listCmdLists)
 
 }
 
-func List(cmd *cobra.Command, args []string) {
+func ListCmds(cmd *cobra.Command, args []string) {
 
 	// setup based on whats passed in:
 	//   - cmds
 	//   - lists
 	//   - if none, list all commands
-	if cmdLists != nil {
-
+	if len(args) > 0 {
+		cmdsToList = args
+	} else {
+		logging.ExitWithMSG("Error: list cmds subcommand needs commands to list", 1, nil)
 	}
+
 	parseS3Config()
 
-	opts := backy.NewOpts(cfgFile)
+	opts := backy.NewOpts(cfgFile, backy.SetLogFile(logFile))
 
 	opts.InitConfig()
 	opts.ReadConfig()
 
-	opts.ListCommand("rm-sn-db")
+	for _, v := range cmdsToList {
+		opts.ListCommand(v)
+	}
+}
+
+func ListCmdLists(cmd *cobra.Command, args []string) {
+
+	parseS3Config()
+
+	if len(args) > 0 {
+		listsToList = args
+	} else {
+		logging.ExitWithMSG("Error: lists subcommand needs lists", 1, nil)
+	}
+
+	opts := backy.NewOpts(cfgFile, backy.SetLogFile(logFile))
+
+	opts.InitConfig()
+	opts.ReadConfig()
+
+	for _, v := range listsToList {
+		opts.ListCommandList(v)
+	}
+
 }

cmd/root.go

@@ -15,6 +15,7 @@ var (
 	// Used for flags.
 	cfgFile    string
 	verbose    bool
+	cmdStdOut  bool
 	logFile    string
 	s3Endpoint string
 
@@ -35,6 +36,7 @@ func Execute() {
 
 func init() {
 	rootCmd.PersistentFlags().StringVar(&logFile, "log-file", "", "log file to write to")
+	rootCmd.PersistentFlags().BoolVar(&cmdStdOut, "cmdStdOut", false, "Pass to print command output to stdout")
 
 	rootCmd.PersistentFlags().StringVarP(&cfgFile, "config", "f", "", "config file to read from")
 	rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "Sets verbose level")

cmd/version.go

@@ -7,7 +7,7 @@ import (
 	"github.com/spf13/cobra"
 )
 
-const versionStr = "0.8.0"
+const versionStr = "0.9.0"
 
 var (
 	versionCmd = &cobra.Command{

docs/content/cli/_index.md

@@ -19,10 +19,11 @@ Available Commands:
   cron        Starts a scheduler that runs lists defined in config file.
   exec        Runs commands defined in config file in order given.
   help        Help about any command
-  list        Lists commands, lists, or hosts defined in config file.
+  list        List commands, lists, or hosts defined in config file.
   version     Prints the version and exits
 
 Flags:
+      --cmdStdOut            Pass to print command output to stdout
   -f, --config string        config file to read from
   -h, --help                 help for backy
       --log-file string      log file to write to
@@ -48,6 +49,7 @@ Flags:
   -l, --lists stringArray   Accepts comma-separated names of command lists to execute.
 
 Global Flags:
+      --cmdStdOut            Pass to print command output to stdout
   -f, --config string        config file to read from
       --log-file string      log file to write to
       --s3-endpoint string   Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable.
@@ -66,6 +68,7 @@ Flags:
   -h, --help   help for cron
 
 Global Flags:
+      --cmdStdOut            Pass to print command output to stdout
   -f, --config string        config file to read from
       --log-file string      log file to write to
       --s3-endpoint string   Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable.
@@ -88,6 +91,7 @@ Flags:
   -h, --help   help for exec
 
 Global Flags:
+      --cmdStdOut            Pass to print command output to stdout
   -f, --config string        config file to read from
       --log-file string      log file to write to
       --s3-endpoint string   Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable.
@@ -111,6 +115,7 @@ Flags:
   -m, --hosts stringArray     Accepts space-separated names of hosts. Specify multiple times for multiple hosts.
 
 Global Flags:
+      --cmdStdOut            Pass to print command output to stdout
   -f, --config string        config file to read from
       --log-file string      log file to write to
       --s3-endpoint string   Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable.
@@ -131,6 +136,7 @@ Flags:
   -V, --vpre   Output the version with v prefixed.
 
 Global Flags:
+      --cmdStdOut            Pass to print command output to stdout
   -f, --config string        config file to read from
       --log-file string      log file to write to
       --s3-endpoint string   Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable.
@@ -140,18 +146,58 @@ Global Flags:
 ## list
 
 ```
-Backup lists commands or groups defined in config file.
-Use the --lists or -l flag to list the specified lists. If not flag is not given, all lists will be executed.
+List commands, lists, or hosts defined in config file
 
 Usage:
-  backy list [--list=list1,list2,... | -l list1, list2,...] [ -cmd cmd1 cmd2 cmd3...] [flags]
+  backy list [command]
+
+Available Commands:
+  cmds        List commands defined in config file.
+  lists       List lists defined in config file.
 
 Flags:
-  -c, --cmds strings    Accepts comma-separated names of commands to list.
-  -h, --help            help for list
-  -l, --lists strings   Accepts comma-separated names of command lists to list.
+  -h, --help   help for list
 
 Global Flags:
+      --cmdStdOut            Pass to print command output to stdout
+  -f, --config string        config file to read from
+      --log-file string      log file to write to
+      --s3-endpoint string   Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable.
+  -v, --verbose              Sets verbose level
+
+Use "backy list [command] --help" for more information about a command.
+```
+## list cmds
+
+```
+List commands defined in config file
+
+Usage:
+  backy list cmds [cmd1 cmd2 cmd3...] [flags]
+
+Flags:
+  -h, --help   help for cmds
+
+Global Flags:
+      --cmdStdOut            Pass to print command output to stdout
+  -f, --config string        config file to read from
+      --log-file string      log file to write to
+      --s3-endpoint string   Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable.
+  -v, --verbose              Sets verbose level
+```
+## list lists
+
+```
+List lists defined in config file
+
+Usage:
+  backy list lists [list1 list2 ...] [flags]
+
+Flags:
+  -h, --help   help for lists
+
+Global Flags:
+      --cmdStdOut            Pass to print command output to stdout
   -f, --config string        config file to read from
       --log-file string      log file to write to
       --s3-endpoint string   Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable.

docs/content/cli/exec.md

@@ -7,13 +7,13 @@ The `exec` subcommand can do some things that the configuration file can't do ye
 `exec host` takes the following arguments:
 
 ```sh
-  -c, --commands strings   Accepts comma-separated names of commands.
+  -c, --commands strings   Accepts space-separated names of commands.
   -h, --help               help for host
-  -m, --hosts strings      Accepts comma-separated names of hosts.
+  -m, --hosts strings      Accepts space-separated names of hosts.
 ```
 
 The commands have to be defined in the config file. The hosts need to at least be in the ssh_config(5) file.
 
 ```sh
-backy exec host [--commands=command1,command2, ... | -c command1,command2, ...] [--hosts=host1,hosts2, ... | -m host1,host2, ...]  [flags]
+backy exec host [--commands=command1 -commands=command2 ... | -c command1 -c command2 ...] [--hosts=host1 --hosts=hosts2 ... | -m host1 -c host2 ...]  [flags]
 ```

docs/content/config/remote-resources.md

@@ -14,4 +14,4 @@ If using S3, you should use the s3 protocol URI: `s3://bucketName/key/path`. You
 
 ## Scripts
 
-Scripts will be coming later.
+Remote script support is currently limited to http/https endpoints.

getCommandHelp

@@ -1,67 +1,83 @@
 #!/bin/bash
 CLI_PAGE="docs/content/cli/_index.md"
+
+BACKYCOMMAND="go run backy.go"
+
+{
 echo "---
-title: "CLI"
+title: CLI
 weight: 4
 ---
 
 This page lists documentation for the CLI.
-" > _index.md
+" 
 
-BACKYCOMMAND="go run backy.go"
-
-echo "## Backy " >> _index.md
-echo " " >> _index.md
-echo "\`\`\`" >> _index.md
-eval "${BACKYCOMMAND} -h >>  _index.md"
-echo "\`\`\`" >> _index.md
-echo " " >> _index.md
+echo "## Backy "
+echo " "
+echo "\`\`\`"
+eval "${BACKYCOMMAND} -h"
+echo "\`\`\`"
+echo " "
 
 
-echo "# Subcommands" >> _index.md
-echo "" >> _index.md
 
-echo "## backup" >> _index.md
-echo "" >> _index.md
-echo "\`\`\`" >> _index.md
-eval "${BACKYCOMMAND} backup -h >>  _index.md"
-echo "\`\`\`" >> _index.md
-echo "" >> _index.md
+echo "# Subcommands" 
+echo "" 
 
-echo "## cron" >> _index.md
-echo "" >> _index.md
-echo "\`\`\`" >> _index.md
-eval "${BACKYCOMMAND} cron -h >>  _index.md"
-echo "\`\`\`" >> _index.md
-echo "" >> _index.md
+echo "## backup" 
+echo "" 
+echo "\`\`\`" 
+eval "${BACKYCOMMAND} backup -h"
+echo "\`\`\`" 
+echo "" 
 
-echo "## exec" >> _index.md
-echo "" >> _index.md
-echo "\`\`\`" >> _index.md
-eval "${BACKYCOMMAND} exec -h >>  _index.md"
-echo "\`\`\`" >> _index.md
-echo "" >> _index.md
+echo "## cron" 
+echo "" 
+echo "\`\`\`" 
+eval "${BACKYCOMMAND} cron -h"
+echo "\`\`\`" 
+echo "" 
 
-echo "### exec host" >> _index.md
-echo "" >> _index.md
-echo "\`\`\`" >> _index.md
-eval "${BACKYCOMMAND} exec host -h >>  _index.md"
-echo "\`\`\`" >> _index.md
-echo "" >> _index.md
+echo "## exec" 
+echo "" 
+echo "\`\`\`" 
+eval "${BACKYCOMMAND} exec -h"
+echo "\`\`\`" 
+echo "" 
+
+echo "### exec host" 
+echo "" 
+echo "\`\`\`" 
+eval "${BACKYCOMMAND} exec host -h"
+echo "\`\`\`" 
+echo "" 
 
 
-echo "## version" >> _index.md
-echo "" >> _index.md
-echo "\`\`\`" >> _index.md
-eval "${BACKYCOMMAND} version -h >>  _index.md"
-echo "\`\`\`" >> _index.md
-echo "" >> _index.md
+echo "## version" 
+echo "" 
+echo "\`\`\`" 
+eval "${BACKYCOMMAND} version -h"
+echo "\`\`\`" 
+echo "" 
 
-echo "## list" >> _index.md
-echo "" >> _index.md
-echo "\`\`\`" >> _index.md
-eval "${BACKYCOMMAND} list -h >>  _index.md"
-echo "\`\`\`" >> _index.md
+echo "## list" 
+echo "" 
+echo "\`\`\`" 
+eval "${BACKYCOMMAND} list -h"
+echo "\`\`\`" 
+
+echo "## list cmds" 
+echo "" 
+echo "\`\`\`" 
+eval "${BACKYCOMMAND} list cmds -h"
+echo "\`\`\`" 
+
+echo "## list lists" 
+echo "" 
+echo "\`\`\`" 
+eval "${BACKYCOMMAND} list lists -h"
+echo "\`\`\`" 
+} >> _index.md
 
 
 mv _index.md "$CLI_PAGE"

go.mod

@@ -8,6 +8,7 @@ replace git.andrewnw.xyz/CyberShell/backy => /home/andrew/Projects/backy
 
 require (
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.76.0
+	github.com/dmarkham/enumer v1.5.11
 	github.com/go-co-op/gocron v1.37.0
 	github.com/hashicorp/vault/api v1.15.0
 	github.com/joho/godotenv v1.5.1
@@ -69,6 +70,7 @@ require (
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
+	github.com/pascaldekloe/name v1.0.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/robfig/cron/v3 v3.0.1 // indirect
 	github.com/rs/xid v1.6.0 // indirect
@@ -83,9 +85,11 @@ require (
 	go.mau.fi/util v0.8.4 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
 	golang.org/x/exp v0.0.0-20250207012021-f9890c6ad9f3 // indirect
-	golang.org/x/net v0.34.0 // indirect
+	golang.org/x/mod v0.23.0 // indirect
+	golang.org/x/net v0.35.0 // indirect
 	golang.org/x/sync v0.11.0 // indirect
 	golang.org/x/sys v0.30.0 // indirect
 	golang.org/x/text v0.22.0 // indirect
 	golang.org/x/time v0.10.0 // indirect
+	golang.org/x/tools v0.30.0 // indirect
 )

go.sum

@@ -30,6 +30,8 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dmarkham/enumer v1.5.11 h1:quorLCaEfzjJ23Pf7PB9lyyaHseh91YfTM/sAD/4Mbo=
+github.com/dmarkham/enumer v1.5.11/go.mod h1:yixql+kDDQRYqcuBM2n9Vlt7NoT9ixgXhaXry8vmRg8=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
@@ -128,6 +130,8 @@ github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zx
 github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
 github.com/nikoksr/notify v1.3.0 h1:UxzfxzAYGQD9a5JYLBTVx0lFMxeHCke3rPCkfWdPgLs=
 github.com/nikoksr/notify v1.3.0/go.mod h1:Xor2hMmkvrCfkCKvXGbcrESez4brac2zQjhd6U2BbeM=
+github.com/pascaldekloe/name v1.0.0 h1:n7LKFgHixETzxpRv2R77YgPUFo85QHGZKrdaYm7eY5U=
+github.com/pascaldekloe/name v1.0.0/go.mod h1:Z//MfYJnH4jVpQ9wkclwu2I2MkHmXTlT9wR5UZScttM=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -184,8 +188,10 @@ golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus=
 golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M=
 golang.org/x/exp v0.0.0-20250207012021-f9890c6ad9f3 h1:qNgPs5exUA+G0C96DrPwNrvLSj7GT/9D+3WMWUcUg34=
 golang.org/x/exp v0.0.0-20250207012021-f9890c6ad9f3/go.mod h1:tujkw807nyEEAamNbDrEGzRav+ilXA7PCRAd6xsmwiU=
-golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0=
-golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k=
+golang.org/x/mod v0.23.0 h1:Zb7khfcRGKk+kqfxFaP5tZqCnDZMjC5VtUBs87Hr6QM=
+golang.org/x/mod v0.23.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY=
+golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8=
+golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk=
 golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w=
 golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -199,6 +205,8 @@ golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM=
 golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
 golang.org/x/time v0.10.0 h1:3usCWA8tQn0L8+hFJQNgzpWbd89begxN66o1Ojdn5L4=
 golang.org/x/time v0.10.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
+golang.org/x/tools v0.30.0 h1:BgcpHewrV5AUp2G9MebG4XPFI1E2W41zU1SaqVA9vJY=
+golang.org/x/tools v0.30.0/go.mod h1:c347cR/OJfw5TI+GfX7RUPNMdDRRbjvYTS0jPyvsVtY=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=

pkg/backy/backy.go

@@ -47,22 +47,19 @@ func (command *Command) RunCmd(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([
 		outputArr []string // holds the output strings returned by processes
 	)
 
-	// Get the command type
-	// This must be done before concatenating the arguments
-	command = getCommandType(command)
+	// Getting the command type must be done before concatenating the arguments
+	command = getCommandTypeAndSetCommandInfo(command)
 
 	for _, v := range command.Args {
 		ArgsStr += fmt.Sprintf(" %s", v)
 	}
 
-	// print the user's password if it is updated
-	if command.Type == "user" {
+	if command.Type == UserCT {
 		if command.UserOperation == "password" {
 			cmdCtxLogger.Info().Str("password", command.UserPassword).Msg("user password to be updated")
 		}
 	}
 
-	// is host defined
 	if command.Host != nil {
 		outputArr, errSSH = command.RunCmdSSH(cmdCtxLogger, opts)
 		if errSSH != nil {
@@ -71,7 +68,7 @@ func (command *Command) RunCmd(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([
 	} else {
 
 		// Handle package operations
-		if command.Type == "package" && command.PackageOperation == "checkVersion" {
+		if command.Type == PackageCT && command.PackageOperation == PackOpCheckVersion {
 			cmdCtxLogger.Info().Str("package", command.PackageName).Msg("Checking package versions")
 
 			// Execute the package version command
@@ -88,6 +85,64 @@ func (command *Command) RunCmd(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([
 		}
 
 		var localCMD *exec.Cmd
+		if command.Type == RemoteScriptCT {
+			script, err := command.Fetcher.Fetch(command.Cmd)
+			if err != nil {
+				return nil, err
+			}
+
+			if command.Shell == "" {
+				command.Shell = "sh"
+			}
+			localCMD = exec.Command(command.Shell, command.Args...)
+			injectEnvIntoLocalCMD(envVars, localCMD, cmdCtxLogger)
+
+			cmdOutWriters = io.MultiWriter(&cmdOutBuf)
+
+			if IsCmdStdOutEnabled() {
+				cmdOutWriters = io.MultiWriter(os.Stdout, &cmdOutBuf)
+			}
+			if command.OutputFile != "" {
+				file, err := os.Create(command.OutputFile)
+				if err != nil {
+					return nil, fmt.Errorf("error creating output file: %w", err)
+				}
+				defer file.Close()
+				cmdOutWriters = io.MultiWriter(file, &cmdOutBuf)
+
+				if IsCmdStdOutEnabled() {
+					cmdOutWriters = io.MultiWriter(os.Stdout, file, &cmdOutBuf)
+				}
+
+			}
+
+			localCMD.Stdin = bytes.NewReader(script)
+			localCMD.Stdout = cmdOutWriters
+			localCMD.Stderr = cmdOutWriters
+
+			cmdCtxLogger.Info().Str("Command", fmt.Sprintf("Running remoteScript %s on local machine in %s", command.Cmd, command.Shell)).Send()
+			err = localCMD.Run()
+			if err != nil {
+				return nil, fmt.Errorf("error running remote script: %w", err)
+			}
+
+			outScanner := bufio.NewScanner(&cmdOutBuf)
+
+			for outScanner.Scan() {
+				outMap := make(map[string]interface{})
+				outMap["cmd"] = command.Cmd
+				outMap["output"] = outScanner.Text()
+
+				if str, ok := outMap["output"].(string); ok {
+					outputArr = append(outputArr, str)
+				}
+				if command.OutputToLog {
+					cmdCtxLogger.Info().Fields(outMap).Send()
+				}
+			}
+			return outputArr, nil
+		}
+
 		var err error
 		if command.Shell != "" {
 			cmdCtxLogger.Info().Str("Command", fmt.Sprintf("Running command %s on local machine in %s", command.Name, command.Shell)).Send()
@@ -101,7 +156,7 @@ func (command *Command) RunCmd(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([
 			cmdCtxLogger.Info().Str("Command", fmt.Sprintf("Running command %s on local machine", command.Name)).Send()
 
 			// execute package commands in a shell
-			if command.Type == "package" {
+			if command.Type == PackageCT {
 				cmdCtxLogger.Info().Str("package", command.PackageName).Msg("Executing package command")
 				ArgsStr = fmt.Sprintf("%s %s", command.Cmd, ArgsStr)
 				localCMD = exec.Command("/bin/sh", "-c", ArgsStr)

pkg/backy/commandtype_enumer.go

@@ -0,0 +1,141 @@
+// Code generated by "enumer -linecomment -yaml -text -json -type=CommandType"; DO NOT EDIT.
+
+package backy
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+)
+
+const _CommandTypeName = "scriptscriptFileremoteScriptpackageuser"
+
+var _CommandTypeIndex = [...]uint8{0, 0, 6, 16, 28, 35, 39}
+
+const _CommandTypeLowerName = "scriptscriptfileremotescriptpackageuser"
+
+func (i CommandType) String() string {
+	if i < 0 || i >= CommandType(len(_CommandTypeIndex)-1) {
+		return fmt.Sprintf("CommandType(%d)", i)
+	}
+	return _CommandTypeName[_CommandTypeIndex[i]:_CommandTypeIndex[i+1]]
+}
+
+// An "invalid array index" compiler error signifies that the constant values have changed.
+// Re-run the stringer command to generate them again.
+func _CommandTypeNoOp() {
+	var x [1]struct{}
+	_ = x[DefaultCT-(0)]
+	_ = x[ScriptCT-(1)]
+	_ = x[ScriptFileCT-(2)]
+	_ = x[RemoteScriptCT-(3)]
+	_ = x[PackageCT-(4)]
+	_ = x[UserCT-(5)]
+}
+
+var _CommandTypeValues = []CommandType{DefaultCT, ScriptCT, ScriptFileCT, RemoteScriptCT, PackageCT, UserCT}
+
+var _CommandTypeNameToValueMap = map[string]CommandType{
+	_CommandTypeName[0:0]:        DefaultCT,
+	_CommandTypeLowerName[0:0]:   DefaultCT,
+	_CommandTypeName[0:6]:        ScriptCT,
+	_CommandTypeLowerName[0:6]:   ScriptCT,
+	_CommandTypeName[6:16]:       ScriptFileCT,
+	_CommandTypeLowerName[6:16]:  ScriptFileCT,
+	_CommandTypeName[16:28]:      RemoteScriptCT,
+	_CommandTypeLowerName[16:28]: RemoteScriptCT,
+	_CommandTypeName[28:35]:      PackageCT,
+	_CommandTypeLowerName[28:35]: PackageCT,
+	_CommandTypeName[35:39]:      UserCT,
+	_CommandTypeLowerName[35:39]: UserCT,
+}
+
+var _CommandTypeNames = []string{
+	_CommandTypeName[0:0],
+	_CommandTypeName[0:6],
+	_CommandTypeName[6:16],
+	_CommandTypeName[16:28],
+	_CommandTypeName[28:35],
+	_CommandTypeName[35:39],
+}
+
+// CommandTypeString retrieves an enum value from the enum constants string name.
+// Throws an error if the param is not part of the enum.
+func CommandTypeString(s string) (CommandType, error) {
+	if val, ok := _CommandTypeNameToValueMap[s]; ok {
+		return val, nil
+	}
+
+	if val, ok := _CommandTypeNameToValueMap[strings.ToLower(s)]; ok {
+		return val, nil
+	}
+	return 0, fmt.Errorf("%s does not belong to CommandType values", s)
+}
+
+// CommandTypeValues returns all values of the enum
+func CommandTypeValues() []CommandType {
+	return _CommandTypeValues
+}
+
+// CommandTypeStrings returns a slice of all String values of the enum
+func CommandTypeStrings() []string {
+	strs := make([]string, len(_CommandTypeNames))
+	copy(strs, _CommandTypeNames)
+	return strs
+}
+
+// IsACommandType returns "true" if the value is listed in the enum definition. "false" otherwise
+func (i CommandType) IsACommandType() bool {
+	for _, v := range _CommandTypeValues {
+		if i == v {
+			return true
+		}
+	}
+	return false
+}
+
+// MarshalJSON implements the json.Marshaler interface for CommandType
+func (i CommandType) MarshalJSON() ([]byte, error) {
+	return json.Marshal(i.String())
+}
+
+// UnmarshalJSON implements the json.Unmarshaler interface for CommandType
+func (i *CommandType) UnmarshalJSON(data []byte) error {
+	var s string
+	if err := json.Unmarshal(data, &s); err != nil {
+		return fmt.Errorf("CommandType should be a string, got %s", data)
+	}
+
+	var err error
+	*i, err = CommandTypeString(s)
+	return err
+}
+
+// MarshalText implements the encoding.TextMarshaler interface for CommandType
+func (i CommandType) MarshalText() ([]byte, error) {
+	return []byte(i.String()), nil
+}
+
+// UnmarshalText implements the encoding.TextUnmarshaler interface for CommandType
+func (i *CommandType) UnmarshalText(text []byte) error {
+	var err error
+	*i, err = CommandTypeString(string(text))
+	return err
+}
+
+// MarshalYAML implements a YAML Marshaler for CommandType
+func (i CommandType) MarshalYAML() (interface{}, error) {
+	return i.String(), nil
+}
+
+// UnmarshalYAML implements a YAML Unmarshaler for CommandType
+func (i *CommandType) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var s string
+	if err := unmarshal(&s); err != nil {
+		return err
+	}
+
+	var err error
+	*i, err = CommandTypeString(s)
+	return err
+}

pkg/backy/config.go

@@ -53,7 +53,7 @@ func (opts *ConfigOpts) InitConfig() {
 	cacheDir := homeCacheDir
 
 	// Load metadata from file
-	opts.CachedData, err = remotefetcher.LoadMetadataFromFile(path.Join(backyHomeConfDir, "cache.yml"))
+	opts.CachedData, err = remotefetcher.LoadMetadataFromFile(path.Join(backyHomeConfDir, "cache", "cache.yml"))
 	if err != nil {
 		fmt.Println("Error loading metadata:", err)
 		logging.ExitWithMSG(err.Error(), 1, &opts.Logger)
@@ -77,11 +77,13 @@ func (opts *ConfigOpts) InitConfig() {
 	if err != nil {
 		logging.ExitWithMSG(fmt.Sprintf("error initializing cache: %v", err), 1, nil)
 	}
-	// Initialize the fetcher
-	// println("Creating new fetcher for source", opts.ConfigFilePath)
-	fetcher, err := remotefetcher.NewRemoteFetcher(opts.ConfigFilePath, opts.Cache)
-	// println("Created new fetcher for source", opts.ConfigFilePath)
 
+	fetcher, err := remotefetcher.NewRemoteFetcher(opts.ConfigFilePath, opts.Cache)
+
+	if isRemoteURL(opts.ConfigFilePath) {
+		p, _ := getRemoteDir(opts.ConfigFilePath)
+		opts.ConfigDir = p
+	}
 	if err != nil {
 		logging.ExitWithMSG(fmt.Sprintf("error initializing config fetcher: %v", err), 1, nil)
 	}
@@ -136,7 +138,12 @@ func (opts *ConfigOpts) ReadConfig() *ConfigOpts {
 	opts.loadEnv()
 
 	if backyKoanf.Bool(getNestedConfig("logging", "cmd-std-out")) {
-		os.Setenv("BACKY_STDOUT", "enabled")
+		os.Setenv("BACKY_CMDSTDOUT", "enabled")
+	}
+
+	// override the default value of cmd-std-out if flag is set
+	if opts.CmdStdOut {
+		os.Setenv("BACKY_CMDSTDOUT", "enabled")
 	}
 
 	CheckConfigValues(backyKoanf, opts.ConfigFilePath)
@@ -269,21 +276,22 @@ func resolveProxyHosts(host *Host, opts *ConfigOpts) {
 }
 
 func loadCommandLists(opts *ConfigOpts, backyKoanf *koanf.Koanf) {
-	var backyConfigFileDir string
 	var listConfigFiles []string
 	var u *url.URL
+	var p string
 	// if config file is remote, use the directory of the remote file
 	if isRemoteURL(opts.ConfigFilePath) {
-		_, u = getRemoteDir(opts.ConfigFilePath)
+		p, u = getRemoteDir(opts.ConfigFilePath)
+		opts.ConfigDir = p
+		println(p)
 		// // Still use local list files if a remote config file is used, but use them last
 		listConfigFiles = []string{u.JoinPath("lists.yml").String(), u.JoinPath("lists.yaml").String()}
 	} else {
-		backyConfigFileDir = path.Dir(opts.ConfigFilePath)
-		// println("backyConfigFileDir", backyConfigFileDir)
+		opts.ConfigDir = path.Dir(opts.ConfigFilePath)
 		listConfigFiles = []string{
 			// "./lists.yml", "./lists.yaml",
-			path.Join(backyConfigFileDir, "lists.yml"),
-			path.Join(backyConfigFileDir, "lists.yaml"),
+			path.Join(opts.ConfigDir, "lists.yml"),
+			path.Join(opts.ConfigDir, "lists.yaml"),
 		}
 	}
 
@@ -323,7 +331,6 @@ func loadListConfigFile(filePath string, k *koanf.Koanf, opts *ConfigOpts) bool
 	if err != nil {
 		// if file not found, ignore
 		if errors.Is(err, remotefetcher.ErrIgnoreFileNotFound) {
-			println("File not found", filePath)
 			return true
 		}
 
@@ -340,6 +347,7 @@ func loadListConfigFile(filePath string, k *koanf.Koanf, opts *ConfigOpts) bool
 	}
 
 	unmarshalConfig(k, "cmdLists", &opts.CmdConfigLists, opts.Logger)
+	keyNotSupported("cmd-lists", "cmdLists", k, opts, true)
 	opts.CmdListFile = filePath
 	return true
 }
@@ -365,6 +373,7 @@ func loadCmdListsFile(backyKoanf *koanf.Koanf, listsConfig *koanf.Koanf, opts *C
 		logging.ExitWithMSG(fmt.Sprintf("error loading config: %v", err), 1, &opts.Logger)
 	}
 
+	keyNotSupported("cmd-lists", "cmdLists", listsConfig, opts, true)
 	unmarshalConfig(listsConfig, "cmdLists", &opts.CmdConfigLists, opts.Logger)
 	opts.Logger.Info().Str("using lists config file", opts.CmdListFile).Send()
 }
@@ -559,14 +568,24 @@ func processCmds(opts *ConfigOpts) error {
 				opts.Hosts[*cmd.Host] = &Host{Host: *cmd.Host}
 				cmd.RemoteHost = &Host{Host: *cmd.Host}
 			}
+		} else {
+
+			if cmd.Dir != nil {
+
+				cmdDir, err := getFullPathWithHomeDir(*cmd.Dir)
+				if err != nil {
+					return err
+				}
+				cmd.Dir = &cmdDir
+			}
 		}
 
 		// Parse package commands
-		if cmd.Type == "package" {
+		if cmd.Type == PackageCT {
 			if cmd.PackageManager == "" {
 				return fmt.Errorf("package manager is required for package command %s", cmd.PackageName)
 			}
-			if cmd.PackageOperation == "" {
+			if cmd.PackageOperation.String() == "" {
 				return fmt.Errorf("package operation is required for package command %s", cmd.PackageName)
 			}
 			if cmd.PackageName == "" {
@@ -575,19 +594,20 @@ func processCmds(opts *ConfigOpts) error {
 			var err error
 
 			// Validate the operation
-			switch cmd.PackageOperation {
-			case "install", "remove", "upgrade", "checkVersion":
+			if cmd.PackageOperation.IsAPackageOperation() {
+
 				cmd.pkgMan, err = pkgman.PackageManagerFactory(cmd.PackageManager, pkgman.WithoutAuth())
 				if err != nil {
 					return err
 				}
-			default:
+			} else {
 				return fmt.Errorf("unsupported package operation %s for command %s", cmd.PackageOperation, cmd.Name)
 			}
+
 		}
 
 		// Parse user commands
-		if cmd.Type == "user" {
+		if cmd.Type == UserCT {
 			if cmd.Username == "" {
 				return fmt.Errorf("username is required for user command %s", cmd.Name)
 			}
@@ -614,12 +634,24 @@ func processCmds(opts *ConfigOpts) error {
 
 		}
 
-		if cmd.Type == "remoteScript" {
+		if cmd.Type == RemoteScriptCT {
+			var fetchErr error
 			if !isRemoteURL(cmd.Cmd) {
 				return fmt.Errorf("remoteScript command %s must be a remote resource", cmdName)
 			}
+			cmd.Fetcher, fetchErr = remotefetcher.NewRemoteFetcher(cmd.Cmd, opts.Cache, remotefetcher.WithFileType("script"))
+			if fetchErr != nil {
+				return fmt.Errorf("error initializing remote fetcher for remoteScript: %v", fetchErr)
+			}
 
 		}
+		if cmd.OutputFile != "" {
+			var err error
+			cmd.OutputFile, err = getFullPathWithHomeDir(cmd.OutputFile)
+			if err != nil {
+				return err
+			}
+		}
 	}
 	return nil
 }
@@ -685,3 +717,14 @@ func detectOSType(cmd *Command, opts *ConfigOpts) error {
 	}
 	return nil
 }
+
+func keyNotSupported(oldKey, newKey string, koanf *koanf.Koanf, opts *ConfigOpts, deprecated bool) {
+
+	if koanf.Exists(oldKey) {
+		if deprecated {
+			opts.Logger.Warn().Str("key", oldKey).Msg("key is deprecated. Use " + newKey + " instead.")
+		} else {
+			opts.Logger.Fatal().Err(fmt.Errorf("key %s found; it has changed to %s", oldKey, newKey)).Send()
+		}
+	}
+}

pkg/backy/list.go

@@ -23,8 +23,7 @@ func (opts *ConfigOpts) ListCommand(cmd string) {
 	var cmdFound bool = false
 	var cmdInfo *Command
 	// check commands in file against cmd
-	for _, cmdInFile := range opts.executeCmds {
-		print(cmdInFile)
+	for cmdInFile := range opts.Cmds {
 		cmdFound = false
 
 		if cmd == cmdInFile {
@@ -37,28 +36,39 @@ func (opts *ConfigOpts) ListCommand(cmd string) {
 	// print the command's information
 	if cmdFound {
 
-		print("Command: ")
+		println("Command: ")
 
 		print(cmdInfo.Cmd)
-		if len(cmdInfo.Args) >= 0 {
 
-			for _, v := range cmdInfo.Args {
-				print(" ") // print space between command and args
-				print(v)   // print command arg
-			}
+		for _, v := range cmdInfo.Args {
+			print(" ") // print space between command and args
+			print(v)   // print command arg
 		}
 
-		// is is remote or local
+		// is it remote or local
 		if cmdInfo.Host != nil {
-
-			print("Host: ", cmdInfo.Host)
+			println()
+			print("Host: ", *cmdInfo.Host)
+			println()
 
 		} else {
 
+			println()
 			print("Host: Runs on Local Machine\n\n")
 
 		}
 
+		if cmdInfo.Dir != nil {
+			println()
+			print("Directory: ", *cmdInfo.Dir)
+			println()
+		}
+
+		if cmdInfo.Type.String() != "" {
+			print("Type: ", cmdInfo.Type.String())
+			println()
+		}
+
 	} else {
 
 		fmt.Printf("Command %s not found. Check spelling.\n", cmd)
@@ -66,3 +76,38 @@ func (opts *ConfigOpts) ListCommand(cmd string) {
 	}
 
 }
+
+func (opts *ConfigOpts) ListCommandList(list string) {
+	// bool for commands not found
+	// gets set to false if a command is not found
+	// set to true if the command is found
+	var listFound bool = false
+	var listInfo *CmdList
+	// check commands in file against cmd
+	for listInFile, l := range opts.CmdConfigLists {
+		listFound = false
+
+		if list == listInFile {
+			listFound = true
+			listInfo = l
+			break
+		}
+	}
+
+	// print the command's information
+	if listFound {
+
+		println("List: ", list)
+		println()
+
+		for _, v := range listInfo.Order {
+			println()
+			opts.ListCommand(v)
+		}
+
+	} else {
+
+		fmt.Printf("List %s not found. Check spelling.\n", list)
+
+	}
+}

pkg/backy/packageoperation_enumer.go

@@ -0,0 +1,145 @@
+// Code generated by "enumer -linecomment -yaml -text -json -type=PackageOperation"; DO NOT EDIT.
+
+package backy
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+)
+
+const _PackageOperationName = "installupgradepurgeremovecheckVersionisInstalled"
+
+var _PackageOperationIndex = [...]uint8{0, 0, 7, 14, 19, 25, 37, 48}
+
+const _PackageOperationLowerName = "installupgradepurgeremovecheckversionisinstalled"
+
+func (i PackageOperation) String() string {
+	if i < 0 || i >= PackageOperation(len(_PackageOperationIndex)-1) {
+		return fmt.Sprintf("PackageOperation(%d)", i)
+	}
+	return _PackageOperationName[_PackageOperationIndex[i]:_PackageOperationIndex[i+1]]
+}
+
+// An "invalid array index" compiler error signifies that the constant values have changed.
+// Re-run the stringer command to generate them again.
+func _PackageOperationNoOp() {
+	var x [1]struct{}
+	_ = x[DefaultPO-(0)]
+	_ = x[PackOpInstall-(1)]
+	_ = x[PackOpUpgrade-(2)]
+	_ = x[PackOpPurge-(3)]
+	_ = x[PackOpRemove-(4)]
+	_ = x[PackOpCheckVersion-(5)]
+	_ = x[PackOpIsInstalled-(6)]
+}
+
+var _PackageOperationValues = []PackageOperation{DefaultPO, PackOpInstall, PackOpUpgrade, PackOpPurge, PackOpRemove, PackOpCheckVersion, PackOpIsInstalled}
+
+var _PackageOperationNameToValueMap = map[string]PackageOperation{
+	_PackageOperationName[0:0]:        DefaultPO,
+	_PackageOperationLowerName[0:0]:   DefaultPO,
+	_PackageOperationName[0:7]:        PackOpInstall,
+	_PackageOperationLowerName[0:7]:   PackOpInstall,
+	_PackageOperationName[7:14]:       PackOpUpgrade,
+	_PackageOperationLowerName[7:14]:  PackOpUpgrade,
+	_PackageOperationName[14:19]:      PackOpPurge,
+	_PackageOperationLowerName[14:19]: PackOpPurge,
+	_PackageOperationName[19:25]:      PackOpRemove,
+	_PackageOperationLowerName[19:25]: PackOpRemove,
+	_PackageOperationName[25:37]:      PackOpCheckVersion,
+	_PackageOperationLowerName[25:37]: PackOpCheckVersion,
+	_PackageOperationName[37:48]:      PackOpIsInstalled,
+	_PackageOperationLowerName[37:48]: PackOpIsInstalled,
+}
+
+var _PackageOperationNames = []string{
+	_PackageOperationName[0:0],
+	_PackageOperationName[0:7],
+	_PackageOperationName[7:14],
+	_PackageOperationName[14:19],
+	_PackageOperationName[19:25],
+	_PackageOperationName[25:37],
+	_PackageOperationName[37:48],
+}
+
+// PackageOperationString retrieves an enum value from the enum constants string name.
+// Throws an error if the param is not part of the enum.
+func PackageOperationString(s string) (PackageOperation, error) {
+	if val, ok := _PackageOperationNameToValueMap[s]; ok {
+		return val, nil
+	}
+
+	if val, ok := _PackageOperationNameToValueMap[strings.ToLower(s)]; ok {
+		return val, nil
+	}
+	return 0, fmt.Errorf("%s does not belong to PackageOperation values", s)
+}
+
+// PackageOperationValues returns all values of the enum
+func PackageOperationValues() []PackageOperation {
+	return _PackageOperationValues
+}
+
+// PackageOperationStrings returns a slice of all String values of the enum
+func PackageOperationStrings() []string {
+	strs := make([]string, len(_PackageOperationNames))
+	copy(strs, _PackageOperationNames)
+	return strs
+}
+
+// IsAPackageOperation returns "true" if the value is listed in the enum definition. "false" otherwise
+func (i PackageOperation) IsAPackageOperation() bool {
+	for _, v := range _PackageOperationValues {
+		if i == v {
+			return true
+		}
+	}
+	return false
+}
+
+// MarshalJSON implements the json.Marshaler interface for PackageOperation
+func (i PackageOperation) MarshalJSON() ([]byte, error) {
+	return json.Marshal(i.String())
+}
+
+// UnmarshalJSON implements the json.Unmarshaler interface for PackageOperation
+func (i *PackageOperation) UnmarshalJSON(data []byte) error {
+	var s string
+	if err := json.Unmarshal(data, &s); err != nil {
+		return fmt.Errorf("PackageOperation should be a string, got %s", data)
+	}
+
+	var err error
+	*i, err = PackageOperationString(s)
+	return err
+}
+
+// MarshalText implements the encoding.TextMarshaler interface for PackageOperation
+func (i PackageOperation) MarshalText() ([]byte, error) {
+	return []byte(i.String()), nil
+}
+
+// UnmarshalText implements the encoding.TextUnmarshaler interface for PackageOperation
+func (i *PackageOperation) UnmarshalText(text []byte) error {
+	var err error
+	*i, err = PackageOperationString(string(text))
+	return err
+}
+
+// MarshalYAML implements a YAML Marshaler for PackageOperation
+func (i PackageOperation) MarshalYAML() (interface{}, error) {
+	return i.String(), nil
+}
+
+// UnmarshalYAML implements a YAML Unmarshaler for PackageOperation
+func (i *PackageOperation) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var s string
+	if err := unmarshal(&s); err != nil {
+		return err
+	}
+
+	var err error
+	*i, err = PackageOperationString(s)
+	return err
+}

pkg/backy/ssh.go

@@ -54,7 +54,7 @@ func (remoteConfig *Host) ConnectToHost(opts *ConfigOpts) error {
 
 	if !remoteConfig.useDefaultConfig {
 		var err error
-		remoteConfig.ConfigFilePath, err = resolveDir(remoteConfig.ConfigFilePath)
+		remoteConfig.ConfigFilePath, err = getFullPathWithHomeDir(remoteConfig.ConfigFilePath)
 		if err != nil {
 			return err
 		}
@@ -63,7 +63,7 @@ func (remoteConfig *Host) ConnectToHost(opts *ConfigOpts) error {
 			return sshConfigFileOpenErr
 		}
 	} else {
-		defaultConfig, _ := resolveDir("~/.ssh/config")
+		defaultConfig, _ := getFullPathWithHomeDir("~/.ssh/config")
 		configFile, sshConfigFileOpenErr = os.Open(defaultConfig)
 		if sshConfigFileOpenErr != nil {
 			return sshConfigFileOpenErr
@@ -242,7 +242,7 @@ func (remoteHost *Host) GetPrivateKeyFileFromConfig() {
 		identityFile = remoteHost.PrivateKeyPath
 	}
 
-	remoteHost.PrivateKeyPath, _ = resolveDir(identityFile)
+	remoteHost.PrivateKeyPath, _ = getFullPathWithHomeDir(identityFile)
 }
 
 // GetPort checks if the port from the config file is 0
@@ -252,7 +252,6 @@ func (remoteHost *Host) GetPort() {
 	// port specifed?
 	// port will be 0 if missing from backy config
 	if port == "0" {
-		// get port from specified SSH config file
 		port, _ = remoteHost.SSHConfigFile.SshConfigFile.Get(remoteHost.Host, "Port")
 
 		if port == "" {
@@ -315,7 +314,6 @@ func (remoteHost *Host) ConnectThroughBastion(log zerolog.Logger) (*ssh.Client,
 		return nil, err
 	}
 
-	// sClient is an ssh client connected to the service host, through the bastion host.
 	sClient := ssh.NewClient(ncc, chans, reqs)
 
 	return sClient, nil
@@ -326,10 +324,10 @@ func (remoteHost *Host) ConnectThroughBastion(log zerolog.Logger) (*ssh.Client,
 func (remotehHost *Host) GetKnownHosts() error {
 	var knownHostsFileErr error
 	if TS(remotehHost.KnownHostsFile) != "" {
-		remotehHost.KnownHostsFile, knownHostsFileErr = resolveDir(remotehHost.KnownHostsFile)
+		remotehHost.KnownHostsFile, knownHostsFileErr = getFullPathWithHomeDir(remotehHost.KnownHostsFile)
 		return knownHostsFileErr
 	}
-	remotehHost.KnownHostsFile, knownHostsFileErr = resolveDir("~/.ssh/known_hosts")
+	remotehHost.KnownHostsFile, knownHostsFileErr = getFullPathWithHomeDir("~/.ssh/known_hosts")
 	return knownHostsFileErr
 }
 
@@ -338,7 +336,7 @@ func GetPrivateKeyPassword(key string, opts *ConfigOpts, log zerolog.Logger) (st
 	var prKeyPassword string
 	if strings.HasPrefix(key, "file:") {
 		privKeyPassFilePath := strings.TrimPrefix(key, "file:")
-		privKeyPassFilePath, _ = resolveDir(privKeyPassFilePath)
+		privKeyPassFilePath, _ = getFullPathWithHomeDir(privKeyPassFilePath)
 		keyFile, keyFileErr := os.Open(privKeyPassFilePath)
 		if keyFileErr != nil {
 			return "", errors.Errorf("Private key password file %s failed to open. \n Make sure it is accessible and correct.", privKeyPassFilePath)
@@ -370,7 +368,7 @@ func GetPassword(pass string, opts *ConfigOpts, log zerolog.Logger) (string, err
 	var password string
 	if strings.HasPrefix(pass, "file:") {
 		passFilePath := strings.TrimPrefix(pass, "file:")
-		passFilePath, _ = resolveDir(passFilePath)
+		passFilePath, _ = getFullPathWithHomeDir(passFilePath)
 		keyFile, keyFileErr := os.Open(passFilePath)
 		if keyFileErr != nil {
 			return "", errors.New("Password file failed to open")
@@ -442,7 +440,7 @@ func (remoteConfig *Host) GetProxyJumpConfig(hosts map[string]*Host, opts *Confi
 			return sshConfigFileOpenErr
 		}
 	} else {
-		defaultConfig, _ := resolveDir("~/.ssh/config")
+		defaultConfig, _ := getFullPathWithHomeDir("~/.ssh/config")
 		configFile, sshConfigFileOpenErr = os.Open(defaultConfig)
 		if sshConfigFileOpenErr != nil {
 			return sshConfigFileOpenErr
@@ -480,7 +478,6 @@ func (remoteConfig *Host) GetProxyJumpConfig(hosts map[string]*Host, opts *Confi
 	return nil
 }
 
-// RunCmdSSH runs commands over SSH.
 func (command *Command) RunCmdSSH(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([]string, error) {
 	var (
 		ArgsStr       string
@@ -492,10 +489,8 @@ func (command *Command) RunCmdSSH(cmdCtxLogger zerolog.Logger, opts *ConfigOpts)
 			env:  command.Environment,
 		}
 	)
-	// Get the command type
-	// This must be done before concatenating the arguments
-	command.Type = strings.TrimSpace(command.Type)
-	command = getCommandType(command)
+	// Getting the command type must be done before concatenating the arguments
+	command = getCommandTypeAndSetCommandInfo(command)
 
 	// Prepare command arguments
 	for _, v := range command.Args {
@@ -505,7 +500,7 @@ func (command *Command) RunCmdSSH(cmdCtxLogger zerolog.Logger, opts *ConfigOpts)
 	cmdCtxLogger.Info().
 		Str("Command", command.Name).
 		Str("Host", *command.Host).
-		Msgf("Running %s on host %s", getCommandTypeLabel(command.Type), *command.Host)
+		Msgf("Running %s on host %s", getCommandTypeAndSetCommandInfoLabel(command.Type), *command.Host)
 
 	// cmdCtxLogger.Debug().Str("cmd", command.Cmd).Strs("args", command.Args).Send()
 
@@ -536,12 +531,14 @@ func (command *Command) RunCmdSSH(cmdCtxLogger zerolog.Logger, opts *ConfigOpts)
 
 	// Handle command execution based on type
 	switch command.Type {
-	case "script":
+	case ScriptCT:
 		return command.runScript(commandSession, cmdCtxLogger, &cmdOutBuf)
-	case "scriptFile":
+	case RemoteScriptCT:
+		return command.runRemoteScript(commandSession, cmdCtxLogger, &cmdOutBuf)
+	case ScriptFileCT:
 		return command.runScriptFile(commandSession, cmdCtxLogger, &cmdOutBuf)
-	case "package":
-		if command.PackageOperation == "checkVersion" {
+	case PackageCT:
+		if command.PackageOperation == PackOpCheckVersion {
 			commandSession.Stderr = nil
 			// Execute the package version command remotely
 			// Parse the output of package version command
@@ -558,7 +555,7 @@ func (command *Command) RunCmdSSH(cmdCtxLogger zerolog.Logger, opts *ConfigOpts)
 			cmdCtxLogger.Debug().Str("cmd + args", ArgsStr).Send()
 			// Run simple command
 			if err := commandSession.Run(ArgsStr); err != nil {
-				return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, true), fmt.Errorf("error running command: %w", err)
+				return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error running command: %w", err)
 			}
 		}
 	default:
@@ -570,11 +567,11 @@ func (command *Command) RunCmdSSH(cmdCtxLogger zerolog.Logger, opts *ConfigOpts)
 		cmdCtxLogger.Debug().Str("cmd + args", ArgsStr).Send()
 		// Run simple command
 		if err := commandSession.Run(ArgsStr); err != nil {
-			return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.GetOutput), fmt.Errorf("error running command: %w", err)
+			return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error running command: %w", err)
 		}
 	}
 
-	return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, true), nil
+	return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), nil
 }
 
 func checkPackageVersion(cmdCtxLogger zerolog.Logger, command *Command, commandSession *ssh.Session, cmdOutBuf bytes.Buffer) ([]string, error) {
@@ -593,17 +590,17 @@ func checkPackageVersion(cmdCtxLogger zerolog.Logger, command *Command, commandS
 
 		_, parseErr := parsePackageVersion(string(cmdOut), cmdCtxLogger, command, cmdOutBuf)
 		if parseErr != nil {
-			return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.GetOutput), fmt.Errorf("error: package %s not listed: %w", command.PackageName, err)
+			return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error: package %s not listed: %w", command.PackageName, err)
 		}
-		return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.GetOutput), fmt.Errorf("error running %s: %w", ArgsStr, err)
+		return collectOutput(&cmdOutBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error running %s: %w", ArgsStr, err)
 	}
 
 	return parsePackageVersion(string(cmdOut), cmdCtxLogger, command, cmdOutBuf)
 }
 
-// getCommandTypeLabel returns a human-readable label for the command type.
-func getCommandTypeLabel(commandType string) string {
-	if commandType == "" {
+// getCommandTypeAndSetCommandInfoLabel returns a human-readable label for the command type.
+func getCommandTypeAndSetCommandInfoLabel(commandType CommandType) string {
+	if !commandType.IsACommandType() {
 		return "command"
 	}
 	return fmt.Sprintf("%s command", commandType)
@@ -644,7 +641,7 @@ func (command *Command) runScriptFile(session *ssh.Session, cmdCtxLogger zerolog
 		return collectOutput(outputBuf, command.Name, cmdCtxLogger, true), fmt.Errorf("error waiting for shell: %w", err)
 	}
 
-	return collectOutput(outputBuf, command.Name, cmdCtxLogger, true), nil
+	return collectOutput(outputBuf, command.Name, cmdCtxLogger, command.OutputToLog), nil
 }
 
 // prepareScriptBuffer prepares a buffer for inline scripts.
@@ -688,9 +685,28 @@ func (command *Command) prepareScriptFileBuffer() (*bytes.Buffer, error) {
 	return &buffer, nil
 }
 
+// runRemoteScript handles the execution of remote scripts
+func (command *Command) runRemoteScript(session *ssh.Session, cmdCtxLogger zerolog.Logger, outputBuf *bytes.Buffer) ([]string, error) {
+	script, err := command.Fetcher.Fetch(command.Cmd)
+	if err != nil {
+		return nil, err
+	}
+	if command.Shell == "" {
+		command.Shell = "sh"
+	}
+	session.Stdin = bytes.NewReader(script)
+	err = session.Run(command.Shell)
+
+	if err != nil {
+		return collectOutput(outputBuf, command.Name, cmdCtxLogger, command.OutputToLog), fmt.Errorf("error running remote script: %w", err)
+	}
+
+	return collectOutput(outputBuf, command.Name, cmdCtxLogger, command.OutputToLog), nil
+}
+
 // readFileToBuffer reads a file into a buffer.
 func readFileToBuffer(filePath string) (*bytes.Buffer, error) {
-	resolvedPath, err := resolveDir(filePath)
+	resolvedPath, err := getFullPathWithHomeDir(filePath)
 	if err != nil {
 		return nil, err
 	}

pkg/backy/tools.go

@@ -0,0 +1,8 @@
+//go:build tools
+
+package backy
+
+import (
+	// Protect this entry in go.mod from being removed by go mod tidy.
+	_ "github.com/dmarkham/enumer"
+)

pkg/backy/types.go

@@ -54,9 +54,8 @@ type (
 		// command to run
 		Cmd string `yaml:"cmd"`
 
-		// Possible values: script, scriptFile
-		// If blank, it is regular command.
-		Type string `yaml:"type,omitempty"`
+		// See CommandType enum further down the page for acceptable values
+		Type CommandType `yaml:"type,omitempty"`
 
 		// host on which to run cmd
 		Host *string `yaml:"host,omitempty"`
@@ -77,7 +76,6 @@ type (
 
 		/*
 			Dir specifies a directory in which to run the command.
-			Ignored if Host is set.
 		*/
 		Dir *string `yaml:"dir,omitempty"`
 
@@ -94,6 +92,10 @@ type (
 
 		ScriptEnvFile string `yaml:"scriptEnvFile"`
 
+		OutputToLog bool `yaml:"outputToLog,omitempty"`
+
+		OutputFile string `yaml:"outputFile,omitempty"`
+
 		// BEGIN PACKAGE COMMAND FIELDS
 
 		PackageManager string `yaml:"packageManager,omitempty"`
@@ -104,7 +106,7 @@ type (
 		PackageVersion string `yaml:"packageVersion,omitempty"`
 
 		// PackageOperation specifies the action for package-related commands (e.g., "install" or "remove")
-		PackageOperation string `yaml:"packageOperation,omitempty"`
+		PackageOperation PackageOperation `yaml:"packageOperation,omitempty"`
 
 		pkgMan pkgman.PackageManager
 
@@ -117,6 +119,8 @@ type (
 		// FetchBeforeExecution determines if the remoteSource should be fetched before running
 		FetchBeforeExecution bool `yaml:"fetchBeforeExecution,omitempty"`
 
+		Fetcher remotefetcher.RemoteFetcher
+
 		// BEGIN USER COMMAND FIELDS
 
 		// Username specifies the username for user creation or related operations
@@ -198,13 +202,14 @@ type (
 		// Global log level
 		BackyLogLvl *string
 
-		// Holds config file
+		CmdStdOut bool
+
 		ConfigFilePath string
 
-		// Holds log file
+		ConfigDir string
+
 		LogFilePath string
 
-		// for command list file
 		CmdListFile string
 
 		// use command lists using cron
@@ -288,4 +293,29 @@ type (
 		ListName string // Name of the command list
 		Error    error  // Error encountered, if any
 	}
+
+	// use ints so we can use enums
+	CommandType      int
+	PackageOperation int
+)
+
+//go:generate go run github.com/dmarkham/enumer -linecomment -yaml -text -json -type=CommandType
+const (
+	DefaultCT      CommandType = iota //
+	ScriptCT                          // script
+	ScriptFileCT                      // scriptFile
+	RemoteScriptCT                    // remoteScript
+	PackageCT                         // package
+	UserCT                            // user
+)
+
+//go:generate go run github.com/dmarkham/enumer -linecomment -yaml -text -json -type=PackageOperation
+const (
+	DefaultPO          PackageOperation = iota //
+	PackOpInstall                              // install
+	PackOpUpgrade                              // upgrade
+	PackOpPurge                                // purge
+	PackOpRemove                               // remove
+	PackOpCheckVersion                         // checkVersion
+	PackOpIsInstalled                          // isInstalled
 )

pkg/backy/utils.go

@@ -15,6 +15,7 @@ import (
 	"strings"
 
 	"git.andrewnw.xyz/CyberShell/backy/pkg/logging"
+	"git.andrewnw.xyz/CyberShell/backy/pkg/remotefetcher"
 	"github.com/joho/godotenv"
 	"github.com/knadh/koanf/v2"
 	"github.com/rs/zerolog"
@@ -64,6 +65,13 @@ func SetLogFile(logFile string) BackyOptionFunc {
 	}
 }
 
+// SetCmdStdOut forces the command output to stdout
+func SetCmdStdOut(setStdOut bool) BackyOptionFunc {
+	return func(bco *ConfigOpts) {
+		bco.CmdStdOut = setStdOut
+	}
+}
+
 // EnableCron enables the execution of command lists at specified times
 func EnableCron() BackyOptionFunc {
 	return func(bco *ConfigOpts) {
@@ -84,7 +92,7 @@ func NewOpts(configFilePath string, opts ...BackyOptionFunc) *ConfigOpts {
 
 func injectEnvIntoSSH(envVarsToInject environmentVars, process *ssh.Session, opts *ConfigOpts, log zerolog.Logger) {
 	if envVarsToInject.file != "" {
-		envPath, envPathErr := resolveDir(envVarsToInject.file)
+		envPath, envPathErr := getFullPathWithHomeDir(envVarsToInject.file)
 		if envPathErr != nil {
 			log.Fatal().Str("envFile", envPath).Err(envPathErr).Send()
 		}
@@ -118,7 +126,7 @@ errEnvFile:
 
 func injectEnvIntoLocalCMD(envVarsToInject environmentVars, process *exec.Cmd, log zerolog.Logger) {
 	if envVarsToInject.file != "" {
-		envPath, _ := resolveDir(envVarsToInject.file)
+		envPath, _ := getFullPathWithHomeDir(envVarsToInject.file)
 
 		file, fileErr := os.Open(envPath)
 		if fileErr != nil {
@@ -182,10 +190,12 @@ func IsTerminalActive() bool {
 }
 
 func IsCmdStdOutEnabled() bool {
-	return os.Getenv("BACKY_STDOUT") == "enabled"
+	return os.Getenv("BACKY_CMDSTDOUT") == "enabled"
 }
 
-func resolveDir(path string) (string, error) {
+func getFullPathWithHomeDir(path string) (string, error) {
+	path = strings.TrimSpace(path)
+
 	if path == "~" {
 		homeDir, err := os.UserHomeDir()
 		if err != nil {
@@ -207,11 +217,31 @@ func resolveDir(path string) (string, error) {
 
 // loadEnv loads a .env file from the config file directory
 func (opts *ConfigOpts) loadEnv() {
-	envFileInConfigDir := fmt.Sprintf("%s/.env", path.Dir(opts.ConfigFilePath))
 	var backyEnv map[string]string
-	backyEnv, envFileErr := godotenv.Read(envFileInConfigDir)
-	if envFileErr != nil {
-		return
+	var envFileInConfigDir string
+	var envFileErr error
+	if isRemoteURL(opts.ConfigFilePath) {
+		_, u := getRemoteDir(opts.ConfigFilePath)
+		envFileInConfigDir = u.JoinPath(".env").String()
+		envFetcher, err := remotefetcher.NewRemoteFetcher(envFileInConfigDir, opts.Cache)
+		if err != nil {
+			return
+		}
+		data, err := envFetcher.Fetch(envFileInConfigDir)
+		if err != nil {
+			return
+		}
+		backyEnv, envFileErr = godotenv.UnmarshalBytes(data)
+		if envFileErr != nil {
+			return
+		}
+
+	} else {
+		envFileInConfigDir = fmt.Sprintf("%s/.env", path.Dir(opts.ConfigFilePath))
+		backyEnv, envFileErr = godotenv.Read(envFileInConfigDir)
+		if envFileErr != nil {
+			return
+		}
 	}
 
 	opts.backyEnv = backyEnv
@@ -242,26 +272,23 @@ func expandEnvVars(backyEnv map[string]string, envVars []string) {
 	}
 }
 
-// getCommandType checks for command type and if the command has already been set
-// Checks for types package and user
-// Returns the modified Command with the package- or userManager command as Cmd and the package- or userOperation as args, plus any additional Args
-func getCommandType(command *Command) *Command {
+func getCommandTypeAndSetCommandInfo(command *Command) *Command {
 
-	if command.Type == "package" && !command.packageCmdSet {
+	if command.Type == PackageCT && !command.packageCmdSet {
 		command.packageCmdSet = true
 		switch command.PackageOperation {
-		case "install":
+		case PackOpInstall:
 			command.Cmd, command.Args = command.pkgMan.Install(command.PackageName, command.PackageVersion, command.Args)
-		case "remove":
+		case PackOpRemove:
 			command.Cmd, command.Args = command.pkgMan.Remove(command.PackageName, command.Args)
-		case "upgrade":
+		case PackOpUpgrade:
 			command.Cmd, command.Args = command.pkgMan.Upgrade(command.PackageName, command.PackageVersion)
-		case "checkVersion":
+		case PackOpCheckVersion:
 			command.Cmd, command.Args = command.pkgMan.CheckVersion(command.PackageName, command.PackageVersion)
 		}
 	}
 
-	if command.Type == "user" && !command.userCmdSet {
+	if command.Type == UserCT && !command.userCmdSet {
 		command.userCmdSet = true
 		switch command.UserOperation {
 		case "add":

pkg/remotefetcher/cache.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"path"
 	"path/filepath"
 	"sync"
 
@@ -16,6 +17,7 @@ type CacheData struct {
 	Hash string `yaml:"hash"`
 	Path string `yaml:"path"`
 	Type string `yaml:"type"`
+	URL  string `yaml:"url"`
 }
 
 type Cache struct {
@@ -71,7 +73,7 @@ func (c *Cache) saveToFile() error {
 	for _, data := range c.store {
 		cacheData = append(cacheData, data)
 	}
-
+	cacheData = unique(cacheData)
 	data, err := yaml.Marshal(cacheData)
 	if err != nil {
 		return err
@@ -101,13 +103,17 @@ func (c *Cache) AddDataToStore(hash string, cacheData CacheData) error {
 	return c.saveToFile()
 }
 
+// Set stores data on disk and in the cache file and returns the cache data
+// The filepath of the data is the file name + a SHA256 hash of the URL
 func (c *Cache) Set(source, hash string, data []byte, dataType string) (CacheData, error) {
 	c.mu.Lock()
 	defer c.mu.Unlock()
 
+	sourceHash := HashURL(source)
+
 	fileName := filepath.Base(source)
 
-	path := filepath.Join(c.dir, fmt.Sprintf("%s-%s", fileName, hash))
+	path := filepath.Join(c.dir, fmt.Sprintf("%s-%s", fileName, sourceHash))
 
 	if _, err := os.Stat(path); os.IsNotExist(err) {
 		os.MkdirAll(c.dir, 0700)
@@ -122,9 +128,10 @@ func (c *Cache) Set(source, hash string, data []byte, dataType string) (CacheDat
 		Hash: hash,
 		Path: path,
 		Type: dataType,
+		URL:  sourceHash,
 	}
 
-	c.store[hash] = cacheData
+	c.store[sourceHash] = cacheData
 
 	// Unlock before calling saveToFile to avoid double-locking
 	c.mu.Unlock()
@@ -164,6 +171,7 @@ func (cf *CachedFetcher) Hash(data []byte) string {
 func LoadMetadataFromFile(filePath string) ([]*CacheData, error) {
 	if _, err := os.Stat(filePath); os.IsNotExist(err) {
 		// Create the file if it does not exist
+		os.MkdirAll(path.Dir(filePath), 0700)
 		emptyData := []byte("[]")
 		err := os.WriteFile(filePath, emptyData, 0644)
 		if err != nil {
@@ -178,9 +186,35 @@ func LoadMetadataFromFile(filePath string) ([]*CacheData, error) {
 
 	var cacheData []*CacheData
 	err = yaml.Unmarshal(data, &cacheData)
+
 	if err != nil {
 		return nil, err
 	}
 
 	return cacheData, nil
 }
+
+func HashURL(url string) string {
+	hash := sha256.Sum256([]byte(url))
+	return hex.EncodeToString(hash[:])
+}
+
+func unique(cache []CacheData) []CacheData {
+	var unique []CacheData
+	type key struct{ value1, value2, value3, value4 string }
+	m := make(map[key]int)
+	for _, v := range cache {
+		k := key{v.URL, v.Hash, v.Path, v.Type}
+		if i, ok := m[k]; ok {
+			// Overwrite previous value per requirement in
+			// question to keep last matching value.
+			unique[i] = v
+		} else {
+			// Unique key found. Record position and collect
+			// in result.
+			m[k] = len(unique)
+			unique = append(unique, v)
+		}
+	}
+	return unique
+}

pkg/remotefetcher/configfetcher.go

@@ -57,11 +57,13 @@ func NewRemoteFetcher(source string, cache *Cache, options ...FetcherOption) (Re
 		return nil, err
 	}
 
-	hash := fetcher.Hash(data)
-	if cachedData, cacheMeta, exists := cache.Get(hash); exists {
+	URLHash := HashURL(source)
+	if cachedData, cacheMeta, exists := cache.Get(URLHash); exists {
+		println(cachedData)
 		return &CachedFetcher{data: cachedData, path: cacheMeta.Path, dataType: cacheMeta.Type}, nil
 	}
 
+	hash := fetcher.Hash(data)
 	cacheData, err := cache.Set(source, hash, data, config.FileType)
 	if err != nil {
 		return nil, err

pkg/remotefetcher/s3.go

@@ -49,6 +49,7 @@ func NewS3Fetcher(endpoint string, options ...FetcherOption) (*S3Fetcher, error)
 		println(err.Error())
 		return nil, err
 	}
+
 	// Initialize S3 client if not provided
 	if cfg.S3Client == nil {
 		s3Client, err = minio.New(s3Endpoint, &minio.Options{
@@ -128,7 +129,6 @@ func (s *S3Fetcher) Hash(data []byte) string {
 }
 
 func getS3Credentials(profile, host string, httpClient *http.Client) (*credentials.Credentials, error) {
-	// println(s3utils.GetRegionFromURL(*u))
 	homeDir, hdirErr := homedir.Dir()
 	if hdirErr != nil {
 		return nil, hdirErr

release

@@ -1,5 +1,6 @@
 #!/bin/bash
 set -eou pipefail
+go generate ./...
 export CURRENT_TAG="$(go run backy.go version -V)"
 goreleaser -f .goreleaser/github.yml check
 goreleaser -f .goreleaser/gitea.yml check

tests/backy.yaml

@@ -0,0 +1,18 @@
+commands:
+  echoTestPass:
+    cmd: echo
+    shell: bash
+    Args: hello world
+
+  runRemoteShellScriptSuccess:
+    cmd: 
+
+
+  packageCommandSuccess:
+    packageName: docker-ce
+    Args:
+      - docker-ce-cli
+    packageManager: apt
+    packageOperation: install
+
+