v0.9.1

.changes/unreleased/Added-20250220-144549.yaml

@@ -1,3 +0,0 @@
-kind: Added
-body: '`list` command with subcommands `cmds` and `lists`'
-time: 2025-02-20T14:45:49.562361581-06:00

.changes/unreleased/Added-20250220-145014.yaml

@@ -1,3 +0,0 @@
-kind: Added
-body: Deprecation and unsupported warnings for old config keys
-time: 2025-02-20T14:50:14.048452348-06:00

.changes/unreleased/Added-20250220-145245.yaml

@@ -1,3 +0,0 @@
-kind: Added
-body: CLI flag `--cmdStdOut` to output command's stdout/stderr to stdout
-time: 2025-02-20T14:52:45.030582408-06:00

.changes/unreleased/Added-20250221-003012.yaml

@@ -1,3 +0,0 @@
-kind: Added
-body: Command type `remoteScript`. See docs for more info.
-time: 2025-02-21T00:30:12.276616792-06:00

.changes/unreleased/Changed-20250223-153012.yaml

@@ -1,3 +0,0 @@
-kind: Changed
-body: change to enums for Command type
-time: 2025-02-23T15:30:12.033359922-06:00

.changes/unreleased/Changed-20250223-153305.yaml

@@ -1,3 +0,0 @@
-kind: Changed
-body: Cache now stores resources by URL hash for ease-of-lookup
-time: 2025-02-23T15:33:05.33444642-06:00

.changes/unreleased/Fixed-20250220-144843.yaml

@@ -1,3 +0,0 @@
-kind: Fixed
-body: Local command's `dir` config key resolved to full path
-time: 2025-02-20T14:48:43.475300515-06:00

.changes/v0.9.0.md

@@ -0,0 +1,12 @@
+## v0.9.0 - 2025-02-28
+### Added
+* `list` command with subcommands `cmds` and `lists`
+* Deprecation and unsupported warnings for old config keys
+* CLI flag `--cmdStdOut` to output command's stdout/stderr to stdout
+* Command type `remoteScript`. See docs for more info.
+### Changed
+* change to enums for Command type
+* Cache now stores resources by URL hash for ease-of-lookup
+* Changed PackageOperation to enums
+### Fixed
+* Local command's `dir` full path is now found with home directory

.changes/v0.9.1.md

@@ -0,0 +1,3 @@
+## v0.9.1 - 2025-03-01
+### Changed
+* Use EnvVar AWS_PROFILE to get S3 profile

CHANGELOG.md

@@ -6,6 +6,23 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html),
 and is generated by [Changie](https://github.com/miniscruff/changie).
 
 
+## v0.9.1 - 2025-03-01
+### Changed
+* Use EnvVar AWS_PROFILE to get S3 profile
+
+## v0.9.0 - 2025-02-28
+### Added
+* `list` command with subcommands `cmds` and `lists`
+* Deprecation and unsupported warnings for old config keys
+* CLI flag `--cmdStdOut` to output command's stdout/stderr to stdout
+* Command type `remoteScript`. See docs for more info.
+### Changed
+* change to enums for Command type
+* Cache now stores resources by URL hash for ease-of-lookup
+* Changed PackageOperation to enums
+### Fixed
+* Local command's `dir` full path is now found with home directory
+
 ## v0.8.0 - 2025-02-15
 ### Changed
 * Breaking: `cmd-lists` key changed to `cmdLists`

backy.code-workspace

@@ -7,8 +7,11 @@
 	],
 	"settings": {
 		"cSpell.words": [
+			"Autorestic",
+			"changie",
 			"Cmds",
 			"CMDSTDOUT",
+			"goreleaser",
 			"knadh",
 			"koanf",
 			"mattn",

cmd/cron.go

@@ -18,7 +18,11 @@ var (
 func cron(cmd *cobra.Command, args []string) {
 	parseS3Config()
 
-	opts := backy.NewOpts(cfgFile, backy.EnableCron(), backy.SetLogFile(logFile), backy.SetCmdStdOut(cmdStdOut))
+	opts := backy.NewOpts(cfgFile,
+		backy.EnableCron(),
+		backy.SetLogFile(logFile),
+		backy.SetCmdStdOut(cmdStdOut))
+
 	opts.InitConfig()
 	opts.ReadConfig()
 

cmd/version.go

@@ -7,7 +7,7 @@ import (
 	"github.com/spf13/cobra"
 )
 
-const versionStr = "0.8.0"
+const versionStr = "0.9.1"
 
 var (
 	versionCmd = &cobra.Command{

docs/content/cli/_index.md

@@ -167,3 +167,39 @@ Global Flags:
 
 Use "backy list [command] --help" for more information about a command.
 ```
+## list cmds
+
+```
+List commands defined in config file
+
+Usage:
+  backy list cmds [cmd1 cmd2 cmd3...] [flags]
+
+Flags:
+  -h, --help   help for cmds
+
+Global Flags:
+      --cmdStdOut            Pass to print command output to stdout
+  -f, --config string        config file to read from
+      --log-file string      log file to write to
+      --s3-endpoint string   Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable.
+  -v, --verbose              Sets verbose level
+```
+## list lists
+
+```
+List lists defined in config file
+
+Usage:
+  backy list lists [list1 list2 ...] [flags]
+
+Flags:
+  -h, --help   help for lists
+
+Global Flags:
+      --cmdStdOut            Pass to print command output to stdout
+  -f, --config string        config file to read from
+      --log-file string      log file to write to
+      --s3-endpoint string   Sets the S3 endpoint used for config file fetching. Overrides S3_ENDPOINT env variable.
+  -v, --verbose              Sets verbose level
+```

docs/content/cli/exec.md

@@ -15,5 +15,5 @@ The `exec` subcommand can do some things that the configuration file can't do ye
 The commands have to be defined in the config file. The hosts need to at least be in the ssh_config(5) file.
 
 ```sh
-backy exec host [--commands command1 -commands command2 ... | -c command1 -c command2 ...] [--hosts host1 --hosts hosts2 ... | -m host1 -c host2 ...]  [flags]
+backy exec host [--commands=command1 -commands=command2 ... | -c command1 -c command2 ...] [--hosts=host1 --hosts=hosts2 ... | -m host1 -c host2 ...]  [flags]
 ```

docs/content/config/remote-resources.md

@@ -12,6 +12,10 @@ For the main config file to be fetched remotely, pass the URL using `-f [url]`.
 
 If using S3, you should use the s3 protocol URI: `s3://bucketName/key/path`. You will also need to set the env variable `S3_ENDPOINT` to the appropriate value. The flag `--s3-endpoint` can be used to override this value or to set this value, if not already set.
 
+## Authentication
+
+Currently, only the AWS authentication credentials file `~/.aws/credentials` is supported. For now, the environment variable `AWS_PROFILE` is used to lookup the profile.
+
 ## Scripts
 
-Scripts will be coming later.
+Remote script support is currently limited to http/https endpoints.

getCommandHelp

@@ -1,67 +1,83 @@
 #!/bin/bash
 CLI_PAGE="docs/content/cli/_index.md"
+
+BACKYCOMMAND="go run backy.go"
+
+{
 echo "---
-title: "CLI"
+title: CLI
 weight: 4
 ---
 
 This page lists documentation for the CLI.
-" > _index.md
+" 
 
-BACKYCOMMAND="go run backy.go"
+echo "## Backy "
-
+echo " "
-echo "## Backy " >> _index.md
+echo "\`\`\`"
-echo " " >> _index.md
+eval "${BACKYCOMMAND} -h"
-echo "\`\`\`" >> _index.md
+echo "\`\`\`"
-eval "${BACKYCOMMAND} -h >>  _index.md"
+echo " "
-echo "\`\`\`" >> _index.md
-echo " " >> _index.md
 
 
-echo "# Subcommands" >> _index.md
-echo "" >> _index.md
 
-echo "## backup" >> _index.md
+echo "# Subcommands" 
-echo "" >> _index.md
+echo "" 
-echo "\`\`\`" >> _index.md
-eval "${BACKYCOMMAND} backup -h >>  _index.md"
-echo "\`\`\`" >> _index.md
-echo "" >> _index.md
 
-echo "## cron" >> _index.md
+echo "## backup" 
-echo "" >> _index.md
+echo "" 
-echo "\`\`\`" >> _index.md
+echo "\`\`\`" 
-eval "${BACKYCOMMAND} cron -h >>  _index.md"
+eval "${BACKYCOMMAND} backup -h"
-echo "\`\`\`" >> _index.md
+echo "\`\`\`" 
-echo "" >> _index.md
+echo "" 
 
-echo "## exec" >> _index.md
+echo "## cron" 
-echo "" >> _index.md
+echo "" 
-echo "\`\`\`" >> _index.md
+echo "\`\`\`" 
-eval "${BACKYCOMMAND} exec -h >>  _index.md"
+eval "${BACKYCOMMAND} cron -h"
-echo "\`\`\`" >> _index.md
+echo "\`\`\`" 
-echo "" >> _index.md
+echo "" 
 
-echo "### exec host" >> _index.md
+echo "## exec" 
-echo "" >> _index.md
+echo "" 
-echo "\`\`\`" >> _index.md
+echo "\`\`\`" 
-eval "${BACKYCOMMAND} exec host -h >>  _index.md"
+eval "${BACKYCOMMAND} exec -h"
-echo "\`\`\`" >> _index.md
+echo "\`\`\`" 
-echo "" >> _index.md
+echo "" 
+
+echo "### exec host" 
+echo "" 
+echo "\`\`\`" 
+eval "${BACKYCOMMAND} exec host -h"
+echo "\`\`\`" 
+echo "" 
 
 
-echo "## version" >> _index.md
+echo "## version" 
-echo "" >> _index.md
+echo "" 
-echo "\`\`\`" >> _index.md
+echo "\`\`\`" 
-eval "${BACKYCOMMAND} version -h >>  _index.md"
+eval "${BACKYCOMMAND} version -h"
-echo "\`\`\`" >> _index.md
+echo "\`\`\`" 
-echo "" >> _index.md
+echo "" 
 
-echo "## list" >> _index.md
+echo "## list" 
-echo "" >> _index.md
+echo "" 
-echo "\`\`\`" >> _index.md
+echo "\`\`\`" 
-eval "${BACKYCOMMAND} list -h >>  _index.md"
+eval "${BACKYCOMMAND} list -h"
-echo "\`\`\`" >> _index.md
+echo "\`\`\`" 
+
+echo "## list cmds" 
+echo "" 
+echo "\`\`\`" 
+eval "${BACKYCOMMAND} list cmds -h"
+echo "\`\`\`" 
+
+echo "## list lists" 
+echo "" 
+echo "\`\`\`" 
+eval "${BACKYCOMMAND} list lists -h"
+echo "\`\`\`" 
+} >> _index.md
 
 
 mv _index.md "$CLI_PAGE"

pkg/backy/backy.go

@@ -54,7 +54,7 @@ func (command *Command) RunCmd(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([
 		ArgsStr += fmt.Sprintf(" %s", v)
 	}
 
-	if command.Type == User {
+	if command.Type == UserCT {
 		if command.UserOperation == "password" {
 			cmdCtxLogger.Info().Str("password", command.UserPassword).Msg("user password to be updated")
 		}
@@ -68,7 +68,7 @@ func (command *Command) RunCmd(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([
 	} else {
 
 		// Handle package operations
-		if command.Type == Package && command.PackageOperation == "checkVersion" {
+		if command.Type == PackageCT && command.PackageOperation == PackOpCheckVersion {
 			cmdCtxLogger.Info().Str("package", command.PackageName).Msg("Checking package versions")
 
 			// Execute the package version command
@@ -85,7 +85,7 @@ func (command *Command) RunCmd(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([
 		}
 
 		var localCMD *exec.Cmd
-		if command.Type == RemoteScript {
+		if command.Type == RemoteScriptCT {
 			script, err := command.Fetcher.Fetch(command.Cmd)
 			if err != nil {
 				return nil, err
@@ -156,7 +156,7 @@ func (command *Command) RunCmd(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([
 			cmdCtxLogger.Info().Str("Command", fmt.Sprintf("Running command %s on local machine", command.Name)).Send()
 
 			// execute package commands in a shell
-			if command.Type == Package {
+			if command.Type == PackageCT {
 				cmdCtxLogger.Info().Str("package", command.PackageName).Msg("Executing package command")
 				ArgsStr = fmt.Sprintf("%s %s", command.Cmd, ArgsStr)
 				localCMD = exec.Command("/bin/sh", "-c", ArgsStr)

pkg/backy/commandtype_enumer.go

@@ -25,29 +25,29 @@ func (i CommandType) String() string {
 // Re-run the stringer command to generate them again.
 func _CommandTypeNoOp() {
 	var x [1]struct{}
-	_ = x[Default-(0)]
+	_ = x[DefaultCT-(0)]
-	_ = x[Script-(1)]
+	_ = x[ScriptCT-(1)]
-	_ = x[ScriptFile-(2)]
+	_ = x[ScriptFileCT-(2)]
-	_ = x[RemoteScript-(3)]
+	_ = x[RemoteScriptCT-(3)]
-	_ = x[Package-(4)]
+	_ = x[PackageCT-(4)]
-	_ = x[User-(5)]
+	_ = x[UserCT-(5)]
 }
 
-var _CommandTypeValues = []CommandType{Default, Script, ScriptFile, RemoteScript, Package, User}
+var _CommandTypeValues = []CommandType{DefaultCT, ScriptCT, ScriptFileCT, RemoteScriptCT, PackageCT, UserCT}
 
 var _CommandTypeNameToValueMap = map[string]CommandType{
-	_CommandTypeName[0:0]:        Default,
+	_CommandTypeName[0:0]:        DefaultCT,
-	_CommandTypeLowerName[0:0]:   Default,
+	_CommandTypeLowerName[0:0]:   DefaultCT,
-	_CommandTypeName[0:6]:        Script,
+	_CommandTypeName[0:6]:        ScriptCT,
-	_CommandTypeLowerName[0:6]:   Script,
+	_CommandTypeLowerName[0:6]:   ScriptCT,
-	_CommandTypeName[6:16]:       ScriptFile,
+	_CommandTypeName[6:16]:       ScriptFileCT,
-	_CommandTypeLowerName[6:16]:  ScriptFile,
+	_CommandTypeLowerName[6:16]:  ScriptFileCT,
-	_CommandTypeName[16:28]:      RemoteScript,
+	_CommandTypeName[16:28]:      RemoteScriptCT,
-	_CommandTypeLowerName[16:28]: RemoteScript,
+	_CommandTypeLowerName[16:28]: RemoteScriptCT,
-	_CommandTypeName[28:35]:      Package,
+	_CommandTypeName[28:35]:      PackageCT,
-	_CommandTypeLowerName[28:35]: Package,
+	_CommandTypeLowerName[28:35]: PackageCT,
-	_CommandTypeName[35:39]:      User,
+	_CommandTypeName[35:39]:      UserCT,
-	_CommandTypeLowerName[35:39]: User,
+	_CommandTypeLowerName[35:39]: UserCT,
 }
 
 var _CommandTypeNames = []string{

pkg/backy/config.go

@@ -53,7 +53,7 @@ func (opts *ConfigOpts) InitConfig() {
 	cacheDir := homeCacheDir
 
 	// Load metadata from file
-	opts.CachedData, err = remotefetcher.LoadMetadataFromFile(path.Join(backyHomeConfDir, "cache.yml"))
+	opts.CachedData, err = remotefetcher.LoadMetadataFromFile(path.Join(backyHomeConfDir, "cache", "cache.yml"))
 	if err != nil {
 		fmt.Println("Error loading metadata:", err)
 		logging.ExitWithMSG(err.Error(), 1, &opts.Logger)
@@ -77,11 +77,13 @@ func (opts *ConfigOpts) InitConfig() {
 	if err != nil {
 		logging.ExitWithMSG(fmt.Sprintf("error initializing cache: %v", err), 1, nil)
 	}
-	// Initialize the fetcher
-	// println("Creating new fetcher for source", opts.ConfigFilePath)
-	fetcher, err := remotefetcher.NewRemoteFetcher(opts.ConfigFilePath, opts.Cache)
-	// println("Created new fetcher for source", opts.ConfigFilePath)
 
+	fetcher, err := remotefetcher.NewRemoteFetcher(opts.ConfigFilePath, opts.Cache)
+
+	if isRemoteURL(opts.ConfigFilePath) {
+		p, _ := getRemoteDir(opts.ConfigFilePath)
+		opts.ConfigDir = p
+	}
 	if err != nil {
 		logging.ExitWithMSG(fmt.Sprintf("error initializing config fetcher: %v", err), 1, nil)
 	}
@@ -274,20 +276,22 @@ func resolveProxyHosts(host *Host, opts *ConfigOpts) {
 }
 
 func loadCommandLists(opts *ConfigOpts, backyKoanf *koanf.Koanf) {
-	var backyConfigFileDir string
 	var listConfigFiles []string
 	var u *url.URL
+	var p string
 	// if config file is remote, use the directory of the remote file
 	if isRemoteURL(opts.ConfigFilePath) {
-		_, u = getRemoteDir(opts.ConfigFilePath)
+		p, u = getRemoteDir(opts.ConfigFilePath)
+		opts.ConfigDir = p
+		println(p)
 		// // Still use local list files if a remote config file is used, but use them last
 		listConfigFiles = []string{u.JoinPath("lists.yml").String(), u.JoinPath("lists.yaml").String()}
 	} else {
-		backyConfigFileDir = path.Dir(opts.ConfigFilePath)
+		opts.ConfigDir = path.Dir(opts.ConfigFilePath)
 		listConfigFiles = []string{
 			// "./lists.yml", "./lists.yaml",
-			path.Join(backyConfigFileDir, "lists.yml"),
+			path.Join(opts.ConfigDir, "lists.yml"),
-			path.Join(backyConfigFileDir, "lists.yaml"),
+			path.Join(opts.ConfigDir, "lists.yaml"),
 		}
 	}
 
@@ -327,7 +331,6 @@ func loadListConfigFile(filePath string, k *koanf.Koanf, opts *ConfigOpts) bool
 	if err != nil {
 		// if file not found, ignore
 		if errors.Is(err, remotefetcher.ErrIgnoreFileNotFound) {
-			println("File not found", filePath)
 			return true
 		}
 
@@ -569,7 +572,7 @@ func processCmds(opts *ConfigOpts) error {
 
 			if cmd.Dir != nil {
 
-				cmdDir, err := resolveDir(*cmd.Dir)
+				cmdDir, err := getFullPathWithHomeDir(*cmd.Dir)
 				if err != nil {
 					return err
 				}
@@ -578,11 +581,11 @@ func processCmds(opts *ConfigOpts) error {
 		}
 
 		// Parse package commands
-		if cmd.Type == Package {
+		if cmd.Type == PackageCT {
 			if cmd.PackageManager == "" {
 				return fmt.Errorf("package manager is required for package command %s", cmd.PackageName)
 			}
-			if cmd.PackageOperation == "" {
+			if cmd.PackageOperation.String() == "" {
 				return fmt.Errorf("package operation is required for package command %s", cmd.PackageName)
 			}
 			if cmd.PackageName == "" {
@@ -591,19 +594,20 @@ func processCmds(opts *ConfigOpts) error {
 			var err error
 
 			// Validate the operation
-			switch cmd.PackageOperation {
+			if cmd.PackageOperation.IsAPackageOperation() {
-			case "install", "remove", "upgrade", "checkVersion":
+
 				cmd.pkgMan, err = pkgman.PackageManagerFactory(cmd.PackageManager, pkgman.WithoutAuth())
 				if err != nil {
 					return err
 				}
-			default:
+			} else {
 				return fmt.Errorf("unsupported package operation %s for command %s", cmd.PackageOperation, cmd.Name)
 			}
+
 		}
 
 		// Parse user commands
-		if cmd.Type == User {
+		if cmd.Type == UserCT {
 			if cmd.Username == "" {
 				return fmt.Errorf("username is required for user command %s", cmd.Name)
 			}
@@ -630,7 +634,7 @@ func processCmds(opts *ConfigOpts) error {
 
 		}
 
-		if cmd.Type == RemoteScript {
+		if cmd.Type == RemoteScriptCT {
 			var fetchErr error
 			if !isRemoteURL(cmd.Cmd) {
 				return fmt.Errorf("remoteScript command %s must be a remote resource", cmdName)
@@ -643,7 +647,7 @@ func processCmds(opts *ConfigOpts) error {
 		}
 		if cmd.OutputFile != "" {
 			var err error
-			cmd.OutputFile, err = resolveDir(cmd.OutputFile)
+			cmd.OutputFile, err = getFullPathWithHomeDir(cmd.OutputFile)
 			if err != nil {
 				return err
 			}

pkg/backy/packageoperation_enumer.go

@@ -0,0 +1,145 @@
+// Code generated by "enumer -linecomment -yaml -text -json -type=PackageOperation"; DO NOT EDIT.
+
+package backy
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+)
+
+const _PackageOperationName = "installupgradepurgeremovecheckVersionisInstalled"
+
+var _PackageOperationIndex = [...]uint8{0, 0, 7, 14, 19, 25, 37, 48}
+
+const _PackageOperationLowerName = "installupgradepurgeremovecheckversionisinstalled"
+
+func (i PackageOperation) String() string {
+	if i < 0 || i >= PackageOperation(len(_PackageOperationIndex)-1) {
+		return fmt.Sprintf("PackageOperation(%d)", i)
+	}
+	return _PackageOperationName[_PackageOperationIndex[i]:_PackageOperationIndex[i+1]]
+}
+
+// An "invalid array index" compiler error signifies that the constant values have changed.
+// Re-run the stringer command to generate them again.
+func _PackageOperationNoOp() {
+	var x [1]struct{}
+	_ = x[DefaultPO-(0)]
+	_ = x[PackOpInstall-(1)]
+	_ = x[PackOpUpgrade-(2)]
+	_ = x[PackOpPurge-(3)]
+	_ = x[PackOpRemove-(4)]
+	_ = x[PackOpCheckVersion-(5)]
+	_ = x[PackOpIsInstalled-(6)]
+}
+
+var _PackageOperationValues = []PackageOperation{DefaultPO, PackOpInstall, PackOpUpgrade, PackOpPurge, PackOpRemove, PackOpCheckVersion, PackOpIsInstalled}
+
+var _PackageOperationNameToValueMap = map[string]PackageOperation{
+	_PackageOperationName[0:0]:        DefaultPO,
+	_PackageOperationLowerName[0:0]:   DefaultPO,
+	_PackageOperationName[0:7]:        PackOpInstall,
+	_PackageOperationLowerName[0:7]:   PackOpInstall,
+	_PackageOperationName[7:14]:       PackOpUpgrade,
+	_PackageOperationLowerName[7:14]:  PackOpUpgrade,
+	_PackageOperationName[14:19]:      PackOpPurge,
+	_PackageOperationLowerName[14:19]: PackOpPurge,
+	_PackageOperationName[19:25]:      PackOpRemove,
+	_PackageOperationLowerName[19:25]: PackOpRemove,
+	_PackageOperationName[25:37]:      PackOpCheckVersion,
+	_PackageOperationLowerName[25:37]: PackOpCheckVersion,
+	_PackageOperationName[37:48]:      PackOpIsInstalled,
+	_PackageOperationLowerName[37:48]: PackOpIsInstalled,
+}
+
+var _PackageOperationNames = []string{
+	_PackageOperationName[0:0],
+	_PackageOperationName[0:7],
+	_PackageOperationName[7:14],
+	_PackageOperationName[14:19],
+	_PackageOperationName[19:25],
+	_PackageOperationName[25:37],
+	_PackageOperationName[37:48],
+}
+
+// PackageOperationString retrieves an enum value from the enum constants string name.
+// Throws an error if the param is not part of the enum.
+func PackageOperationString(s string) (PackageOperation, error) {
+	if val, ok := _PackageOperationNameToValueMap[s]; ok {
+		return val, nil
+	}
+
+	if val, ok := _PackageOperationNameToValueMap[strings.ToLower(s)]; ok {
+		return val, nil
+	}
+	return 0, fmt.Errorf("%s does not belong to PackageOperation values", s)
+}
+
+// PackageOperationValues returns all values of the enum
+func PackageOperationValues() []PackageOperation {
+	return _PackageOperationValues
+}
+
+// PackageOperationStrings returns a slice of all String values of the enum
+func PackageOperationStrings() []string {
+	strs := make([]string, len(_PackageOperationNames))
+	copy(strs, _PackageOperationNames)
+	return strs
+}
+
+// IsAPackageOperation returns "true" if the value is listed in the enum definition. "false" otherwise
+func (i PackageOperation) IsAPackageOperation() bool {
+	for _, v := range _PackageOperationValues {
+		if i == v {
+			return true
+		}
+	}
+	return false
+}
+
+// MarshalJSON implements the json.Marshaler interface for PackageOperation
+func (i PackageOperation) MarshalJSON() ([]byte, error) {
+	return json.Marshal(i.String())
+}
+
+// UnmarshalJSON implements the json.Unmarshaler interface for PackageOperation
+func (i *PackageOperation) UnmarshalJSON(data []byte) error {
+	var s string
+	if err := json.Unmarshal(data, &s); err != nil {
+		return fmt.Errorf("PackageOperation should be a string, got %s", data)
+	}
+
+	var err error
+	*i, err = PackageOperationString(s)
+	return err
+}
+
+// MarshalText implements the encoding.TextMarshaler interface for PackageOperation
+func (i PackageOperation) MarshalText() ([]byte, error) {
+	return []byte(i.String()), nil
+}
+
+// UnmarshalText implements the encoding.TextUnmarshaler interface for PackageOperation
+func (i *PackageOperation) UnmarshalText(text []byte) error {
+	var err error
+	*i, err = PackageOperationString(string(text))
+	return err
+}
+
+// MarshalYAML implements a YAML Marshaler for PackageOperation
+func (i PackageOperation) MarshalYAML() (interface{}, error) {
+	return i.String(), nil
+}
+
+// UnmarshalYAML implements a YAML Unmarshaler for PackageOperation
+func (i *PackageOperation) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var s string
+	if err := unmarshal(&s); err != nil {
+		return err
+	}
+
+	var err error
+	*i, err = PackageOperationString(s)
+	return err
+}

pkg/backy/ssh.go

@@ -54,7 +54,7 @@ func (remoteConfig *Host) ConnectToHost(opts *ConfigOpts) error {
 
 	if !remoteConfig.useDefaultConfig {
 		var err error
-		remoteConfig.ConfigFilePath, err = resolveDir(remoteConfig.ConfigFilePath)
+		remoteConfig.ConfigFilePath, err = getFullPathWithHomeDir(remoteConfig.ConfigFilePath)
 		if err != nil {
 			return err
 		}
@@ -63,7 +63,7 @@ func (remoteConfig *Host) ConnectToHost(opts *ConfigOpts) error {
 			return sshConfigFileOpenErr
 		}
 	} else {
-		defaultConfig, _ := resolveDir("~/.ssh/config")
+		defaultConfig, _ := getFullPathWithHomeDir("~/.ssh/config")
 		configFile, sshConfigFileOpenErr = os.Open(defaultConfig)
 		if sshConfigFileOpenErr != nil {
 			return sshConfigFileOpenErr
@@ -242,7 +242,7 @@ func (remoteHost *Host) GetPrivateKeyFileFromConfig() {
 		identityFile = remoteHost.PrivateKeyPath
 	}
 
-	remoteHost.PrivateKeyPath, _ = resolveDir(identityFile)
+	remoteHost.PrivateKeyPath, _ = getFullPathWithHomeDir(identityFile)
 }
 
 // GetPort checks if the port from the config file is 0
@@ -324,10 +324,10 @@ func (remoteHost *Host) ConnectThroughBastion(log zerolog.Logger) (*ssh.Client,
 func (remotehHost *Host) GetKnownHosts() error {
 	var knownHostsFileErr error
 	if TS(remotehHost.KnownHostsFile) != "" {
-		remotehHost.KnownHostsFile, knownHostsFileErr = resolveDir(remotehHost.KnownHostsFile)
+		remotehHost.KnownHostsFile, knownHostsFileErr = getFullPathWithHomeDir(remotehHost.KnownHostsFile)
 		return knownHostsFileErr
 	}
-	remotehHost.KnownHostsFile, knownHostsFileErr = resolveDir("~/.ssh/known_hosts")
+	remotehHost.KnownHostsFile, knownHostsFileErr = getFullPathWithHomeDir("~/.ssh/known_hosts")
 	return knownHostsFileErr
 }
 
@@ -336,7 +336,7 @@ func GetPrivateKeyPassword(key string, opts *ConfigOpts, log zerolog.Logger) (st
 	var prKeyPassword string
 	if strings.HasPrefix(key, "file:") {
 		privKeyPassFilePath := strings.TrimPrefix(key, "file:")
-		privKeyPassFilePath, _ = resolveDir(privKeyPassFilePath)
+		privKeyPassFilePath, _ = getFullPathWithHomeDir(privKeyPassFilePath)
 		keyFile, keyFileErr := os.Open(privKeyPassFilePath)
 		if keyFileErr != nil {
 			return "", errors.Errorf("Private key password file %s failed to open. \n Make sure it is accessible and correct.", privKeyPassFilePath)
@@ -368,7 +368,7 @@ func GetPassword(pass string, opts *ConfigOpts, log zerolog.Logger) (string, err
 	var password string
 	if strings.HasPrefix(pass, "file:") {
 		passFilePath := strings.TrimPrefix(pass, "file:")
-		passFilePath, _ = resolveDir(passFilePath)
+		passFilePath, _ = getFullPathWithHomeDir(passFilePath)
 		keyFile, keyFileErr := os.Open(passFilePath)
 		if keyFileErr != nil {
 			return "", errors.New("Password file failed to open")
@@ -440,7 +440,7 @@ func (remoteConfig *Host) GetProxyJumpConfig(hosts map[string]*Host, opts *Confi
 			return sshConfigFileOpenErr
 		}
 	} else {
-		defaultConfig, _ := resolveDir("~/.ssh/config")
+		defaultConfig, _ := getFullPathWithHomeDir("~/.ssh/config")
 		configFile, sshConfigFileOpenErr = os.Open(defaultConfig)
 		if sshConfigFileOpenErr != nil {
 			return sshConfigFileOpenErr
@@ -531,14 +531,14 @@ func (command *Command) RunCmdSSH(cmdCtxLogger zerolog.Logger, opts *ConfigOpts)
 
 	// Handle command execution based on type
 	switch command.Type {
-	case Script:
+	case ScriptCT:
 		return command.runScript(commandSession, cmdCtxLogger, &cmdOutBuf)
-	case RemoteScript:
+	case RemoteScriptCT:
 		return command.runRemoteScript(commandSession, cmdCtxLogger, &cmdOutBuf)
-	case ScriptFile:
+	case ScriptFileCT:
 		return command.runScriptFile(commandSession, cmdCtxLogger, &cmdOutBuf)
-	case Package:
+	case PackageCT:
-		if command.PackageOperation == "checkVersion" {
+		if command.PackageOperation == PackOpCheckVersion {
 			commandSession.Stderr = nil
 			// Execute the package version command remotely
 			// Parse the output of package version command
@@ -706,7 +706,7 @@ func (command *Command) runRemoteScript(session *ssh.Session, cmdCtxLogger zerol
 
 // readFileToBuffer reads a file into a buffer.
 func readFileToBuffer(filePath string) (*bytes.Buffer, error) {
-	resolvedPath, err := resolveDir(filePath)
+	resolvedPath, err := getFullPathWithHomeDir(filePath)
 	if err != nil {
 		return nil, err
 	}

pkg/backy/types.go

@@ -106,7 +106,7 @@ type (
 		PackageVersion string `yaml:"packageVersion,omitempty"`
 
 		// PackageOperation specifies the action for package-related commands (e.g., "install" or "remove")
-		PackageOperation string `yaml:"packageOperation,omitempty"`
+		PackageOperation PackageOperation `yaml:"packageOperation,omitempty"`
 
 		pkgMan pkgman.PackageManager
 
@@ -204,13 +204,12 @@ type (
 
 		CmdStdOut bool
 
-		// Holds config file
 		ConfigFilePath string
 
-		// Holds log file
+		ConfigDir string
+
 		LogFilePath string
 
-		// for command list file
 		CmdListFile string
 
 		// use command lists using cron
@@ -294,15 +293,29 @@ type (
 		ListName string // Name of the command list
 		Error    error  // Error encountered, if any
 	}
-	CommandType int
+
+	// use ints so we can use enums
+	CommandType      int
+	PackageOperation int
 )
 
 //go:generate go run github.com/dmarkham/enumer -linecomment -yaml -text -json -type=CommandType
 const (
-	Default      CommandType = iota //
+	DefaultCT      CommandType = iota //
-	Script                          // script
+	ScriptCT                          // script
-	ScriptFile                      // scriptFile
+	ScriptFileCT                      // scriptFile
-	RemoteScript                    // remoteScript
+	RemoteScriptCT                    // remoteScript
-	Package                         // package
+	PackageCT                         // package
-	User                            // user
+	UserCT                            // user
+)
+
+//go:generate go run github.com/dmarkham/enumer -linecomment -yaml -text -json -type=PackageOperation
+const (
+	DefaultPO          PackageOperation = iota //
+	PackOpInstall                              // install
+	PackOpUpgrade                              // upgrade
+	PackOpPurge                                // purge
+	PackOpRemove                               // remove
+	PackOpCheckVersion                         // checkVersion
+	PackOpIsInstalled                          // isInstalled
 )

pkg/backy/utils.go

@@ -15,6 +15,7 @@ import (
 	"strings"
 
 	"git.andrewnw.xyz/CyberShell/backy/pkg/logging"
+	"git.andrewnw.xyz/CyberShell/backy/pkg/remotefetcher"
 	"github.com/joho/godotenv"
 	"github.com/knadh/koanf/v2"
 	"github.com/rs/zerolog"
@@ -91,7 +92,7 @@ func NewOpts(configFilePath string, opts ...BackyOptionFunc) *ConfigOpts {
 
 func injectEnvIntoSSH(envVarsToInject environmentVars, process *ssh.Session, opts *ConfigOpts, log zerolog.Logger) {
 	if envVarsToInject.file != "" {
-		envPath, envPathErr := resolveDir(envVarsToInject.file)
+		envPath, envPathErr := getFullPathWithHomeDir(envVarsToInject.file)
 		if envPathErr != nil {
 			log.Fatal().Str("envFile", envPath).Err(envPathErr).Send()
 		}
@@ -125,7 +126,7 @@ errEnvFile:
 
 func injectEnvIntoLocalCMD(envVarsToInject environmentVars, process *exec.Cmd, log zerolog.Logger) {
 	if envVarsToInject.file != "" {
-		envPath, _ := resolveDir(envVarsToInject.file)
+		envPath, _ := getFullPathWithHomeDir(envVarsToInject.file)
 
 		file, fileErr := os.Open(envPath)
 		if fileErr != nil {
@@ -192,7 +193,7 @@ func IsCmdStdOutEnabled() bool {
 	return os.Getenv("BACKY_CMDSTDOUT") == "enabled"
 }
 
-func resolveDir(path string) (string, error) {
+func getFullPathWithHomeDir(path string) (string, error) {
 	path = strings.TrimSpace(path)
 
 	if path == "~" {
@@ -216,11 +217,31 @@ func resolveDir(path string) (string, error) {
 
 // loadEnv loads a .env file from the config file directory
 func (opts *ConfigOpts) loadEnv() {
-	envFileInConfigDir := fmt.Sprintf("%s/.env", path.Dir(opts.ConfigFilePath))
 	var backyEnv map[string]string
-	backyEnv, envFileErr := godotenv.Read(envFileInConfigDir)
+	var envFileInConfigDir string
-	if envFileErr != nil {
+	var envFileErr error
-		return
+	if isRemoteURL(opts.ConfigFilePath) {
+		_, u := getRemoteDir(opts.ConfigFilePath)
+		envFileInConfigDir = u.JoinPath(".env").String()
+		envFetcher, err := remotefetcher.NewRemoteFetcher(envFileInConfigDir, opts.Cache)
+		if err != nil {
+			return
+		}
+		data, err := envFetcher.Fetch(envFileInConfigDir)
+		if err != nil {
+			return
+		}
+		backyEnv, envFileErr = godotenv.UnmarshalBytes(data)
+		if envFileErr != nil {
+			return
+		}
+
+	} else {
+		envFileInConfigDir = fmt.Sprintf("%s/.env", path.Dir(opts.ConfigFilePath))
+		backyEnv, envFileErr = godotenv.Read(envFileInConfigDir)
+		if envFileErr != nil {
+			return
+		}
 	}
 
 	opts.backyEnv = backyEnv
@@ -251,26 +272,23 @@ func expandEnvVars(backyEnv map[string]string, envVars []string) {
 	}
 }
 
-// getCommandTypeAndSetCommandInfo checks for command type and if the command has already been set
-// Checks for types package and user
-// Returns the modified Command with the package- or userManager command as Cmd and the package- or userOperation as args, plus any additional Args
 func getCommandTypeAndSetCommandInfo(command *Command) *Command {
 
-	if command.Type == Package && !command.packageCmdSet {
+	if command.Type == PackageCT && !command.packageCmdSet {
 		command.packageCmdSet = true
 		switch command.PackageOperation {
-		case "install":
+		case PackOpInstall:
 			command.Cmd, command.Args = command.pkgMan.Install(command.PackageName, command.PackageVersion, command.Args)
-		case "remove":
+		case PackOpRemove:
 			command.Cmd, command.Args = command.pkgMan.Remove(command.PackageName, command.Args)
-		case "upgrade":
+		case PackOpUpgrade:
 			command.Cmd, command.Args = command.pkgMan.Upgrade(command.PackageName, command.PackageVersion)
-		case "checkVersion":
+		case PackOpCheckVersion:
 			command.Cmd, command.Args = command.pkgMan.CheckVersion(command.PackageName, command.PackageVersion)
 		}
 	}
 
-	if command.Type == User && !command.userCmdSet {
+	if command.Type == UserCT && !command.userCmdSet {
 		command.userCmdSet = true
 		switch command.UserOperation {
 		case "add":

pkg/remotefetcher/cache.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"path"
 	"path/filepath"
 	"sync"
 
@@ -72,7 +73,7 @@ func (c *Cache) saveToFile() error {
 	for _, data := range c.store {
 		cacheData = append(cacheData, data)
 	}
-
+	cacheData = unique(cacheData)
 	data, err := yaml.Marshal(cacheData)
 	if err != nil {
 		return err
@@ -170,6 +171,7 @@ func (cf *CachedFetcher) Hash(data []byte) string {
 func LoadMetadataFromFile(filePath string) ([]*CacheData, error) {
 	if _, err := os.Stat(filePath); os.IsNotExist(err) {
 		// Create the file if it does not exist
+		os.MkdirAll(path.Dir(filePath), 0700)
 		emptyData := []byte("[]")
 		err := os.WriteFile(filePath, emptyData, 0644)
 		if err != nil {
@@ -184,6 +186,7 @@ func LoadMetadataFromFile(filePath string) ([]*CacheData, error) {
 
 	var cacheData []*CacheData
 	err = yaml.Unmarshal(data, &cacheData)
+
 	if err != nil {
 		return nil, err
 	}
@@ -195,3 +198,23 @@ func HashURL(url string) string {
 	hash := sha256.Sum256([]byte(url))
 	return hex.EncodeToString(hash[:])
 }
+
+func unique(cache []CacheData) []CacheData {
+	var unique []CacheData
+	type key struct{ value1, value2, value3, value4 string }
+	m := make(map[key]int)
+	for _, v := range cache {
+		k := key{v.URL, v.Hash, v.Path, v.Type}
+		if i, ok := m[k]; ok {
+			// Overwrite previous value per requirement in
+			// question to keep last matching value.
+			unique[i] = v
+		} else {
+			// Unique key found. Record position and collect
+			// in result.
+			m[k] = len(unique)
+			unique = append(unique, v)
+		}
+	}
+	return unique
+}

pkg/remotefetcher/configfetcher.go

@@ -59,6 +59,7 @@ func NewRemoteFetcher(source string, cache *Cache, options ...FetcherOption) (Re
 
 	URLHash := HashURL(source)
 	if cachedData, cacheMeta, exists := cache.Get(URLHash); exists {
+		println(cachedData)
 		return &CachedFetcher{data: cachedData, path: cacheMeta.Path, dataType: cacheMeta.Type}, nil
 	}
 

pkg/remotefetcher/s3.go

@@ -44,11 +44,11 @@ func NewS3Fetcher(endpoint string, options ...FetcherOption) (*S3Fetcher, error)
 	*/
 
 	s3Endpoint := os.Getenv("S3_ENDPOINT")
-	creds, err := getS3Credentials("default", s3Endpoint, cfg.HTTPClient)
+	creds, err := getS3Credentials(os.Getenv("AWS_PROFILE"), s3Endpoint, cfg.HTTPClient)
 	if err != nil {
-		println(err.Error())
 		return nil, err
 	}
+
 	// Initialize S3 client if not provided
 	if cfg.S3Client == nil {
 		s3Client, err = minio.New(s3Endpoint, &minio.Options{
@@ -128,12 +128,11 @@ func (s *S3Fetcher) Hash(data []byte) string {
 }
 
 func getS3Credentials(profile, host string, httpClient *http.Client) (*credentials.Credentials, error) {
-	// println(s3utils.GetRegionFromURL(*u))
 	homeDir, hdirErr := homedir.Dir()
 	if hdirErr != nil {
 		return nil, hdirErr
 	}
-	s3Creds := credentials.NewFileAWSCredentials(path.Join(homeDir, ".aws", "credentials"), "default")
+	s3Creds := credentials.NewFileAWSCredentials(path.Join(homeDir, ".aws", "credentials"), profile)
 	credVals, credErr := s3Creds.GetWithContext(&credentials.CredContext{Endpoint: host, Client: httpClient})
 	if credErr != nil {
 		return nil, credErr

release

@@ -1,7 +1,7 @@
 #!/bin/bash
 set -eou pipefail
 go generate ./...
-export CURRENT_TAG="$(go run backy.go version -V)"
+CURRENT_TAG="$(go run backy.go version -V)"
 goreleaser -f .goreleaser/github.yml check
 goreleaser -f .goreleaser/gitea.yml check
 changie batch $CURRENT_TAG

tests/backy.yaml

@@ -0,0 +1,18 @@
+commands:
+  echoTestPass:
+    cmd: echo
+    shell: bash
+    Args: hello world
+
+  runRemoteShellScriptSuccess:
+    cmd: 
+
+
+  packageCommandSuccess:
+    packageName: docker-ce
+    Args:
+      - docker-ce-cli
+    packageManager: apt
+    packageOperation: install
+
+