[WIP] v0.7.0 fixes and changes to cache and remotefetcher

.changes/unreleased/Added-20250128-153524.yaml

@@ -0,0 +1,3 @@
+kind: Added
+body: Cache functionality - still a WIP
+time: 2025-01-28T15:35:24.512485671-06:00

.changes/unreleased/Changed-20250128-154204.yaml

@@ -0,0 +1,3 @@
+kind: Changed
+body: "name of `configfetcher` to `remotefetcher`"
+time: 2025-01-28T15:42:04.282668058-06:00

.changes/unreleased/Fixed-20250128-153806.yaml

@@ -0,0 +1,3 @@
+kind: Fixed
+body: Parsing of remote URLs when determining list config file path
+time: 2025-01-28T15:38:06.957506929-06:00

.gitignore

@@ -1,4 +1,4 @@
 
-dist/
-
 .codegpt
+
+*.log

.vscode/settings.json

@@ -1,7 +1,7 @@
 {
     "cSpell.words": [
         "Cmds",
-        "configfetcher",
+        "remotefetcher",
         "knadh",
         "koanf",
         "mattn",

docs/content/config/_index.md

@@ -14,9 +14,20 @@ If you leave the config path blank, the following paths will be searched in orde
 
 1. `./backy.yml`
 2. `./backy.yaml`
-3. `~/.config/backy.yml`
-4. `~/.config/backy.yaml`
+3. The same two files above contained in a `backy` subdirectory under in what is returned by Go's `os` package function `UserConfigDir()`.
 
-Create a file at `~/.config/backy.yml`.
+{{% expand title="`UserConfigDir()` documentation:" %}}
 
-See the rest of the documentation in this section to configure it.
+Up-to date documentation for this function may be found on [GoDoc](https://pkg.go.dev/os#UserConfigDir).
+
+>UserConfigDir returns the default root directory to use for user-specific configuration data. Users should create their own application-specific subdirectory within this one and use that.
+
+>On Unix systems, it returns $XDG_CONFIG_HOME as specified by https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html if non-empty, else $HOME/.config. On Darwin, it returns $HOME/Library/Application Support. On Windows, it returns %AppData%. On Plan 9, it returns $home/lib.
+
+>If the location cannot be determined (for example, $HOME is not defined), then it will return an error.
+
+{{% /expand %}}
+
+See the rest of the documentation, titles included below, in this section to configure it.
+
+{{% children description="true" %}}

docs/content/config/command-lists.md

@@ -2,7 +2,7 @@
 title: "Command Lists"
 weight: 2
 description: >
-  This page tells you how to get started with Backy.
+  This page tells you how to get use command lists.
 ---
 
 Command lists are for executing commands in sequence and getting notifications from them.
@@ -14,7 +14,7 @@ Lists can go in a separate file. Command lists should be in a separate file if:
 1. key 'cmd-lists.file' is found
 2. hosts.yml or hosts.yaml is found in the same directory as the backy config file
 
-```yaml
+```yaml {lineNos="true" wrap="true" title="yaml"}
   test2:
     name: test2
     order:
@@ -65,10 +65,10 @@ Backy also has a cron mode, so one can run `backy cron` and start a process that
 Adding `cron: 0 0 1 * * *` to a `cmd-lists` object will schedule the list at 1 in the morning. See [https://crontab.guru/](https://crontab.guru/) for reference.
 
 {{% notice tip %}}
-Note: Backy uses the second field of cron, so add anything except * to the beginning of a regular cron expression.
+Note: Backy uses the second field of cron, so add anything except `*` to the beginning of a regular cron expression.
 {{% /notice %}}
 
-```yaml
+```yaml {lineNos="true" wrap="true" title="yaml"}
 cmd-lists:
   docker-container-backup: # this can be any name you want
     # all commands have to be defined

docs/content/config/commands.md

@@ -1,5 +1,6 @@
 ---
 title: "Commands"
+description: Commands are just that, commands
 weight: 1
 ---
 

docs/content/config/packages.md

@@ -1,6 +1,7 @@
 ---
 title: "Packages"
 weight: 2
+description: This is dedicated to package commands.
 ---
 
 This is dedicated to `package` commands. The command `type` field must be `package`. Package is a type that allows one to perform package operations. There are several additional options available when `type` is `package`:

docs/content/config/vault.md

@@ -1,6 +1,7 @@
 ---
 title: "Vault"
 weight: 4
+description: Set up and configure vault.
 ---
 
 [Vault](https://www.vaultproject.io/) is a tool for storing secrets and other data securely.

examples/backy.yaml

@@ -28,12 +28,23 @@ commands:
     cmd: hostname
   update-docker:
     type: package
-    # shell: zsh
+    shell: zsh # best to run package commands in a shell
     packageName: docker-ce
     Args:
       - docker-ce-cli
     packageManager: apt
-    packageOperation: upgrade
+    packageOperation: install
+  update-dockerApt:
+    # type: package
+    shell: zsh
+    cmd: apt
+    Args:
+      - update
+      - "&&"
+      - apt install -y docker-ce
+      - docker-ce-cli
+    packageManager: apt
+    packageOperation: install
 
 cmd-lists:
   cmds-to-run: # this can be any name you want

pkg/backy/backy.go

@@ -98,8 +98,14 @@ func (command *Command) RunCmd(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([
 
 			cmdCtxLogger.Info().Str("Command", fmt.Sprintf("Running command %s on local machine", command.Name)).Send()
 
+			// execute package commands in a shell
+			if command.Type == "package" {
+				cmdCtxLogger.Info().Str("package", command.PackageName).Msg("Executing package command")
+				ArgsStr = fmt.Sprintf("%s %s", command.Cmd, ArgsStr)
+				localCMD = exec.Command("/bin/sh", "-c", ArgsStr)
+			} else {
 				localCMD = exec.Command(command.Cmd, command.Args...)
-
+			}
 		}
 
 		if command.Dir != nil {

pkg/backy/config.go

@@ -2,15 +2,17 @@ package backy
 
 import (
 	"context"
+	"errors"
 	"fmt"
+	"net/url"
 	"os"
 	"path"
 	"runtime"
 	"strings"
 
-	"git.andrewnw.xyz/CyberShell/backy/pkg/configfetcher"
 	"git.andrewnw.xyz/CyberShell/backy/pkg/logging"
 	"git.andrewnw.xyz/CyberShell/backy/pkg/pkgman"
+	"git.andrewnw.xyz/CyberShell/backy/pkg/remotefetcher"
 	"git.andrewnw.xyz/CyberShell/backy/pkg/usermanager"
 	vault "github.com/hashicorp/vault/api"
 	"github.com/knadh/koanf/parsers/yaml"
@@ -20,23 +22,23 @@ import (
 	"github.com/rs/zerolog"
 )
 
-var homeDir string
-var homeDirErr error
-var backyHomeConfDir string
-var configFiles []string
-
 const macroStart string = "%{"
 const macroEnd string = "}%"
 const envMacroStart string = "%{env:"
 const vaultMacroStart string = "%{vault:"
 
 func (opts *ConfigOpts) InitConfig() {
-	homeDir, err := os.UserHomeDir()
+	var err error
+	homeConfigDir, err := os.UserConfigDir()
+	if err != nil {
+		logging.ExitWithMSG(err.Error(), 1, nil)
+	}
+	homeCacheDir, err := os.UserCacheDir()
 	if err != nil {
 		logging.ExitWithMSG(err.Error(), 1, nil)
 	}
 
-	backyHomeConfDir := path.Join(homeDir, ".config/backy/")
+	backyHomeConfDir := path.Join(homeConfigDir, "backy")
 	configFiles := []string{
 		"./backy.yml", "./backy.yaml",
 		path.Join(backyHomeConfDir, "backy.yml"),
@@ -46,8 +48,36 @@ func (opts *ConfigOpts) InitConfig() {
 	backyKoanf := koanf.New(".")
 	opts.ConfigFilePath = strings.TrimSpace(opts.ConfigFilePath)
 
+	metadataFile := "hashMetadataSample.yml"
+	cacheDir := homeCacheDir
+
+	// Load metadata from file
+	opts.CachedData, err = remotefetcher.LoadMetadataFromFile(metadataFile)
+	if err != nil {
+		fmt.Println("Error loading metadata:", err)
+		panic(err)
+	}
+
+	// Initialize cache with loaded metadata
+	cache, err := remotefetcher.NewCache(metadataFile, cacheDir)
+	if err != nil {
+		fmt.Println("Error initializing cache:", err)
+		panic(err)
+	}
+
+	// Populate cache with loaded metadata
+	for _, data := range opts.CachedData {
+		cache.AddDataToStore(data.Hash, *data)
+	}
+
+	opts.Cache, err = remotefetcher.NewCache(path.Join(backyHomeConfDir, "cache.yml"), backyHomeConfDir)
+	if err != nil {
+		logging.ExitWithMSG(fmt.Sprintf("error initializing cache: %v", err), 1, nil)
+	}
 	// Initialize the fetcher
-	fetcher, err := configfetcher.NewConfigFetcher(opts.ConfigFilePath)
+	println("Creating new fetcher for source", opts.ConfigFilePath)
+	fetcher, err := remotefetcher.NewConfigFetcher(opts.ConfigFilePath, opts.Cache)
+	println("Created new fetcher for source", opts.ConfigFilePath)
 
 	if err != nil {
 		logging.ExitWithMSG(fmt.Sprintf("error initializing config fetcher: %v", err), 1, nil)
@@ -62,7 +92,7 @@ func (opts *ConfigOpts) InitConfig() {
 	opts.koanf = backyKoanf
 }
 
-func loadConfigFile(fetcher configfetcher.ConfigFetcher, filePath string, k *koanf.Koanf, opts *ConfigOpts) {
+func loadConfigFile(fetcher remotefetcher.ConfigFetcher, filePath string, k *koanf.Koanf, opts *ConfigOpts) {
 	data, err := fetcher.Fetch(filePath)
 	if err != nil {
 		logging.ExitWithMSG(fmt.Sprintf("Could not fetch config file %s: %v", filePath, err), 1, nil)
@@ -73,7 +103,7 @@ func loadConfigFile(fetcher configfetcher.ConfigFetcher, filePath string, k *koa
 	}
 }
 
-func loadDefaultConfigFiles(fetcher configfetcher.ConfigFetcher, configFiles []string, k *koanf.Koanf, opts *ConfigOpts) {
+func loadDefaultConfigFiles(fetcher remotefetcher.ConfigFetcher, configFiles []string, k *koanf.Koanf, opts *ConfigOpts) {
 	cFileFailures := 0
 	for _, c := range configFiles {
 		data, err := fetcher.Fetch(c)
@@ -236,12 +266,22 @@ func resolveProxyHosts(host *Host, opts *ConfigOpts) {
 }
 
 func loadCommandLists(opts *ConfigOpts, backyKoanf *koanf.Koanf) {
-	backyConfigFileDir := path.Dir(opts.ConfigFilePath)
-	listsConfig := koanf.New(".")
-	listConfigFiles := []string{
+	var backyConfigFileDir string
+	var listConfigFiles []string
+	var u *url.URL
+	// if config file is remote, use the directory of the remote file
+	if isRemoteURL(opts.ConfigFilePath) {
+		_, u = getRemoteDir(opts.ConfigFilePath)
+		listConfigFiles = []string{u.JoinPath("lists.yml").String(), u.JoinPath("lists.yaml").String()}
+	} else {
+		backyConfigFileDir = path.Dir(opts.ConfigFilePath)
+		listConfigFiles = []string{
 			path.Join(backyConfigFileDir, "lists.yml"),
 			path.Join(backyConfigFileDir, "lists.yaml"),
 		}
+	}
+
+	listsConfig := koanf.New(".")
 
 	for _, l := range listConfigFiles {
 		if loadListConfigFile(l, listsConfig, opts) {
@@ -257,9 +297,29 @@ func loadCommandLists(opts *ConfigOpts, backyKoanf *koanf.Koanf) {
 	}
 }
 
-func loadListConfigFile(filePath string, k *koanf.Koanf, opts *ConfigOpts) bool {
-	fetcher, err := configfetcher.NewConfigFetcher(filePath)
+func isRemoteURL(filePath string) bool {
+	return strings.HasPrefix(filePath, "http://") || strings.HasPrefix(filePath, "https://") || strings.HasPrefix(filePath, "s3://")
+}
+
+func getRemoteDir(filePath string) (string, *url.URL) {
+	u, err := url.Parse(filePath)
 	if err != nil {
+		return "", nil
+	}
+	// u.Path is the path to the file, stripped of scheme and hostname
+	u.Path = path.Dir(u.Path)
+
+	return u.String(), u
+}
+
+func loadListConfigFile(filePath string, k *koanf.Koanf, opts *ConfigOpts) bool {
+	fetcher, err := remotefetcher.NewConfigFetcher(filePath, opts.Cache, remotefetcher.IgnoreFileNotFound())
+	if err != nil {
+		// if file not found, ignore
+		if errors.Is(err, remotefetcher.ErrFileNotFound) {
+			return true
+		}
+
 		logging.ExitWithMSG(fmt.Sprintf("error initializing config fetcher: %v", err), 1, nil)
 	}
 
@@ -282,7 +342,8 @@ func loadCmdListsFile(backyKoanf *koanf.Koanf, listsConfig *koanf.Koanf, opts *C
 		opts.CmdListFile = path.Join(path.Dir(opts.ConfigFilePath), opts.CmdListFile)
 	}
 
-	fetcher, err := configfetcher.NewConfigFetcher(opts.CmdListFile)
+	fetcher, err := remotefetcher.NewConfigFetcher(opts.CmdListFile, opts.Cache)
+
 	if err != nil {
 		logging.ExitWithMSG(fmt.Sprintf("error initializing config fetcher: %v", err), 1, nil)
 	}

pkg/backy/types.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 
 	"git.andrewnw.xyz/CyberShell/backy/pkg/pkgman"
+	"git.andrewnw.xyz/CyberShell/backy/pkg/remotefetcher"
 	"git.andrewnw.xyz/CyberShell/backy/pkg/usermanager"
 	vaultapi "github.com/hashicorp/vault/api"
 	"github.com/kevinburke/ssh_config"
@@ -217,6 +218,9 @@ type (
 		koanf *koanf.Koanf
 
 		NotificationConf *Notifications `yaml:"notifications"`
+
+		Cache      *remotefetcher.Cache
+		CachedData []*remotefetcher.CacheData
 	}
 
 	outStruct struct {

pkg/backy/utils.go

@@ -275,7 +275,7 @@ func getCommandType(command *Command) *Command {
 		case "modify":
 			command.Cmd, command.Args = command.userMan.ModifyUser(
 				command.Username,
-				homeDir,
+				command.UserHome,
 				command.UserShell,
 				command.UserGroups)
 		case "checkIfExists":

pkg/configfetcher/configfetcher.go

@@ -1,27 +0,0 @@
-package configfetcher
-
-import "strings"
-
-type ConfigFetcher interface {
-	// Fetch retrieves the configuration from the specified URL or source
-	// Returns the raw data as bytes or an error
-	Fetch(source string) ([]byte, error)
-
-	// Parse decodes the raw data into a Go structure (e.g., Commands, CommandLists)
-	// Takes the raw data as input and populates the target interface
-	Parse(data []byte, target interface{}) error
-}
-
-func NewConfigFetcher(source string, options ...Option) (ConfigFetcher, error) {
-	if strings.HasPrefix(source, "http") || strings.HasPrefix(source, "https") {
-		return NewHTTPFetcher(options...), nil
-	} else if strings.HasPrefix(source, "s3") {
-		fetcher, err := NewS3Fetcher(options...)
-		if err != nil {
-			return nil, err
-		}
-		return fetcher, nil
-	} else {
-		return &LocalFetcher{}, nil
-	}
-}

pkg/configfetcher/local.go

@@ -1,26 +0,0 @@
-package configfetcher
-
-import (
-	"io"
-	"os"
-
-	"gopkg.in/yaml.v3"
-)
-
-type LocalFetcher struct{}
-
-// Fetch retrieves the configuration from the specified local file path
-func (l *LocalFetcher) Fetch(source string) ([]byte, error) {
-	file, err := os.Open(source)
-	if err != nil {
-		return nil, err
-	}
-	defer file.Close()
-
-	return io.ReadAll(file)
-}
-
-// Parse decodes the raw data into the provided target structure
-func (l *LocalFetcher) Parse(data []byte, target interface{}) error {
-	return yaml.Unmarshal(data, target)
-}

pkg/remotefetcher/cache.go

@@ -0,0 +1,191 @@
+package remotefetcher
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sync"
+
+	"gopkg.in/yaml.v3"
+)
+
+type CacheData struct {
+	Hash string `yaml:"hash"`
+	Path string `yaml:"path"`
+	Type string `yaml:"type"`
+}
+
+type Cache struct {
+	mu    sync.Mutex
+	store map[string]CacheData
+	file  string
+	dir   string
+}
+
+func NewCache(file, dir string) (*Cache, error) {
+	cache := &Cache{
+		store: make(map[string]CacheData),
+		file:  file,
+		dir:   dir,
+	}
+	err := cache.loadFromFile()
+	if err != nil {
+		return nil, err
+	}
+	return cache, nil
+}
+
+func (c *Cache) loadFromFile() error {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	if _, err := os.Stat(c.file); os.IsNotExist(err) {
+		return nil
+	}
+
+	data, err := os.ReadFile(c.file)
+	if err != nil {
+		return err
+	}
+
+	var cacheData []CacheData
+	err = yaml.Unmarshal(data, &cacheData)
+	if err != nil {
+		return err
+	}
+
+	for _, item := range cacheData {
+		c.store[item.Hash] = item
+	}
+
+	return nil
+}
+
+func (c *Cache) saveToFile() error {
+	// println("Saving cache to file:", c.file)
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	var cacheData []CacheData
+	for _, data := range c.store {
+		cacheData = append(cacheData, data)
+	}
+
+	data, err := yaml.Marshal(cacheData)
+	if err != nil {
+		return err
+	}
+
+	return os.WriteFile(c.file, data, 0644)
+}
+
+func (c *Cache) Get(hash string) ([]byte, CacheData, bool) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	println("Getting cache data for hash:", hash)
+	cacheData, exists := c.store[hash]
+	if !exists {
+		println("Cache data does not exist for hash:", hash)
+		return nil, CacheData{}, false
+	}
+
+	data, err := os.ReadFile(cacheData.Path)
+	if err != nil {
+		return nil, CacheData{}, false
+	}
+
+	return data, cacheData, true
+}
+
+func (c *Cache) AddDataToStore(hash string, cacheData CacheData) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	c.store[hash] = cacheData
+}
+
+func (c *Cache) Set(source, hash string, data []byte, dataType string) (CacheData, error) {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	fileName := filepath.Base(source)
+
+	path := filepath.Join(c.dir, fmt.Sprintf("%s-%s", fileName, hash))
+
+	if _, err := os.Stat(path); os.IsNotExist(err) {
+		os.MkdirAll(c.dir, 0700)
+	}
+
+	err := os.WriteFile(path, data, 0644)
+	if err != nil {
+		return CacheData{}, err
+	}
+
+	cacheData := CacheData{
+		Hash: hash,
+		Path: path,
+		Type: dataType,
+	}
+
+	c.store[hash] = cacheData
+
+	// Unlock before calling saveToFile to avoid double-locking
+	c.mu.Unlock()
+	err = c.saveToFile()
+	c.mu.Lock()
+	if err != nil {
+		return CacheData{}, err
+	}
+
+	// fmt.Printf("Cache data: %v", cacheData)
+	return cacheData, nil
+}
+
+type CachedFetcher struct {
+	data     []byte
+	path     string
+	dataType string
+}
+
+func (cf *CachedFetcher) Fetch(source string) ([]byte, error) {
+	return cf.data, nil
+}
+
+func (cf *CachedFetcher) Parse(data []byte, target interface{}) error {
+	if cf.dataType == "yaml" {
+		return yaml.Unmarshal(data, target)
+	}
+	return errors.New("parse not supported on cached fetcher for scripts")
+}
+
+func (cf *CachedFetcher) Hash(data []byte) string {
+	hash := sha256.Sum256(data)
+	return hex.EncodeToString(hash[:])
+}
+
+// Function to read and parse the hashMetadataSample.yml file
+func LoadMetadataFromFile(filePath string) ([]*CacheData, error) {
+	// fmt.Println("Loading metadata from file:", filePath)
+	if _, err := os.Stat(filePath); os.IsNotExist(err) {
+		// Create the file if it does not exist
+		emptyData := []byte("[]")
+		err := os.WriteFile(filePath, emptyData, 0644)
+		if err != nil {
+			return nil, err
+		}
+		return nil, nil
+	}
+	data, err := os.ReadFile(filePath)
+	if err != nil {
+		return nil, err
+	}
+
+	var cacheData []*CacheData
+	err = yaml.Unmarshal(data, &cacheData)
+	if err != nil {
+		return nil, err
+	}
+
+	return cacheData, nil
+}

pkg/remotefetcher/configfetcher.go

@@ -0,0 +1,75 @@
+package remotefetcher
+
+import (
+	"errors"
+	"strings"
+)
+
+type ConfigFetcher interface {
+	// Fetch retrieves the configuration from the specified URL or source
+	// Returns the raw data as bytes or an error
+	Fetch(source string) ([]byte, error)
+
+	// Parse decodes the raw data into a Go structure (e.g., Commands, CommandLists)
+	// Takes the raw data as input and populates the target interface
+	Parse(data []byte, target interface{}) error
+
+	// Hash returns the hash of the configuration data
+	Hash(data []byte) string
+}
+
+// ErrFileNotFound is returned when the file is not found and should be ignored
+var ErrFileNotFound = errors.New("remotefetcher: file not found")
+
+func NewConfigFetcher(source string, cache *Cache, options ...Option) (ConfigFetcher, error) {
+	var fetcher ConfigFetcher
+	var dataType string
+
+	config := FetcherConfig{}
+	for _, option := range options {
+		option(&config)
+	}
+	if strings.HasPrefix(source, "http") || strings.HasPrefix(source, "https") {
+		fetcher = NewHTTPFetcher(options...)
+		dataType = "yaml"
+	} else if strings.HasPrefix(source, "s3") {
+		var err error
+		fetcher, err = NewS3Fetcher(options...)
+		if err != nil {
+			return nil, err
+		}
+		dataType = "yaml"
+	} else {
+		fetcher = &LocalFetcher{}
+		dataType = "yaml"
+
+		return fetcher, nil
+	}
+
+	//TODO: should local files be cached?
+
+	data, err := fetcher.Fetch(source)
+	if err != nil {
+		if config.IgnoreFileNotFound && isFileNotFoundError(err) {
+			return nil, ErrFileNotFound
+		}
+		return nil, err
+	}
+
+	hash := fetcher.Hash(data)
+	if cachedData, cacheMeta, exists := cache.Get(hash); exists {
+		return &CachedFetcher{data: cachedData, path: cacheMeta.Path, dataType: cacheMeta.Type}, nil
+	}
+
+	cacheData, err := cache.Set(source, hash, data, dataType)
+	if err != nil {
+		return nil, err
+	}
+	return &CachedFetcher{data: data, path: cacheData.Path, dataType: cacheData.Type}, nil
+}
+
+func isFileNotFoundError(err error) bool {
+	// Implement logic to check if the error is a "file not found" error
+	// This can be based on the error type or message
+	return strings.Contains(err.Error(), "file not found")
+}

pkg/remotefetcher/http.go

@@ -1,6 +1,8 @@
-package configfetcher
+package remotefetcher
 
 import (
+	"crypto/sha256"
+	"encoding/hex"
 	"errors"
 	"io"
 	"net/http"
@@ -10,6 +12,7 @@ import (
 
 type HTTPFetcher struct {
 	HTTPClient *http.Client
+	config     FetcherConfig
 }
 
 // NewHTTPFetcher creates a new instance of HTTPFetcher with the provided options.
@@ -24,10 +27,10 @@ func NewHTTPFetcher(options ...Option) *HTTPFetcher {
 		cfg.HTTPClient = http.DefaultClient
 	}
 
-	return &HTTPFetcher{HTTPClient: cfg.HTTPClient}
+	return &HTTPFetcher{HTTPClient: cfg.HTTPClient, config: *cfg}
 }
 
-// Fetch retrieves the configuration from the specified URL
+// Fetch retrieves the file from the specified source URL
 func (h *HTTPFetcher) Fetch(source string) ([]byte, error) {
 	resp, err := http.Get(source)
 	if err != nil {
@@ -35,6 +38,10 @@ func (h *HTTPFetcher) Fetch(source string) ([]byte, error) {
 	}
 	defer resp.Body.Close()
 
+	if resp.StatusCode == http.StatusNotFound && h.config.IgnoreFileNotFound {
+		return nil, ErrFileNotFound
+	}
+
 	if resp.StatusCode != http.StatusOK {
 		return nil, errors.New("failed to fetch remote config: " + resp.Status)
 	}
@@ -46,3 +53,8 @@ func (h *HTTPFetcher) Fetch(source string) ([]byte, error) {
 func (h *HTTPFetcher) Parse(data []byte, target interface{}) error {
 	return yaml.Unmarshal(data, target)
 }
+
+func (h *HTTPFetcher) Hash(data []byte) string {
+	hash := sha256.Sum256(data)
+	return hex.EncodeToString(hash[:])
+}

pkg/remotefetcher/local.go

@@ -0,0 +1,42 @@
+package remotefetcher
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"io"
+	"os"
+
+	"gopkg.in/yaml.v3"
+)
+
+type LocalFetcher struct {
+	config FetcherConfig
+}
+
+// Fetch retrieves the file from the specified local file path
+func (l *LocalFetcher) Fetch(source string) ([]byte, error) {
+	// Check if the file exists
+	if _, err := os.Stat(source); os.IsNotExist(err) {
+		if l.config.IgnoreFileNotFound {
+			return nil, ErrFileNotFound
+		}
+		return nil, nil
+	}
+	file, err := os.Open(source)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+
+	return io.ReadAll(file)
+}
+
+// Parse decodes the raw data into the provided target structure
+func (l *LocalFetcher) Parse(data []byte, target interface{}) error {
+	return yaml.Unmarshal(data, target)
+}
+
+func (l *LocalFetcher) Hash(data []byte) string {
+	hash := sha256.Sum256(data)
+	return hex.EncodeToString(hash[:])
+}

pkg/remotefetcher/options.go

@@ -1,4 +1,4 @@
-package configfetcher
+package remotefetcher
 
 import (
 	"net/http"
@@ -13,6 +13,7 @@ type Option func(*FetcherConfig)
 type FetcherConfig struct {
 	S3Client           *s3.Client
 	HTTPClient         *http.Client
+	IgnoreFileNotFound bool
 }
 
 // WithS3Client sets the S3 client for the fetcher.
@@ -28,3 +29,9 @@ func WithHTTPClient(client *http.Client) Option {
 		cfg.HTTPClient = client
 	}
 }
+
+func IgnoreFileNotFound() Option {
+	return func(cfg *FetcherConfig) {
+		cfg.IgnoreFileNotFound = true
+	}
+}

pkg/remotefetcher/s3.go

@@ -1,18 +1,22 @@
-package configfetcher
+package remotefetcher
 
 import (
 	"bytes"
 	"context"
+	"crypto/sha256"
+	"encoding/hex"
 	"errors"
 	"strings"
 
 	"github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"gopkg.in/yaml.v3"
 )
 
 type S3Fetcher struct {
 	S3Client *s3.Client
+	config   FetcherConfig
 }
 
 // NewS3Fetcher creates a new instance of S3Fetcher with the provided options.
@@ -31,7 +35,7 @@ func NewS3Fetcher(options ...Option) (*S3Fetcher, error) {
 		cfg.S3Client = s3.NewFromConfig(awsCfg)
 	}
 
-	return &S3Fetcher{S3Client: cfg.S3Client}, nil
+	return &S3Fetcher{S3Client: cfg.S3Client, config: *cfg}, nil
 }
 
 // Fetch retrieves the configuration from an S3 bucket
@@ -47,6 +51,13 @@ func (s *S3Fetcher) Fetch(source string) ([]byte, error) {
 		Key:    &key,
 	})
 	if err != nil {
+		if err != nil {
+			var notFound *types.NoSuchKey
+			if errors.As(err, &notFound) && s.config.IgnoreFileNotFound {
+				return nil, ErrFileNotFound
+			}
+			return nil, err
+		}
 		return nil, err
 	}
 	defer resp.Body.Close()
@@ -73,3 +84,8 @@ func parseS3Source(source string) (bucket, key string, err error) {
 	}
 	return parts[0], parts[1], nil
 }
+
+func (s *S3Fetcher) Hash(data []byte) string {
+	hash := sha256.Sum256(data)
+	return hex.EncodeToString(hash[:])
+}