fetch env file where config file is stored

pkg/backy/config.go

@@ -53,7 +53,7 @@ func (opts *ConfigOpts) InitConfig() {
 	cacheDir := homeCacheDir
 
 	// Load metadata from file
-	opts.CachedData, err = remotefetcher.LoadMetadataFromFile(path.Join(backyHomeConfDir, "cache.yml"))
+	opts.CachedData, err = remotefetcher.LoadMetadataFromFile(path.Join(backyHomeConfDir, "cache", "cache.yml"))
 	if err != nil {
 		fmt.Println("Error loading metadata:", err)
 		logging.ExitWithMSG(err.Error(), 1, &opts.Logger)
@@ -77,11 +77,13 @@ func (opts *ConfigOpts) InitConfig() {
 	if err != nil {
 		logging.ExitWithMSG(fmt.Sprintf("error initializing cache: %v", err), 1, nil)
 	}
-	// Initialize the fetcher
-	// println("Creating new fetcher for source", opts.ConfigFilePath)
-	fetcher, err := remotefetcher.NewRemoteFetcher(opts.ConfigFilePath, opts.Cache)
-	// println("Created new fetcher for source", opts.ConfigFilePath)
 
+	fetcher, err := remotefetcher.NewRemoteFetcher(opts.ConfigFilePath, opts.Cache)
+
+	if isRemoteURL(opts.ConfigFilePath) {
+		p, _ := getRemoteDir(opts.ConfigFilePath)
+		opts.ConfigDir = p
+	}
 	if err != nil {
 		logging.ExitWithMSG(fmt.Sprintf("error initializing config fetcher: %v", err), 1, nil)
 	}
@@ -274,20 +276,22 @@ func resolveProxyHosts(host *Host, opts *ConfigOpts) {
 }
 
 func loadCommandLists(opts *ConfigOpts, backyKoanf *koanf.Koanf) {
-	var backyConfigFileDir string
 	var listConfigFiles []string
 	var u *url.URL
+	var p string
 	// if config file is remote, use the directory of the remote file
 	if isRemoteURL(opts.ConfigFilePath) {
-		_, u = getRemoteDir(opts.ConfigFilePath)
+		p, u = getRemoteDir(opts.ConfigFilePath)
+		opts.ConfigDir = p
+		println(p)
 		// // Still use local list files if a remote config file is used, but use them last
 		listConfigFiles = []string{u.JoinPath("lists.yml").String(), u.JoinPath("lists.yaml").String()}
 	} else {
-		backyConfigFileDir = path.Dir(opts.ConfigFilePath)
+		opts.ConfigDir = path.Dir(opts.ConfigFilePath)
 		listConfigFiles = []string{
 			// "./lists.yml", "./lists.yaml",
-			path.Join(backyConfigFileDir, "lists.yml"),
-			path.Join(backyConfigFileDir, "lists.yaml"),
+			path.Join(opts.ConfigDir, "lists.yml"),
+			path.Join(opts.ConfigDir, "lists.yaml"),
 		}
 	}
 
@@ -327,7 +331,6 @@ func loadListConfigFile(filePath string, k *koanf.Koanf, opts *ConfigOpts) bool
 	if err != nil {
 		// if file not found, ignore
 		if errors.Is(err, remotefetcher.ErrIgnoreFileNotFound) {
-			println("File not found", filePath)
 			return true
 		}
 

pkg/backy/types.go

@@ -204,13 +204,12 @@ type (
 
 		CmdStdOut bool
 
-		// Holds config file
 		ConfigFilePath string
 
-		// Holds log file
+		ConfigDir string
+
 		LogFilePath string
 
-		// for command list file
 		CmdListFile string
 
 		// use command lists using cron

pkg/backy/utils.go

@@ -15,6 +15,7 @@ import (
 	"strings"
 
 	"git.andrewnw.xyz/CyberShell/backy/pkg/logging"
+	"git.andrewnw.xyz/CyberShell/backy/pkg/remotefetcher"
 	"github.com/joho/godotenv"
 	"github.com/knadh/koanf/v2"
 	"github.com/rs/zerolog"
@@ -216,11 +217,31 @@ func getFullPathWithHomeDir(path string) (string, error) {
 
 // loadEnv loads a .env file from the config file directory
 func (opts *ConfigOpts) loadEnv() {
-	envFileInConfigDir := fmt.Sprintf("%s/.env", path.Dir(opts.ConfigFilePath))
 	var backyEnv map[string]string
-	backyEnv, envFileErr := godotenv.Read(envFileInConfigDir)
-	if envFileErr != nil {
-		return
+	var envFileInConfigDir string
+	var envFileErr error
+	if isRemoteURL(opts.ConfigFilePath) {
+		_, u := getRemoteDir(opts.ConfigFilePath)
+		envFileInConfigDir = u.JoinPath(".env").String()
+		envFetcher, err := remotefetcher.NewRemoteFetcher(envFileInConfigDir, opts.Cache)
+		if err != nil {
+			return
+		}
+		data, err := envFetcher.Fetch(envFileInConfigDir)
+		if err != nil {
+			return
+		}
+		backyEnv, envFileErr = godotenv.UnmarshalBytes(data)
+		if envFileErr != nil {
+			return
+		}
+
+	} else {
+		envFileInConfigDir = fmt.Sprintf("%s/.env", path.Dir(opts.ConfigFilePath))
+		backyEnv, envFileErr = godotenv.Read(envFileInConfigDir)
+		if envFileErr != nil {
+			return
+		}
 	}
 
 	opts.backyEnv = backyEnv

pkg/remotefetcher/cache.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"path"
 	"path/filepath"
 	"sync"
 
@@ -72,7 +73,7 @@ func (c *Cache) saveToFile() error {
 	for _, data := range c.store {
 		cacheData = append(cacheData, data)
 	}
-
+	cacheData = unique(cacheData)
 	data, err := yaml.Marshal(cacheData)
 	if err != nil {
 		return err
@@ -170,6 +171,7 @@ func (cf *CachedFetcher) Hash(data []byte) string {
 func LoadMetadataFromFile(filePath string) ([]*CacheData, error) {
 	if _, err := os.Stat(filePath); os.IsNotExist(err) {
 		// Create the file if it does not exist
+		os.MkdirAll(path.Dir(filePath), 0700)
 		emptyData := []byte("[]")
 		err := os.WriteFile(filePath, emptyData, 0644)
 		if err != nil {
@@ -184,6 +186,7 @@ func LoadMetadataFromFile(filePath string) ([]*CacheData, error) {
 
 	var cacheData []*CacheData
 	err = yaml.Unmarshal(data, &cacheData)
+
 	if err != nil {
 		return nil, err
 	}
@@ -195,3 +198,23 @@ func HashURL(url string) string {
 	hash := sha256.Sum256([]byte(url))
 	return hex.EncodeToString(hash[:])
 }
+
+func unique(cache []CacheData) []CacheData {
+	var unique []CacheData
+	type key struct{ value1, value2, value3, value4 string }
+	m := make(map[key]int)
+	for _, v := range cache {
+		k := key{v.URL, v.Hash, v.Path, v.Type}
+		if i, ok := m[k]; ok {
+			// Overwrite previous value per requirement in
+			// question to keep last matching value.
+			unique[i] = v
+		} else {
+			// Unique key found. Record position and collect
+			// in result.
+			m[k] = len(unique)
+			unique = append(unique, v)
+		}
+	}
+	return unique
+}

pkg/remotefetcher/configfetcher.go

@@ -59,6 +59,7 @@ func NewRemoteFetcher(source string, cache *Cache, options ...FetcherOption) (Re
 
 	URLHash := HashURL(source)
 	if cachedData, cacheMeta, exists := cache.Get(URLHash); exists {
+		println(cachedData)
 		return &CachedFetcher{data: cachedData, path: cacheMeta.Path, dataType: cacheMeta.Type}, nil
 	}