diff --git a/pkg/backy/config.go b/pkg/backy/config.go index 6ccc9bb..b37c0d3 100644 --- a/pkg/backy/config.go +++ b/pkg/backy/config.go @@ -53,7 +53,7 @@ func (opts *ConfigOpts) InitConfig() { cacheDir := homeCacheDir // Load metadata from file - opts.CachedData, err = remotefetcher.LoadMetadataFromFile(path.Join(backyHomeConfDir, "cache.yml")) + opts.CachedData, err = remotefetcher.LoadMetadataFromFile(path.Join(backyHomeConfDir, "cache", "cache.yml")) if err != nil { fmt.Println("Error loading metadata:", err) logging.ExitWithMSG(err.Error(), 1, &opts.Logger) @@ -77,11 +77,13 @@ func (opts *ConfigOpts) InitConfig() { if err != nil { logging.ExitWithMSG(fmt.Sprintf("error initializing cache: %v", err), 1, nil) } - // Initialize the fetcher - // println("Creating new fetcher for source", opts.ConfigFilePath) - fetcher, err := remotefetcher.NewRemoteFetcher(opts.ConfigFilePath, opts.Cache) - // println("Created new fetcher for source", opts.ConfigFilePath) + fetcher, err := remotefetcher.NewRemoteFetcher(opts.ConfigFilePath, opts.Cache) + + if isRemoteURL(opts.ConfigFilePath) { + p, _ := getRemoteDir(opts.ConfigFilePath) + opts.ConfigDir = p + } if err != nil { logging.ExitWithMSG(fmt.Sprintf("error initializing config fetcher: %v", err), 1, nil) } @@ -274,20 +276,22 @@ func resolveProxyHosts(host *Host, opts *ConfigOpts) { } func loadCommandLists(opts *ConfigOpts, backyKoanf *koanf.Koanf) { - var backyConfigFileDir string var listConfigFiles []string var u *url.URL + var p string // if config file is remote, use the directory of the remote file if isRemoteURL(opts.ConfigFilePath) { - _, u = getRemoteDir(opts.ConfigFilePath) + p, u = getRemoteDir(opts.ConfigFilePath) + opts.ConfigDir = p + println(p) // // Still use local list files if a remote config file is used, but use them last listConfigFiles = []string{u.JoinPath("lists.yml").String(), u.JoinPath("lists.yaml").String()} } else { - backyConfigFileDir = path.Dir(opts.ConfigFilePath) + opts.ConfigDir = path.Dir(opts.ConfigFilePath) listConfigFiles = []string{ // "./lists.yml", "./lists.yaml", - path.Join(backyConfigFileDir, "lists.yml"), - path.Join(backyConfigFileDir, "lists.yaml"), + path.Join(opts.ConfigDir, "lists.yml"), + path.Join(opts.ConfigDir, "lists.yaml"), } } @@ -327,7 +331,6 @@ func loadListConfigFile(filePath string, k *koanf.Koanf, opts *ConfigOpts) bool if err != nil { // if file not found, ignore if errors.Is(err, remotefetcher.ErrIgnoreFileNotFound) { - println("File not found", filePath) return true } diff --git a/pkg/backy/types.go b/pkg/backy/types.go index e26af8d..74a719b 100644 --- a/pkg/backy/types.go +++ b/pkg/backy/types.go @@ -204,13 +204,12 @@ type ( CmdStdOut bool - // Holds config file ConfigFilePath string - // Holds log file + ConfigDir string + LogFilePath string - // for command list file CmdListFile string // use command lists using cron diff --git a/pkg/backy/utils.go b/pkg/backy/utils.go index 1c83970..ffe1724 100644 --- a/pkg/backy/utils.go +++ b/pkg/backy/utils.go @@ -15,6 +15,7 @@ import ( "strings" "git.andrewnw.xyz/CyberShell/backy/pkg/logging" + "git.andrewnw.xyz/CyberShell/backy/pkg/remotefetcher" "github.com/joho/godotenv" "github.com/knadh/koanf/v2" "github.com/rs/zerolog" @@ -216,11 +217,31 @@ func getFullPathWithHomeDir(path string) (string, error) { // loadEnv loads a .env file from the config file directory func (opts *ConfigOpts) loadEnv() { - envFileInConfigDir := fmt.Sprintf("%s/.env", path.Dir(opts.ConfigFilePath)) var backyEnv map[string]string - backyEnv, envFileErr := godotenv.Read(envFileInConfigDir) - if envFileErr != nil { - return + var envFileInConfigDir string + var envFileErr error + if isRemoteURL(opts.ConfigFilePath) { + _, u := getRemoteDir(opts.ConfigFilePath) + envFileInConfigDir = u.JoinPath(".env").String() + envFetcher, err := remotefetcher.NewRemoteFetcher(envFileInConfigDir, opts.Cache) + if err != nil { + return + } + data, err := envFetcher.Fetch(envFileInConfigDir) + if err != nil { + return + } + backyEnv, envFileErr = godotenv.UnmarshalBytes(data) + if envFileErr != nil { + return + } + + } else { + envFileInConfigDir = fmt.Sprintf("%s/.env", path.Dir(opts.ConfigFilePath)) + backyEnv, envFileErr = godotenv.Read(envFileInConfigDir) + if envFileErr != nil { + return + } } opts.backyEnv = backyEnv diff --git a/pkg/remotefetcher/cache.go b/pkg/remotefetcher/cache.go index ec5abde..09a9f22 100644 --- a/pkg/remotefetcher/cache.go +++ b/pkg/remotefetcher/cache.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "os" + "path" "path/filepath" "sync" @@ -72,7 +73,7 @@ func (c *Cache) saveToFile() error { for _, data := range c.store { cacheData = append(cacheData, data) } - + cacheData = unique(cacheData) data, err := yaml.Marshal(cacheData) if err != nil { return err @@ -170,6 +171,7 @@ func (cf *CachedFetcher) Hash(data []byte) string { func LoadMetadataFromFile(filePath string) ([]*CacheData, error) { if _, err := os.Stat(filePath); os.IsNotExist(err) { // Create the file if it does not exist + os.MkdirAll(path.Dir(filePath), 0700) emptyData := []byte("[]") err := os.WriteFile(filePath, emptyData, 0644) if err != nil { @@ -184,6 +186,7 @@ func LoadMetadataFromFile(filePath string) ([]*CacheData, error) { var cacheData []*CacheData err = yaml.Unmarshal(data, &cacheData) + if err != nil { return nil, err } @@ -195,3 +198,23 @@ func HashURL(url string) string { hash := sha256.Sum256([]byte(url)) return hex.EncodeToString(hash[:]) } + +func unique(cache []CacheData) []CacheData { + var unique []CacheData + type key struct{ value1, value2, value3, value4 string } + m := make(map[key]int) + for _, v := range cache { + k := key{v.URL, v.Hash, v.Path, v.Type} + if i, ok := m[k]; ok { + // Overwrite previous value per requirement in + // question to keep last matching value. + unique[i] = v + } else { + // Unique key found. Record position and collect + // in result. + m[k] = len(unique) + unique = append(unique, v) + } + } + return unique +} diff --git a/pkg/remotefetcher/configfetcher.go b/pkg/remotefetcher/configfetcher.go index f453198..43d8407 100644 --- a/pkg/remotefetcher/configfetcher.go +++ b/pkg/remotefetcher/configfetcher.go @@ -59,6 +59,7 @@ func NewRemoteFetcher(source string, cache *Cache, options ...FetcherOption) (Re URLHash := HashURL(source) if cachedData, cacheMeta, exists := cache.Get(URLHash); exists { + println(cachedData) return &CachedFetcher{data: cachedData, path: cacheMeta.Path, dataType: cacheMeta.Type}, nil }