CyberShell/backy
1
1
Fork 0
Code Issues 10 Pull Requests Packages Projects 1 Releases 20 Wiki Activity

[WIP] v0.7.0 almost ready to release

Browse Source
This commit is contained in:
Andrew Woodlee
2025-02-08 15:17:34 -06:00
parent 8788d473a5
commit 11ec1a98d8
34 changed files with 556 additions and 196 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
pkg/remotefetcher/cache.go
View File

@ -64,7 +64,6 @@ func (c *Cache) loadFromFile() error {
}
func (c *Cache) saveToFile() error {
// println("Saving cache to file:", c.file)
c.mu.Lock()
defer c.mu.Unlock()
@ -84,10 +83,8 @@ func (c *Cache) saveToFile() error {
func (c *Cache) Get(hash string) ([]byte, CacheData, bool) {
c.mu.Lock()
defer c.mu.Unlock()
println("Getting cache data for hash:", hash)
cacheData, exists := c.store[hash]
if !exists {
println("Cache data does not exist for hash:", hash)
return nil, CacheData{}, false
}
@ -99,10 +96,9 @@ func (c *Cache) Get(hash string) ([]byte, CacheData, bool) {
return data, cacheData, true
}
func (c *Cache) AddDataToStore(hash string, cacheData CacheData) {
c.mu.Lock()
defer c.mu.Unlock()
func (c *Cache) AddDataToStore(hash string, cacheData CacheData) error {
c.store[hash] = cacheData
return c.saveToFile()
}
func (c *Cache) Set(source, hash string, data []byte, dataType string) (CacheData, error) {
@ -164,9 +160,8 @@ func (cf *CachedFetcher) Hash(data []byte) string {
return hex.EncodeToString(hash[:])
}
// Function to read and parse the hashMetadataSample.yml file
// Function to read and parse the metadata file
func LoadMetadataFromFile(filePath string) ([]*CacheData, error) {
// fmt.Println("Loading metadata from file:", filePath)
if _, err := os.Stat(filePath); os.IsNotExist(err) {
// Create the file if it does not exist
emptyData := []byte("[]")

25
pkg/remotefetcher/configfetcher.go
View File

@ -5,7 +5,7 @@ import (
"strings"
)
type ConfigFetcher interface {
type RemoteFetcher interface {
// Fetch retrieves the configuration from the specified URL or source
// Returns the raw data as bytes or an error
Fetch(source string) ([]byte, error)
@ -18,30 +18,31 @@ type ConfigFetcher interface {
Hash(data []byte) string
}
// ErrFileNotFound is returned when the file is not found and should be ignored
var ErrFileNotFound = errors.New("remotefetcher: file not found")
// ErrIgnoreFileNotFound is returned when the file is not found and should be ignored
var ErrIgnoreFileNotFound = errors.New("remotefetcher: file not found")
func NewConfigFetcher(source string, cache *Cache, options ...Option) (ConfigFetcher, error) {
var fetcher ConfigFetcher
var dataType string
func NewRemoteFetcher(source string, cache *Cache, options ...FetcherOption) (RemoteFetcher, error) {
var fetcher RemoteFetcher
config := FetcherConfig{}
for _, option := range options {
option(&config)
}
// If FileType is empty (i.e. WithFileType was not called), yaml is the default file type
if strings.TrimSpace(config.FileType) == "" {
config.FileType = "yaml"
}
if strings.HasPrefix(source, "http") || strings.HasPrefix(source, "https") {
fetcher = NewHTTPFetcher(options...)
dataType = "yaml"
} else if strings.HasPrefix(source, "s3") {
var err error
fetcher, err = NewS3Fetcher(options...)
fetcher, err = NewS3Fetcher(source, options...)
if err != nil {
return nil, err
}
dataType = "yaml"
} else {
fetcher = &LocalFetcher{}
dataType = "yaml"
return fetcher, nil
}
@ -51,7 +52,7 @@ func NewConfigFetcher(source string, cache *Cache, options ...Option) (ConfigFet
data, err := fetcher.Fetch(source)
if err != nil {
if config.IgnoreFileNotFound && isFileNotFoundError(err) {
return nil, ErrFileNotFound
return nil, ErrIgnoreFileNotFound
}
return nil, err
}
@ -61,7 +62,7 @@ func NewConfigFetcher(source string, cache *Cache, options ...Option) (ConfigFet
return &CachedFetcher{data: cachedData, path: cacheMeta.Path, dataType: cacheMeta.Type}, nil
}
cacheData, err := cache.Set(source, hash, data, dataType)
cacheData, err := cache.Set(source, hash, data, config.FileType)
if err != nil {
return nil, err
}

4
pkg/remotefetcher/http.go
View File

@ -16,7 +16,7 @@ type HTTPFetcher struct {
}
// NewHTTPFetcher creates a new instance of HTTPFetcher with the provided options.
func NewHTTPFetcher(options ...Option) *HTTPFetcher {
func NewHTTPFetcher(options ...FetcherOption) *HTTPFetcher {
cfg := &FetcherConfig{}
for _, opt := range options {
opt(cfg)
@ -39,7 +39,7 @@ func (h *HTTPFetcher) Fetch(source string) ([]byte, error) {
defer resp.Body.Close()
if resp.StatusCode == http.StatusNotFound && h.config.IgnoreFileNotFound {
return nil, ErrFileNotFound
return nil, ErrIgnoreFileNotFound
}
if resp.StatusCode != http.StatusOK {

2
pkg/remotefetcher/local.go
View File

@ -18,7 +18,7 @@ func (l *LocalFetcher) Fetch(source string) ([]byte, error) {
// Check if the file exists
if _, err := os.Stat(source); os.IsNotExist(err) {
if l.config.IgnoreFileNotFound {
return nil, ErrFileNotFound
return nil, ErrIgnoreFileNotFound
}
return nil, nil
}

20
pkg/remotefetcher/options.go
View File

@ -2,36 +2,48 @@ package remotefetcher
import (
"net/http"
"strings"
"github.com/aws/aws-sdk-go-v2/service/s3"
)
// Option is a function that configures a fetcher.
type Option func(*FetcherConfig)
type FetcherOption func(*FetcherConfig)
// FetcherConfig holds the configuration for a fetcher.
type FetcherConfig struct {
S3Client *s3.Client
HTTPClient *http.Client
FileType string
IgnoreFileNotFound bool
}
// WithS3Client sets the S3 client for the fetcher.
func WithS3Client(client *s3.Client) Option {
func WithS3Client(client *s3.Client) FetcherOption {
return func(cfg *FetcherConfig) {
cfg.S3Client = client
}
}
// WithHTTPClient sets the HTTP client for the fetcher.
func WithHTTPClient(client *http.Client) Option {
func WithHTTPClient(client *http.Client) FetcherOption {
return func(cfg *FetcherConfig) {
cfg.HTTPClient = client
}
}
func IgnoreFileNotFound() Option {
func IgnoreFileNotFound() FetcherOption {
return func(cfg *FetcherConfig) {
cfg.IgnoreFileNotFound = true
}
}
// WithFileType ensures the default FileType will be yaml
func WithFileType(fileType string) FetcherOption {
return func(cfg *FetcherConfig) {
cfg.FileType = fileType
if strings.TrimSpace(fileType) == "" {
cfg.FileType = "yaml"
}
}
}

120
pkg/remotefetcher/s3.go
View File

@ -1,71 +1,109 @@
package remotefetcher
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"io"
"net/http"
"net/url"
"os"
"path"
"strings"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/mitchellh/go-homedir"
"gopkg.in/yaml.v3"
)
type S3Fetcher struct {
S3Client *s3.Client
S3Client *minio.Client
config FetcherConfig
}
// NewS3Fetcher creates a new instance of S3Fetcher with the provided options.
func NewS3Fetcher(options ...Option) (*S3Fetcher, error) {
func NewS3Fetcher(endpoint string, options ...FetcherOption) (*S3Fetcher, error) {
cfg := &FetcherConfig{}
var s3Client *minio.Client
var err error
for _, opt := range options {
opt(cfg)
}
/*
options for S3 urls:
1. s3://bucket.region.endpoint.tld/path/to/object
2. alias with path and rest is looked up in file - add FetcherOptions
options for S3 credentials:
1. from file ($HOME/.aws/credentials)
2. env vars (AWS_SECRET_KEY, etc.)
*/
s3Endpoint := os.Getenv("S3_ENDPOINT")
creds, err := getS3Credentials("default", s3Endpoint, cfg.HTTPClient)
if err != nil {
println(err.Error())
return nil, err
}
// Initialize S3 client if not provided
if cfg.S3Client == nil {
awsCfg, err := config.LoadDefaultConfig(context.TODO())
s3Client, err = minio.New(s3Endpoint, &minio.Options{
Creds: creds,
Secure: true,
})
if err != nil {
return nil, err
}
cfg.S3Client = s3.NewFromConfig(awsCfg)
}
return &S3Fetcher{S3Client: cfg.S3Client, config: *cfg}, nil
return &S3Fetcher{S3Client: s3Client, config: *cfg}, nil
}
// Fetch retrieves the configuration from an S3 bucket
// Source should be in the format "bucket-name/object-key"
func (s *S3Fetcher) Fetch(source string) ([]byte, error) {
bucket, key, err := parseS3Source(source)
bucket, object, err := parseS3Source(source)
if err != nil {
return nil, err
}
resp, err := s.S3Client.GetObject(context.TODO(), &s3.GetObjectInput{
Bucket: &bucket,
Key: &key,
})
if err != nil {
var notFound *types.NoSuchKey
if errors.As(err, &notFound) && s.config.IgnoreFileNotFound {
return nil, ErrFileNotFound
doesObjectExist, objErr := objectExists(bucket, object, s.S3Client)
if !doesObjectExist {
if objErr != nil {
return nil, err
}
if s.config.IgnoreFileNotFound {
return nil, ErrIgnoreFileNotFound
}
}
fileObject, err := s.S3Client.GetObject(context.TODO(), bucket, object, minio.GetObjectOptions{})
if err != nil {
println(err.Error())
return nil, err
}
defer resp.Body.Close()
defer fileObject.Close()
fileObjectStats, statErr := fileObject.Stat()
if statErr != nil {
return nil, statErr
}
buffer := make([]byte, fileObjectStats.Size)
buf := new(bytes.Buffer)
_, err = buf.ReadFrom(resp.Body)
// Read the object into the buffer
_, err = io.ReadFull(fileObject, buffer)
if err != nil {
return nil, err
}
return buf.Bytes(), nil
return buffer, nil
}
// Parse decodes the raw data into the provided target structure
@ -79,10 +117,46 @@ func parseS3Source(source string) (bucket, key string, err error) {
if len(parts) != 2 {
return "", "", errors.New("invalid S3 source format, expected bucket-name/object-key")
}
return parts[0], parts[1], nil
u, _ := url.Parse(source)
u.Path = strings.TrimPrefix(u.Path, "/")
return u.Host, u.Path, nil
}
func (s *S3Fetcher) Hash(data []byte) string {
hash := sha256.Sum256(data)
return hex.EncodeToString(hash[:])
}
func getS3Credentials(profile, host string, httpClient *http.Client) (*credentials.Credentials, error) {
// println(s3utils.GetRegionFromURL(*u))
homeDir, hdirErr := homedir.Dir()
if hdirErr != nil {
return nil, hdirErr
}
s3Creds := credentials.NewFileAWSCredentials(path.Join(homeDir, ".aws", "credentials"), "default")
credVals, credErr := s3Creds.GetWithContext(&credentials.CredContext{Endpoint: host, Client: httpClient})
if credErr != nil {
return nil, credErr
}
creds := credentials.NewStaticV4(credVals.AccessKeyID, credVals.SecretAccessKey, "")
return creds, nil
}
var (
doesNotExist = "The specified key does not exist."
)
// objectExists checks for name in bucket using client.
// It returns false and nil if the key does not exist
func objectExists(bucket, name string, client *minio.Client) (bool, error) {
_, err := client.StatObject(context.TODO(), bucket, name, minio.StatObjectOptions{})
if err != nil {
switch err.Error() {
case doesNotExist:
return false, nil
default:
return false, errors.Join(err, errors.New("error stating object"))
}
}
return true, nil
}