backy/pkg/remotefetcher/s3.go

package remotefetcher

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"errors"
	"io"
	"net/http"
	"net/url"
	"os"
	"path"
	"strings"

	"github.com/minio/minio-go/v7"
	"github.com/minio/minio-go/v7/pkg/credentials"
	"github.com/mitchellh/go-homedir"
	"gopkg.in/yaml.v3"
)

type S3Fetcher struct {
	S3Client *minio.Client
	config   FetcherConfig
}

// NewS3Fetcher creates a new instance of S3Fetcher with the provided options.
func NewS3Fetcher(endpoint string, options ...FetcherOption) (*S3Fetcher, error) {
	cfg := &FetcherConfig{}
	var s3Client *minio.Client
	var err error
	for _, opt := range options {
		opt(cfg)
	}

	/*
		options for S3 urls:
			1. s3://bucket.region.endpoint.tld/path/to/object
			2. alias with path and rest is looked up in file - add FetcherOptions


		options for S3 credentials:
			1. from file ($HOME/.aws/credentials)
			2. env vars (AWS_SECRET_KEY, etc.)
	*/

	s3Endpoint := os.Getenv("S3_ENDPOINT")
	creds, err := getS3Credentials("default", s3Endpoint, cfg.HTTPClient)
	if err != nil {
		println(err.Error())
		return nil, err
	}
	// Initialize S3 client if not provided
	if cfg.S3Client == nil {
		s3Client, err = minio.New(s3Endpoint, &minio.Options{
			Creds:  creds,
			Secure: true,
		})

		if err != nil {
			return nil, err
		}

	}

	return &S3Fetcher{S3Client: s3Client, config: *cfg}, nil
}

// Fetch retrieves the configuration from an S3 bucket
// Source should be in the format "bucket-name/object-key"
func (s *S3Fetcher) Fetch(source string) ([]byte, error) {
	bucket, object, err := parseS3Source(source)
	if err != nil {
		return nil, err
	}

	doesObjectExist, objErr := objectExists(bucket, object, s.S3Client)
	if !doesObjectExist {
		if objErr != nil {
			return nil, err
		}

		if s.config.IgnoreFileNotFound {
			return nil, ErrIgnoreFileNotFound
		}
	}

	fileObject, err := s.S3Client.GetObject(context.TODO(), bucket, object, minio.GetObjectOptions{})
	if err != nil {
		println(err.Error())

		return nil, err
	}
	defer fileObject.Close()
	fileObjectStats, statErr := fileObject.Stat()
	if statErr != nil {
		return nil, statErr
	}
	buffer := make([]byte, fileObjectStats.Size)

	// Read the object into the buffer
	_, err = io.ReadFull(fileObject, buffer)
	if err != nil {
		return nil, err
	}

	return buffer, nil
}

// Parse decodes the raw data into the provided target structure
func (s *S3Fetcher) Parse(data []byte, target interface{}) error {
	return yaml.Unmarshal(data, target)
}

// Helper function to parse S3 source into bucket and key
func parseS3Source(source string) (bucket, key string, err error) {
	parts := strings.SplitN(source, "/", 2)
	if len(parts) != 2 {
		return "", "", errors.New("invalid S3 source format, expected bucket-name/object-key")
	}
	u, _ := url.Parse(source)
	u.Path = strings.TrimPrefix(u.Path, "/")
	return u.Host, u.Path, nil
}

func (s *S3Fetcher) Hash(data []byte) string {
	hash := sha256.Sum256(data)
	return hex.EncodeToString(hash[:])
}

func getS3Credentials(profile, host string, httpClient *http.Client) (*credentials.Credentials, error) {
	// println(s3utils.GetRegionFromURL(*u))
	homeDir, hdirErr := homedir.Dir()
	if hdirErr != nil {
		return nil, hdirErr
	}
	s3Creds := credentials.NewFileAWSCredentials(path.Join(homeDir, ".aws", "credentials"), "default")
	credVals, credErr := s3Creds.GetWithContext(&credentials.CredContext{Endpoint: host, Client: httpClient})
	if credErr != nil {
		return nil, credErr
	}
	creds := credentials.NewStaticV4(credVals.AccessKeyID, credVals.SecretAccessKey, "")
	return creds, nil
}

var (
	doesNotExist = "The specified key does not exist."
)

// objectExists checks for name in bucket using client.
// It returns false and nil if the key does not exist
func objectExists(bucket, name string, client *minio.Client) (bool, error) {
	_, err := client.StatObject(context.TODO(), bucket, name, minio.StatObjectOptions{})
	if err != nil {
		switch err.Error() {
		case doesNotExist:
			return false, nil
		default:
			return false, errors.Join(err, errors.New("error stating object"))
		}
	}
	return true, nil
}