[WIP] v0.7.0 almost ready to release

pkg/remotefetcher/s3.go

@@ -1,71 +1,109 @@
 package remotefetcher
 
 import (
-	"bytes"
 	"context"
 	"crypto/sha256"
 	"encoding/hex"
 	"errors"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"path"
 	"strings"
 
-	"github.com/aws/aws-sdk-go-v2/config"
-	"github.com/aws/aws-sdk-go-v2/service/s3"
-	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/mitchellh/go-homedir"
 	"gopkg.in/yaml.v3"
 )
 
 type S3Fetcher struct {
-	S3Client *s3.Client
+	S3Client *minio.Client
 	config   FetcherConfig
 }
 
 // NewS3Fetcher creates a new instance of S3Fetcher with the provided options.
-func NewS3Fetcher(options ...Option) (*S3Fetcher, error) {
+func NewS3Fetcher(endpoint string, options ...FetcherOption) (*S3Fetcher, error) {
 	cfg := &FetcherConfig{}
+	var s3Client *minio.Client
+	var err error
 	for _, opt := range options {
 		opt(cfg)
 	}
 
+	/*
+		options for S3 urls:
+			1. s3://bucket.region.endpoint.tld/path/to/object
+			2. alias with path and rest is looked up in file - add FetcherOptions
+
+
+		options for S3 credentials:
+			1. from file ($HOME/.aws/credentials)
+			2. env vars (AWS_SECRET_KEY, etc.)
+	*/
+
+	s3Endpoint := os.Getenv("S3_ENDPOINT")
+	creds, err := getS3Credentials("default", s3Endpoint, cfg.HTTPClient)
+	if err != nil {
+		println(err.Error())
+		return nil, err
+	}
 	// Initialize S3 client if not provided
 	if cfg.S3Client == nil {
-		awsCfg, err := config.LoadDefaultConfig(context.TODO())
+		s3Client, err = minio.New(s3Endpoint, &minio.Options{
+			Creds:  creds,
+			Secure: true,
+		})
+
 		if err != nil {
 			return nil, err
 		}
-		cfg.S3Client = s3.NewFromConfig(awsCfg)
+
 	}
 
-	return &S3Fetcher{S3Client: cfg.S3Client, config: *cfg}, nil
+	return &S3Fetcher{S3Client: s3Client, config: *cfg}, nil
 }
 
 // Fetch retrieves the configuration from an S3 bucket
 // Source should be in the format "bucket-name/object-key"
 func (s *S3Fetcher) Fetch(source string) ([]byte, error) {
-	bucket, key, err := parseS3Source(source)
+	bucket, object, err := parseS3Source(source)
 	if err != nil {
 		return nil, err
 	}
 
-	resp, err := s.S3Client.GetObject(context.TODO(), &s3.GetObjectInput{
-		Bucket: &bucket,
-		Key:    &key,
-	})
-	if err != nil {
-		var notFound *types.NoSuchKey
-		if errors.As(err, &notFound) && s.config.IgnoreFileNotFound {
-			return nil, ErrFileNotFound
+	doesObjectExist, objErr := objectExists(bucket, object, s.S3Client)
+	if !doesObjectExist {
+		if objErr != nil {
+			return nil, err
 		}
+
+		if s.config.IgnoreFileNotFound {
+			return nil, ErrIgnoreFileNotFound
+		}
+	}
+
+	fileObject, err := s.S3Client.GetObject(context.TODO(), bucket, object, minio.GetObjectOptions{})
+	if err != nil {
+		println(err.Error())
+
 		return nil, err
 	}
-	defer resp.Body.Close()
+	defer fileObject.Close()
+	fileObjectStats, statErr := fileObject.Stat()
+	if statErr != nil {
+		return nil, statErr
+	}
+	buffer := make([]byte, fileObjectStats.Size)
 
-	buf := new(bytes.Buffer)
-	_, err = buf.ReadFrom(resp.Body)
+	// Read the object into the buffer
+	_, err = io.ReadFull(fileObject, buffer)
 	if err != nil {
 		return nil, err
 	}
 
-	return buf.Bytes(), nil
+	return buffer, nil
 }
 
 // Parse decodes the raw data into the provided target structure
@@ -79,10 +117,46 @@ func parseS3Source(source string) (bucket, key string, err error) {
 	if len(parts) != 2 {
 		return "", "", errors.New("invalid S3 source format, expected bucket-name/object-key")
 	}
-	return parts[0], parts[1], nil
+	u, _ := url.Parse(source)
+	u.Path = strings.TrimPrefix(u.Path, "/")
+	return u.Host, u.Path, nil
 }
 
 func (s *S3Fetcher) Hash(data []byte) string {
 	hash := sha256.Sum256(data)
 	return hex.EncodeToString(hash[:])
 }
+
+func getS3Credentials(profile, host string, httpClient *http.Client) (*credentials.Credentials, error) {
+	// println(s3utils.GetRegionFromURL(*u))
+	homeDir, hdirErr := homedir.Dir()
+	if hdirErr != nil {
+		return nil, hdirErr
+	}
+	s3Creds := credentials.NewFileAWSCredentials(path.Join(homeDir, ".aws", "credentials"), "default")
+	credVals, credErr := s3Creds.GetWithContext(&credentials.CredContext{Endpoint: host, Client: httpClient})
+	if credErr != nil {
+		return nil, credErr
+	}
+	creds := credentials.NewStaticV4(credVals.AccessKeyID, credVals.SecretAccessKey, "")
+	return creds, nil
+}
+
+var (
+	doesNotExist = "The specified key does not exist."
+)
+
+// objectExists checks for name in bucket using client.
+// It returns false and nil if the key does not exist
+func objectExists(bucket, name string, client *minio.Client) (bool, error) {
+	_, err := client.StatObject(context.TODO(), bucket, name, minio.StatObjectOptions{})
+	if err != nil {
+		switch err.Error() {
+		case doesNotExist:
+			return false, nil
+		default:
+			return false, errors.Join(err, errors.New("error stating object"))
+		}
+	}
+	return true, nil
+}