CyberShell/backy
1
1
Fork 0
Code Issues 10 Pull Requests Packages Projects 1 Releases 20 Wiki Activity

Merge branch 'remoteResources' into develop
Some checks failed
ci/woodpecker/push/go-lint Pipeline failed
Details

Browse Source
This commit is contained in:
Andrew Woodlee
2025-02-08 16:14:37 -06:00
parent 5d3c265ce9 11ec1a98d8
commit e20141043c
45 changed files with 1101 additions and 327 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
pkg/backy/backy.go
View File

@ -100,8 +100,14 @@ func (command *Command) RunCmd(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([
cmdCtxLogger.Info().Str("Command", fmt.Sprintf("Running command %s on local machine", command.Name)).Send()
localCMD = exec.Command(command.Cmd, command.Args...)
// execute package commands in a shell
if command.Type == "package" {
cmdCtxLogger.Info().Str("package", command.PackageName).Msg("Executing package command")
ArgsStr = fmt.Sprintf("%s %s", command.Cmd, ArgsStr)
localCMD = exec.Command("/bin/sh", "-c", ArgsStr)
} else {
localCMD = exec.Command(command.Cmd, command.Args...)
}
}
if command.Dir != nil {
@ -232,7 +238,8 @@ func notifyError(logger zerolog.Logger, templates *msgTemplates, list *CmdList,
"CmdsRan": cmdsRan,
"CmdOutput": outStructArr,
"Err": err,
"Command": cmd.Name,
"CmdName": cmd.Name,
"Command": cmd.Cmd,
"Args": cmd.Args,
}
var errMsg bytes.Buffer

121
pkg/backy/config.go
View File

@ -2,15 +2,17 @@ package backy
import (
"context"
"errors"
"fmt"
"net/url"
"os"
"path"
"runtime"
"strings"
"git.andrewnw.xyz/CyberShell/backy/pkg/configfetcher"
"git.andrewnw.xyz/CyberShell/backy/pkg/logging"
"git.andrewnw.xyz/CyberShell/backy/pkg/pkgman"
"git.andrewnw.xyz/CyberShell/backy/pkg/remotefetcher"
"git.andrewnw.xyz/CyberShell/backy/pkg/usermanager"
vault "github.com/hashicorp/vault/api"
"github.com/knadh/koanf/parsers/yaml"
@ -20,23 +22,23 @@ import (
"github.com/rs/zerolog"
)
var homeDir string
var homeDirErr error
var backyHomeConfDir string
var configFiles []string
const macroStart string = "%{"
const macroEnd string = "}%"
const envMacroStart string = "%{env:"
const vaultMacroStart string = "%{vault:"
func (opts *ConfigOpts) InitConfig() {
homeDir, err := os.UserHomeDir()
var err error
homeConfigDir, err := os.UserConfigDir()
if err != nil {
logging.ExitWithMSG(err.Error(), 1, nil)
}
homeCacheDir, err := os.UserCacheDir()
if err != nil {
logging.ExitWithMSG(err.Error(), 1, nil)
}
backyHomeConfDir := path.Join(homeDir, ".config/backy/")
backyHomeConfDir := path.Join(homeConfigDir, "backy")
configFiles := []string{
"./backy.yml", "./backy.yaml",
path.Join(backyHomeConfDir, "backy.yml"),
@ -46,8 +48,39 @@ func (opts *ConfigOpts) InitConfig() {
backyKoanf := koanf.New(".")
opts.ConfigFilePath = strings.TrimSpace(opts.ConfigFilePath)
// metadataFile := "hashMetadataSample.yml"
cacheDir := homeCacheDir
// Load metadata from file
opts.CachedData, err = remotefetcher.LoadMetadataFromFile(path.Join(backyHomeConfDir, "cache.yml"))
if err != nil {
fmt.Println("Error loading metadata:", err)
logging.ExitWithMSG(err.Error(), 1, &opts.Logger)
}
// Initialize cache with loaded metadata
cache, err := remotefetcher.NewCache(path.Join(backyHomeConfDir, "cache.yml"), cacheDir)
if err != nil {
fmt.Println("Error initializing cache:", err)
logging.ExitWithMSG(err.Error(), 1, &opts.Logger)
}
// Populate cache with loaded metadata
for _, data := range opts.CachedData {
if err := cache.AddDataToStore(data.Hash, *data); err != nil {
logging.ExitWithMSG(err.Error(), 1, &opts.Logger)
}
}
opts.Cache, err = remotefetcher.NewCache(path.Join(backyHomeConfDir, "cache.yml"), backyHomeConfDir)
if err != nil {
logging.ExitWithMSG(fmt.Sprintf("error initializing cache: %v", err), 1, nil)
}
// Initialize the fetcher
fetcher, err := configfetcher.NewConfigFetcher(opts.ConfigFilePath)
// println("Creating new fetcher for source", opts.ConfigFilePath)
fetcher, err := remotefetcher.NewRemoteFetcher(opts.ConfigFilePath, opts.Cache)
// println("Created new fetcher for source", opts.ConfigFilePath)
if err != nil {
logging.ExitWithMSG(fmt.Sprintf("error initializing config fetcher: %v", err), 1, nil)
@ -62,7 +95,7 @@ func (opts *ConfigOpts) InitConfig() {
opts.koanf = backyKoanf
}
func loadConfigFile(fetcher configfetcher.ConfigFetcher, filePath string, k *koanf.Koanf, opts *ConfigOpts) {
func loadConfigFile(fetcher remotefetcher.RemoteFetcher, filePath string, k *koanf.Koanf, opts *ConfigOpts) {
data, err := fetcher.Fetch(filePath)
if err != nil {
logging.ExitWithMSG(fmt.Sprintf("Could not fetch config file %s: %v", filePath, err), 1, nil)
@ -73,7 +106,7 @@ func loadConfigFile(fetcher configfetcher.ConfigFetcher, filePath string, k *koa
}
}
func loadDefaultConfigFiles(fetcher configfetcher.ConfigFetcher, configFiles []string, k *koanf.Koanf, opts *ConfigOpts) {
func loadDefaultConfigFiles(fetcher remotefetcher.RemoteFetcher, configFiles []string, k *koanf.Koanf, opts *ConfigOpts) {
cFileFailures := 0
for _, c := range configFiles {
data, err := fetcher.Fetch(c)
@ -236,13 +269,23 @@ func resolveProxyHosts(host *Host, opts *ConfigOpts) {
}
func loadCommandLists(opts *ConfigOpts, backyKoanf *koanf.Koanf) {
backyConfigFileDir := path.Dir(opts.ConfigFilePath)
listsConfig := koanf.New(".")
listConfigFiles := []string{
path.Join(backyConfigFileDir, "lists.yml"),
path.Join(backyConfigFileDir, "lists.yaml"),
var backyConfigFileDir string
var listConfigFiles []string
var u *url.URL
// if config file is remote, use the directory of the remote file
if isRemoteURL(opts.ConfigFilePath) {
_, u = getRemoteDir(opts.ConfigFilePath)
listConfigFiles = []string{u.JoinPath("lists.yml").String(), u.JoinPath("lists.yaml").String()}
} else {
backyConfigFileDir = path.Dir(opts.ConfigFilePath)
listConfigFiles = []string{
path.Join(backyConfigFileDir, "lists.yml"),
path.Join(backyConfigFileDir, "lists.yaml"),
}
}
listsConfig := koanf.New(".")
for _, l := range listConfigFiles {
if loadListConfigFile(l, listsConfig, opts) {
break
@ -257,9 +300,29 @@ func loadCommandLists(opts *ConfigOpts, backyKoanf *koanf.Koanf) {
}
}
func loadListConfigFile(filePath string, k *koanf.Koanf, opts *ConfigOpts) bool {
fetcher, err := configfetcher.NewConfigFetcher(filePath)
func isRemoteURL(filePath string) bool {
return strings.HasPrefix(filePath, "http://") || strings.HasPrefix(filePath, "https://") || strings.HasPrefix(filePath, "s3://")
}
func getRemoteDir(filePath string) (string, *url.URL) {
u, err := url.Parse(filePath)
if err != nil {
return "", nil
}
// u.Path is the path to the file, stripped of scheme and hostname
u.Path = path.Dir(u.Path)
return u.String(), u
}
func loadListConfigFile(filePath string, k *koanf.Koanf, opts *ConfigOpts) bool {
fetcher, err := remotefetcher.NewRemoteFetcher(filePath, opts.Cache, remotefetcher.IgnoreFileNotFound())
if err != nil {
// if file not found, ignore
if errors.Is(err, remotefetcher.ErrIgnoreFileNotFound) {
return true
}
logging.ExitWithMSG(fmt.Sprintf("error initializing config fetcher: %v", err), 1, nil)
}
@ -282,7 +345,8 @@ func loadCmdListsFile(backyKoanf *koanf.Koanf, listsConfig *koanf.Koanf, opts *C
opts.CmdListFile = path.Join(path.Dir(opts.ConfigFilePath), opts.CmdListFile)
}
fetcher, err := configfetcher.NewConfigFetcher(opts.CmdListFile)
fetcher, err := remotefetcher.NewRemoteFetcher(opts.CmdListFile, opts.Cache)
if err != nil {
logging.ExitWithMSG(fmt.Sprintf("error initializing config fetcher: %v", err), 1, nil)
}
@ -542,23 +606,22 @@ func processCmds(opts *ConfigOpts) error {
}
}
if cmd.Type == "remoteScript" {
if !isRemoteURL(cmd.Cmd) {
return fmt.Errorf("remoteScript command %s must be a remote resource", cmdName)
}
}
}
return nil
}
// processHooks evaluates if hooks are valid Commands
//
// Takes the following arguments:
// The cmd.hookRefs[hookType] is created with any hooks found.
//
// 1. a []string of hooks
// 2. a map of Commands as arguments
// 3. a string hookType, must be the hook type
//
// The cmds.hookRef is modified in this function.
//
// Returns the following:
//
// An error, if any, if the command is not found
// Returns an error, if any, if the hook command is not found
func processHooks(cmd *Command, hooks []string, opts *ConfigOpts, hookType string) error {
// initialize hook type

4
pkg/backy/templates/error.txt
View File

@ -1,12 +1,12 @@
Command list {{.listName }} failed.
The command run was {{.Cmd}}.
The command run was {{.CmdName}}.
The command executed was {{.Command}} {{ if .Args }} {{- range .Args}} {{.}} {{end}} {{end}}
{{ if .Err }} The error was {{ .Err }}{{ end }}
{{ if .Output }} The output was {{- range .Output}} {{.}} {{end}} {{end}}
{{ if .Output }} The output was: {{- range .Output}} {{.}} {{end}} {{end}}
{{ if .CmdsRan }}
The following commands ran:

26
pkg/backy/types.go
View File

@ -7,6 +7,7 @@ import (
"strings"
"git.andrewnw.xyz/CyberShell/backy/pkg/pkgman"
"git.andrewnw.xyz/CyberShell/backy/pkg/remotefetcher"
"git.andrewnw.xyz/CyberShell/backy/pkg/usermanager"
vaultapi "github.com/hashicorp/vault/api"
"github.com/kevinburke/ssh_config"
@ -66,10 +67,7 @@ type (
// hook refs are internal references of commands for each hook type
hookRefs map[string]map[string]*Command
/*
Shell specifies which shell to run the command in, if any.
Not applicable when host is defined.
*/
// Shell specifies which shell to run the command in, if any.
Shell string `yaml:"shell,omitempty"`
RemoteHost *Host `yaml:"-"`
@ -90,11 +88,14 @@ type (
Environment []string `yaml:"environment,omitempty"`
// Output determines if output is requested.
// Only works if command is in a list.
//
// Only for when command is in a list.
GetOutput bool `yaml:"getOutput,omitempty"`
ScriptEnvFile string `yaml:"scriptEnvFile"`
// BEGIN PACKAGE COMMAND FIELDS
PackageManager string `yaml:"packageManager,omitempty"`
PackageName string `yaml:"packageName,omitempty"`
@ -108,6 +109,7 @@ type (
pkgMan pkgman.PackageManager
packageCmdSet bool
// END PACKAGE COMMAND FIELDS
// RemoteSource specifies a URL to fetch the command or configuration remotely
RemoteSource string `yaml:"remoteSource,omitempty"`
@ -115,8 +117,12 @@ type (
// FetchBeforeExecution determines if the remoteSource should be fetched before running
FetchBeforeExecution bool `yaml:"fetchBeforeExecution,omitempty"`
// BEGIN USER COMMAND FIELDS
// Username specifies the username for user creation or related operations
Username string `yaml:"username,omitempty"`
Username string `yaml:"userName,omitempty"`
UserID string `yaml:"userID,omitempty"`
// UserGroups specifies the groups to add the user to
UserGroups []string `yaml:"userGroups,omitempty"`
@ -143,12 +149,15 @@ type (
userCmdSet bool
// stdin only for userOperation = password (for now)
stdin *strings.Reader
// END USER STRUCT FIELDS
}
RemoteSource struct {
URL string `yaml:"url"`
Type string `yaml:"type"` // e.g., yaml
Type string `yaml:"type"` // e.g., s3, http
Auth struct {
AccessKey string `yaml:"accessKey"`
SecretKey string `yaml:"secretKey"`
@ -217,6 +226,9 @@ type (
koanf *koanf.Koanf
NotificationConf *Notifications `yaml:"notifications"`
Cache *remotefetcher.Cache
CachedData []*remotefetcher.CacheData
}
outStruct struct {

8
pkg/backy/utils.go
View File

@ -43,7 +43,7 @@ func AddCommandLists(lists []string) BackyOptionFunc {
}
}
// AddPrintLists adds lists to print out
// SetListsToSearch adds lists to search
func SetListsToSearch(lists []string) BackyOptionFunc {
return func(bco *ConfigOpts) {
bco.List.Lists = append(bco.List.Lists, lists...)
@ -64,8 +64,8 @@ func SetLogFile(logFile string) BackyOptionFunc {
}
}
// cronEnabled enables the execution of command lists at specified times
func CronEnabled() BackyOptionFunc {
// EnableCron enables the execution of command lists at specified times
func EnableCron() BackyOptionFunc {
return func(bco *ConfigOpts) {
bco.cronEnabled = true
}
@ -275,7 +275,7 @@ func getCommandType(command *Command) *Command {
case "modify":
command.Cmd, command.Args = command.userMan.ModifyUser(
command.Username,
homeDir,
command.UserHome,
command.UserShell,
command.UserGroups)
case "checkIfExists":

27
pkg/configfetcher/configfetcher.go
View File

@ -1,27 +0,0 @@
package configfetcher
import "strings"
type ConfigFetcher interface {
// Fetch retrieves the configuration from the specified URL or source
// Returns the raw data as bytes or an error
Fetch(source string) ([]byte, error)
// Parse decodes the raw data into a Go structure (e.g., Commands, CommandLists)
// Takes the raw data as input and populates the target interface
Parse(data []byte, target interface{}) error
}
func NewConfigFetcher(source string, options ...Option) (ConfigFetcher, error) {
if strings.HasPrefix(source, "http") || strings.HasPrefix(source, "https") {
return NewHTTPFetcher(options...), nil
} else if strings.HasPrefix(source, "s3") {
fetcher, err := NewS3Fetcher(options...)
if err != nil {
return nil, err
}
return fetcher, nil
} else {
return &LocalFetcher{}, nil
}
}

26
pkg/configfetcher/local.go
View File

@ -1,26 +0,0 @@
package configfetcher
import (
"io"
"os"
"gopkg.in/yaml.v3"
)
type LocalFetcher struct{}
// Fetch retrieves the configuration from the specified local file path
func (l *LocalFetcher) Fetch(source string) ([]byte, error) {
file, err := os.Open(source)
if err != nil {
return nil, err
}
defer file.Close()
return io.ReadAll(file)
}
// Parse decodes the raw data into the provided target structure
func (l *LocalFetcher) Parse(data []byte, target interface{}) error {
return yaml.Unmarshal(data, target)
}

30
pkg/configfetcher/options.go
View File

@ -1,30 +0,0 @@
package configfetcher
import (
"net/http"
"github.com/aws/aws-sdk-go-v2/service/s3"
)
// Option is a function that configures a fetcher.
type Option func(*FetcherConfig)
// FetcherConfig holds the configuration for a fetcher.
type FetcherConfig struct {
S3Client *s3.Client
HTTPClient *http.Client
}
// WithS3Client sets the S3 client for the fetcher.
func WithS3Client(client *s3.Client) Option {
return func(cfg *FetcherConfig) {
cfg.S3Client = client
}
}
// WithHTTPClient sets the HTTP client for the fetcher.
func WithHTTPClient(client *http.Client) Option {
return func(cfg *FetcherConfig) {
cfg.HTTPClient = client
}
}

75
pkg/configfetcher/s3.go
View File

@ -1,75 +0,0 @@
package configfetcher
import (
"bytes"
"context"
"errors"
"strings"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/service/s3"
"gopkg.in/yaml.v3"
)
type S3Fetcher struct {
S3Client *s3.Client
}
// NewS3Fetcher creates a new instance of S3Fetcher with the provided options.
func NewS3Fetcher(options ...Option) (*S3Fetcher, error) {
cfg := &FetcherConfig{}
for _, opt := range options {
opt(cfg)
}
// Initialize S3 client if not provided
if cfg.S3Client == nil {
awsCfg, err := config.LoadDefaultConfig(context.TODO())
if err != nil {
return nil, err
}
cfg.S3Client = s3.NewFromConfig(awsCfg)
}
return &S3Fetcher{S3Client: cfg.S3Client}, nil
}
// Fetch retrieves the configuration from an S3 bucket
// Source should be in the format "bucket-name/object-key"
func (s *S3Fetcher) Fetch(source string) ([]byte, error) {
bucket, key, err := parseS3Source(source)
if err != nil {
return nil, err
}
resp, err := s.S3Client.GetObject(context.TODO(), &s3.GetObjectInput{
Bucket: &bucket,
Key: &key,
})
if err != nil {
return nil, err
}
defer resp.Body.Close()
buf := new(bytes.Buffer)
_, err = buf.ReadFrom(resp.Body)
if err != nil {
return nil, err
}
return buf.Bytes(), nil
}
// Parse decodes the raw data into the provided target structure
func (s *S3Fetcher) Parse(data []byte, target interface{}) error {
return yaml.Unmarshal(data, target)
}
// Helper function to parse S3 source into bucket and key
func parseS3Source(source string) (bucket, key string, err error) {
parts := strings.SplitN(source, "/", 2)
if len(parts) != 2 {
return "", "", errors.New("invalid S3 source format, expected bucket-name/object-key")
}
return parts[0], parts[1], nil
}

1
pkg/pkgman/apt/apt.go
View File

@ -40,7 +40,6 @@ func (a *AptManager) Install(pkg, version string, args []string) (string, []stri
if args != nil {
baseArgs = append(baseArgs, args...)
}
fmt.Printf("baseArgs: %v\n", baseArgs)
return baseCmd, baseArgs
}

186
pkg/remotefetcher/cache.go Normal file
View File

@ -0,0 +1,186 @@
package remotefetcher
import (
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"os"
"path/filepath"
"sync"
"gopkg.in/yaml.v3"
)
type CacheData struct {
Hash string `yaml:"hash"`
Path string `yaml:"path"`
Type string `yaml:"type"`
}
type Cache struct {
mu sync.Mutex
store map[string]CacheData
file string
dir string
}
func NewCache(file, dir string) (*Cache, error) {
cache := &Cache{
store: make(map[string]CacheData),
file: file,
dir: dir,
}
err := cache.loadFromFile()
if err != nil {
return nil, err
}
return cache, nil
}
func (c *Cache) loadFromFile() error {
c.mu.Lock()
defer c.mu.Unlock()
if _, err := os.Stat(c.file); os.IsNotExist(err) {
return nil
}
data, err := os.ReadFile(c.file)
if err != nil {
return err
}
var cacheData []CacheData
err = yaml.Unmarshal(data, &cacheData)
if err != nil {
return err
}
for _, item := range cacheData {
c.store[item.Hash] = item
}
return nil
}
func (c *Cache) saveToFile() error {
c.mu.Lock()
defer c.mu.Unlock()
var cacheData []CacheData
for _, data := range c.store {
cacheData = append(cacheData, data)
}
data, err := yaml.Marshal(cacheData)
if err != nil {
return err
}
return os.WriteFile(c.file, data, 0644)
}
func (c *Cache) Get(hash string) ([]byte, CacheData, bool) {
c.mu.Lock()
defer c.mu.Unlock()
cacheData, exists := c.store[hash]
if !exists {
return nil, CacheData{}, false
}
data, err := os.ReadFile(cacheData.Path)
if err != nil {
return nil, CacheData{}, false
}
return data, cacheData, true
}
func (c *Cache) AddDataToStore(hash string, cacheData CacheData) error {
c.store[hash] = cacheData
return c.saveToFile()
}
func (c *Cache) Set(source, hash string, data []byte, dataType string) (CacheData, error) {
c.mu.Lock()
defer c.mu.Unlock()
fileName := filepath.Base(source)
path := filepath.Join(c.dir, fmt.Sprintf("%s-%s", fileName, hash))
if _, err := os.Stat(path); os.IsNotExist(err) {
os.MkdirAll(c.dir, 0700)
}
err := os.WriteFile(path, data, 0644)
if err != nil {
return CacheData{}, err
}
cacheData := CacheData{
Hash: hash,
Path: path,
Type: dataType,
}
c.store[hash] = cacheData
// Unlock before calling saveToFile to avoid double-locking
c.mu.Unlock()
err = c.saveToFile()
c.mu.Lock()
if err != nil {
return CacheData{}, err
}
// fmt.Printf("Cache data: %v", cacheData)
return cacheData, nil
}
type CachedFetcher struct {
data []byte
path string
dataType string
}
func (cf *CachedFetcher) Fetch(source string) ([]byte, error) {
return cf.data, nil
}
func (cf *CachedFetcher) Parse(data []byte, target interface{}) error {
if cf.dataType == "yaml" {
return yaml.Unmarshal(data, target)
}
return errors.New("parse not supported on cached fetcher for scripts")
}
func (cf *CachedFetcher) Hash(data []byte) string {
hash := sha256.Sum256(data)
return hex.EncodeToString(hash[:])
}
// Function to read and parse the metadata file
func LoadMetadataFromFile(filePath string) ([]*CacheData, error) {
if _, err := os.Stat(filePath); os.IsNotExist(err) {
// Create the file if it does not exist
emptyData := []byte("[]")
err := os.WriteFile(filePath, emptyData, 0644)
if err != nil {
return nil, err
}
return nil, nil
}
data, err := os.ReadFile(filePath)
if err != nil {
return nil, err
}
var cacheData []*CacheData
err = yaml.Unmarshal(data, &cacheData)
if err != nil {
return nil, err
}
return cacheData, nil
}

76
pkg/remotefetcher/configfetcher.go Normal file
View File

@ -0,0 +1,76 @@
package remotefetcher
import (
"errors"
"strings"
)
type RemoteFetcher interface {
// Fetch retrieves the configuration from the specified URL or source
// Returns the raw data as bytes or an error
Fetch(source string) ([]byte, error)
// Parse decodes the raw data into a Go structure (e.g., Commands, CommandLists)
// Takes the raw data as input and populates the target interface
Parse(data []byte, target interface{}) error
// Hash returns the hash of the configuration data
Hash(data []byte) string
}
// ErrIgnoreFileNotFound is returned when the file is not found and should be ignored
var ErrIgnoreFileNotFound = errors.New("remotefetcher: file not found")
func NewRemoteFetcher(source string, cache *Cache, options ...FetcherOption) (RemoteFetcher, error) {
var fetcher RemoteFetcher
config := FetcherConfig{}
for _, option := range options {
option(&config)
}
// If FileType is empty (i.e. WithFileType was not called), yaml is the default file type
if strings.TrimSpace(config.FileType) == "" {
config.FileType = "yaml"
}
if strings.HasPrefix(source, "http") || strings.HasPrefix(source, "https") {
fetcher = NewHTTPFetcher(options...)
} else if strings.HasPrefix(source, "s3") {
var err error
fetcher, err = NewS3Fetcher(source, options...)
if err != nil {
return nil, err
}
} else {
fetcher = &LocalFetcher{}
return fetcher, nil
}
//TODO: should local files be cached?
data, err := fetcher.Fetch(source)
if err != nil {
if config.IgnoreFileNotFound && isFileNotFoundError(err) {
return nil, ErrIgnoreFileNotFound
}
return nil, err
}
hash := fetcher.Hash(data)
if cachedData, cacheMeta, exists := cache.Get(hash); exists {
return &CachedFetcher{data: cachedData, path: cacheMeta.Path, dataType: cacheMeta.Type}, nil
}
cacheData, err := cache.Set(source, hash, data, config.FileType)
if err != nil {
return nil, err
}
return &CachedFetcher{data: data, path: cacheData.Path, dataType: cacheData.Type}, nil
}
func isFileNotFoundError(err error) bool {
// Implement logic to check if the error is a "file not found" error
// This can be based on the error type or message
return strings.Contains(err.Error(), "file not found")
}

20
pkg/configfetcher/http.go → pkg/remotefetcher/http.go
View File

@ -1,6 +1,8 @@
package configfetcher
package remotefetcher
import (
"crypto/sha256"
"encoding/hex"
"errors"
"io"
"net/http"
@ -10,10 +12,11 @@ import (
type HTTPFetcher struct {
HTTPClient *http.Client
config FetcherConfig
}
// NewHTTPFetcher creates a new instance of HTTPFetcher with the provided options.
func NewHTTPFetcher(options ...Option) *HTTPFetcher {
func NewHTTPFetcher(options ...FetcherOption) *HTTPFetcher {
cfg := &FetcherConfig{}
for _, opt := range options {
opt(cfg)
@ -24,10 +27,10 @@ func NewHTTPFetcher(options ...Option) *HTTPFetcher {
cfg.HTTPClient = http.DefaultClient
}
return &HTTPFetcher{HTTPClient: cfg.HTTPClient}
return &HTTPFetcher{HTTPClient: cfg.HTTPClient, config: *cfg}
}
// Fetch retrieves the configuration from the specified URL
// Fetch retrieves the file from the specified source URL
func (h *HTTPFetcher) Fetch(source string) ([]byte, error) {
resp, err := http.Get(source)
if err != nil {
@ -35,6 +38,10 @@ func (h *HTTPFetcher) Fetch(source string) ([]byte, error) {
}
defer resp.Body.Close()
if resp.StatusCode == http.StatusNotFound && h.config.IgnoreFileNotFound {
return nil, ErrIgnoreFileNotFound
}
if resp.StatusCode != http.StatusOK {
return nil, errors.New("failed to fetch remote config: " + resp.Status)
}
@ -46,3 +53,8 @@ func (h *HTTPFetcher) Fetch(source string) ([]byte, error) {
func (h *HTTPFetcher) Parse(data []byte, target interface{}) error {
return yaml.Unmarshal(data, target)
}
func (h *HTTPFetcher) Hash(data []byte) string {
hash := sha256.Sum256(data)
return hex.EncodeToString(hash[:])
}

42
pkg/remotefetcher/local.go Normal file
View File

@ -0,0 +1,42 @@
package remotefetcher
import (
"crypto/sha256"
"encoding/hex"
"io"
"os"
"gopkg.in/yaml.v3"
)
type LocalFetcher struct {
config FetcherConfig
}
// Fetch retrieves the file from the specified local file path
func (l *LocalFetcher) Fetch(source string) ([]byte, error) {
// Check if the file exists
if _, err := os.Stat(source); os.IsNotExist(err) {
if l.config.IgnoreFileNotFound {
return nil, ErrIgnoreFileNotFound
}
return nil, nil
}
file, err := os.Open(source)
if err != nil {
return nil, err
}
defer file.Close()
return io.ReadAll(file)
}
// Parse decodes the raw data into the provided target structure
func (l *LocalFetcher) Parse(data []byte, target interface{}) error {
return yaml.Unmarshal(data, target)
}
func (l *LocalFetcher) Hash(data []byte) string {
hash := sha256.Sum256(data)
return hex.EncodeToString(hash[:])
}

49
pkg/remotefetcher/options.go Normal file
View File

@ -0,0 +1,49 @@
package remotefetcher
import (
"net/http"
"strings"
"github.com/aws/aws-sdk-go-v2/service/s3"
)
// Option is a function that configures a fetcher.
type FetcherOption func(*FetcherConfig)
// FetcherConfig holds the configuration for a fetcher.
type FetcherConfig struct {
S3Client *s3.Client
HTTPClient *http.Client
FileType string
IgnoreFileNotFound bool
}
// WithS3Client sets the S3 client for the fetcher.
func WithS3Client(client *s3.Client) FetcherOption {
return func(cfg *FetcherConfig) {
cfg.S3Client = client
}
}
// WithHTTPClient sets the HTTP client for the fetcher.
func WithHTTPClient(client *http.Client) FetcherOption {
return func(cfg *FetcherConfig) {
cfg.HTTPClient = client
}
}
func IgnoreFileNotFound() FetcherOption {
return func(cfg *FetcherConfig) {
cfg.IgnoreFileNotFound = true
}
}
// WithFileType ensures the default FileType will be yaml
func WithFileType(fileType string) FetcherOption {
return func(cfg *FetcherConfig) {
cfg.FileType = fileType
if strings.TrimSpace(fileType) == "" {
cfg.FileType = "yaml"
}
}
}

162
pkg/remotefetcher/s3.go Normal file
View File

@ -0,0 +1,162 @@
package remotefetcher
import (
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"io"
"net/http"
"net/url"
"os"
"path"
"strings"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/mitchellh/go-homedir"
"gopkg.in/yaml.v3"
)
type S3Fetcher struct {
S3Client *minio.Client
config FetcherConfig
}
// NewS3Fetcher creates a new instance of S3Fetcher with the provided options.
func NewS3Fetcher(endpoint string, options ...FetcherOption) (*S3Fetcher, error) {
cfg := &FetcherConfig{}
var s3Client *minio.Client
var err error
for _, opt := range options {
opt(cfg)
}
/*
options for S3 urls:
1. s3://bucket.region.endpoint.tld/path/to/object
2. alias with path and rest is looked up in file - add FetcherOptions
options for S3 credentials:
1. from file ($HOME/.aws/credentials)
2. env vars (AWS_SECRET_KEY, etc.)
*/
s3Endpoint := os.Getenv("S3_ENDPOINT")
creds, err := getS3Credentials("default", s3Endpoint, cfg.HTTPClient)
if err != nil {
println(err.Error())
return nil, err
}
// Initialize S3 client if not provided
if cfg.S3Client == nil {
s3Client, err = minio.New(s3Endpoint, &minio.Options{
Creds: creds,
Secure: true,
})
if err != nil {
return nil, err
}
}
return &S3Fetcher{S3Client: s3Client, config: *cfg}, nil
}
// Fetch retrieves the configuration from an S3 bucket
// Source should be in the format "bucket-name/object-key"
func (s *S3Fetcher) Fetch(source string) ([]byte, error) {
bucket, object, err := parseS3Source(source)
if err != nil {
return nil, err
}
doesObjectExist, objErr := objectExists(bucket, object, s.S3Client)
if !doesObjectExist {
if objErr != nil {
return nil, err
}
if s.config.IgnoreFileNotFound {
return nil, ErrIgnoreFileNotFound
}
}
fileObject, err := s.S3Client.GetObject(context.TODO(), bucket, object, minio.GetObjectOptions{})
if err != nil {
println(err.Error())
return nil, err
}
defer fileObject.Close()
fileObjectStats, statErr := fileObject.Stat()
if statErr != nil {
return nil, statErr
}
buffer := make([]byte, fileObjectStats.Size)
// Read the object into the buffer
_, err = io.ReadFull(fileObject, buffer)
if err != nil {
return nil, err
}
return buffer, nil
}
// Parse decodes the raw data into the provided target structure
func (s *S3Fetcher) Parse(data []byte, target interface{}) error {
return yaml.Unmarshal(data, target)
}
// Helper function to parse S3 source into bucket and key
func parseS3Source(source string) (bucket, key string, err error) {
parts := strings.SplitN(source, "/", 2)
if len(parts) != 2 {
return "", "", errors.New("invalid S3 source format, expected bucket-name/object-key")
}
u, _ := url.Parse(source)
u.Path = strings.TrimPrefix(u.Path, "/")
return u.Host, u.Path, nil
}
func (s *S3Fetcher) Hash(data []byte) string {
hash := sha256.Sum256(data)
return hex.EncodeToString(hash[:])
}
func getS3Credentials(profile, host string, httpClient *http.Client) (*credentials.Credentials, error) {
// println(s3utils.GetRegionFromURL(*u))
homeDir, hdirErr := homedir.Dir()
if hdirErr != nil {
return nil, hdirErr
}
s3Creds := credentials.NewFileAWSCredentials(path.Join(homeDir, ".aws", "credentials"), "default")
credVals, credErr := s3Creds.GetWithContext(&credentials.CredContext{Endpoint: host, Client: httpClient})
if credErr != nil {
return nil, credErr
}
creds := credentials.NewStaticV4(credVals.AccessKeyID, credVals.SecretAccessKey, "")
return creds, nil
}
var (
doesNotExist = "The specified key does not exist."
)
// objectExists checks for name in bucket using client.
// It returns false and nil if the key does not exist
func objectExists(bucket, name string, client *minio.Client) (bool, error) {
_, err := client.StatObject(context.TODO(), bucket, name, minio.StatObjectOptions{})
if err != nil {
switch err.Error() {
case doesNotExist:
return false, nil
default:
return false, errors.Join(err, errors.New("error stating object"))
}
}
return true, nil
}

7
pkg/usermanager/common/options.go Normal file
View File

@ -0,0 +1,7 @@
package common
// ConfigurablePackageManager defines methods for setting configuration options.
type ConfigurableUserManager interface {
SetUseAuth(useAuth bool)
SetAuthCommand(authCommand string)
}

1
pkg/usermanager/userman.go
View File

@ -20,6 +20,7 @@ type UserManager interface {
UserExists(username string) (string, []string)
}
// NewUserManager returns a UserManager-compatible struct
func NewUserManager(system string) (UserManager, error) {
var manager UserManager