CyberShell/backy
1
1
Fork 0
Code Issues 10 Pull Requests Packages Projects 1 Releases 20 Wiki Activity

[WIP] v0.7.0 almost ready to release

Browse Source
This commit is contained in:
Andrew Woodlee
2025-02-08 15:17:34 -06:00
parent 8788d473a5
commit 11ec1a98d8
34 changed files with 556 additions and 196 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
pkg/backy/backy.go
View File

@ -236,7 +236,8 @@ func notifyError(logger zerolog.Logger, templates *msgTemplates, list *CmdList,
"CmdsRan": cmdsRan,
"CmdOutput": outStructArr,
"Err": err,
"Command": cmd.Name,
"CmdName": cmd.Name,
"Command": cmd.Cmd,
"Args": cmd.Args,
}
var errMsg bytes.Buffer

50
pkg/backy/config.go
View File

@ -48,26 +48,29 @@ func (opts *ConfigOpts) InitConfig() {
backyKoanf := koanf.New(".")
opts.ConfigFilePath = strings.TrimSpace(opts.ConfigFilePath)
metadataFile := "hashMetadataSample.yml"
// metadataFile := "hashMetadataSample.yml"
cacheDir := homeCacheDir
// Load metadata from file
opts.CachedData, err = remotefetcher.LoadMetadataFromFile(metadataFile)
opts.CachedData, err = remotefetcher.LoadMetadataFromFile(path.Join(backyHomeConfDir, "cache.yml"))
if err != nil {
fmt.Println("Error loading metadata:", err)
panic(err)
logging.ExitWithMSG(err.Error(), 1, &opts.Logger)
}
// Initialize cache with loaded metadata
cache, err := remotefetcher.NewCache(metadataFile, cacheDir)
cache, err := remotefetcher.NewCache(path.Join(backyHomeConfDir, "cache.yml"), cacheDir)
if err != nil {
fmt.Println("Error initializing cache:", err)
panic(err)
logging.ExitWithMSG(err.Error(), 1, &opts.Logger)
}
// Populate cache with loaded metadata
for _, data := range opts.CachedData {
cache.AddDataToStore(data.Hash, *data)
if err := cache.AddDataToStore(data.Hash, *data); err != nil {
logging.ExitWithMSG(err.Error(), 1, &opts.Logger)
}
}
opts.Cache, err = remotefetcher.NewCache(path.Join(backyHomeConfDir, "cache.yml"), backyHomeConfDir)
@ -75,9 +78,9 @@ func (opts *ConfigOpts) InitConfig() {
logging.ExitWithMSG(fmt.Sprintf("error initializing cache: %v", err), 1, nil)
}
// Initialize the fetcher
println("Creating new fetcher for source", opts.ConfigFilePath)
fetcher, err := remotefetcher.NewConfigFetcher(opts.ConfigFilePath, opts.Cache)
println("Created new fetcher for source", opts.ConfigFilePath)
// println("Creating new fetcher for source", opts.ConfigFilePath)
fetcher, err := remotefetcher.NewRemoteFetcher(opts.ConfigFilePath, opts.Cache)
// println("Created new fetcher for source", opts.ConfigFilePath)
if err != nil {
logging.ExitWithMSG(fmt.Sprintf("error initializing config fetcher: %v", err), 1, nil)
@ -92,7 +95,7 @@ func (opts *ConfigOpts) InitConfig() {
opts.koanf = backyKoanf
}
func loadConfigFile(fetcher remotefetcher.ConfigFetcher, filePath string, k *koanf.Koanf, opts *ConfigOpts) {
func loadConfigFile(fetcher remotefetcher.RemoteFetcher, filePath string, k *koanf.Koanf, opts *ConfigOpts) {
data, err := fetcher.Fetch(filePath)
if err != nil {
logging.ExitWithMSG(fmt.Sprintf("Could not fetch config file %s: %v", filePath, err), 1, nil)
@ -103,7 +106,7 @@ func loadConfigFile(fetcher remotefetcher.ConfigFetcher, filePath string, k *koa
}
}
func loadDefaultConfigFiles(fetcher remotefetcher.ConfigFetcher, configFiles []string, k *koanf.Koanf, opts *ConfigOpts) {
func loadDefaultConfigFiles(fetcher remotefetcher.RemoteFetcher, configFiles []string, k *koanf.Koanf, opts *ConfigOpts) {
cFileFailures := 0
for _, c := range configFiles {
data, err := fetcher.Fetch(c)
@ -313,10 +316,10 @@ func getRemoteDir(filePath string) (string, *url.URL) {
}
func loadListConfigFile(filePath string, k *koanf.Koanf, opts *ConfigOpts) bool {
fetcher, err := remotefetcher.NewConfigFetcher(filePath, opts.Cache, remotefetcher.IgnoreFileNotFound())
fetcher, err := remotefetcher.NewRemoteFetcher(filePath, opts.Cache, remotefetcher.IgnoreFileNotFound())
if err != nil {
// if file not found, ignore
if errors.Is(err, remotefetcher.ErrFileNotFound) {
if errors.Is(err, remotefetcher.ErrIgnoreFileNotFound) {
return true
}
@ -342,7 +345,7 @@ func loadCmdListsFile(backyKoanf *koanf.Koanf, listsConfig *koanf.Koanf, opts *C
opts.CmdListFile = path.Join(path.Dir(opts.ConfigFilePath), opts.CmdListFile)
}
fetcher, err := remotefetcher.NewConfigFetcher(opts.CmdListFile, opts.Cache)
fetcher, err := remotefetcher.NewRemoteFetcher(opts.CmdListFile, opts.Cache)
if err != nil {
logging.ExitWithMSG(fmt.Sprintf("error initializing config fetcher: %v", err), 1, nil)
@ -603,23 +606,22 @@ func processCmds(opts *ConfigOpts) error {
}
}
if cmd.Type == "remoteScript" {
if !isRemoteURL(cmd.Cmd) {
return fmt.Errorf("remoteScript command %s must be a remote resource", cmdName)
}
}
}
return nil
}
// processHooks evaluates if hooks are valid Commands
//
// Takes the following arguments:
// The cmd.hookRefs[hookType] is created with any hooks found.
//
// 1. a []string of hooks
// 2. a map of Commands as arguments
// 3. a string hookType, must be the hook type
//
// The cmds.hookRef is modified in this function.
//
// Returns the following:
//
// An error, if any, if the command is not found
// Returns an error, if any, if the hook command is not found
func processHooks(cmd *Command, hooks []string, opts *ConfigOpts, hookType string) error {
// initialize hook type

4
pkg/backy/templates/error.txt
View File

@ -1,12 +1,12 @@
Command list {{.listName }} failed.
The command run was {{.Cmd}}.
The command run was {{.CmdName}}.
The command executed was {{.Command}} {{ if .Args }} {{- range .Args}} {{.}} {{end}} {{end}}
{{ if .Err }} The error was {{ .Err }}{{ end }}
{{ if .Output }} The output was {{- range .Output}} {{.}} {{end}} {{end}}
{{ if .Output }} The output was: {{- range .Output}} {{.}} {{end}} {{end}}
{{ if .CmdsRan }}
The following commands ran:

22
pkg/backy/types.go
View File

@ -67,10 +67,7 @@ type (
// hook refs are internal references of commands for each hook type
hookRefs map[string]map[string]*Command
/*
Shell specifies which shell to run the command in, if any.
Not applicable when host is defined.
*/
// Shell specifies which shell to run the command in, if any.
Shell string `yaml:"shell,omitempty"`
RemoteHost *Host `yaml:"-"`
@ -91,11 +88,14 @@ type (
Environment []string `yaml:"environment,omitempty"`
// Output determines if output is requested.
// Only works if command is in a list.
//
// Only for when command is in a list.
GetOutput bool `yaml:"getOutput,omitempty"`
ScriptEnvFile string `yaml:"scriptEnvFile"`
// BEGIN PACKAGE COMMAND FIELDS
PackageManager string `yaml:"packageManager,omitempty"`
PackageName string `yaml:"packageName,omitempty"`
@ -109,6 +109,7 @@ type (
pkgMan pkgman.PackageManager
packageCmdSet bool
// END PACKAGE COMMAND FIELDS
// RemoteSource specifies a URL to fetch the command or configuration remotely
RemoteSource string `yaml:"remoteSource,omitempty"`
@ -116,8 +117,12 @@ type (
// FetchBeforeExecution determines if the remoteSource should be fetched before running
FetchBeforeExecution bool `yaml:"fetchBeforeExecution,omitempty"`
// BEGIN USER COMMAND FIELDS
// Username specifies the username for user creation or related operations
Username string `yaml:"username,omitempty"`
Username string `yaml:"userName,omitempty"`
UserID string `yaml:"userID,omitempty"`
// UserGroups specifies the groups to add the user to
UserGroups []string `yaml:"userGroups,omitempty"`
@ -144,12 +149,15 @@ type (
userCmdSet bool
// stdin only for userOperation = password (for now)
stdin *strings.Reader
// END USER STRUCT FIELDS
}
RemoteSource struct {
URL string `yaml:"url"`
Type string `yaml:"type"` // e.g., yaml
Type string `yaml:"type"` // e.g., s3, http
Auth struct {
AccessKey string `yaml:"accessKey"`
SecretKey string `yaml:"secretKey"`

6
pkg/backy/utils.go
View File

@ -43,7 +43,7 @@ func AddCommandLists(lists []string) BackyOptionFunc {
}
}
// AddPrintLists adds lists to print out
// SetListsToSearch adds lists to search
func SetListsToSearch(lists []string) BackyOptionFunc {
return func(bco *ConfigOpts) {
bco.List.Lists = append(bco.List.Lists, lists...)
@ -64,8 +64,8 @@ func SetLogFile(logFile string) BackyOptionFunc {
}
}
// cronEnabled enables the execution of command lists at specified times
func CronEnabled() BackyOptionFunc {
// EnableCron enables the execution of command lists at specified times
func EnableCron() BackyOptionFunc {
return func(bco *ConfigOpts) {
bco.cronEnabled = true
}

1
pkg/pkgman/apt/apt.go
View File

@ -40,7 +40,6 @@ func (a *AptManager) Install(pkg, version string, args []string) (string, []stri
if args != nil {
baseArgs = append(baseArgs, args...)
}
fmt.Printf("baseArgs: %v\n", baseArgs)
return baseCmd, baseArgs
}

11
pkg/remotefetcher/cache.go
View File

@ -64,7 +64,6 @@ func (c *Cache) loadFromFile() error {
}
func (c *Cache) saveToFile() error {
// println("Saving cache to file:", c.file)
c.mu.Lock()
defer c.mu.Unlock()
@ -84,10 +83,8 @@ func (c *Cache) saveToFile() error {
func (c *Cache) Get(hash string) ([]byte, CacheData, bool) {
c.mu.Lock()
defer c.mu.Unlock()
println("Getting cache data for hash:", hash)
cacheData, exists := c.store[hash]
if !exists {
println("Cache data does not exist for hash:", hash)
return nil, CacheData{}, false
}
@ -99,10 +96,9 @@ func (c *Cache) Get(hash string) ([]byte, CacheData, bool) {
return data, cacheData, true
}
func (c *Cache) AddDataToStore(hash string, cacheData CacheData) {
c.mu.Lock()
defer c.mu.Unlock()
func (c *Cache) AddDataToStore(hash string, cacheData CacheData) error {
c.store[hash] = cacheData
return c.saveToFile()
}
func (c *Cache) Set(source, hash string, data []byte, dataType string) (CacheData, error) {
@ -164,9 +160,8 @@ func (cf *CachedFetcher) Hash(data []byte) string {
return hex.EncodeToString(hash[:])
}
// Function to read and parse the hashMetadataSample.yml file
// Function to read and parse the metadata file
func LoadMetadataFromFile(filePath string) ([]*CacheData, error) {
// fmt.Println("Loading metadata from file:", filePath)
if _, err := os.Stat(filePath); os.IsNotExist(err) {
// Create the file if it does not exist
emptyData := []byte("[]")

25
pkg/remotefetcher/configfetcher.go
View File

@ -5,7 +5,7 @@ import (
"strings"
)
type ConfigFetcher interface {
type RemoteFetcher interface {
// Fetch retrieves the configuration from the specified URL or source
// Returns the raw data as bytes or an error
Fetch(source string) ([]byte, error)
@ -18,30 +18,31 @@ type ConfigFetcher interface {
Hash(data []byte) string
}
// ErrFileNotFound is returned when the file is not found and should be ignored
var ErrFileNotFound = errors.New("remotefetcher: file not found")
// ErrIgnoreFileNotFound is returned when the file is not found and should be ignored
var ErrIgnoreFileNotFound = errors.New("remotefetcher: file not found")
func NewConfigFetcher(source string, cache *Cache, options ...Option) (ConfigFetcher, error) {
var fetcher ConfigFetcher
var dataType string
func NewRemoteFetcher(source string, cache *Cache, options ...FetcherOption) (RemoteFetcher, error) {
var fetcher RemoteFetcher
config := FetcherConfig{}
for _, option := range options {
option(&config)
}
// If FileType is empty (i.e. WithFileType was not called), yaml is the default file type
if strings.TrimSpace(config.FileType) == "" {
config.FileType = "yaml"
}
if strings.HasPrefix(source, "http") || strings.HasPrefix(source, "https") {
fetcher = NewHTTPFetcher(options...)
dataType = "yaml"
} else if strings.HasPrefix(source, "s3") {
var err error
fetcher, err = NewS3Fetcher(options...)
fetcher, err = NewS3Fetcher(source, options...)
if err != nil {
return nil, err
}
dataType = "yaml"
} else {
fetcher = &LocalFetcher{}
dataType = "yaml"
return fetcher, nil
}
@ -51,7 +52,7 @@ func NewConfigFetcher(source string, cache *Cache, options ...Option) (ConfigFet
data, err := fetcher.Fetch(source)
if err != nil {
if config.IgnoreFileNotFound && isFileNotFoundError(err) {
return nil, ErrFileNotFound
return nil, ErrIgnoreFileNotFound
}
return nil, err
}
@ -61,7 +62,7 @@ func NewConfigFetcher(source string, cache *Cache, options ...Option) (ConfigFet
return &CachedFetcher{data: cachedData, path: cacheMeta.Path, dataType: cacheMeta.Type}, nil
}
cacheData, err := cache.Set(source, hash, data, dataType)
cacheData, err := cache.Set(source, hash, data, config.FileType)
if err != nil {
return nil, err
}

4
pkg/remotefetcher/http.go
View File

@ -16,7 +16,7 @@ type HTTPFetcher struct {
}
// NewHTTPFetcher creates a new instance of HTTPFetcher with the provided options.
func NewHTTPFetcher(options ...Option) *HTTPFetcher {
func NewHTTPFetcher(options ...FetcherOption) *HTTPFetcher {
cfg := &FetcherConfig{}
for _, opt := range options {
opt(cfg)
@ -39,7 +39,7 @@ func (h *HTTPFetcher) Fetch(source string) ([]byte, error) {
defer resp.Body.Close()
if resp.StatusCode == http.StatusNotFound && h.config.IgnoreFileNotFound {
return nil, ErrFileNotFound
return nil, ErrIgnoreFileNotFound
}
if resp.StatusCode != http.StatusOK {

2
pkg/remotefetcher/local.go
View File

@ -18,7 +18,7 @@ func (l *LocalFetcher) Fetch(source string) ([]byte, error) {
// Check if the file exists
if _, err := os.Stat(source); os.IsNotExist(err) {
if l.config.IgnoreFileNotFound {
return nil, ErrFileNotFound
return nil, ErrIgnoreFileNotFound
}
return nil, nil
}

20
pkg/remotefetcher/options.go
View File

@ -2,36 +2,48 @@ package remotefetcher
import (
"net/http"
"strings"
"github.com/aws/aws-sdk-go-v2/service/s3"
)
// Option is a function that configures a fetcher.
type Option func(*FetcherConfig)
type FetcherOption func(*FetcherConfig)
// FetcherConfig holds the configuration for a fetcher.
type FetcherConfig struct {
S3Client *s3.Client
HTTPClient *http.Client
FileType string
IgnoreFileNotFound bool
}
// WithS3Client sets the S3 client for the fetcher.
func WithS3Client(client *s3.Client) Option {
func WithS3Client(client *s3.Client) FetcherOption {
return func(cfg *FetcherConfig) {
cfg.S3Client = client
}
}
// WithHTTPClient sets the HTTP client for the fetcher.
func WithHTTPClient(client *http.Client) Option {
func WithHTTPClient(client *http.Client) FetcherOption {
return func(cfg *FetcherConfig) {
cfg.HTTPClient = client
}
}
func IgnoreFileNotFound() Option {
func IgnoreFileNotFound() FetcherOption {
return func(cfg *FetcherConfig) {
cfg.IgnoreFileNotFound = true
}
}
// WithFileType ensures the default FileType will be yaml
func WithFileType(fileType string) FetcherOption {
return func(cfg *FetcherConfig) {
cfg.FileType = fileType
if strings.TrimSpace(fileType) == "" {
cfg.FileType = "yaml"
}
}
}

120
pkg/remotefetcher/s3.go
View File

@ -1,71 +1,109 @@
package remotefetcher
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"errors"
"io"
"net/http"
"net/url"
"os"
"path"
"strings"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/service/s3"
"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/mitchellh/go-homedir"
"gopkg.in/yaml.v3"
)
type S3Fetcher struct {
S3Client *s3.Client
S3Client *minio.Client
config FetcherConfig
}
// NewS3Fetcher creates a new instance of S3Fetcher with the provided options.
func NewS3Fetcher(options ...Option) (*S3Fetcher, error) {
func NewS3Fetcher(endpoint string, options ...FetcherOption) (*S3Fetcher, error) {
cfg := &FetcherConfig{}
var s3Client *minio.Client
var err error
for _, opt := range options {
opt(cfg)
}
/*
options for S3 urls:
1. s3://bucket.region.endpoint.tld/path/to/object
2. alias with path and rest is looked up in file - add FetcherOptions
options for S3 credentials:
1. from file ($HOME/.aws/credentials)
2. env vars (AWS_SECRET_KEY, etc.)
*/
s3Endpoint := os.Getenv("S3_ENDPOINT")
creds, err := getS3Credentials("default", s3Endpoint, cfg.HTTPClient)
if err != nil {
println(err.Error())
return nil, err
}
// Initialize S3 client if not provided
if cfg.S3Client == nil {
awsCfg, err := config.LoadDefaultConfig(context.TODO())
s3Client, err = minio.New(s3Endpoint, &minio.Options{
Creds: creds,
Secure: true,
})
if err != nil {
return nil, err
}
cfg.S3Client = s3.NewFromConfig(awsCfg)
}
return &S3Fetcher{S3Client: cfg.S3Client, config: *cfg}, nil
return &S3Fetcher{S3Client: s3Client, config: *cfg}, nil
}
// Fetch retrieves the configuration from an S3 bucket
// Source should be in the format "bucket-name/object-key"
func (s *S3Fetcher) Fetch(source string) ([]byte, error) {
bucket, key, err := parseS3Source(source)
bucket, object, err := parseS3Source(source)
if err != nil {
return nil, err
}
resp, err := s.S3Client.GetObject(context.TODO(), &s3.GetObjectInput{
Bucket: &bucket,
Key: &key,
})
if err != nil {
var notFound *types.NoSuchKey
if errors.As(err, &notFound) && s.config.IgnoreFileNotFound {
return nil, ErrFileNotFound
doesObjectExist, objErr := objectExists(bucket, object, s.S3Client)
if !doesObjectExist {
if objErr != nil {
return nil, err
}
if s.config.IgnoreFileNotFound {
return nil, ErrIgnoreFileNotFound
}
}
fileObject, err := s.S3Client.GetObject(context.TODO(), bucket, object, minio.GetObjectOptions{})
if err != nil {
println(err.Error())
return nil, err
}
defer resp.Body.Close()
defer fileObject.Close()
fileObjectStats, statErr := fileObject.Stat()
if statErr != nil {
return nil, statErr
}
buffer := make([]byte, fileObjectStats.Size)
buf := new(bytes.Buffer)
_, err = buf.ReadFrom(resp.Body)
// Read the object into the buffer
_, err = io.ReadFull(fileObject, buffer)
if err != nil {
return nil, err
}
return buf.Bytes(), nil
return buffer, nil
}
// Parse decodes the raw data into the provided target structure
@ -79,10 +117,46 @@ func parseS3Source(source string) (bucket, key string, err error) {
if len(parts) != 2 {
return "", "", errors.New("invalid S3 source format, expected bucket-name/object-key")
}
return parts[0], parts[1], nil
u, _ := url.Parse(source)
u.Path = strings.TrimPrefix(u.Path, "/")
return u.Host, u.Path, nil
}
func (s *S3Fetcher) Hash(data []byte) string {
hash := sha256.Sum256(data)
return hex.EncodeToString(hash[:])
}
func getS3Credentials(profile, host string, httpClient *http.Client) (*credentials.Credentials, error) {
// println(s3utils.GetRegionFromURL(*u))
homeDir, hdirErr := homedir.Dir()
if hdirErr != nil {
return nil, hdirErr
}
s3Creds := credentials.NewFileAWSCredentials(path.Join(homeDir, ".aws", "credentials"), "default")
credVals, credErr := s3Creds.GetWithContext(&credentials.CredContext{Endpoint: host, Client: httpClient})
if credErr != nil {
return nil, credErr
}
creds := credentials.NewStaticV4(credVals.AccessKeyID, credVals.SecretAccessKey, "")
return creds, nil
}
var (
doesNotExist = "The specified key does not exist."
)
// objectExists checks for name in bucket using client.
// It returns false and nil if the key does not exist
func objectExists(bucket, name string, client *minio.Client) (bool, error) {
_, err := client.StatObject(context.TODO(), bucket, name, minio.StatObjectOptions{})
if err != nil {
switch err.Error() {
case doesNotExist:
return false, nil
default:
return false, errors.Join(err, errors.New("error stating object"))
}
}
return true, nil
}

7
pkg/usermanager/common/options.go Normal file
View File

@ -0,0 +1,7 @@
package common
// ConfigurablePackageManager defines methods for setting configuration options.
type ConfigurableUserManager interface {
SetUseAuth(useAuth bool)
SetAuthCommand(authCommand string)
}

1
pkg/usermanager/userman.go
View File

@ -20,6 +20,7 @@ type UserManager interface {
UserExists(username string) (string, []string)
}
// NewUserManager returns a UserManager-compatible struct
func NewUserManager(system string) (UserManager, error) {
var manager UserManager