CyberShell/backy
1
1
Fork 0
Code Issues 11 Pull Requests Packages Projects 1 Releases 20 Wiki Activity

Merge branch 'remoteResources' into develop
Some checks failed
ci/woodpecker/push/go-lint Pipeline failed
Details

Browse Source
This commit is contained in:
Andrew Woodlee
2025-02-08 16:14:37 -06:00
parent 5d3c265ce9 11ec1a98d8
commit e20141043c
45 changed files with 1101 additions and 327 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
pkg/backy/backy.go
View File

@@ -100,8 +100,14 @@ func (command *Command) RunCmd(cmdCtxLogger zerolog.Logger, opts *ConfigOpts) ([
cmdCtxLogger.Info().Str("Command", fmt.Sprintf("Running command %s on local machine", command.Name)).Send()
localCMD = exec.Command(command.Cmd, command.Args...)
// execute package commands in a shell
if command.Type == "package" {
cmdCtxLogger.Info().Str("package", command.PackageName).Msg("Executing package command")
ArgsStr = fmt.Sprintf("%s %s", command.Cmd, ArgsStr)
localCMD = exec.Command("/bin/sh", "-c", ArgsStr)
} else {
localCMD = exec.Command(command.Cmd, command.Args...)
}
}
if command.Dir != nil {
@@ -232,7 +238,8 @@ func notifyError(logger zerolog.Logger, templates *msgTemplates, list *CmdList,
"CmdsRan": cmdsRan,
"CmdOutput": outStructArr,
"Err": err,
"Command": cmd.Name,
"CmdName": cmd.Name,
"Command": cmd.Cmd,
"Args": cmd.Args,
}
var errMsg bytes.Buffer

121
pkg/backy/config.go
View File

@@ -2,15 +2,17 @@ package backy
import (
"context"
"errors"
"fmt"
"net/url"
"os"
"path"
"runtime"
"strings"
"git.andrewnw.xyz/CyberShell/backy/pkg/configfetcher"
"git.andrewnw.xyz/CyberShell/backy/pkg/logging"
"git.andrewnw.xyz/CyberShell/backy/pkg/pkgman"
"git.andrewnw.xyz/CyberShell/backy/pkg/remotefetcher"
"git.andrewnw.xyz/CyberShell/backy/pkg/usermanager"
vault "github.com/hashicorp/vault/api"
"github.com/knadh/koanf/parsers/yaml"
@@ -20,23 +22,23 @@ import (
"github.com/rs/zerolog"
)
var homeDir string
var homeDirErr error
var backyHomeConfDir string
var configFiles []string
const macroStart string = "%{"
const macroEnd string = "}%"
const envMacroStart string = "%{env:"
const vaultMacroStart string = "%{vault:"
func (opts *ConfigOpts) InitConfig() {
homeDir, err := os.UserHomeDir()
var err error
homeConfigDir, err := os.UserConfigDir()
if err != nil {
logging.ExitWithMSG(err.Error(), 1, nil)
}
homeCacheDir, err := os.UserCacheDir()
if err != nil {
logging.ExitWithMSG(err.Error(), 1, nil)
}
backyHomeConfDir := path.Join(homeDir, ".config/backy/")
backyHomeConfDir := path.Join(homeConfigDir, "backy")
configFiles := []string{
"./backy.yml", "./backy.yaml",
path.Join(backyHomeConfDir, "backy.yml"),
@@ -46,8 +48,39 @@ func (opts *ConfigOpts) InitConfig() {
backyKoanf := koanf.New(".")
opts.ConfigFilePath = strings.TrimSpace(opts.ConfigFilePath)
// metadataFile := "hashMetadataSample.yml"
cacheDir := homeCacheDir
// Load metadata from file
opts.CachedData, err = remotefetcher.LoadMetadataFromFile(path.Join(backyHomeConfDir, "cache.yml"))
if err != nil {
fmt.Println("Error loading metadata:", err)
logging.ExitWithMSG(err.Error(), 1, &opts.Logger)
}
// Initialize cache with loaded metadata
cache, err := remotefetcher.NewCache(path.Join(backyHomeConfDir, "cache.yml"), cacheDir)
if err != nil {
fmt.Println("Error initializing cache:", err)
logging.ExitWithMSG(err.Error(), 1, &opts.Logger)
}
// Populate cache with loaded metadata
for _, data := range opts.CachedData {
if err := cache.AddDataToStore(data.Hash, *data); err != nil {
logging.ExitWithMSG(err.Error(), 1, &opts.Logger)
}
}
opts.Cache, err = remotefetcher.NewCache(path.Join(backyHomeConfDir, "cache.yml"), backyHomeConfDir)
if err != nil {
logging.ExitWithMSG(fmt.Sprintf("error initializing cache: %v", err), 1, nil)
}
// Initialize the fetcher
fetcher, err := configfetcher.NewConfigFetcher(opts.ConfigFilePath)
// println("Creating new fetcher for source", opts.ConfigFilePath)
fetcher, err := remotefetcher.NewRemoteFetcher(opts.ConfigFilePath, opts.Cache)
// println("Created new fetcher for source", opts.ConfigFilePath)
if err != nil {
logging.ExitWithMSG(fmt.Sprintf("error initializing config fetcher: %v", err), 1, nil)
@@ -62,7 +95,7 @@ func (opts *ConfigOpts) InitConfig() {
opts.koanf = backyKoanf
}
func loadConfigFile(fetcher configfetcher.ConfigFetcher, filePath string, k *koanf.Koanf, opts *ConfigOpts) {
func loadConfigFile(fetcher remotefetcher.RemoteFetcher, filePath string, k *koanf.Koanf, opts *ConfigOpts) {
data, err := fetcher.Fetch(filePath)
if err != nil {
logging.ExitWithMSG(fmt.Sprintf("Could not fetch config file %s: %v", filePath, err), 1, nil)
@@ -73,7 +106,7 @@ func loadConfigFile(fetcher configfetcher.ConfigFetcher, filePath string, k *koa
}
}
func loadDefaultConfigFiles(fetcher configfetcher.ConfigFetcher, configFiles []string, k *koanf.Koanf, opts *ConfigOpts) {
func loadDefaultConfigFiles(fetcher remotefetcher.RemoteFetcher, configFiles []string, k *koanf.Koanf, opts *ConfigOpts) {
cFileFailures := 0
for _, c := range configFiles {
data, err := fetcher.Fetch(c)
@@ -236,13 +269,23 @@ func resolveProxyHosts(host *Host, opts *ConfigOpts) {
}
func loadCommandLists(opts *ConfigOpts, backyKoanf *koanf.Koanf) {
backyConfigFileDir := path.Dir(opts.ConfigFilePath)
listsConfig := koanf.New(".")
listConfigFiles := []string{
path.Join(backyConfigFileDir, "lists.yml"),
path.Join(backyConfigFileDir, "lists.yaml"),
var backyConfigFileDir string
var listConfigFiles []string
var u *url.URL
// if config file is remote, use the directory of the remote file
if isRemoteURL(opts.ConfigFilePath) {
_, u = getRemoteDir(opts.ConfigFilePath)
listConfigFiles = []string{u.JoinPath("lists.yml").String(), u.JoinPath("lists.yaml").String()}
} else {
backyConfigFileDir = path.Dir(opts.ConfigFilePath)
listConfigFiles = []string{
path.Join(backyConfigFileDir, "lists.yml"),
path.Join(backyConfigFileDir, "lists.yaml"),
}
}
listsConfig := koanf.New(".")
for _, l := range listConfigFiles {
if loadListConfigFile(l, listsConfig, opts) {
break
@@ -257,9 +300,29 @@ func loadCommandLists(opts *ConfigOpts, backyKoanf *koanf.Koanf) {
}
}
func loadListConfigFile(filePath string, k *koanf.Koanf, opts *ConfigOpts) bool {
fetcher, err := configfetcher.NewConfigFetcher(filePath)
func isRemoteURL(filePath string) bool {
return strings.HasPrefix(filePath, "http://") || strings.HasPrefix(filePath, "https://") || strings.HasPrefix(filePath, "s3://")
}
func getRemoteDir(filePath string) (string, *url.URL) {
u, err := url.Parse(filePath)
if err != nil {
return "", nil
}
// u.Path is the path to the file, stripped of scheme and hostname
u.Path = path.Dir(u.Path)
return u.String(), u
}
func loadListConfigFile(filePath string, k *koanf.Koanf, opts *ConfigOpts) bool {
fetcher, err := remotefetcher.NewRemoteFetcher(filePath, opts.Cache, remotefetcher.IgnoreFileNotFound())
if err != nil {
// if file not found, ignore
if errors.Is(err, remotefetcher.ErrIgnoreFileNotFound) {
return true
}
logging.ExitWithMSG(fmt.Sprintf("error initializing config fetcher: %v", err), 1, nil)
}
@@ -282,7 +345,8 @@ func loadCmdListsFile(backyKoanf *koanf.Koanf, listsConfig *koanf.Koanf, opts *C
opts.CmdListFile = path.Join(path.Dir(opts.ConfigFilePath), opts.CmdListFile)
}
fetcher, err := configfetcher.NewConfigFetcher(opts.CmdListFile)
fetcher, err := remotefetcher.NewRemoteFetcher(opts.CmdListFile, opts.Cache)
if err != nil {
logging.ExitWithMSG(fmt.Sprintf("error initializing config fetcher: %v", err), 1, nil)
}
@@ -542,23 +606,22 @@ func processCmds(opts *ConfigOpts) error {
}
}
if cmd.Type == "remoteScript" {
if !isRemoteURL(cmd.Cmd) {
return fmt.Errorf("remoteScript command %s must be a remote resource", cmdName)
}
}
}
return nil
}
// processHooks evaluates if hooks are valid Commands
//
// Takes the following arguments:
// The cmd.hookRefs[hookType] is created with any hooks found.
//
// 1. a []string of hooks
// 2. a map of Commands as arguments
// 3. a string hookType, must be the hook type
//
// The cmds.hookRef is modified in this function.
//
// Returns the following:
//
// An error, if any, if the command is not found
// Returns an error, if any, if the hook command is not found
func processHooks(cmd *Command, hooks []string, opts *ConfigOpts, hookType string) error {
// initialize hook type

4
pkg/backy/templates/error.txt
View File

@@ -1,12 +1,12 @@
Command list {{.listName }} failed.
The command run was {{.Cmd}}.
The command run was {{.CmdName}}.
The command executed was {{.Command}} {{ if .Args }} {{- range .Args}} {{.}} {{end}} {{end}}
{{ if .Err }} The error was {{ .Err }}{{ end }}
{{ if .Output }} The output was {{- range .Output}} {{.}} {{end}} {{end}}
{{ if .Output }} The output was: {{- range .Output}} {{.}} {{end}} {{end}}
{{ if .CmdsRan }}
The following commands ran:

26
pkg/backy/types.go
View File

@@ -7,6 +7,7 @@ import (
"strings"
"git.andrewnw.xyz/CyberShell/backy/pkg/pkgman"
"git.andrewnw.xyz/CyberShell/backy/pkg/remotefetcher"
"git.andrewnw.xyz/CyberShell/backy/pkg/usermanager"
vaultapi "github.com/hashicorp/vault/api"
"github.com/kevinburke/ssh_config"
@@ -66,10 +67,7 @@ type (
// hook refs are internal references of commands for each hook type
hookRefs map[string]map[string]*Command
/*
Shell specifies which shell to run the command in, if any.
Not applicable when host is defined.
*/
// Shell specifies which shell to run the command in, if any.
Shell string `yaml:"shell,omitempty"`
RemoteHost *Host `yaml:"-"`
@@ -90,11 +88,14 @@ type (
Environment []string `yaml:"environment,omitempty"`
// Output determines if output is requested.
// Only works if command is in a list.
//
// Only for when command is in a list.
GetOutput bool `yaml:"getOutput,omitempty"`
ScriptEnvFile string `yaml:"scriptEnvFile"`
// BEGIN PACKAGE COMMAND FIELDS
PackageManager string `yaml:"packageManager,omitempty"`
PackageName string `yaml:"packageName,omitempty"`
@@ -108,6 +109,7 @@ type (
pkgMan pkgman.PackageManager
packageCmdSet bool
// END PACKAGE COMMAND FIELDS
// RemoteSource specifies a URL to fetch the command or configuration remotely
RemoteSource string `yaml:"remoteSource,omitempty"`
@@ -115,8 +117,12 @@ type (
// FetchBeforeExecution determines if the remoteSource should be fetched before running
FetchBeforeExecution bool `yaml:"fetchBeforeExecution,omitempty"`
// BEGIN USER COMMAND FIELDS
// Username specifies the username for user creation or related operations
Username string `yaml:"username,omitempty"`
Username string `yaml:"userName,omitempty"`
UserID string `yaml:"userID,omitempty"`
// UserGroups specifies the groups to add the user to
UserGroups []string `yaml:"userGroups,omitempty"`
@@ -143,12 +149,15 @@ type (
userCmdSet bool
// stdin only for userOperation = password (for now)
stdin *strings.Reader
// END USER STRUCT FIELDS
}
RemoteSource struct {
URL string `yaml:"url"`
Type string `yaml:"type"` // e.g., yaml
Type string `yaml:"type"` // e.g., s3, http
Auth struct {
AccessKey string `yaml:"accessKey"`
SecretKey string `yaml:"secretKey"`
@@ -217,6 +226,9 @@ type (
koanf *koanf.Koanf
NotificationConf *Notifications `yaml:"notifications"`
Cache *remotefetcher.Cache
CachedData []*remotefetcher.CacheData
}
outStruct struct {

8
pkg/backy/utils.go
View File

@@ -43,7 +43,7 @@ func AddCommandLists(lists []string) BackyOptionFunc {
}
}
// AddPrintLists adds lists to print out
// SetListsToSearch adds lists to search
func SetListsToSearch(lists []string) BackyOptionFunc {
return func(bco *ConfigOpts) {
bco.List.Lists = append(bco.List.Lists, lists...)
@@ -64,8 +64,8 @@ func SetLogFile(logFile string) BackyOptionFunc {
}
}
// cronEnabled enables the execution of command lists at specified times
func CronEnabled() BackyOptionFunc {
// EnableCron enables the execution of command lists at specified times
func EnableCron() BackyOptionFunc {
return func(bco *ConfigOpts) {
bco.cronEnabled = true
}
@@ -275,7 +275,7 @@ func getCommandType(command *Command) *Command {
case "modify":
command.Cmd, command.Args = command.userMan.ModifyUser(
command.Username,
homeDir,
command.UserHome,
command.UserShell,
command.UserGroups)
case "checkIfExists":