diff --git a/Dockerfile b/Dockerfile index 66de853..9b7bb0e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,8 +11,8 @@ RUN go build FROM debian COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ -COPY --from=build /app/cmd/geneos/geneos /bin RUN apt update && apt install -y fontconfig +COPY --from=build /app/cmd/geneos/geneos /bin RUN useradd -ms /bin/bash geneos USER geneos WORKDIR /home/geneos diff --git a/cmd/geneos/README.md b/cmd/geneos/README.md index c020822..b9bdf72 100644 --- a/cmd/geneos/README.md +++ b/cmd/geneos/README.md @@ -3,9 +3,10 @@ The `geneos` program will help you manage your Geneos environment, one server at a time. Some of it's features, existing and planned, include: * Set-up a new environment with a series of simple commands +* Remote systems support (very much early testing) * Add new instances of common componments with sensible defaults * Check the status of components -* Stop, start, restart components (without unnecessary delays) +* Stop, start, restart components (without minimal delays) * Support fo existing gatewayctl/netprobectl/licdctl scripts and their file and configuration layout * Convert existing set-ups to JSON based configs with more options * Edit individual settings of instances @@ -64,6 +65,49 @@ You still have to configure the Gateway to connect to the Netprobe, but all thre This program has been written in such a way that is *should* be safe to install SETUID root or run using `sudo` for almost all cases. The program will refuse to accidentally run an instance as root unless the `User` config parameter is explicitly set - for example when a Netprobe needs to run as root. As with many complex programs, care should be taken and privileged execution should be used when required. +## Remote Management (NEW!) + +The `geneos` command can now transparently manage instances across multiple systems using SSH. Some things works well, some work with minor issues and some features do not work at all. + +This feature is still very much under development there will be changes coming. + +### What does this mean? + +See if these commands give you a hint: + +```bash +$ geneos add remote server2 ssh://geneos@myotherserver.example.com/opt/geneos +$ geneos add gateway newgateway@server2 +$ geneos start +``` + +Command like `ls` and `ps` will works transparently and merge all instances together, showing you where they are runnng (or not). + +A remote is a psuedo-instance and you add and manage it with the normal commands. At the moment the only supported transport is SSH and the URL is a slightly extended version of the RFC standard to include the Geneos home directory. The format, for the `add` command is: + +`ssh://[USER@]HOST[:PORT][/PATH]` + +If not set, USER defaults to the current username. Similarly PORT defaults to 22. PATH defaults to the local ITRSHome path. The most basic SSH URL of the form `ssh://hostname` results in a remote accessed as the current user on the default SSH port and rooted in the same directory as the local set-up. Is the remote directory is empty (dot files are ignored) then the standard file layout is created. + +### How does it work? + +There are a number of prerequisites for remote support: + +1. Linux on amd64 for all servers +2. Passwordless SSH access, either via an `ssh-agent` or unprotected private keys +3. At this time the only private keys supported are those in your `.ssh` directory beginning `id_` - later updates will allow you to set the name of the key to load, but using an agent is recommended. +4. The remote user must be confiugured to use a `bash` shell or similar. See limitations below. + +If you can log in to a remote Linux server using `ssh user@server` and no be prompted for a password or passphrase then you are set to go. It's beyond the scope of this README to explain how to set-up `ssh-agent` or how to create an unprotected private key file, so please search online. + +### Limitations + +The remote connections over SSH mean there are limitations to the features available on remote servers: + +1. No following logs (i.e. the `-f` option). The program is written to use `fsnotify` and that only works on local filesystems and not over sftp. This may be added using a more primitive polling mecahnism later. +2. Control over instance processes is done via shell commands and little error checking is done, so it is possible to cause damage and/or processes not to to start or stop as expected. Contributions of fixes are welcomed. +3. All actions are taken as the user given in the SSH URL (which should NEVER be `root`!) and so instances that are meant to run as other users cannot be controlled. Files and directories may not be available if the user does not have suitable permissions. + ## Usage Please note that the full list of commands and parameters is changing all the time. This list below is mostly, but not completely, up-to-date. @@ -215,7 +259,7 @@ The `geneos tls` command provides a number of subcommands to create and manage c Once enabled then all new instances will also have certificates created and configuration set to use secure (encrypted) connections where possible. * `geneos tls init` - Initialised the TLS environment by creating a `tls` directory in ITRSHome and populkating it with a new root and intermediate (signing) certificate and keys as well as a `chain.pem` which includes both CA certificates. The keys are only readable by the user running the command. + Initialised the TLS environment by creating a `tls` directory in ITRSHome and populkating it with a new root and intermediate (signing) certificate and keys as well as a `chain.pem` which includes both CA certificates. The keys are only readable by the user running the command. Also does a `sync` if remotes are configured. Any existing instances have certificates created and their configurations updated to reference them. This means that any legacy `.rc` configurations will be migrated to `.json` files. @@ -231,6 +275,9 @@ Once enabled then all new instances will also have certificates created and conf * `geneos tls ls [-c | -j | -i | -l] [TYPE] [NAME...]` List instance certificate information. Options are the same as for the main `ls` command but the data shown is specific to certificates. +* `geneos tls sync` + Copies chain.pem to all remotes + ## Configuration Files ### General Configuration diff --git a/cmd/geneos/add.go b/cmd/geneos/add.go index f54c4d6..dc0c930 100644 --- a/cmd/geneos/add.go +++ b/cmd/geneos/add.go @@ -18,7 +18,7 @@ func init() { commands["add"] = Command{ Function: commandAdd, ParseFlags: defaultFlag, - ParseArgs: parseArgs, + ParseArgs: parseArgsNoWildcard, CommandLine: "geneos add TYPE NAME", Summary: `Add a new instance`, Description: `Add a new instance called NAME with the TYPE supplied. The details will depends on the @@ -48,6 +48,12 @@ user in the instance configuration or the default user. Currently only one file time.`} } +// Add a single instance +// +// XXX argument validation is minimal +// +// remote support would be of the form name@remotename +// func commandAdd(ct ComponentType, args []string, params []string) (err error) { if len(args) == 0 { logError.Fatalln("not enough args") @@ -72,7 +78,7 @@ func commandAdd(ct ComponentType, args []string, params []string) (err error) { if err != nil { return } - log.Printf("new %s %q added, listening port %s\n", Type(c), Name(c), getIntAsString(c, Prefix(c)+"Port")) + log.Printf("new %s %q added, port %s\n", Type(c), Name(c), getIntAsString(c, Prefix(c)+"Port")) return } @@ -227,8 +233,8 @@ func uploadFile(c Instance, source string) (err error) { logError.Fatalln("dest path must be relative to (and in) instance directory") } // if the destination exists is it a directory? - if st, err := os.Stat(filepath.Join(Home(c), destfile)); err == nil { - if st.IsDir() { + if s, err := statFile(Location(c), filepath.Join(Home(c), destfile)); err == nil { + if s.st.IsDir() { destdir = filepath.Join(Home(c), destfile) destfile = "" } @@ -274,7 +280,7 @@ func uploadFile(c Instance, source string) (err error) { default: // support globbing later - from, err = os.Open(source) + from, _, err = openStatFile(LOCAL, source) if err != nil { return err } @@ -286,49 +292,73 @@ func uploadFile(c Instance, source string) (err error) { destfile = filepath.Join(destdir, destfile) - if _, err := os.Stat(filepath.Dir(destfile)); err != nil { - err = os.MkdirAll(filepath.Dir(destfile), 0775) + if _, err := statFile(Location(c), filepath.Dir(destfile)); err != nil { + err = mkdirAll(Location(c), filepath.Dir(destfile), 0775) if err != nil && !errors.Is(err, fs.ErrExist) { logError.Fatalln(err) } // if created, chown the last element if err == nil { - if err = os.Chown(filepath.Dir(destfile), int(uid), int(gid)); err != nil { + if err = chown(Location(c), filepath.Dir(destfile), int(uid), int(gid)); err != nil { return err } } } // xxx - wrong way around. create tmp first, move over later - if st, err := os.Stat(destfile); err == nil { - if !st.Mode().IsRegular() { + if s, err := statFile(Location(c), destfile); err == nil { + if !s.st.Mode().IsRegular() { logError.Fatalln("dest exists and is not a plain file") } datetime := time.Now().UTC().Format("20060102150405") backuppath = destfile + "." + datetime + ".old" - if err = os.Rename(destfile, backuppath); err != nil { + if err = renameFile(Location(c), destfile, backuppath); err != nil { return err } } - out, err := os.Create(destfile) - if err != nil { - return err - } - defer out.Close() - if err = out.Chown(int(uid), int(gid)); err != nil { - os.Remove(out.Name()) - if backuppath != "" { - if err = os.Rename(backuppath, destfile); err != nil { + var out io.Writer + + switch Location(c) { + case LOCAL: + cf, err := os.Create(destfile) + if err != nil { + return err + } + out = cf + defer cf.Close() + + if err = cf.Chown(int(uid), int(gid)); err != nil { + removeFile(Location(c), destfile) + if backuppath != "" { + if err = renameFile(Location(c), backuppath, destfile); err != nil { + return err + } return err } + } + default: + cf, err := createRemoteFile(Location(c), destfile) + if err != nil { return err } + out = cf + defer cf.Close() + + if err = cf.Chown(int(uid), int(gid)); err != nil { + removeFile(Location(c), destfile) + if backuppath != "" { + if err = renameFile(Location(c), backuppath, destfile); err != nil { + return err + } + return err + } + } } if _, err = io.Copy(out, from); err != nil { return err } - log.Println("uploaded", source, "to", out.Name()) + log.Println("uploaded", source, "to", destfile) return nil } diff --git a/cmd/geneos/command.go b/cmd/geneos/command.go index a9b70bf..4d2db92 100644 --- a/cmd/geneos/command.go +++ b/cmd/geneos/command.go @@ -4,7 +4,6 @@ import ( "bufio" "bytes" "fmt" - "os" "os/exec" "path/filepath" "strings" @@ -38,26 +37,30 @@ type Commands map[string]Command // return a single slice of all instances, ordered and grouped // configuration are not loaded, just the defaults ready for overlay func allInstances() (confs []Instance) { - for _, ct := range componentTypes() { - confs = append(confs, instances(ct)...) + for _, ct := range realComponentTypes() { + for _, remote := range allRemotes() { + confs = append(confs, instancesOfComponent(Name(remote), ct)...) + } } return } -// return a slice of instance for a given ComponentType -func instances(ct ComponentType) (confs []Instance) { - for _, name := range instanceDirs(ct) { +// return a slice of instancesOfComponent for a given ComponentType +func instancesOfComponent(remote string, ct ComponentType) (confs []Instance) { + for _, name := range instanceDirsForComponent(remote, ct) { confs = append(confs, newComponent(ct, name)...) } return } +// return a slice of component types that exist for this name func findInstances(name string) (cts []ComponentType) { - for _, t := range componentTypes() { - compdirs := instanceDirs(t) - for _, dir := range compdirs { + local, remote := splitInstanceName(name) + for _, t := range realComponentTypes() { + for _, dir := range instanceDirsForComponent(remote, t) { // for case insensitive match change to EqualFold here - if filepath.Base(dir) == name { + ldir, _ := splitInstanceName(dir) + if filepath.Base(ldir) == local { cts = append(cts, t) } } @@ -72,7 +75,7 @@ func loadConfig(c Instance, update bool) (err error) { baseconf := filepath.Join(Home(c), Type(c).String()) j := baseconf + ".json" - if err = readConfigFile(j, &c); err == nil { + if err = readConfigFile(Location(c), j, &c); err == nil { // return if NO error, else drop through return } @@ -84,7 +87,7 @@ func loadConfig(c Instance, update bool) (err error) { logError.Println("failed to wrtite config file:", err) return } - if err = os.Rename(baseconf+".rc", baseconf+".rc.orig"); err != nil { + if err = renameFile(Location(c), baseconf+".rc", baseconf+".rc.orig"); err != nil { logError.Println("failed to rename old config:", err) } logDebug.Println(Type(c), Name(c), "migrated to JSON config") @@ -121,7 +124,7 @@ func buildCmd(c Instance) (cmd *exec.Cmd, env []string) { // save off extra env too // XXX - scan file line by line, protect memory func readRCConfig(c Instance) (err error) { - rcdata, err := os.ReadFile(filepath.Join(Home(c), Type(c).String()+".rc")) + rcdata, err := readFile(Location(c), filepath.Join(Home(c), Type(c).String()+".rc")) if err != nil { return } diff --git a/cmd/geneos/component.go b/cmd/geneos/component.go index ee44bc3..8824502 100644 --- a/cmd/geneos/component.go +++ b/cmd/geneos/component.go @@ -2,7 +2,6 @@ package main import ( "io/fs" - "os" "path/filepath" "sort" "strings" @@ -24,6 +23,8 @@ const ( Remote ) +// XXX this should become an interface +// but that involves lots of rebuilding. type ComponentFuncs struct { Instance func(string) interface{} Command func(Instance) ([]string, []string) @@ -49,6 +50,9 @@ type Common struct { // The Name of an instance. This may be different to the instance // directory name during certain operations, e.g. rename Name string `json:"Name"` + // The potential remote name (this is a remote component and not + // a server name) + Location string `default:"local" json:"Location"` // The ComponentType of an instance Type string `json:"-"` // The root directory of the Geneos installation. Used in template @@ -60,8 +64,8 @@ type Common struct { // currently supported real component types, for looping // (go doesn't allow const slices, a function is the workaround) -// not including Remote for now -func componentTypes() []ComponentType { +// not including Remote - this is special +func realComponentTypes() []ComponentType { return []ComponentType{Gateway, Netprobe, Licd, Webserver} } @@ -102,21 +106,21 @@ func parseComponentName(component string) ComponentType { } } -// Return a slice of all directories for a given ComponentType. No checking is done +// Return a slice of all instances for a given ComponentType. No checking is done // to validate that the directory is a populated instance. // // No side-effects -func instanceDirs(ct ComponentType) []string { - return sortedDirs(componentDir(ct)) +func instanceDirsForComponent(remote string, ct ComponentType) []string { + return sortedInstancesInDir(remote, componentDir(remote, ct)) } // Return the base directory for a ComponentType -func componentDir(ct ComponentType) string { +func componentDir(remote string, ct ComponentType) string { switch ct { case Remote: return filepath.Join(RunningConfig.ITRSHome, ct.String()+"s") default: - return filepath.Join(RunningConfig.ITRSHome, ct.String(), ct.String()+"s") + return filepath.Join(remoteRoot(remote), ct.String(), ct.String()+"s") } } @@ -131,6 +135,10 @@ func Name(c Instance) string { return getString(c, "Name") } +func Location(c Instance) string { + return getString(c, "Location") +} + func Home(c Instance) string { return getString(c, Prefix(c)+"Home") } @@ -155,13 +163,13 @@ func sortDirEntries(files []fs.DirEntry) { } // Return a sorted list of sub-directories -func sortedDirs(dir string) []string { - files, _ := os.ReadDir(dir) +func sortedInstancesInDir(remote string, dir string) []string { + files, _ := readDir(remote, dir) sortDirEntries(files) components := make([]string, 0, len(files)) for _, file := range files { if file.IsDir() { - components = append(components, file.Name()) + components = append(components, file.Name()+"@"+remote) } } return components @@ -176,6 +184,8 @@ func sortedDirs(dir string) []string { // have to exist on disk. func newComponent(ct ComponentType, name string) (c []Instance) { if ct == None { + // for _, cts := realComponentTypes() { + // } cs := findInstances(name) for _, cm := range cs { c = append(c, newComponent(cm, name)...) diff --git a/cmd/geneos/config.go b/cmd/geneos/config.go index d661f55..a8a02e0 100644 --- a/cmd/geneos/config.go +++ b/cmd/geneos/config.go @@ -11,6 +11,8 @@ import ( "path/filepath" "regexp" "strings" + + "github.com/pkg/sftp" ) func init() { @@ -223,12 +225,12 @@ var initDirs = []string{ // load system config from global and user JSON files and process any // environment variables we choose func loadSysConfig() { - readConfigFile(globalConfig, &RunningConfig) + readConfigFile(LOCAL, globalConfig, &RunningConfig) // root should not have a per-user config, but if sun by sudo the // HOME dir is conserved, so allow for now userConfDir, _ := os.UserConfigDir() - err := readConfigFile(filepath.Join(userConfDir, "geneos.json"), &RunningConfig) + err := readConfigFile(LOCAL, filepath.Join(userConfDir, "geneos.json"), &RunningConfig) if err != nil && !errors.Is(err, os.ErrNotExist) { log.Println(err) } @@ -366,9 +368,9 @@ func initAsRoot(c *ConfigType, args []string) (err error) { } // dir must first not exist (or be empty) and then be createable - if _, err := os.Stat(dir); err == nil { + if _, err := statFile(LOCAL, dir); err == nil { // check empty - dirs, err := os.ReadDir(dir) + dirs, err := readDir(LOCAL, dir) if err != nil { logError.Fatalln(err) } @@ -377,32 +379,31 @@ func initAsRoot(c *ConfigType, args []string) (err error) { } } else { // need to create out own, chown base directory only - if err = os.MkdirAll(dir, 0775); err != nil { + if err = mkdirAll(LOCAL, dir, 0775); err != nil { logError.Fatalln(err) } } - if err = os.Chown(dir, int(uid), int(gid)); err != nil { + if err = chown(LOCAL, dir, int(uid), int(gid)); err != nil { logError.Fatalln(err) } c.ITRSHome = dir c.DefaultUser = username - if err = writeConfigFile(globalConfig, c); err != nil { + if err = writeConfigFile(LOCAL, globalConfig, c); err != nil { logError.Fatalln("cannot write global config", err) } // if everything else worked, remove any existing user config - _ = os.Remove(filepath.Join(u.HomeDir, ".config", "geneos.json")) + _ = removeFile(LOCAL, filepath.Join(u.HomeDir, ".config", "geneos.json")) // create directories for _, d := range initDirs { dir := filepath.Join(c.ITRSHome, d) - if err = os.MkdirAll(dir, 0775); err != nil { + if err = mkdirAll(LOCAL, dir, 0775); err != nil { logError.Fatalln(err) } } err = filepath.WalkDir(c.ITRSHome, func(path string, dir fs.DirEntry, err error) error { if err == nil { - logDebug.Println("chown", path, uid, gid) - err = os.Chown(path, int(uid), int(gid)) + err = chown(LOCAL, path, int(uid), int(gid)) } return err }) @@ -426,9 +427,9 @@ func initAsUser(c *ConfigType, args []string) (err error) { } // dir must first not exist (or be empty) and then be createable - if _, err = os.Stat(dir); err == nil { + if _, err = statFile(LOCAL, dir); err == nil { // check empty - dirs, err := os.ReadDir(dir) + dirs, err := readDir(LOCAL, dir) if err != nil { logError.Fatalln(err) } @@ -440,7 +441,7 @@ func initAsUser(c *ConfigType, args []string) (err error) { } } else { // need to create out own, chown base directory only - if err = os.MkdirAll(dir, 0775); err != nil { + if err = mkdirAll(LOCAL, dir, 0775); err != nil { logError.Fatalln(err) } } @@ -452,13 +453,13 @@ func initAsUser(c *ConfigType, args []string) (err error) { userConfFile := filepath.Join(userConfDir, "geneos.json") c.ITRSHome = dir c.DefaultUser = u.Username - if err = writeConfigFile(userConfFile, c); err != nil { + if err = writeConfigFile(LOCAL, userConfFile, c); err != nil { return } // create directories for _, d := range initDirs { dir := filepath.Join(c.ITRSHome, d) - if err = os.MkdirAll(dir, 0775); err != nil { + if err = mkdirAll(LOCAL, dir, 0775); err != nil { logError.Fatalln(err) } } @@ -484,19 +485,19 @@ func revertInstance(c Instance, params []string) (err error) { baseconf := filepath.Join(Home(c), Type(c).String()) // if *.rc file exists, remove rc.orig+JSON, continue - if _, err := os.Stat(baseconf + ".rc"); err == nil { + if _, err := statFile(Location(c), baseconf+".rc"); err == nil { // ignore errors - if os.Remove(baseconf+".rc.orig") == nil || os.Remove(baseconf+".json") == nil { + if removeFile(Location(c), baseconf+".rc.orig") == nil || removeFile(Location(c), baseconf+".json") == nil { logDebug.Println(Type(c), Name(c), "removed extra config file(s)") } return err } - if err = os.Rename(baseconf+".rc.orig", baseconf+".rc"); err != nil { + if err = renameFile(Location(c), baseconf+".rc.orig", baseconf+".rc"); err != nil { return } - if err = os.Remove(baseconf + ".json"); err != nil { + if err = removeFile(Location(c), baseconf+".json"); err != nil { return } @@ -518,13 +519,13 @@ func commandShow(ct ComponentType, names []string, params []string) (err error) switch names[0] { case "global": var c ConfigType - readConfigFile(globalConfig, &c) + readConfigFile(LOCAL, globalConfig, &c) printConfigStructJSON(c) return case "user": var c ConfigType userConfDir, _ := os.UserConfigDir() - readConfigFile(filepath.Join(userConfDir, "geneos.json"), &c) + readConfigFile(LOCAL, filepath.Join(userConfDir, "geneos.json"), &c) printConfigStructJSON(c) return } @@ -688,7 +689,7 @@ func commandSet(ct ComponentType, args []string, params []string) (err error) { // now loop through the collected results anbd write out for _, c := range instances { conffile := filepath.Join(Home(c), Type(c).String()+".json") - if err = writeConfigFile(conffile, c); err != nil { + if err = writeConfigFile(Location(c), conffile, c); err != nil { log.Println(err) } } @@ -699,7 +700,7 @@ func commandSet(ct ComponentType, args []string, params []string) (err error) { func writeConfigParams(filename string, params []string) (err error) { var c ConfigType // ignore err - config may not exist, but that's OK - _ = readConfigFile(filename, &c) + _ = readConfigFile(LOCAL, filename, &c) // change here for _, set := range params { // skip all non '=' args @@ -712,27 +713,27 @@ func writeConfigParams(filename string, params []string) (err error) { return } } - return writeConfigFile(filename, c) + return writeConfigFile(LOCAL, filename, c) } func writeInstanceConfig(c Instance) (err error) { - err = writeConfigFile(filepath.Join(Home(c), Type(c).String()+".json"), c) + err = writeConfigFile(Location(c), filepath.Join(Home(c), Type(c).String()+".json"), c) return } -func readConfigFile(file string, config interface{}) (err error) { - jsonFile, err := os.Open(file) +func readConfigFile(remote, file string, config interface{}) (err error) { + jsonFile, err := readFile(remote, file) if err != nil { return } - dec := json.NewDecoder(jsonFile) - return dec.Decode(&config) + // dec := json.NewDecoder(jsonFile) + return json.Unmarshal(jsonFile, &config) } // try to be atomic, lots of edge cases, UNIX/Linux only // we know the size of config structs is typicall small, so just marshal // in memory -func writeConfigFile(file string, config interface{}) (err error) { +func writeConfigFile(remote, file string, config interface{}) (err error) { buffer, err := json.MarshalIndent(config, "", " ") if err != nil { return @@ -751,35 +752,59 @@ func writeConfigFile(file string, config interface{}) (err error) { uid, gid = int(ux), int(gx) } - dir, name := filepath.Split(file) - dir = strings.TrimSuffix(dir, "/") + dir := filepath.Dir(file) // try to ensure directory exists - if err = os.MkdirAll(dir, 0775); err != nil { + if err = mkdirAll(remote, dir, 0775); err != nil { return } // change final directory ownership - _ = os.Chown(dir, uid, gid) + _ = chown(remote, dir, uid, gid) - f, err := os.CreateTemp(dir, name) - if err != nil { - return fmt.Errorf("cannot create %q: %w", file, errors.Unwrap(err)) - } - defer os.Remove(f.Name()) - // use Println to get a final newline - if _, err = fmt.Fprintln(f, string(buffer)); err != nil { - return - } + switch remote { + case LOCAL: + f, err := os.CreateTemp(dir, filepath.Base(file)) + if err != nil { + return fmt.Errorf("cannot create %q: %w", file, errors.Unwrap(err)) + } + defer removeFile(remote, f.Name()) + // use Println to get a final newline + if _, err = fmt.Fprintln(f, string(buffer)); err != nil { + return err + } - // update file perms and owner before final rename to overwrite - // existing file - if err = f.Chmod(0664); err != nil { - return - } - if err = f.Chown(uid, gid); err != nil { - return - } + // update file perms and owner before final rename to overwrite + // existing file + if err = f.Chmod(0664); err != nil { + return err + } + if err = f.Chown(uid, gid); err != nil { + return err + } + + return renameFile(remote, f.Name(), file) + default: + var f *sftp.File + f, err = createRemoteTemp(remote, file) + if err != nil { + return fmt.Errorf("cannot create %q: %w", file, errors.Unwrap(err)) + } + defer removeFile(remote, f.Name()) + // use Println to get a final newline + if _, err = fmt.Fprintln(f, string(buffer)); err != nil { + return err + } - return os.Rename(f.Name(), file) + // update file perms and owner before final rename to overwrite + // existing file + if err = f.Chmod(0664); err != nil { + return err + } + if err = f.Chown(uid, gid); err != nil { + return err + } + + return renameFile(remote, f.Name(), file) + } } func commandRename(ct ComponentType, args []string, params []string) (err error) { @@ -810,26 +835,26 @@ func commandRename(ct ComponentType, args []string, params []string) (err error) oldhome := Home(oldconf) newhome := Home(newconf) - if err = os.Rename(oldhome, newhome); err != nil { + if err = renameFile(Location(oldhome), oldhome, newhome); err != nil { logDebug.Println("rename failed:", oldhome, newhome, err) return } if err = setField(oldconf, "Name", newname); err != nil { // try to recover - _ = os.Rename(newhome, oldhome) + _ = renameFile(Location(newhome), newhome, oldhome) return } - if err = setField(oldconf, Prefix(oldconf)+"Home", filepath.Join(componentDir(ct), newname)); err != nil { + if err = setField(oldconf, Prefix(oldconf)+"Home", filepath.Join(componentDir(Location(newhome), ct), newname)); err != nil { // try to recover - _ = os.Rename(newhome, oldhome) + _ = renameFile(Location(newhome), newhome, oldhome) return // } // config changes don't matter until writing config succeeds - if err = writeConfigFile(filepath.Join(newhome, ct.String()+".json"), oldconf); err != nil { - _ = os.Rename(newhome, oldhome) + if err = writeConfigFile(Location(newconf), filepath.Join(newhome, ct.String()+".json"), oldconf); err != nil { + _ = renameFile(Location(newhome), newhome, oldhome) return } log.Println(ct, oldname, "renamed to", newname) @@ -842,7 +867,7 @@ func commandDelete(ct ComponentType, args []string, params []string) (err error) func deleteInstance(c Instance, params []string) (err error) { if isDisabled(c) { - if err = os.RemoveAll(Home(c)); err != nil { + if err = removeAll(Location(c), Home(c)); err != nil { logError.Fatalln(err) } return nil diff --git a/cmd/geneos/control.go b/cmd/geneos/control.go index 865c092..7264217 100644 --- a/cmd/geneos/control.go +++ b/cmd/geneos/control.go @@ -111,10 +111,12 @@ func commandStart(ct ComponentType, args []string, params []string) (err error) return } +// XXX remote support required func startInstance(c Instance, params []string) (err error) { - pid, _, err := findInstanceProc(c) + pid, err := findInstancePID(c) if err == nil { - log.Println(Type(c), Name(c), "already running with PID", pid) + log.Printf("%s %s@%s already running with PID %d", Type(c), Name(c), Location(c), pid) + return nil } @@ -123,7 +125,7 @@ func startInstance(c Instance, params []string) (err error) { } binary := getString(c, Prefix(c)+"Exec") - if _, err = os.Stat(binary); err != nil { + if _, err = statFile(Location(c), binary); err != nil { return } @@ -139,6 +141,57 @@ func startInstance(c Instance, params []string) (err error) { // set underlying user for child proc username := getString(c, Prefix(c)+"User") + errfile := filepath.Join(Home(c), Type(c).String()+".txt") + + if Location(c) != LOCAL { + r := loadRemoteConfig(Location(c)) + rUsername := getString(r, "Username") + if rUsername != username { + log.Fatalf("cannot run remote process as a different user (%q != %q)", rUsername, username) + } + rem, err := sshOpenRemote(Location(c)) + if err != nil { + log.Fatalln(err) + } + sess, err := rem.NewSession() + if err != nil { + log.Fatalln(err) + } + + // we have to convert cmd to a string ourselves as we have to quote any args + // with spaces (like "Demo Gateway") + // + // given this is sent to a shell, we can quote everything blindly ? + var cmdstr = "" + for _, a := range cmd.Args { + cmdstr = fmt.Sprintf("%s %q", cmdstr, a) + } + pipe, err := sess.StdinPipe() + if err != nil { + log.Fatalln() + } + + if err = sess.Shell(); err != nil { + log.Fatalln(err) + } + fmt.Fprintln(pipe, "cd", Home(c)) + for _, e := range env { + fmt.Fprintln(pipe, "export", e) + } + fmt.Fprintf(pipe, "%s > %q 2>&1 &", cmdstr, errfile) + fmt.Fprintln(pipe, "exit") + sess.Close() + // wait a short while for remote to catch-up + time.Sleep(250 * time.Millisecond) + + pid, err := findInstancePID(c) + if err != nil { + log.Fatalln(err) + } + log.Printf("%s %s@%s started with PID %d", Type(c), Name(c), Location(c), pid) + return nil + } + // pass possibly empty string down to setuser - it handles defaults if err = setUser(cmd, username); err != nil { return @@ -146,8 +199,6 @@ func startInstance(c Instance, params []string) (err error) { cmd.Env = append(os.Environ(), env...) - errfile := filepath.Join(Home(c), Type(c).String()+".txt") - out, err := os.OpenFile(errfile, os.O_CREATE|os.O_WRONLY, 0644) if err != nil { return err @@ -166,8 +217,7 @@ func startInstance(c Instance, params []string) (err error) { if err = cmd.Start(); err != nil { return } - log.Println(Type(c), Name(c), "started with PID", cmd.Process.Pid) - + log.Printf("%s %s@%s started with PID %d", Type(c), Name(c), Location(c), cmd.Process.Pid) if cmd.Process != nil { // detach from control cmd.Process.Release() @@ -187,25 +237,72 @@ func commandStop(ct ComponentType, args []string, params []string) (err error) { } func stopInstance(c Instance, params []string) (err error) { - pid, st, err := findInstanceProc(c) + pid, err := findInstancePID(c) if err != nil && errors.Is(err, ErrProcNotExist) { // not found is fine return } + if Location(c) != LOCAL { + rem, err := sshOpenRemote(Location(c)) + if err != nil { + log.Fatalln(err) + } + sess, err := rem.NewSession() + if err != nil { + log.Fatalln(err) + } + pipe, err := sess.StdinPipe() + if err != nil { + log.Fatalln() + } + + if err = sess.Shell(); err != nil { + log.Fatalln(err) + } + + if !stopKill { + fmt.Fprintln(pipe, "kill", pid) + for i := 0; i < 10; i++ { + time.Sleep(250 * time.Millisecond) + _, err = findInstancePID(c) + if err == ErrProcNotExist { + break + } + fmt.Fprintln(pipe, "kill", pid) + } + _, err = findInstancePID(c) + if err != ErrProcNotExist { + log.Printf("%s %s@%s stopped", Type(c), Name(c), Location(c)) + fmt.Fprintln(pipe, "exit") + sess.Close() + return nil + } + } + + fmt.Fprintln(pipe, "kill -KILL", pid) + fmt.Fprintln(pipe, "exit") + sess.Close() + + _, err = findInstancePID(c) + if err == ErrProcNotExist { + log.Printf("%s %s@%s killed", Type(c), Name(c), Location(c)) + return nil + } + + logDebug.Println("process still running as", pid) + return ErrProcExists + } + if !canControl(c) { return ErrPermission } - logDebug.Println("process running as", st.Uid, st.Gid) - proc, _ := os.FindProcess(pid) if !stopKill { - log.Println("stopping", Type(c), Name(c), "with PID", pid) - if err = proc.Signal(syscall.SIGTERM); err != nil { - log.Println("sending SIGTERM failed:", err) + logError.Println("sending SIGTERM failed:", err) return } @@ -213,7 +310,7 @@ func stopInstance(c Instance, params []string) (err error) { for i := 0; i < 10; i++ { time.Sleep(250 * time.Millisecond) if err = proc.Signal(syscall.Signal(0)); err != nil { - logDebug.Println(Type(c), "terminated") + log.Printf("%s %s@%s stopped", Type(c), Name(c), Location(c)) return nil } } @@ -223,7 +320,8 @@ func stopInstance(c Instance, params []string) (err error) { if err = proc.Signal(syscall.SIGKILL); err != nil { log.Println("sending SIGKILL failed:", err) } - log.Println("killed", Type(c), Name(c), "with PID", pid) + + log.Printf("%s %s@%s killed", Type(c), Name(c), Location(c)) return } @@ -278,14 +376,29 @@ func disableInstance(c Instance, params []string) (err error) { return } - f, err := os.Create(filepath.Join(Home(c), Type(c).String()+disableExtension)) - if err != nil { - return - } - defer f.Close() + disablePath := filepath.Join(Home(c), Type(c).String()+disableExtension) - if err = f.Chown(int(uid), int(gid)); err != nil { - os.Remove(f.Name()) + switch Location(c) { + case LOCAL: + f, err := os.Create(disablePath) + if err != nil { + return err + } + defer f.Close() + + if err = f.Chown(int(uid), int(gid)); err != nil { + removeFile(Location(c), f.Name()) + } + default: + f, err := createRemoteFile(Location(c), disablePath) + if err != nil { + return err + } + defer f.Close() + + if err = f.Chown(int(uid), int(gid)); err != nil { + removeFile(Location(c), f.Name()) + } } return } @@ -297,7 +410,7 @@ func commandEneable(ct ComponentType, args []string, params []string) (err error } func enableInstance(c Instance, params []string) (err error) { - if err = os.Remove(filepath.Join(Home(c), Type(c).String()+disableExtension)); err == nil || errors.Is(err, os.ErrNotExist) { + if err = removeFile(Location(c), filepath.Join(Home(c), Type(c).String()+disableExtension)); err == nil || errors.Is(err, os.ErrNotExist) { err = startInstance(c, params) } return @@ -305,7 +418,7 @@ func enableInstance(c Instance, params []string) (err error) { func isDisabled(c Instance) bool { d := filepath.Join(Home(c), Type(c).String()+disableExtension) - if f, err := os.Stat(d); err == nil && f.Mode().IsRegular() { + if f, err := statFile(Location(c), d); err == nil && f.st.Mode().IsRegular() { return true } return false diff --git a/cmd/geneos/edit.go b/cmd/geneos/edit.go index e1334c0..df4687e 100644 --- a/cmd/geneos/edit.go +++ b/cmd/geneos/edit.go @@ -67,6 +67,9 @@ func commandEdit(ct ComponentType, args []string, params []string) (err error) { var cs []string for _, name := range args { for _, c := range newComponent(ct, name) { + if Location(c) != LOCAL { + logError.Fatalln(ErrNotSupported) + } // try to migrate the config, which will not work if empty if err = loadConfig(c, true); err != nil { log.Println(Type(c), Name(c), "cannot load configuration, check syntax") diff --git a/cmd/geneos/gateway.go b/cmd/geneos/gateway.go index 9fa266a..e297bbd 100644 --- a/cmd/geneos/gateway.go +++ b/cmd/geneos/gateway.go @@ -3,6 +3,7 @@ package main import ( _ "embed" "errors" + "io" "os" "path/filepath" "strconv" @@ -47,11 +48,12 @@ func init() { } func gatewayInstance(name string) interface{} { - // Bootstrap + local, remote := splitInstanceName(name) c := &Gateways{} - c.Root = RunningConfig.ITRSHome + c.Root = remoteRoot(remote) c.Type = Gateway.String() - c.Name = name + c.Name = local + c.Location = remote setDefaults(&c) return c } @@ -130,15 +132,38 @@ func gatewayAdd(name string, username string, params []string) (c Instance, err if err != nil { logError.Fatalln(err) } - cf, err := os.OpenFile(filepath.Join(Home(c), "gateway.setup.xml"), os.O_CREATE|os.O_WRONLY, 0664) - if err != nil { - log.Println(err) - return + + var out io.Writer + + switch Location(c) { + case LOCAL: + cf, err := os.Create(filepath.Join(Home(c), "gateway.setup.xml")) + out = cf + if err != nil { + log.Println(err) + return nil, err + } + defer cf.Close() + if err = cf.Chmod(0664); err != nil { + logError.Fatalln(err) + } + default: + cf, err := createRemoteFile(Location(c), filepath.Join(Home(c), "gateway.setup.xml")) + out = cf + if err != nil { + log.Println(err) + return nil, err + } + defer cf.Close() + if err = cf.Chmod(0664); err != nil { + logError.Fatalln(err) + } } - defer cf.Close() - if err = t.Execute(cf, c); err != nil { + + if err = t.Execute(out, c); err != nil { logError.Fatalln(err) } + return } @@ -170,7 +195,11 @@ func gatewayClean(c Instance, purge bool, params []string) (err error) { } func gatewayReload(c Instance, params []string) (err error) { - pid, _, err := findInstanceProc(c) + if Location(c) != LOCAL { + logError.Fatalln(ErrNotSupported) + } + + pid, err := findInstancePID(c) if err != nil { return } diff --git a/cmd/geneos/licd.go b/cmd/geneos/licd.go index 154de6d..b0dbd5b 100644 --- a/cmd/geneos/licd.go +++ b/cmd/geneos/licd.go @@ -36,11 +36,12 @@ func init() { } func licdInstance(name string) interface{} { - // Bootstrap + local, remote := splitInstanceName(name) c := &Licds{} - c.Root = RunningConfig.ITRSHome + c.Root = remoteRoot(remote) c.Type = Licd.String() - c.Name = name + c.Name = local + c.Location = remote setDefaults(&c) return c } diff --git a/cmd/geneos/list.go b/cmd/geneos/list.go index 00bab39..d4dc2f9 100644 --- a/cmd/geneos/list.go +++ b/cmd/geneos/list.go @@ -67,14 +67,6 @@ func flagsList(command string, args []string) []string { } func commandLS(ct ComponentType, args []string, params []string) (err error) { - if ct == Remote { - // geneos ls remote [NAME] - if len(args) == 0 { - // list remotes - - } - } - switch { case listJSON: jsonEncoder = json.NewEncoder(log.Writer()) @@ -84,12 +76,12 @@ func commandLS(ct ComponentType, args []string, params []string) (err error) { err = loopCommand(lsInstanceJSON, ct, args, params) case listCSV: csvWriter = csv.NewWriter(log.Writer()) - csvWriter.Write([]string{"Type", "Name", "Disabled", "Home"}) + csvWriter.Write([]string{"Type", "Name", "Disabled", "Location", "Home"}) err = loopCommand(lsInstanceCSV, ct, args, params) csvWriter.Flush() default: lsTabWriter = tabwriter.NewWriter(log.Writer(), 3, 8, 2, ' ', 0) - fmt.Fprintf(lsTabWriter, "Type\tName\tHome\n") + fmt.Fprintf(lsTabWriter, "Type\tName\tLocation\tHome\n") err = loopCommand(lsInstancePlain, ct, args, params) lsTabWriter.Flush() } @@ -102,7 +94,7 @@ func lsInstancePlain(c Instance, params []string) (err error) { if isDisabled(c) { suffix = "*" } - fmt.Fprintf(lsTabWriter, "%s\t%s\t%s\n", Type(c), Name(c)+suffix, Home(c)) + fmt.Fprintf(lsTabWriter, "%s\t%s\t%s\t%s\n", Type(c), Name(c)+suffix, Location(c), Home(c)) return } @@ -111,7 +103,7 @@ func lsInstanceCSV(c Instance, params []string) (err error) { if isDisabled(c) { dis = "Y" } - csvWriter.Write([]string{Type(c).String(), Name(c), dis, Home(c)}) + csvWriter.Write([]string{Type(c).String(), Name(c), dis, Location(c), Home(c)}) return } @@ -119,6 +111,7 @@ type lsType struct { Type string Name string Disabled string + Location string Home string } @@ -127,7 +120,7 @@ func lsInstanceJSON(c Instance, params []string) (err error) { if isDisabled(c) { dis = "Y" } - jsonEncoder.Encode(lsType{Type(c).String(), Name(c), dis, Home(c)}) + jsonEncoder.Encode(lsType{Type(c).String(), Name(c), dis, Location(c), Home(c)}) return } @@ -141,6 +134,7 @@ var psTabWriter *tabwriter.Writer type psType struct { Type string Name string + Remote string PID string User string Group string @@ -156,12 +150,12 @@ func commandPS(ct ComponentType, args []string, params []string) (err error) { err = loopCommand(psInstanceJSON, ct, args, params) case listCSV: csvWriter = csv.NewWriter(log.Writer()) - csvWriter.Write([]string{"Type:Name", "PID", "User", "Group", "Starttime", "Home"}) + csvWriter.Write([]string{"Type:Name@Location", "PID", "User", "Group", "Starttime", "Home"}) err = loopCommand(psInstanceCSV, ct, args, params) csvWriter.Flush() default: psTabWriter = tabwriter.NewWriter(log.Writer(), 3, 8, 2, ' ', 0) - fmt.Fprintf(psTabWriter, "Type:Name\tPID\tUser\tGroup\tStarttime\tHome\n") + fmt.Fprintf(psTabWriter, "Type:Name@Location\tPID\tUser\tGroup\tStarttime\tHome\n") err = loopCommand(psInstancePlain, ct, args, params) psTabWriter.Flush() } @@ -172,7 +166,7 @@ func psInstancePlain(c Instance, params []string) (err error) { if isDisabled(c) { return nil } - pid, st, err := findInstanceProc(c) + pid, uid, gid, mtime, err := findInstanceProc(c) if err != nil { return nil } @@ -180,17 +174,17 @@ func psInstancePlain(c Instance, params []string) (err error) { var u *user.User var g *user.Group - username := fmt.Sprint(st.Uid) - groupname := fmt.Sprint(st.Gid) + username := fmt.Sprint(uid) + groupname := fmt.Sprint(gid) - if u, err = user.LookupId(fmt.Sprint(st.Uid)); err == nil { + if u, err = user.LookupId(username); err == nil { username = u.Username } - if g, err = user.LookupGroupId(fmt.Sprint(st.Gid)); err == nil { + if g, err = user.LookupGroupId(groupname); err == nil { groupname = g.Name } - fmt.Fprintf(psTabWriter, "%s:%s\t%d\t%s\t%s\t%s\t%s\n", Type(c), Name(c), pid, username, groupname, time.Unix(st.Ctim.Sec, st.Ctim.Nsec).Local().Format(time.RFC3339), Home(c)) + fmt.Fprintf(psTabWriter, "%s:%s@%s\t%d\t%s\t%s\t%s\t%s\n", Type(c), Name(c), Location(c), pid, username, groupname, time.Unix(mtime, 0).Local().Format(time.RFC3339), Home(c)) return } @@ -199,7 +193,7 @@ func psInstanceCSV(c Instance, params []string) (err error) { if isDisabled(c) { return nil } - pid, st, err := findInstanceProc(c) + pid, uid, gid, mtime, err := findInstanceProc(c) if err != nil { return nil } @@ -207,17 +201,17 @@ func psInstanceCSV(c Instance, params []string) (err error) { var u *user.User var g *user.Group - username := fmt.Sprint(st.Uid) - groupname := fmt.Sprint(st.Gid) + username := fmt.Sprint(uid) + groupname := fmt.Sprint(gid) - if u, err = user.LookupId(fmt.Sprint(st.Uid)); err == nil { + if u, err = user.LookupId(username); err == nil { username = u.Username } - if g, err = user.LookupGroupId(fmt.Sprint(st.Gid)); err == nil { + if g, err = user.LookupGroupId(groupname); err == nil { groupname = g.Name } - csvWriter.Write([]string{Type(c).String() + ":" + Name(c), fmt.Sprint(pid), username, groupname, time.Unix(st.Ctim.Sec, st.Ctim.Nsec).Local().Format(time.RFC3339), Home(c)}) + csvWriter.Write([]string{Type(c).String() + ":" + Name(c) + "@" + Location(c), fmt.Sprint(pid), username, groupname, time.Unix(mtime, 0).Local().Format(time.RFC3339), Home(c)}) return } @@ -226,7 +220,7 @@ func psInstanceJSON(c Instance, params []string) (err error) { if isDisabled(c) { return nil } - pid, st, err := findInstanceProc(c) + pid, uid, gid, mtime, err := findInstanceProc(c) if err != nil { return nil } @@ -234,17 +228,17 @@ func psInstanceJSON(c Instance, params []string) (err error) { var u *user.User var g *user.Group - username := fmt.Sprint(st.Uid) - groupname := fmt.Sprint(st.Gid) + username := fmt.Sprint(uid) + groupname := fmt.Sprint(gid) - if u, err = user.LookupId(fmt.Sprint(st.Uid)); err == nil { + if u, err = user.LookupId(username); err == nil { username = u.Username } - if g, err = user.LookupGroupId(fmt.Sprint(st.Gid)); err == nil { + if g, err = user.LookupGroupId(groupname); err == nil { groupname = g.Name } - jsonEncoder.Encode(psType{Type(c).String(), Name(c), fmt.Sprint(pid), username, groupname, time.Unix(st.Ctim.Sec, st.Ctim.Nsec).Local().Format(time.RFC3339), Home(c)}) + jsonEncoder.Encode(psType{Type(c).String(), Name(c), Location(c), fmt.Sprint(pid), username, groupname, time.Unix(mtime, 0).Local().Format(time.RFC3339), Home(c)}) return } @@ -254,13 +248,17 @@ func commandCommand(ct ComponentType, args []string, params []string) (err error } func commandInstance(c Instance, params []string) (err error) { + log.Printf("=== %s %s@%s ===", Type(c), Name(c), Location(c)) cmd, env := buildCmd(c) if cmd != nil { - log.Printf("command: %q\n", cmd.String()) - log.Println("env:") + log.Println("command line:") + log.Println("\t", cmd.String()) + log.Println() + log.Println("environment:") for _, e := range env { - log.Println(e) + log.Println("\t", e) } + log.Println() } return } diff --git a/cmd/geneos/logs.go b/cmd/geneos/logs.go index db8fc67..f663488 100644 --- a/cmd/geneos/logs.go +++ b/cmd/geneos/logs.go @@ -57,9 +57,9 @@ var watcher *fsnotify.Watcher // struct to hold logfile details type tail struct { - f *os.File - t ComponentType - n string + f io.ReadSeekCloser + ct ComponentType + name string } // map of log file path to File set to the last position read @@ -114,21 +114,21 @@ func outHeader(logfile string) { if lastout != "" { log.Println() } - log.Printf("==> %s:%s %s <==\n", tails[logfile].t, tails[logfile].n, logfile) + log.Printf("==> %s:%s %s <==\n", tails[logfile].ct, tails[logfile].name, logfile) lastout = logfile } func logTailInstance(c Instance, params []string) (err error) { logfile := getLogfilePath(c) - lines, err := os.Open(logfile) + lines, st, err := openStatFile(Location(c), logfile) if err != nil { return } defer lines.Close() - tails[logfile] = &tail{lines, Type(c), Name(c)} + tails[logfile] = &tail{lines, Type(c), Name(c) + "@" + Location(c)} - text, err := tailLines(lines, logsLines) + text, err := tailLines(lines, st, logsLines) if err != nil && !errors.Is(err, io.EOF) { log.Println(err) } @@ -138,7 +138,7 @@ func logTailInstance(c Instance, params []string) (err error) { return nil } -func tailLines(file *os.File, linecount int) (text string, err error) { +func tailLines(f io.ReadSeekCloser, st fileStat, linecount int) (text string, err error) { // reasonable guess at bytes per line to use as a multiplier const charsPerLine = 132 var chunk int64 = int64(linecount * charsPerLine) @@ -148,18 +148,19 @@ func tailLines(file *os.File, linecount int) (text string, err error) { if linecount == 0 { // seek to end and return - _, err = file.Seek(0, os.SEEK_END) + _, err = f.Seek(0, os.SEEK_END) return } - st, err := file.Stat() + // st, err := f.Stat() if err != nil { return } - end := st.Size() + end := st.st.Size() for i = 1 + end/chunk; i > 0; i-- { - n, err := file.ReadAt(buf, (i-1)*chunk) + f.Seek((i-1)*chunk, 0) + n, err := f.Read(buf) if err != nil && !errors.Is(err, io.EOF) { logError.Fatalln(err) } @@ -172,13 +173,13 @@ func tailLines(file *os.File, linecount int) (text string, err error) { alllines = append(newlines, alllines[1:]...) if len(alllines) > linecount { text = strings.Join(alllines[len(alllines)-linecount:], "\n") - file.Seek(end, io.SeekStart) + f.Seek(end, io.SeekStart) return text, err } } text = strings.Join(alllines, "\n") - file.Seek(end, io.SeekStart) + f.Seek(end, io.SeekStart) return } @@ -222,11 +223,11 @@ func filterOutput(logfile string, reader io.Reader) { func logCatInstance(c Instance, params []string) (err error) { logfile := getLogfilePath(c) - lines, err := os.Open(logfile) + lines, _, err := openStatFile(Location(c), logfile) if err != nil { return } - tails[logfile] = &tail{lines, Type(c), Name(c)} + tails[logfile] = &tail{lines, Type(c), Name(c) + "@" + Location(c)} defer lines.Close() filterOutput(logfile, lines) @@ -234,14 +235,18 @@ func logCatInstance(c Instance, params []string) (err error) { } func logFollowInstance(c Instance, params []string) (err error) { + if Location(c) != LOCAL { + logError.Fatalln("remote!=local", ErrNotSupported) + } logfile := getLogfilePath(c) f, _ := os.Open(logfile) + st, _ := statFile(LOCAL, logfile) // perfectly valid to not have a file to watch at start - tails[logfile] = &tail{f, Type(c), Name(c)} + tails[logfile] = &tail{f, Type(c), Name(c) + "@" + Location(c)} // output up to this point - text, _ := tailLines(tails[logfile].f, logsLines) + text, _ := tailLines(tails[logfile].f, st, logsLines) if len(text) != 0 { filterOutput(logfile, strings.NewReader(text+"\n")) diff --git a/cmd/geneos/main.go b/cmd/geneos/main.go index 38cc2f0..2d5cdf4 100644 --- a/cmd/geneos/main.go +++ b/cmd/geneos/main.go @@ -9,7 +9,6 @@ import ( "os" "os/user" "strings" - "syscall" "wonderland.org/geneos/pkg/logger" ) @@ -80,16 +79,15 @@ func main() { log.SetOutput(ioutil.Discard) } else if debug { logger.EnableDebugLog() - } else if verbose { - log.Println("look at ME!") } - loadSysConfig() - if len(leftargs) == 0 { - logError.Fatalln("[usage here]: not enough args") + commandHelp(None, nil, nil) + os.Exit(0) } + loadSysConfig() + var command = strings.ToLower(leftargs[0]) var ct ComponentType = None var args []string = leftargs[1:] @@ -140,24 +138,42 @@ func main() { default: // test home dir, stop if invalid if RunningConfig.ITRSHome == "" { - logError.Fatalln("home directory is not set") + log.Fatalln(` +Installation directory is not set. + +You can fix this by doing one of the following: + +1. Create a new Geneos environment: + + $ geneos init /path/to/geneos + +2. Set the ITRS_HOME environment: + + $ export ITRS_HOME=/path/to/geneos + +3. Set the ITRSHome user parameter: + + $ geneos set user ITRSHome=/path/to/geneos + +3. Set the ITRSHome parameter in the global configuration file: + + $ echo '{ "ITRSHome": "/path/to/geneos" }' >` + globalConfig) } - s, err := os.Stat(RunningConfig.ITRSHome) + s, err := statFile(LOCAL, RunningConfig.ITRSHome) if err != nil { logError.Fatalf("home directory %q: %s", RunningConfig.ITRSHome, errors.Unwrap(err)) } - if !s.IsDir() { + if !s.st.IsDir() { logError.Fatalln(RunningConfig.ITRSHome, "is not a directory") } // we have a valid home directory, now set default user if // not set elsewhere if RunningConfig.DefaultUser == "" { - s2 := s.Sys().(*syscall.Stat_t) - if s2.Uid == 0 { + if s.uid == 0 { logError.Fatalf("home directory %q: owned by root and no default user configured", RunningConfig.ITRSHome) } - u, err := user.LookupId(fmt.Sprint(s2.Uid)) + u, err := user.LookupId(fmt.Sprint(s.uid)) if err != nil { logError.Fatalln(RunningConfig.ITRSHome, err) } diff --git a/cmd/geneos/netprobe.go b/cmd/geneos/netprobe.go index b33e222..1c6a963 100644 --- a/cmd/geneos/netprobe.go +++ b/cmd/geneos/netprobe.go @@ -36,11 +36,12 @@ func init() { } func netprobeInstance(name string) interface{} { - // Bootstrap + local, remote := splitInstanceName(name) c := &Netprobes{} - c.Root = RunningConfig.ITRSHome + c.Root = remoteRoot(remote) c.Type = Netprobe.String() - c.Name = name + c.Name = local + c.Location = remote setDefaults(&c) return c } diff --git a/cmd/geneos/packages.go b/cmd/geneos/packages.go new file mode 100644 index 0000000..73a7335 --- /dev/null +++ b/cmd/geneos/packages.go @@ -0,0 +1,636 @@ +package main + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "io/fs" + "net/http" + "net/url" + "os" + "path/filepath" + "regexp" + "strings" +) + +func init() { + commands["extract"] = Command{ + Function: commandExtract, + ParseFlags: flagsExtract, + ParseArgs: checkComponentArg, + CommandLine: "geneos extract [-r REMOTE] [TYPE] | FILE [FILE...]", + Summary: `Extract files from downloaded Geneos packages. Intended for sites without Internet access.`, + Description: `Extracts files from FILE(s) in to the packages/ directory. The filename(s) must of of the form: + + geneos-TYPE-VERSION*.tar.gz + +The directory for the package is created using the VERSION from the archive +filename. + +If a TYPE is given then the latest version from the packages/archives +directory for that TYPE is extracted, otherwise it is treated as a +normal file path. This is primarily for extracting to remote locations. + +FLAGS: + -r REMOTE - extract from local archive to remote. default is local. all means all remotes and local. +`} + + extractFlags = flag.NewFlagSet("extract", flag.ExitOnError) + extractFlags.StringVar(&extractRemote, "r", ALL, "Perform on a remote. \"all\" means all remotes and locally") + + commands["download"] = Command{ + Function: commandDownload, + ParseFlags: flagsDownload, + ParseArgs: checkComponentArg, + CommandLine: "geneos download [-n] [-r REMOTE] [TYPE] [latest|FILTER|URL...]", + Summary: `Download and extract Geneos software archive.`, + Description: `Download and extract the sources in the packages directory or latest version(s) from +the official download site. The filename must of of the format: + + geneos-TYPE-VERSION*.tar.gz + +The TYPE, if supplied, limits the selection of downloaded archive(s). The directory +for the package is created using the VERSION from the archive filename. + +The downloaded file is saved in the packages/archives/ direcvtory for +future re-use, especially for remote support. + +FLAGS: + -n - Do not save download archive + -r REMOTE - download and extract from local archive to remote. default is local. all means all remotes and local. + +`} + + downloadFlags = flag.NewFlagSet("download", flag.ExitOnError) + downloadFlags.BoolVar(&downloadNosave, "n", false, "Do not save download") + downloadFlags.BoolVar(&helpFlag, "h", false, helpUsage) + downloadFlags.StringVar(&downloadRemote, "r", ALL, "Perform on a remote. \"all\" means all remotes and locally") + + commands["update"] = Command{ + Function: commandUpdate, + ParseFlags: flagsUpdate, + ParseArgs: checkComponentArg, + CommandLine: "geneos update [-r REMOTE] [TYPE] VERSION", + Summary: `Update the active version of Geneos software.`, + Description: `Update the symlink for the default base name of the package used to VERSION. The base directory, + for historical reasons, is 'active_prod' and is usally linked to the latest version of a component type +in the packages directory. VERSION can either be a directory name or the literal 'latest'. If TYPE is not +supplied, all supported component types are updated to VERSION. + +Update will stop all matching instances of the each type before updating the link and starting them up +again, but only if the instance uses the 'active_prod' basename. + +The 'latest' version is based on directory names of the form: + +[GA]X.Y.Z + +Where X, Y, Z are each ordered in ascending numerical order. If a directory starts 'GA' it will be selected +over a directory with the same numerical versions. All other directories name formats will result in unexpected +behaviour. + +Future version may support selecting a base other than 'active_prod'. + +FLAGS: + -r REMOTE - update remote. default is local. all means all remotes and local. + +`} + + updateFlags = flag.NewFlagSet("update", flag.ExitOnError) + updateFlags.StringVar(&updateBase, "b", "active_prod", "Override the base active_prod link name") + updateFlags.StringVar(&updateRemote, "r", ALL, "Perform on a remote. \"all\" means all remotes and locally") +} + +var extractFlags *flag.FlagSet +var extractRemote string + +var downloadFlags *flag.FlagSet +var downloadNosave bool +var downloadRemote string + +var updateFlags *flag.FlagSet +var updateBase string +var updateRemote string + +func flagsExtract(command string, args []string) []string { + extractFlags.Parse(args) + checkHelpFlag(command) + return extractFlags.Args() +} + +func flagsDownload(command string, args []string) []string { + downloadFlags.Parse(args) + checkHelpFlag(command) + return downloadFlags.Args() +} + +func flagsUpdate(command string, args []string) []string { + updateFlags.Parse(args) + checkHelpFlag(command) + return updateFlags.Args() +} + +// +// if there is no 'active_prod' link then attach it to the latest version +// installed +// +func commandExtract(ct ComponentType, files []string, params []string) (err error) { + if ct != None { + logDebug.Println(ct.String()) + // archive directory is local? + archiveDir := filepath.Join(remoteRoot(LOCAL), "packages", "archives") + archiveFile := latestMatch(LOCAL, archiveDir, func(v os.DirEntry) bool { + switch ct { + default: + logDebug.Println(v.Name(), ct.String()) + return !strings.Contains(v.Name(), ct.String()) + case Webserver: + return !strings.Contains(v.Name(), "web-server") + } + }) + gz, _, err := openStatFile(LOCAL, filepath.Join(archiveDir, archiveFile)) + if err != nil { + return err + } + defer gz.Close() + if _, err = unarchive(extractRemote, ct, archiveFile, gz); err != nil { + log.Println("location:", extractRemote, err) + return err + } + return nil + } + + for _, file := range files { + filename := filepath.Base(file) + gz, _, err := openStatFile(LOCAL, file) + if err != nil { + return err + } + defer gz.Close() + + if extractRemote == ALL { + for _, remote := range allRemotes() { + if _, err = unarchive(Name(remote), ct, filename, gz); err != nil { + log.Println("location:", Name(remote), err) + continue + } + } + } else { + if _, err = unarchive(extractRemote, ct, filename, gz); err != nil { + log.Println("location:", LOCAL, err) + return err + } + } + } + + // create a symlink only if one doesn't exist + return updateToVersion(extractRemote, ct, "latest", false) +} + +func commandDownload(ct ComponentType, files []string, params []string) (err error) { + version := "latest" + if len(files) > 0 { + version = files[0] + } + + if downloadRemote == ALL { + for _, remote := range allRemotes() { + if err = downloadComponent(Name(remote), ct, version); err != nil { + logError.Println("location:", Name(remote), err) + continue + } + } + } else { + if err = downloadComponent(downloadRemote, ct, version); err != nil { + logError.Println("location:", downloadRemote, err) + return err + } + } + return +} + +func downloadComponent(remote string, ct ComponentType, version string) (err error) { + switch ct { + case Remote: + // do nothing + return nil + case None: + for _, t := range realComponentTypes() { + if err = downloadComponent(remote, t, version); err != nil { + if errors.Is(err, fs.ErrExist) { + continue + } + logError.Println(err) + return + } + } + return nil + default: + filename, gz, err := downloadArchive(ct, version) + if err != nil { + return err + } + defer gz.Close() + + logDebug.Println("downloaded", ct.String(), filename) + + var finalVersion string + if finalVersion, err = unarchive(remote, ct, filename, gz); err != nil { + if errors.Is(err, fs.ErrExist) { + return nil + } + return err + } + if version == "latest" { + return updateToVersion(remote, ct, finalVersion, true) + } + return updateToVersion(remote, ct, finalVersion, false) + } +} + +// how to split an archive name into type and version +var archiveRE = regexp.MustCompile(`^geneos-(web-server|\w+)-([\w\.-]+?)[\.-]?linux`) + +func unarchive(remote string, ct ComponentType, filename string, gz io.Reader) (finalVersion string, err error) { + parts := archiveRE.FindStringSubmatch(filename) + if len(parts) == 0 { + logError.Fatalf("invalid archive name format: %q", filename) + } + version := parts[2] + filect := parseComponentName(parts[1]) + switch ct { + case None: + ct = filect + case filect: + break + default: + // mismatch + logError.Fatalf("component type and archive mismatch: %q is not a %q", filename, ct) + } + + basedir := filepath.Join(remoteRoot(remote), "packages", ct.String(), version) + if _, err = statFile(remote, basedir); err == nil { + return // "", fmt.Errorf("%s: %w", basedir, fs.ErrExist) + } + if err = mkdirAll(remote, basedir, 0775); err != nil { + return + } + + t, err := gzip.NewReader(gz) + if err != nil { + return + } + defer t.Close() + + tr := tar.NewReader(t) + for { + var hdr *tar.Header + hdr, err = tr.Next() + if err == io.EOF { + err = nil + break + } + if err != nil { + return + } + // log.Println("file:", hdr.Name, "size", hdr.Size) + // strip leading component name (XXX - except webserver) + // do not trust tar archives to contain safe paths + var name string + switch ct { + case Webserver: + name = hdr.Name + default: + name = strings.TrimPrefix(hdr.Name, ct.String()+"/") + } + if name == "" { + continue + } + name, err = cleanRelativePath(name) + if err != nil { + logError.Fatalln(err) + } + fullpath := filepath.Join(basedir, name) + switch hdr.Typeflag { + case tar.TypeReg: + // check (and created) containing directories - account for munged tar files + dir := filepath.Dir(fullpath) + if err = mkdirAll(remote, dir, 0777); err != nil { + return + } + + switch remote { + case LOCAL: + // XXX + out, err := os.OpenFile(fullpath, os.O_CREATE|os.O_WRONLY, hdr.FileInfo().Mode()) + if err != nil { + return "", err + } + n, err := io.Copy(out, tr) + if err != nil { + out.Close() + return "", err + } + if n != hdr.Size { + log.Println("lengths different:", hdr.Size, n) + } + out.Close() + default: + // XXX + out, err := createRemoteFile(remote, fullpath) // , os.O_CREATE|os.O_WRONLY, hdr.FileInfo().Mode()) + if err != nil { + return "", err + } + out.Chmod(hdr.FileInfo().Mode()) + n, err := io.Copy(out, tr) + if err != nil { + out.Close() + return "", err + } + if n != hdr.Size { + log.Println("lengths different:", hdr.Size, n) + } + out.Close() + } + + case tar.TypeDir: + if err = mkdirAll(remote, fullpath, hdr.FileInfo().Mode()); err != nil { + return + } + case tar.TypeSymlink, tar.TypeGNULongLink: + if filepath.IsAbs(hdr.Linkname) { + logError.Fatalln("archive contains absolute symlink target") + } + if _, err = statFile(remote, fullpath); err != nil { + if err = symlink(remote, hdr.Linkname, fullpath); err != nil { + logError.Fatalln(err) + } + } + default: + log.Printf("unsupported file type %c\n", hdr.Typeflag) + } + } + log.Printf("extracted %q to %q\n", filename, basedir) + finalVersion = filepath.Base(basedir) + return +} + +func commandUpdate(ct ComponentType, args []string, params []string) (err error) { + version := "latest" + if len(args) > 0 { + version = args[0] + } + if updateRemote == ALL { + for _, remote := range allRemotes() { + if err = updateToVersion(Name(remote), ct, version, true); err != nil { + log.Println("could not update", Name(remote), err) + } + } + return nil + } + return updateToVersion(updateRemote, ct, version, true) +} + +// check selected version exists first +func updateToVersion(remote string, ct ComponentType, version string, overwrite bool) (err error) { + basedir := filepath.Join(remoteRoot(remote), "packages", ct.String()) + basepath := filepath.Join(basedir, updateBase) + + logDebug.Printf("checking and updating %s %q to %q", remote, updateBase, version) + + switch ct { + case None: + for _, t := range realComponentTypes() { + if err = updateToVersion(remote, t, version, overwrite); err != nil { + log.Println(err) + } + } + case Gateway, Netprobe, Licd, Webserver: + if version == "" || version == "latest" { + version = latestMatch(remote, basedir, func(d os.DirEntry) bool { + return !d.IsDir() + }) + } + // does the version directory exist? + current, err := readlink(remote, basepath) + if err != nil { + log.Println("cannot read link for existing version", basepath) + return nil + } + if _, err = statFile(remote, filepath.Join(basedir, version)); err != nil { + err = fmt.Errorf("update %s@%s to version %s failed, left on %s", ct, remote, version, current) + return err + } + if current != "" && !overwrite { + return nil + } + // empty current is fine + if current == version { + logDebug.Println(ct, updateBase, "is already linked to", version) + return nil + } + // check remote only + insts := matchComponents(remote, ct, "Base", updateBase) + // stop matching instances + for _, i := range insts { + stopInstance(i, nil) + defer startInstance(i, nil) + } + if err = removeFile(remote, basepath); err != nil && !errors.Is(err, fs.ErrNotExist) { + return err + } + if err = symlink(remote, version, basepath); err != nil { + return err + } + log.Println(ct, "on", remote, updateBase, "updated to", version) + default: + return ErrNotSupported + } + return nil +} + +var versRE = regexp.MustCompile(`(\d+(\.\d+){0,2})`) + +// given a directory find the "latest" version of the form +// [GA]M.N.P[-DATE] M, N, P are numbers, DATE is treated as a string +func latestMatch(remote, dir string, fn func(os.DirEntry) bool) (latest string) { + dirs, err := readDir(remote, dir) + if err != nil { + logError.Fatalln(err) + } + max := make([]int, 3) + for _, v := range dirs { + if fn(v) { + continue + } + // strip 'GA' prefix and get name + d := strings.TrimPrefix(v.Name(), "GA") + x := versRE.FindString(d) + if x == "" { + logDebug.Println(d, "does not match a valid directory pattern") + continue + } + s := strings.SplitN(x, ".", 3) + next := slicetoi(s) + + OUTER: + for i := range max { + switch { + case next[i] < max[i]: + break OUTER + case next[i] > max[i]: + // do a final lexical scan for suffixes? + latest = v.Name() + max[i] = next[i] + default: + // if equal and we are on last number, lexical comparison + // to pick up suffixes + if len(max) == i+1 && v.Name() > latest { + latest = v.Name() + } + } + } + } + return +} + +// given a component type and a key/value pair, return matching +// instances +func matchComponents(remote string, ct ComponentType, k, v string) (insts []Instance) { + for _, i := range instancesOfComponent(remote, ct) { + if v == getString(i, Prefix(i)+k) { + if err := loadConfig(&i, false); err != nil { + log.Println(Type(i), Name(i), "cannot load config") + } + insts = append(insts, i) + } + } + return +} + +// fetch a (the latest) component from a URL, but the URLs +// are special and the resultant redirection contains the filename +// etc. +// +// URL is +// https://resources.itrsgroup.com/download/latest/[COMPONENT]?os=linux +// is RHEL8 is required, add ?title=el8 +// +// there is a mapping of our compoent types to the URLs too. +// +// Gateways -> Gateway+2 +// Netprobes -> Netprobe +// Licds -> Licence+Daemon +// Webservers -> Web+Dashboard +// +// auth requires a POST with a JSON body of +// { "username": "EMAIL", "password": "PASSWORD" } +// until anon access is allowed +// + +const defaultURL = "https://resources.itrsgroup.com/download/latest/" + +type DownloadAuth struct { + Username string `json:"username"` + Password string `json:"password"` +} + +var downloadMap = map[ComponentType]string{ + Gateway: "Gateway+2", + Netprobe: "Netprobe", + Licd: "Licence+Daemon", + Webserver: "Web+Dashboard", +} + +// XXX use HEAD to check match and compare to on disk versions +func downloadArchive(ct ComponentType, version string) (filename string, body io.ReadCloser, err error) { + baseurl := RunningConfig.DownloadURL + if baseurl == "" { + baseurl = defaultURL + } + + var resp *http.Response + + downloadURL, _ := url.Parse(baseurl) + realpath, _ := url.Parse(downloadMap[ct]) + v := url.Values{} + v.Set("os", "linux") + if version != "latest" { + v.Set("title", version) + } + realpath.RawQuery = v.Encode() + source := downloadURL.ResolveReference(realpath).String() + logDebug.Println("source url:", source) + + // if a download user is set then issue a POST with username and password + // in a JSON body, else just try the GET + if RunningConfig.DownloadUser != "" { + var authbody DownloadAuth + authbody.Username = RunningConfig.DownloadUser + authbody.Password = RunningConfig.DownloadPass + + var authjson []byte + authjson, err = json.Marshal(authbody) + if err != nil { + logError.Fatalln(err) + } + + resp, err = http.Post(source, "application/json", bytes.NewBuffer(authjson)) + } else { + resp, err = http.Get(source) + } + if err != nil { + logError.Fatalln(err) + } + if resp.StatusCode > 299 { + err = fmt.Errorf("cannot download %s package version %s: %s", ct, version, resp.Status) + resp.Body.Close() + return + } + + filename, err = filenameFromHTTPResp(resp, downloadURL) + if err != nil { + logError.Fatalln(err) + } + + // check size against downloaded archive and serve local instead, regardless + // of -n flag + archiveDir := filepath.Join(RunningConfig.ITRSHome, "packages", "archives") + mkdirAll(LOCAL, archiveDir, 0775) + archivePath := filepath.Join(archiveDir, filename) + s, err := statFile(LOCAL, archivePath) + if err == nil && s.st.Size() == resp.ContentLength { + logDebug.Println("file with same size already exists, skipping save") + f, _, err := openStatFile(LOCAL, archivePath) + if err != nil { + return filename, body, nil + } + resp.Body.Close() + return filename, f, nil + } + + // transient download + if downloadNosave { + body = resp.Body + return + } + + // save the file archive and rewind, return + f, err := os.Create(archivePath) + if err != nil { + log.Fatalln(err) + } + if _, err = io.Copy(f, resp.Body); err != nil { + log.Fatalln(err) + } + resp.Body.Close() + if _, err = f.Seek(0, 0); err != nil { + log.Fatalln(err) + } + body = f + return +} diff --git a/cmd/geneos/remote.go b/cmd/geneos/remote.go index 7a8923f..a20c3e8 100644 --- a/cmd/geneos/remote.go +++ b/cmd/geneos/remote.go @@ -1,44 +1,20 @@ package main -import "net/url" +import ( + "fmt" + "io" + "io/fs" + "math/rand" + "net/url" + "os" + "path/filepath" + "strings" + "syscall" -// remote support - -// "remote" is another component type, + "github.com/pkg/sftp" +) -// e.g. -// geneos new remote X URL -// -// below is out of date - -// examples: -// -// geneos add remote name URL -// URL here may be ssh://user@host etc. -// URL can include path to ITRS_HOME / ITRSHome, e.g. -// ssh://user@server/home/geneos -// else it default to same as local -// -// non ssh schemes to follow -// ssh support for agents and private key files - no passwords -// known_hosts will be checked, no changes made, missing keys will -// result in error. user must add hosts before use (ssh-keyscan) -// -// geneos ls remote NAME -// XXX - geneos init remote NAME -// -// remote 'localhost' is always implied -// -// geneos ls remote -// ... list remote locations -// -// geneos start gateway [name]@remote -// -// XXX support gateway pairs for standby - how ? -// -// XXX remote netprobes, auto configure with gateway for SANs etc.? -// -// support existing geneos-utils installs on remote +// remote support type Remotes struct { Common @@ -60,24 +36,51 @@ func init() { } func remoteInstance(name string) interface{} { + local, remote := splitInstanceName(name) + if remote != LOCAL { + logError.Fatalln("remote remotes not suported") + } // Bootstrap c := &Remotes{} c.Root = RunningConfig.ITRSHome c.Type = Remote.String() - c.Name = name + c.Name = local + c.Location = remote setDefaults(&c) return c } +func loadRemoteConfig(remote string) (c Instance) { + c = remoteInstance(remote).(Instance) + if err := loadConfig(c, false); err != nil { + logError.Fatalf("cannot open remote %q configuration file", remote) + } + return +} + +// Return the base directory for a ComponentType +func remoteRoot(remote string) string { + switch remote { + case LOCAL: + return RunningConfig.ITRSHome + default: + i := loadRemoteConfig(remote) + if err := loadConfig(i, false); err != nil { + logError.Fatalf("cannot open remote %q configuration file", remote) + } + return getString(i, "ITRSHome") + } +} + // // 'geneos add remote NAME SSH-URL' // -func remoteAdd(name string, username string, params []string) (c Instance, err error) { +func remoteAdd(remote string, username string, params []string) (c Instance, err error) { if len(params) == 0 { - log.Fatalln("remote destination must be provided in the form of a URL") + logError.Fatalln("remote destination must be provided in the form of a URL") } - c = remoteInstance(name) + c = remoteInstance(remote) u, err := url.Parse(params[0]) if err != nil { @@ -85,24 +88,329 @@ func remoteAdd(name string, username string, params []string) (c Instance, err e return } - switch { - case u.Scheme == "ssh": - if u.Host == "" { - log.Fatalln("hostname must be provided") + if u.Scheme != "ssh" { + logError.Fatalln("unsupport scheme (only ssh at the moment):", u.Scheme) + } + + if u.Host == "" { + logError.Fatalln("hostname must be provided") + } + setField(c, "Hostname", u.Host) + + if u.Port() != "" { + setField(c, "Port", u.Port()) + } + + if u.User.Username() != "" { + username = u.User.Username() + } + setField(c, "Username", username) + + homepath := RunningConfig.ITRSHome + if u.Path != "" { + homepath = u.Path + } + setField(c, "ITRSHome", homepath) + + err = writeInstanceConfig(c) + if err != nil { + logError.Fatalln(err) + } + + // now check and created file layout + // s := sftpOpenSession(name) + if _, err = statFile(remote, homepath); err == nil { + + dirs, err := readDir(remote, homepath) + if err != nil { + logError.Fatalln(err) + } + // ignore dot files + for _, entry := range dirs { + if !strings.HasPrefix(entry.Name(), ".") { + // directory exists and contains non dot files/dirs - so return + return c, nil + } + } + } else { + // need to create out own, chown base directory only + if err = mkdirAll(remote, homepath, 0775); err != nil { + logError.Fatalln(err) + } + } + + // create dirs + // create directories - initDirs is global, in main.go + for _, d := range initDirs { + dir := filepath.Join(homepath, d) + if err = mkdirAll(remote, dir, 0775); err != nil { + logError.Fatalln(err) + } + } + return +} + +// global to indicate current remote target. default to "local" which is a special case +// var remoteTarget = "local" +const LOCAL = "local" +const ALL = "all" + +// given an instance name, split on an '@' and return left and right parts, using +// "local" as a default +func splitInstanceName(in string) (name, remote string) { + remote = "local" + parts := strings.SplitN(in, "@", 2) + name = parts[0] + if len(parts) > 1 { + remote = parts[1] + } + return +} + +// this is not recursive, +// but we include a special LOCAL instance +func allRemotes() (remotes []Instance) { + remotes = newComponent(Remote, LOCAL) + remotes = append(remotes, instancesOfComponent(LOCAL, Remote)...) + return +} + +// shim methods that test remote and direct to ssh / sftp / os + +func symlink(remote string, oldname, newname string) error { + switch remote { + case LOCAL: + return os.Symlink(oldname, newname) + default: + s := sftpOpenSession(remote) + return s.Symlink(oldname, newname) + } +} + +func readlink(remote, file string) (link string, err error) { + switch remote { + case LOCAL: + return os.Readlink(file) + default: + s := sftpOpenSession(remote) + return s.ReadLink(file) + } +} + +func mkdirAll(remote string, path string, perm os.FileMode) error { + switch remote { + case LOCAL: + return os.MkdirAll(path, perm) + default: + s := sftpOpenSession(remote) + return s.MkdirAll(path) + } +} + +func chown(remote string, name string, uid, gid int) error { + switch remote { + case LOCAL: + return os.Chown(name, uid, gid) + default: + s := sftpOpenSession(remote) + return s.Chown(name, uid, gid) + } +} + +func createRemoteFile(remote string, path string) (*sftp.File, error) { + switch remote { + case LOCAL: + return nil, ErrNotSupported + default: + s := sftpOpenSession(remote) + return s.Create(path) + } +} + +func removeFile(remote string, name string) error { + switch remote { + case LOCAL: + return os.Remove(name) + default: + s := sftpOpenSession(remote) + return s.Remove(name) + } +} + +func removeAll(remote string, name string) (err error) { + switch remote { + case LOCAL: + return os.RemoveAll(name) + default: + s := sftpOpenSession(remote) + + // walk, reverse order by prepending and remove + files := []string{} + w := s.Walk(name) + for w.Step() { + if w.Err() != nil { + continue + } + files = append([]string{w.Path()}, files...) + } + for _, file := range files { + if err = s.Remove(file); err != nil { + log.Println("remove failed", err) + return + } + } + return + } +} + +func renameFile(remote string, oldpath, newpath string) error { + switch remote { + case LOCAL: + return os.Rename(oldpath, newpath) + default: + s := sftpOpenSession(remote) + // use PosixRename to overwrite oldpath + return s.PosixRename(oldpath, newpath) + } +} + +// massaged file stats +type fileStat struct { + st os.FileInfo + uid uint32 + gid uint32 + mtime int64 +} + +// stat() a local or remote file and normalise common values +func statFile(remote string, name string) (s fileStat, err error) { + switch remote { + case LOCAL: + s.st, err = os.Stat(name) + if err != nil { + return + } + s.uid = s.st.Sys().(*syscall.Stat_t).Uid + s.gid = s.st.Sys().(*syscall.Stat_t).Gid + s.mtime = s.st.Sys().(*syscall.Stat_t).Mtim.Sec + default: + sf := sftpOpenSession(remote) + s.st, err = sf.Stat(name) + if err != nil { + return + } + s.uid = s.st.Sys().(*sftp.FileStat).UID + s.gid = s.st.Sys().(*sftp.FileStat).GID + s.mtime = int64(s.st.Sys().(*sftp.FileStat).Mtime) + } + return +} + +func globPath(remote string, pattern string) ([]string, error) { + switch remote { + case LOCAL: + return filepath.Glob(pattern) + default: + s := sftpOpenSession(remote) + return s.Glob(pattern) + } +} + +func writeFile(remote string, name string, b []byte, perm os.FileMode) (err error) { + switch remote { + case LOCAL: + return os.WriteFile(name, b, perm) + default: + s := sftpOpenSession(remote) + var f *sftp.File + f, err = s.Create(name) + if err != nil { + return + } + defer f.Close() + f.Chmod(perm) + _, err = f.Write(b) + return + } +} + +func readFile(remote string, name string) ([]byte, error) { + switch remote { + case LOCAL: + return os.ReadFile(name) + default: + s := sftpOpenSession(remote) + f, err := s.Open(name) + if err != nil { + // logError.Fatalln(err) + return nil, err } - setField(c, "Hostname", u.Host) - if u.Port() != "" { - setField(c, "Port", u.Port()) + defer f.Close() + + st, err := f.Stat() + if err != nil { + // logError.Fatalln(err) + return nil, err } - if u.User.Username() != "" { - setField(c, "Username", u.User.Username()) + // force a block read as /proc doesn't give sizes + sz := st.Size() + if sz == 0 { + sz = 8192 } - if u.Path != "" { - setField(c, "ITRSHome", u.Path) + return io.ReadAll(f) + } +} + +func readDir(remote string, name string) (dirs []os.DirEntry, err error) { + switch remote { + case LOCAL: + return os.ReadDir(name) + default: + s := sftpOpenSession(remote) + f, err := s.ReadDir(name) + if err != nil { + return nil, err } - return c, writeInstanceConfig(c) + for _, d := range f { + dirs = append(dirs, fs.FileInfoToDirEntry(d)) + } + } + return +} + +func openStatFile(remote string, name string) (f io.ReadSeekCloser, st fileStat, err error) { + st, err = statFile(remote, name) + if err != nil { + return + } + switch remote { + case LOCAL: + f, err = os.Open(name) default: - log.Fatalln("unsupport scheme (only ssh at the moment):", u.Scheme) + s := sftpOpenSession(remote) + f, err = s.Open(name) } return } + +func nextRandom() string { + return fmt.Sprint(rand.Uint32()) +} + +// based on os.CreatTemp, but allows for remotes and much simplified +// given a remote and a full path, create a file with a suffix +// and return an io.File +func createRemoteTemp(remote string, path string) (*sftp.File, error) { + try := 0 + for { + name := path + nextRandom() + f, err := createRemoteFile(remote, name) + if os.IsExist(err) { + if try++; try < 100 { + continue + } + return nil, fs.ErrExist + } + return f, err + } +} diff --git a/cmd/geneos/ssh.go b/cmd/geneos/ssh.go index 640ae40..7aa293c 100644 --- a/cmd/geneos/ssh.go +++ b/cmd/geneos/ssh.go @@ -1,11 +1,10 @@ package main import ( - "bytes" - "fmt" "net" "os" "path/filepath" + "time" "github.com/pkg/sftp" "golang.org/x/crypto/ssh" @@ -13,7 +12,7 @@ import ( "golang.org/x/crypto/ssh/knownhosts" ) -var userSSHdir = ".ssh" +const userSSHdir = ".ssh" var privateKeyFiles = []string{ "id_rsa", @@ -24,11 +23,18 @@ var privateKeyFiles = []string{ "id_dsa", } -// ssh utilities for remote connections +var signers []ssh.Signer +var khCallback ssh.HostKeyCallback +var agentClient agent.ExtendedAgent +// cache SSH connections +var remoteSSHClients = make(map[string]*ssh.Client) +var remoteSFTPClients = make(map[string]*sftp.Client) + +// load all the known private keys with no passphrase func readSSHkeys(homedir string) (signers []ssh.Signer) { for _, keyfile := range privateKeyFiles { - path := filepath.Join(homedir, ".ssh", keyfile) + path := filepath.Join(homedir, userSSHdir, keyfile) key, err := os.ReadFile(path) if err != nil { logDebug.Println(err) @@ -45,65 +51,105 @@ func readSSHkeys(homedir string) (signers []ssh.Signer) { return } -func sshTest(username string, host string) { - socket := os.Getenv("SSH_AUTH_SOCK") - sshAgent, err := net.Dial("unix", socket) +// this is not an init() func as we do late initialisation in case we +// don't need ssh +func sshInit() (err error) { + var homedir string + homedir, err = os.UserHomeDir() if err != nil { - log.Fatalf("Failed to open SSH_AUTH_SOCK: %v", err) + logError.Fatalln(err) } - - agentClient := agent.NewClient(sshAgent) - - homedir, err := os.UserHomeDir() - if err != nil { - log.Fatalln(err) + if khCallback == nil { + k := filepath.Join(homedir, userSSHdir, "known_hosts") + khCallback, err = knownhosts.New(k) + if err != nil { + logDebug.Println("cannot load ssh known_hosts file, ssh will not be available.") + return + } } - knownHostsFile := filepath.Join(homedir, ".ssh", "known_hosts") - logDebug.Println(knownHostsFile) - khcallback, err := knownhosts.New(knownHostsFile) - if err != nil { - log.Fatalln(err) + if signers == nil { + signers = readSSHkeys(homedir) + } + if agentClient == nil { + socket := os.Getenv("SSH_AUTH_SOCK") + sshAgent, err := net.Dial("unix", socket) + if err != nil { + log.Printf("Failed to open SSH_AUTH_SOCK: %v", err) + } else { + agentClient = agent.NewClient(sshAgent) + } } - signers := readSSHkeys(homedir) + return +} + +func sshConnect(dest, user string) (client *ssh.Client, err error) { + sshInit() config := &ssh.ClientConfig{ - User: username, + User: user, Auth: []ssh.AuthMethod{ ssh.PublicKeysCallback(agentClient.Signers), ssh.PublicKeys(signers...), }, - HostKeyCallback: khcallback, + HostKeyCallback: khCallback, + Timeout: 5 * time.Second, } - conn, err := ssh.Dial("tcp", host, config) + client, err = ssh.Dial("tcp", dest, config) if err != nil { - log.Fatal("unable to connect: ", err) + logError.Fatalln("unable to connect:", err) } - defer conn.Close() + return +} - session, err := conn.NewSession() - if err != nil { - log.Fatalln(err) +func sshOpenRemote(remote string) (client *ssh.Client, err error) { + client, ok := remoteSSHClients[remote] + if !ok { + i := remoteInstance(remote).(Instance) + if err = loadConfig(i, false); err != nil { + logError.Fatalln(err) + } + dest := getString(i, "Hostname") + ":" + getIntAsString(i, "Port") + user := getString(i, "Username") + client, err = sshConnect(dest, user) + if err != nil { + logError.Fatalln(err) + } + logDebug.Println("remote opened", remote, dest, user) + remoteSSHClients[remote] = client } - defer session.Close() + return +} - var b bytes.Buffer - session.Stdout = &b - if err := session.Run("ls -l"); err != nil { - log.Fatal("Failed to run: " + err.Error()) +func sshCloseRemote(remote string) { + sftpCloseSession(remote) + c, ok := remoteSSHClients[remote] + if ok { + c.Close() + delete(remoteSSHClients, remote) } - fmt.Println(b.String()) +} - client, err := sftp.NewClient(conn) - if err != nil { - log.Fatal(err) +// succeed or fatal +func sftpOpenSession(remote string) (s *sftp.Client) { + s, ok := remoteSFTPClients[remote] + if !ok { + c, err := sshOpenRemote(remote) + if err != nil { + logError.Fatalln(err) + } + s, err = sftp.NewClient(c) + if err != nil { + logError.Fatalln(err) + } + logDebug.Println("remote opened", remote) + remoteSFTPClients[remote] = s } - defer client.Close() + return +} - // walk a directory - w := client.Walk("/home/pi") - for w.Step() { - if w.Err() != nil { - continue - } - log.Println(w.Path()) +func sftpCloseSession(remote string) { + s, ok := remoteSFTPClients[remote] + if ok { + s.Close() + delete(remoteSFTPClients, remote) } } diff --git a/cmd/geneos/tls.go b/cmd/geneos/tls.go index 976e04e..41cab4e 100644 --- a/cmd/geneos/tls.go +++ b/cmd/geneos/tls.go @@ -42,6 +42,10 @@ func init() { geneos tls ls [TYPE] [NAME] list certificates for matcing instances, including the root and intermediate CA certs. same options as for the main 'ls' command + + geneos tls sync + copy the current chain.pem to all known remotes + this is also done by 'init' if remotes are configured at that point `} TLSFlags = flag.NewFlagSet("tls", flag.ExitOnError) @@ -73,7 +77,7 @@ func flagsTLS(command string, args []string) (ret []string) { // pop subcommand, parse args, put subcommand back onto params? func TLSArgs(rawargs []string) (ct ComponentType, args []string, params []string) { if len(rawargs) == 0 { - log.Fatalln("command requires more arguments - help text here") + logError.Fatalln("command requires more arguments - help text here") } subcommand := rawargs[0] ct, args, params = parseArgs(rawargs[1:]) @@ -90,13 +94,15 @@ func commandTLS(ct ComponentType, args []string, params []string) (err error) { switch subcommand { case "init": if err = TLSInit(); err != nil { - log.Fatalln(err) + logError.Fatalln(err) } return loopSubcommand(TLSInstance, "new", ct, args, params) case "import": return TLSImport(args) case "ls": return listCertsCommand(ct, args, params) + case "sync": + return TLSSync() } return loopSubcommand(TLSInstance, subcommand, ct, args, params) @@ -105,6 +111,7 @@ func commandTLS(ct ComponentType, args []string, params []string) (err error) { type lsCertType struct { Type string Name string + Location string Remaining time.Duration Expires time.Time CommonName string @@ -114,14 +121,8 @@ type lsCertType struct { } func listCertsCommand(ct ComponentType, args []string, params []string) (err error) { - rootCert, err := readRootCert() - if err != nil { - return - } - geneosCert, err := readSigningCert() - if err != nil { - return - } + rootCert, _ := readRootCert() + geneosCert, _ := readSigningCert() switch { case TLSlistJSON: @@ -129,32 +130,39 @@ func listCertsCommand(ct ComponentType, args []string, params []string) (err err if TLSlistJSONIndent { jsonEncoder.SetIndent("", " ") } - jsonEncoder.Encode(lsCertType{ - "global", - rootCAFile, - time.Duration(time.Until(rootCert.NotAfter).Seconds()), - rootCert.NotAfter, - rootCert.Subject.CommonName, - rootCert.Issuer.CommonName, - nil, - nil, - }) - jsonEncoder.Encode(lsCertType{ - "global", - signingCertFile, - time.Duration(time.Until(geneosCert.NotAfter).Seconds()), - geneosCert.NotAfter, - geneosCert.Subject.CommonName, - geneosCert.Issuer.CommonName, - nil, - nil, - }) + if rootCert != nil { + jsonEncoder.Encode(lsCertType{ + "global", + rootCAFile, + LOCAL, + time.Duration(time.Until(rootCert.NotAfter).Seconds()), + rootCert.NotAfter, + rootCert.Subject.CommonName, + rootCert.Issuer.CommonName, + nil, + nil, + }) + } + if geneosCert != nil { + jsonEncoder.Encode(lsCertType{ + "global", + signingCertFile, + LOCAL, + time.Duration(time.Until(geneosCert.NotAfter).Seconds()), + geneosCert.NotAfter, + geneosCert.Subject.CommonName, + geneosCert.Issuer.CommonName, + nil, + nil, + }) + } err = loopCommand(lsInstanceCertJSON, ct, args, params) case TLSlistCSV: csvWriter = csv.NewWriter(log.Writer()) csvWriter.Write([]string{ "Type", "Name", + "Location", "Remaining", "Expires", "CommonName", @@ -162,37 +170,47 @@ func listCertsCommand(ct ComponentType, args []string, params []string) (err err "SubjAltNames", "IPs", }) - csvWriter.Write([]string{ - "global", - rootCAFile, - fmt.Sprintf("%0f", time.Until(rootCert.NotAfter).Seconds()), - rootCert.NotAfter.String(), - rootCert.Subject.CommonName, - rootCert.Issuer.CommonName, - "[]", - "[]", - }) - csvWriter.Write([]string{ - "global", - signingCertFile, - fmt.Sprintf("%0f", time.Until(geneosCert.NotAfter).Seconds()), - geneosCert.NotAfter.String(), - geneosCert.Subject.CommonName, - geneosCert.Issuer.CommonName, - "[]", - "[]", - }) + if rootCert != nil { + csvWriter.Write([]string{ + "global", + rootCAFile, + LOCAL, + fmt.Sprintf("%0f", time.Until(rootCert.NotAfter).Seconds()), + rootCert.NotAfter.String(), + rootCert.Subject.CommonName, + rootCert.Issuer.CommonName, + "[]", + "[]", + }) + } + if geneosCert != nil { + csvWriter.Write([]string{ + "global", + signingCertFile, + LOCAL, + fmt.Sprintf("%0f", time.Until(geneosCert.NotAfter).Seconds()), + geneosCert.NotAfter.String(), + geneosCert.Subject.CommonName, + geneosCert.Issuer.CommonName, + "[]", + "[]", + }) + } err = loopCommand(lsInstanceCertCSV, ct, args, params) csvWriter.Flush() default: lsTabWriter = tabwriter.NewWriter(log.Writer(), 3, 8, 2, ' ', 0) - fmt.Fprintf(lsTabWriter, "Type\tName\tRemaining\tExpires\tCommonName\tIssuer\tSubjAltNames\tIPs\n") - fmt.Fprintf(lsTabWriter, "global\t%s\t%.f\t%q\t%q\t%q\t\t\t\n", rootCAFile, - time.Until(rootCert.NotAfter).Seconds(), rootCert.NotAfter, - rootCert.Subject.CommonName, rootCert.Issuer.CommonName) - fmt.Fprintf(lsTabWriter, "global\t%s\t%.f\t%q\t%q\t%q\t\t\t\n", signingCertFile, - time.Until(geneosCert.NotAfter).Seconds(), geneosCert.NotAfter, - geneosCert.Subject.CommonName, geneosCert.Issuer.CommonName) + fmt.Fprintf(lsTabWriter, "Type\tName\tLocation\tRemaining\tExpires\tCommonName\tIssuer\tSubjAltNames\tIPs\n") + if rootCert != nil { + fmt.Fprintf(lsTabWriter, "global\t%s\t%s\t%.f\t%q\t%q\t%q\t\t\t\n", rootCAFile, LOCAL, + time.Until(rootCert.NotAfter).Seconds(), rootCert.NotAfter, + rootCert.Subject.CommonName, rootCert.Issuer.CommonName) + } + if geneosCert != nil { + fmt.Fprintf(lsTabWriter, "global\t%s\t%s\t%.f\t%q\t%q\t%q\t\t\t\n", signingCertFile, LOCAL, + time.Until(geneosCert.NotAfter).Seconds(), geneosCert.NotAfter, + geneosCert.Subject.CommonName, geneosCert.Issuer.CommonName) + } err = loopCommand(lsInstanceCert, ct, args, params) lsTabWriter.Flush() } @@ -200,7 +218,7 @@ func listCertsCommand(ct ComponentType, args []string, params []string) (err err } func TLSInstance(c Instance, subcommand string, params []string) (err error) { - logDebug.Println("TLSInstance:", Type(c), Name(c), subcommand, params) + logDebug.Println("TLSInstance:", Type(c), Name(c), Location(c), subcommand, params) switch subcommand { case "new": // create a cert, DO NOT overwrite any existing unless renewing @@ -218,7 +236,7 @@ func lsInstanceCert(c Instance, params []string) (err error) { return } expires := cert.NotAfter - fmt.Fprintf(lsTabWriter, "%s\t%s\t%.f\t%q\t%q\t%q\t", Type(c), Name(c), time.Until(expires).Seconds(), expires, cert.Subject.CommonName, cert.Issuer.CommonName) + fmt.Fprintf(lsTabWriter, "%s\t%s\t%s\t%.f\t%q\t%q\t%q\t", Type(c), Name(c), Location(c), time.Until(expires).Seconds(), expires, cert.Subject.CommonName, cert.Issuer.CommonName) if len(cert.DNSNames) > 0 { fmt.Fprintf(lsTabWriter, "%v", cert.DNSNames) } @@ -237,7 +255,7 @@ func lsInstanceCertCSV(c Instance, params []string) (err error) { } expires := cert.NotAfter until := fmt.Sprintf("%0f", time.Until(expires).Seconds()) - cols := []string{Type(c).String(), Name(c), until, expires.String(), cert.Subject.CommonName, cert.Issuer.CommonName} + cols := []string{Type(c).String(), Name(c), Location(c), until, expires.String(), cert.Subject.CommonName, cert.Issuer.CommonName} cols = append(cols, fmt.Sprintf("%v", cert.DNSNames)) cols = append(cols, fmt.Sprintf("%v", cert.IPAddresses)) @@ -250,7 +268,7 @@ func lsInstanceCertJSON(c Instance, params []string) (err error) { if err != nil { return } - jsonEncoder.Encode(lsCertType{Type(c).String(), Name(c), time.Duration(time.Until(cert.NotAfter).Seconds()), + jsonEncoder.Encode(lsCertType{Type(c).String(), Name(c), Location(c), time.Duration(time.Until(cert.NotAfter).Seconds()), cert.NotAfter, cert.Subject.CommonName, cert.Issuer.CommonName, cert.DNSNames, cert.IPAddresses}) return } @@ -261,27 +279,53 @@ func lsInstanceCertJSON(c Instance, params []string) (err error) { func TLSInit() (err error) { tlsPath := filepath.Join(RunningConfig.ITRSHome, "tls") // directory permissions do not need to be restrictive - err = os.MkdirAll(tlsPath, 0777) + err = mkdirAll(LOCAL, tlsPath, 0777) if err != nil { - log.Fatalln(err) + logError.Fatalln(err) } rootCert, err := newRootCA(tlsPath) if err != nil { - log.Fatalln(err) + logError.Fatalln(err) } interCert, err := newIntrCA(tlsPath) if err != nil { - log.Fatalln(err) + logError.Fatalln(err) } // concatenate a chain - if err = writeCerts(filepath.Join(tlsPath, "chain.pem"), rootCert, interCert); err != nil { - log.Fatalln(err) + if err = writeCerts(LOCAL, filepath.Join(tlsPath, "chain.pem"), rootCert, interCert); err != nil { + logError.Fatalln(err) } log.Println("created chain.pem") + return TLSSync() +} + +// if there is a local tls/chain.pem file then copy it to all remotes +// overwriting any existing versions +func TLSSync() (err error) { + rootCert, _ := readRootCert() + geneosCert, _ := readSigningCert() + + if rootCert == nil && geneosCert == nil { + return + } + + for _, remote := range allRemotes() { + if Name(remote) == LOCAL { + continue + } + tlsPath := filepath.Join(remoteRoot(Name(remote)), "tls") + if err = mkdirAll(Name(remote), tlsPath, 0775); err != nil { + logError.Fatalln(err) + } + if err = writeCerts(Name(remote), filepath.Join(tlsPath, "chain.pem"), rootCert, geneosCert); err != nil { + logError.Fatalln(err) + } + log.Println("Updated chain.pem on", Name(remote)) + } return } @@ -293,7 +337,7 @@ func TLSImport(files []string) (err error) { for _, source := range files { f, err := readSource(source) if err != nil { - log.Fatalln(err) + logError.Fatalln(err) } for { block, rest := pem.Decode(f) @@ -304,17 +348,17 @@ func TLSImport(files []string) (err error) { case "CERTIFICATE": cert, err := x509.ParseCertificate(block.Bytes) if err != nil { - log.Fatalln(err) + logError.Fatalln(err) } - writeCert(filepath.Join(tlsPath, signingCertFile+".pem"), cert) + writeCert(LOCAL, filepath.Join(tlsPath, signingCertFile+".pem"), cert) case "RSA PRIVATE KEY": key, err := x509.ParsePKCS1PrivateKey(block.Bytes) if err != nil { - log.Fatalln(err) + logError.Fatalln(err) } - writeKey(filepath.Join(tlsPath, signingCertFile+".key"), key) + writeKey(LOCAL, filepath.Join(tlsPath, signingCertFile+".key"), key) default: - log.Fatalln("unknown PEM type:", block.Type) + logError.Fatalln("unknown PEM type:", block.Type) } f = rest @@ -328,7 +372,7 @@ func newRootCA(dir string) (cert *x509.Certificate, err error) { rootCertPath := filepath.Join(dir, rootCAFile+".pem") rootKeyPath := filepath.Join(dir, rootCAFile+".key") - if cert, err = readCert(rootCertPath); err == nil { + if cert, err = readRootCert(); err == nil { log.Println(rootCAFile, "already exists") return } @@ -347,16 +391,16 @@ func newRootCA(dir string) (cert *x509.Certificate, err error) { cert, key, err := createCert(&template, &template, nil, nil) if err != nil { - log.Fatalln(err) + logError.Fatalln(err) } - err = writeCert(rootCertPath, cert) + err = writeCert(LOCAL, rootCertPath, cert) if err != nil { - log.Fatalln(err) + logError.Fatalln(err) } - err = writeKey(rootKeyPath, key) + err = writeKey(LOCAL, rootKeyPath, key) if err != nil { - log.Fatalln(err) + logError.Fatalln(err) } log.Println("CA certificate created for", rootCAFile) @@ -367,7 +411,7 @@ func newIntrCA(dir string) (cert *x509.Certificate, err error) { intrCertPath := filepath.Join(dir, signingCertFile+".pem") intrKeyPath := filepath.Join(dir, signingCertFile+".key") - if cert, err = readCert(intrCertPath); err == nil { + if cert, err = readSigningCert(); err == nil { log.Println(signingCertFile, "already exists") return } @@ -385,27 +429,27 @@ func newIntrCA(dir string) (cert *x509.Certificate, err error) { MaxPathLen: 1, } - rootCert, err := readCert(filepath.Join(dir, rootCAFile+".pem")) + rootCert, err := readRootCert() if err != nil { - log.Fatalln(err) + logError.Fatalln(err) } - rootKey, err := readKey(filepath.Join(dir, rootCAFile+".key")) + rootKey, err := readKey(LOCAL, filepath.Join(dir, rootCAFile+".key")) if err != nil { - log.Fatalln(err) + logError.Fatalln(err) } cert, key, err := createCert(&template, rootCert, rootKey, nil) if err != nil { - log.Fatalln(err) + logError.Fatalln(err) } - err = writeCert(intrCertPath, cert) + err = writeCert(LOCAL, intrCertPath, cert) if err != nil { - log.Fatalln(err) + logError.Fatalln(err) } - err = writeKey(intrKeyPath, key) + err = writeKey(LOCAL, intrKeyPath, key) if err != nil { - log.Fatalln(err) + logError.Fatalln(err) } log.Println("CA certificate created for", signingCertFile) @@ -413,143 +457,161 @@ func newIntrCA(dir string) (cert *x509.Certificate, err error) { return } -// renew an instance certificate, leave the private key untouched +// create a new certificate for an instance // -// if private key doesn't exist, do we error? -func renewInstanceCert(c Instance) (err error) { +// this also creates a new private key +// +// skip if certificate exists (no expiry check) +func createInstanceCert(c Instance) (err error) { tlsDir := filepath.Join(RunningConfig.ITRSHome, "tls") + // skip if we can load an existing certificate + if _, err = readInstanceCert(c); err == nil { + return + } + host, _ := os.Hostname() + if Location(c) != LOCAL { + r := loadRemoteConfig(Location(c)) + host = getString(r, "Hostname") + } + serial, err := rand.Prime(rand.Reader, 64) if err != nil { - log.Fatalln(err) + logError.Fatalln(err) } + expires := time.Now().AddDate(1, 0, 0) template := x509.Certificate{ SerialNumber: serial, Subject: pkix.Name{ CommonName: fmt.Sprintf("geneos %s %s", Type(c), Name(c)), }, NotBefore: time.Now().Add(-60 * time.Second), - NotAfter: time.Now().AddDate(1, 0, 0), + NotAfter: expires, KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, MaxPathLenZero: true, - DNSNames: []string{"localhost", host}, - IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + DNSNames: []string{host}, + // IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, } - intrCert, err := readCert(filepath.Join(tlsDir, signingCertFile+".pem")) + intrCert, err := readSigningCert() if err != nil { - log.Fatalln(err) + logError.Fatalln(err) } - intrKey, err := readKey(filepath.Join(tlsDir, signingCertFile+".key")) + intrKey, err := readKey(LOCAL, filepath.Join(tlsDir, signingCertFile+".key")) if err != nil { - log.Fatalln(err) + logError.Fatalln(err) } - existingKey, _ := readInstanceKey(c) - cert, key, err := createCert(&template, intrCert, intrKey, existingKey) + cert, key, err := createCert(&template, intrCert, intrKey, nil) if err != nil { - log.Fatalln(err) + logError.Fatalln(err) } err = writeInstanceCert(c, cert) if err != nil { - log.Fatalln(err) + logError.Fatalln(err) } - if existingKey == nil { - err = writeInstanceKey(c, key) - if err != nil { - log.Fatalln(err) - } + err = writeInstanceKey(c, key) + if err != nil { + logError.Fatalln(err) } - log.Println("certificate renewed for", Type(c), Name(c)) + log.Printf("certificate created for %s %s@%s (expires %s)", Type(c), Name(c), Location(c), expires) + return } -// create a new certificate for an instance -// -// this also creates a new private key +// renew an instance certificate, leave the private key untouched // -// skip if certificate exists (no expiry check) -func createInstanceCert(c Instance) (err error) { +// if private key doesn't exist, do we error? +func renewInstanceCert(c Instance) (err error) { tlsDir := filepath.Join(RunningConfig.ITRSHome, "tls") - // skip if we can load an existing certificate - if _, err = readInstanceCert(c); err == nil { - return + host, _ := os.Hostname() + if Location(c) != LOCAL { + r := loadRemoteConfig(Location(c)) + host = getString(r, "Hostname") } - host, _ := os.Hostname() serial, err := rand.Prime(rand.Reader, 64) if err != nil { - log.Fatalln(err) + logError.Fatalln(err) } + expires := time.Now().AddDate(1, 0, 0) template := x509.Certificate{ SerialNumber: serial, Subject: pkix.Name{ CommonName: fmt.Sprintf("geneos %s %s", Type(c), Name(c)), }, NotBefore: time.Now().Add(-60 * time.Second), - NotAfter: time.Now().AddDate(1, 0, 0), + NotAfter: expires, KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, MaxPathLenZero: true, - DNSNames: []string{"localhost", host}, - IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + DNSNames: []string{host}, + // IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, } - intrCert, err := readCert(filepath.Join(tlsDir, signingCertFile+".pem")) + intrCert, err := readCert(LOCAL, filepath.Join(tlsDir, signingCertFile+".pem")) if err != nil { - log.Fatalln(err) + logError.Fatalln(err) } - intrKey, err := readKey(filepath.Join(tlsDir, signingCertFile+".key")) + intrKey, err := readKey(LOCAL, filepath.Join(tlsDir, signingCertFile+".key")) if err != nil { - log.Fatalln(err) + logError.Fatalln(err) } - cert, key, err := createCert(&template, intrCert, intrKey, nil) + existingKey, _ := readInstanceKey(c) + cert, key, err := createCert(&template, intrCert, intrKey, existingKey) if err != nil { - log.Fatalln(err) + logError.Fatalln(err) } err = writeInstanceCert(c, cert) if err != nil { - log.Fatalln(err) + logError.Fatalln(err) } - err = writeInstanceKey(c, key) - if err != nil { - log.Fatalln(err) + if existingKey == nil { + err = writeInstanceKey(c, key) + if err != nil { + logError.Fatalln(err) + } } - log.Println("certificate created for", Type(c), Name(c)) + log.Printf("certificate renewed for %s %s@%s (expires %s)", Type(c), Name(c), Location(c), expires) + return } func writeInstanceCert(c Instance, cert *x509.Certificate) (err error) { if c == nil || Type(c) == None { - log.Fatalln(err) + logError.Fatalln(err) } certfile := Type(c).String() + ".pem" - if err = writeCert(filepath.Join(Home(c), certfile), cert); err != nil { + if err = writeCert(Location(c), filepath.Join(Home(c), certfile), cert); err != nil { return } if err = setField(c, Prefix(c)+"Cert", certfile); err != nil { return } - return writeInstanceConfig(c) + + if err = writeInstanceConfig(c); err != nil { + log.Fatalln("here:", err) + } + return } func writeInstanceKey(c Instance, key *rsa.PrivateKey) (err error) { if Type(c) == None { - log.Fatalln(err) + logError.Fatalln(err) } keyfile := Type(c).String() + ".key" - if err = writeKey(filepath.Join(Home(c), keyfile), key); err != nil { + if err = writeKey(Location(c), filepath.Join(Home(c), keyfile), key); err != nil { return } if err = setField(c, Prefix(c)+"Key", keyfile); err != nil { @@ -558,34 +620,36 @@ func writeInstanceKey(c Instance, key *rsa.PrivateKey) (err error) { return writeInstanceConfig(c) } -func writeKey(path string, key *rsa.PrivateKey) (err error) { +func writeKey(remote, path string, key *rsa.PrivateKey) (err error) { logDebug.Println("write key to", path) keyPEM := pem.EncodeToMemory(&pem.Block{ Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key), }) - err = os.WriteFile(path, keyPEM, 0640) + err = writeFile(remote, path, keyPEM, 0640) if err != nil { - log.Fatalln(err) + logError.Fatalln(err) } return } -func writeCert(path string, cert *x509.Certificate) (err error) { +func writeCert(remote, path string, cert *x509.Certificate) (err error) { logDebug.Println("write cert to", path) certPEM := pem.EncodeToMemory(&pem.Block{ Type: "CERTIFICATE", Bytes: cert.Raw, }) - err = os.WriteFile(path, certPEM, 0644) + + err = writeFile(remote, path, certPEM, 0644) + if err != nil { - log.Fatalln(err) + logError.Fatalln(err) } return } -func writeCerts(path string, certs ...*x509.Certificate) (err error) { +func writeCerts(remote string, path string, certs ...*x509.Certificate) (err error) { logDebug.Println("write certs to", path) var certsPEM []byte for _, cert := range certs { @@ -595,15 +659,15 @@ func writeCerts(path string, certs ...*x509.Certificate) (err error) { }) certsPEM = append(certsPEM, p...) } - err = os.WriteFile(path, certsPEM, 0644) + err = writeFile(remote, path, certsPEM, 0644) if err != nil { - log.Fatalln(err) + logError.Fatalln(err) } return } -func readCert(path string) (cert *x509.Certificate, err error) { - certPEM, err := os.ReadFile(path) +func readCert(remote string, path string) (cert *x509.Certificate, err error) { + certPEM, err := readFile(remote, path) if err != nil { return } @@ -618,24 +682,24 @@ func readCert(path string) (cert *x509.Certificate, err error) { func readRootCert() (cert *x509.Certificate, err error) { tlsDir := filepath.Join(RunningConfig.ITRSHome, "tls") - return readCert(filepath.Join(tlsDir, rootCAFile+".pem")) + return readCert(LOCAL, filepath.Join(tlsDir, rootCAFile+".pem")) } func readSigningCert() (cert *x509.Certificate, err error) { tlsDir := filepath.Join(RunningConfig.ITRSHome, "tls") - return readCert(filepath.Join(tlsDir, signingCertFile+".pem")) + return readCert(LOCAL, filepath.Join(tlsDir, signingCertFile+".pem")) } func readInstanceCert(c Instance) (cert *x509.Certificate, err error) { if Type(c) == None { - log.Fatalln(err) + logError.Fatalln(err) } - return readCert(filepathForInstance(c, getString(c, Prefix(c)+"Cert"))) + return readCert(Location(c), filepathForInstance(c, getString(c, Prefix(c)+"Cert"))) } -func readKey(path string) (key *rsa.PrivateKey, err error) { - keyPEM, err := os.ReadFile(path) +func readKey(remote string, path string) (key *rsa.PrivateKey, err error) { + keyPEM, err := readFile(remote, path) if err != nil { return } @@ -650,10 +714,10 @@ func readKey(path string) (key *rsa.PrivateKey, err error) { func readInstanceKey(c Instance) (key *rsa.PrivateKey, err error) { if Type(c) == None { - log.Fatalln(err) + logError.Fatalln(err) } - return readKey(filepathForInstance(c, getString(c, Prefix(c)+"Key"))) + return readKey(Location(c), filepathForInstance(c, getString(c, Prefix(c)+"Key"))) } func createCert(template, parent *x509.Certificate, parentKey *rsa.PrivateKey, existingKey *rsa.PrivateKey) (cert *x509.Certificate, key *rsa.PrivateKey, err error) { @@ -662,7 +726,7 @@ func createCert(template, parent *x509.Certificate, parentKey *rsa.PrivateKey, e } else { key, err = rsa.GenerateKey(rand.Reader, 4096) if err != nil { - log.Fatalln(err) + logError.Fatalln(err) } } @@ -673,12 +737,12 @@ func createCert(template, parent *x509.Certificate, parentKey *rsa.PrivateKey, e certBytes, err := x509.CreateCertificate(rand.Reader, template, parent, &key.PublicKey, privKey) if err != nil { - log.Fatalln(err) + logError.Fatalln(err) } cert, err = x509.ParseCertificate(certBytes) if err != nil { - log.Fatalln(err) + logError.Fatalln(err) } return diff --git a/cmd/geneos/unpack.go b/cmd/geneos/unpack.go deleted file mode 100644 index 2038e83..0000000 --- a/cmd/geneos/unpack.go +++ /dev/null @@ -1,453 +0,0 @@ -package main - -import ( - "archive/tar" - "bytes" - "compress/gzip" - "encoding/json" - "errors" - "fmt" - "io" - "io/fs" - "net/http" - "net/url" - "os" - "path/filepath" - "regexp" - "strings" -) - -func init() { - commands["extract"] = Command{ - Function: commandExtract, - ParseFlags: defaultFlag, - ParseArgs: checkComponentArg, - CommandLine: "geneos extract FILE [FILE...]", - Summary: `Extract files from downloaded Geneos packages. Intended for sites without Internet access.`, - Description: `Extracts files from FILE(s) in to the packages/ directory. The filename(s) must of of the form: - - geneos-TYPE-VERSION*.tar.gz - -The directory for the package is created using the VERSION from the archive filename.`} - - commands["download"] = Command{ - Function: commandDownload, - ParseFlags: defaultFlag, - ParseArgs: checkComponentArg, - CommandLine: "geneos download [TYPE] [latest|FILTER|URL...]", - Summary: `Download and extract Geneos software archive.`, - Description: `Download and extract the sources in the packages directory or latest version(s) from -the official download site. The filename must of of the format: - - geneos-TYPE-VERSION*.tar.gz - -The TYPE, if supplied, limits the selection of downloaded archive(s). The directory -for the package is created using the VERSION from the archive filename.`} - - commands["update"] = Command{ - Function: commandUpdate, - ParseFlags: defaultFlag, - ParseArgs: checkComponentArg, - CommandLine: "geneos update [TYPE] VERSION", - Summary: `Update the active version of Geneos software.`, - Description: `Update the symlink for the default base name of the package used to VERSION. The base directory, -for historical reasons, is 'active_prod' and is usally linked to the latest version of a component type -in the packages directory. VERSION can either be a directory name or the literal 'latest'. If TYPE is not -supplied, all supported component types are updated to VERSION. - -Update will stop all matching instances of the each type before updating the link and starting them up -again, but only if the instance uses the 'active_prod' basename. - -The 'latest' version is based on directory names of the form: - - [GA]X.Y.Z - -Where X, Y, Z are each ordered in ascending numerical order. If a directory starts 'GA' it will be selected -over a directory with the same numerical versions. All other directories name formats will result in unexpected -behaviour. - -Future version may support selecting a base other than 'active_prod'.`} - - // need overwrite flags -} - -// -// if there is no 'active_prod' link then attach it to the latest version -// installed -// -func commandExtract(ct ComponentType, files []string, params []string) (err error) { - if ct != None { - logError.Fatalln("Must not specify a component type, only archive files") - } - - for _, archive := range files { - filename := filepath.Base(archive) - gz, err := os.Open(archive) - if err != nil { - return err - } - defer gz.Close() - - if _, err = unarchive(ct, filename, gz); err != nil { - log.Println(err) - return err - } - } - // create a symlink only if one doesn't exist - return updateToVersion(ct, "latest", false) -} - -func commandDownload(ct ComponentType, files []string, params []string) (err error) { - version := "latest" - if len(files) > 0 { - version = files[0] - } - return downloadComponent(ct, version) -} - -func downloadComponent(ct ComponentType, version string) (err error) { - switch ct { - case None: - for _, t := range componentTypes() { - if err = downloadComponent(t, version); err != nil { - if errors.Is(err, fs.ErrExist) { - log.Println(err) - continue - } - return - } - } - return nil - default: - filename, gz, err := downloadArchive(ct, version) - if err != nil { - return err - } - defer gz.Close() - - log.Println("downloaded", ct.String(), filename) - - var finalVersion string - if finalVersion, err = unarchive(ct, filename, gz); err != nil { - return err - } - if version == "latest" { - return updateToVersion(ct, finalVersion, true) - } - return updateToVersion(ct, finalVersion, false) - } -} - -var archiveRE = regexp.MustCompile(`^geneos-(web-server|\w+)-([\w\.-]+?)[\.-]?linux`) - -func unarchive(ct ComponentType, filename string, gz io.Reader) (finalVersion string, err error) { - parts := archiveRE.FindStringSubmatch(filename) - if len(parts) == 0 { - logError.Fatalf("invalid archive name format: %q", filename) - } - version := parts[2] - filect := parseComponentName(parts[1]) - switch ct { - case None: - ct = filect - case filect: - break - default: - // mismatch - logError.Fatalf("component type and archive mismatch: %q is not a %q", filename, ct) - } - basedir := filepath.Join(RunningConfig.ITRSHome, "packages", ct.String(), version) - if _, err = os.Stat(basedir); err == nil { - return "", fmt.Errorf("%s: %w", basedir, fs.ErrExist) - } - if err = os.MkdirAll(basedir, 0775); err != nil { - return - } - - t, err := gzip.NewReader(gz) - if err != nil { - return - } - defer t.Close() - - tr := tar.NewReader(t) - for { - var hdr *tar.Header - hdr, err = tr.Next() - if err == io.EOF { - err = nil - break - } - if err != nil { - return - } - // log.Println("file:", hdr.Name, "size", hdr.Size) - // strip leading component name (XXX - except webserver) - // do not trust tar archives to contain safe paths - var name string - switch ct { - case Webserver: - name = hdr.Name - default: - name = strings.TrimPrefix(hdr.Name, ct.String()+"/") - } - if name == "" { - continue - } - name, err = cleanRelativePath(name) - if err != nil { - logError.Fatalln(err) - } - fullpath := filepath.Join(basedir, name) - switch hdr.Typeflag { - case tar.TypeReg: - // check (and created) containing directories - account for munged tar files - dir := filepath.Dir(fullpath) - if err = os.MkdirAll(dir, 0777); err != nil { - return - } - - out, err := os.OpenFile(fullpath, os.O_CREATE|os.O_WRONLY, hdr.FileInfo().Mode()) - if err != nil { - return "", err - } - n, err := io.Copy(out, tr) - if err != nil { - out.Close() - return "", err - } - if n != hdr.Size { - log.Println("lengths different:", hdr.Size, n) - } - out.Close() - case tar.TypeDir: - if err = os.MkdirAll(fullpath, hdr.FileInfo().Mode()); err != nil { - return - } - case tar.TypeSymlink, tar.TypeGNULongLink: - if filepath.IsAbs(hdr.Linkname) { - logError.Fatalln("archive contains absolute symlink target") - } - if _, err = os.Stat(fullpath); err != nil { - if err = os.Symlink(hdr.Linkname, fullpath); err != nil { - logError.Fatalln(err) - } - } - default: - log.Printf("unsupported file type %c\n", hdr.Typeflag) - } - } - log.Printf("extracted %q to %q\n", filename, basedir) - finalVersion = filepath.Base(basedir) - return -} - -func commandUpdate(ct ComponentType, args []string, params []string) (err error) { - version := "latest" - if len(args) > 0 { - version = args[0] - } - return updateToVersion(ct, version, true) -} - -// check selected version exists first -func updateToVersion(ct ComponentType, version string, overwrite bool) (err error) { - base := "active_prod" - basedir := filepath.Join(RunningConfig.ITRSHome, "packages", ct.String()) - basepath := filepath.Join(basedir, base) - - logDebug.Printf("checking and updating %q to %q", base, version) - - switch ct { - case None: - for _, t := range componentTypes() { - if err = updateToVersion(t, version, overwrite); err != nil { - log.Println(err) - } - } - case Gateway, Netprobe, Licd, Webserver: - if version == "" || version == "latest" { - version = latestDir(basedir) - } - // does the version directory exist? - if _, err = os.Stat(filepath.Join(basedir, version)); err != nil { - err = fmt.Errorf("update %s to version %s failed: %w", ct, version, err) - return err - } - current, err := os.Readlink(basepath) - if err != nil && errors.Is(err, &fs.PathError{}) { - log.Println("cannot read link", basepath) - } - if current != "" && !overwrite { - return nil - } - // empty current is fine - if current == version { - logDebug.Println(ct, base, "is already linked to", version) - return nil - } - insts := matchComponents(ct, "Base", base) - // stop matching instances - for _, i := range insts { - stopInstance(i, nil) - defer startInstance(i, nil) - } - if err = os.Remove(basepath); err != nil && !errors.Is(err, fs.ErrNotExist) { - return err - } - if err = os.Symlink(version, basepath); err != nil { - return err - } - log.Println(ct, base, "updated to", version) - default: - return ErrNotSupported - } - return nil -} - -var versRE = regexp.MustCompile(`^\d+(\.\d+){0,2}`) - -// given a directory find the "latest" version of the form -// [GA]M.N.P[-DATE] M, N, P are numbers, DATE is treated as a string -func latestDir(dir string) (latest string) { - dirs, err := os.ReadDir(dir) - if err != nil { - logError.Fatalln(err) - } - max := make([]int, 3) - for _, v := range dirs { - if !v.IsDir() { - continue - } - // strip 'GA' prefix and get name - d := strings.TrimPrefix(v.Name(), "GA") - if !versRE.MatchString(d) { - logDebug.Println(d, "does not match a valid directory pattern") - continue - } - s := strings.SplitN(d, ".", 3) - next := slicetoi(s) - - OUTER: - for i := range max { - switch { - case next[i] < max[i]: - break OUTER - case next[i] > max[i]: - // do a final lexical scan for suffixes? - latest = v.Name() - max[i] = next[i] - default: - // if equal and we are on last number, lexical comparison - // to pick up suffixes - if len(max) == i+1 && v.Name() > latest { - latest = v.Name() - } - } - } - } - return -} - -// given a component type and a key/value pair, return matching -// instances -func matchComponents(ct ComponentType, k, v string) (insts []Instance) { - for _, i := range instances(ct) { - if v == getString(i, Prefix(i)+k) { - if err := loadConfig(&i, false); err != nil { - log.Println(Type(i), Name(i), "cannot load config") - } - insts = append(insts, i) - } - } - return -} - -// fetch a (the latest) component from a URL, but the URLs -// are special and the resultant redirection contains the filename -// etc. -// -// URL is -// https://resources.itrsgroup.com/download/latest/[COMPONENT]?os=linux -// is RHEL8 is required, add ?title=el8 -// -// there is a mapping of our compoent types to the URLs too. -// -// Gateways -> Gateway+2 -// Netprobes -> Netprobe -// Licds -> Licence+Daemon -// Webservers -> Web+Dashboard -// -// auth requires a POST with a JSON body of -// { "username": "EMAIL", "password": "PASSWORD" } -// until anon access is allowed -// - -const defaultURL = "https://resources.itrsgroup.com/download/latest/" - -type DownloadAuth struct { - Username string `json:"username"` - Password string `json:"password"` -} - -var downloadMap = map[ComponentType]string{ - Gateway: "Gateway+2", - Netprobe: "Netprobe", - Licd: "Licence+Daemon", - Webserver: "Web+Dashboard", -} - -// XXX use HEAD to check match and compare to on disk versions -func downloadArchive(ct ComponentType, version string) (filename string, body io.ReadCloser, err error) { - baseurl := RunningConfig.DownloadURL - if baseurl == "" { - baseurl = defaultURL - } - - var resp *http.Response - - downloadURL, _ := url.Parse(baseurl) - realpath, _ := url.Parse(downloadMap[ct]) - v := url.Values{} - v.Set("os", "linux") - if version != "latest" { - v.Set("title", version) - } - realpath.RawQuery = v.Encode() - source := downloadURL.ResolveReference(realpath).String() - logDebug.Println("source url:", source) - - // if a download user is set then issue a POST with username and password - // in a JSON body, else just try the GET - if RunningConfig.DownloadUser != "" { - var authbody DownloadAuth - authbody.Username = RunningConfig.DownloadUser - authbody.Password = RunningConfig.DownloadPass - - var authjson []byte - authjson, err = json.Marshal(authbody) - if err != nil { - logError.Fatalln(err) - } - - resp, err = http.Post(source, "application/json", bytes.NewBuffer(authjson)) - } else { - resp, err = http.Get(source) - } - if err != nil { - logError.Fatalln(err) - } - if resp.StatusCode > 299 { - err = fmt.Errorf("cannot download %s package version %s: %s", ct, version, resp.Status) - resp.Body.Close() - return - // logError.Fatalln(resp.Status) - } - - filename, err = filenameFromHTTPResp(resp, downloadURL) - if err != nil { - logError.Fatalln(err) - } - body = resp.Body - return -} diff --git a/cmd/geneos/utils.go b/cmd/geneos/utils.go index 18e8f5a..1602b97 100644 --- a/cmd/geneos/utils.go +++ b/cmd/geneos/utils.go @@ -25,38 +25,36 @@ import ( "text/template" ) -// locate a process by compoent type and name -// -// the component type must be part of the basename of the executable and -// the component name must be on the command line as an exact and -// standalone args -// -func findInstanceProc(c Instance) (pid int, st *syscall.Stat_t, err error) { +// walk the /proc directory (local or remote) and find the matching pid +// this is subject to races, but... +func findInstancePID(c Instance) (pid int, err error) { var pids []int // safe to ignore error as it can only be bad pattern, // which means no matches to range over - dirs, _ := filepath.Glob("/proc/[0-9]*") + dirs, _ := globPath(Location(c), "/proc/[0-9]*") for _, dir := range dirs { - pid, _ := strconv.Atoi(filepath.Base(dir)) - pids = append(pids, pid) + p, _ := strconv.Atoi(filepath.Base(dir)) + pids = append(pids, p) } sort.Ints(pids) for _, pid = range pids { var data []byte - data, err = os.ReadFile(fmt.Sprintf("/proc/%d/cmdline", pid)) + data, err = readFile(Location(c), fmt.Sprintf("/proc/%d/cmdline", pid)) if err != nil { + // process may disappear by this point, ignore error + logDebug.Println(err) continue } args := bytes.Split(data, []byte("\000")) - bin := filepath.Base(string(args[0])) + execfile := filepath.Base(string(args[0])) switch Type(c) { case Webserver: var wdOK, jarOK bool - if bin != "java" { + if execfile != "java" { continue } for _, arg := range args[1:] { @@ -67,26 +65,38 @@ func findInstanceProc(c Instance) (pid int, st *syscall.Stat_t, err error) { jarOK = true } if wdOK && jarOK { - if s, err := os.Stat(fmt.Sprintf("/proc/%d", pid)); err == nil { - st = s.Sys().(*syscall.Stat_t) - } return } } default: - if strings.HasPrefix(bin, Type(c).String()) { + if strings.HasPrefix(execfile, Type(c).String()) { for _, arg := range args[1:] { + // very simplistic - we look for a bare arg that matches the instance name if string(arg) == Name(c) { - if s, err := os.Stat(fmt.Sprintf("/proc/%d", pid)); err == nil { - st = s.Sys().(*syscall.Stat_t) - } + // found return } } } } } - return 0, nil, ErrProcNotExist + return 0, ErrProcNotExist +} + +// locate a process by compoent type and name +// +// the component type must be part of the basename of the executable and +// the component name must be on the command line as an exact and +// standalone args +// +func findInstanceProc(c Instance) (pid int, uid uint32, gid uint32, mtime int64, err error) { + pid, err = findInstancePID(c) + if err == nil { + var s fileStat + s, err = statFile(Location(c), fmt.Sprintf("/proc/%d", pid)) + return pid, s.uid, s.uid, s.mtime, err + } + return 0, 0, 0, 0, ErrProcNotExist } func getUser(username string) (uid, gid uint32, gids []uint32, err error) { @@ -236,9 +246,11 @@ func parseArgs(rawargs []string) (ct ComponentType, args []string, params []stri // empty list of names = all names for that ct if len(args) == 0 { - args = emptyArgs(ct) + args = allArgsForComponent(ct) } + logDebug.Println("ct, args, params", ct, args, params) + // make sure names/args are unique but retain order // check for reserved names here? // do space exchange inbound here @@ -266,24 +278,68 @@ func parseArgs(rawargs []string) (ct ComponentType, args []string, params []stri // repeat if args is now empty (all params) if len(args) == 0 { - args = emptyArgs(ct) + args = allArgsForComponent(ct) } logDebug.Println("params:", params) return } -func emptyArgs(ct ComponentType) (args []string) { +// for commands (like 'add') that don't want to know about existing matches +func parseArgsNoWildcard(rawargs []string) (ct ComponentType, args []string, params []string) { + var newnames []string + + if len(rawargs) == 0 { + return + } + if ct = parseComponentName(rawargs[0]); ct == Unknown { + return + } + args = rawargs[1:] + + logDebug.Println("ct, args, params", ct, args, params) + + m := make(map[string]bool, len(args)) + for _, name := range args { + // filter name here + if reservedName(args[0]) { + logError.Fatalf("%q is reserved instance name", args[0]) + } + if !validInstanceName(args[0]) { + logDebug.Printf("%q is not a valid instance name", args[0]) + break + } + if m[name] { + continue + } + newnames = append(newnames, name) + args = args[1:] + m[name] = true + } + params = args + args = newnames + + logDebug.Println("params:", params) + return +} + +func allArgsForComponent(ct ComponentType) (args []string) { var confs []Instance switch ct { case None, Unknown: // wildcard again - sort oder matters, fix confs = allInstances() + case Remote: + confs = append(confs, instancesOfComponent(LOCAL, ct)...) default: - confs = instances(ct) + for _, remote := range allRemotes() { + logDebug.Println("checking remote:", Name(remote)) + confs = append(confs, instancesOfComponent(Name(remote), ct)...) + } } for _, c := range confs { - args = append(args, Name(c)) + // XXX + args = append(args, Name(c)+"@"+Location(c)) } return } @@ -309,16 +365,17 @@ func reservedName(in string) (ok bool) { } // spaces are valid - dumb, but valid - for now -var validStringRE = regexp.MustCompile(`^\w[\w -]*$`) +var validStringRE = regexp.MustCompile(`^\w[@\w -]*$`) // return true while a string is considered a valid instance name // // used to consume instance names until parameters are then passed down // func validInstanceName(in string) (ok bool) { - logDebug.Printf("checking %q", in) ok = validStringRE.MatchString(in) - logDebug.Println("rexexp match", ok) + if !ok { + logDebug.Println("no rexexp match:", in) + } return } @@ -438,13 +495,13 @@ func removePathList(c Instance, paths string) (err error) { return fmt.Errorf("%s %w", p, err) } // glob here - m, err := filepath.Glob(filepath.Join(Home(c), p)) + m, err := globPath(Location(c), filepath.Join(Home(c), p)) if err != nil { return err } for _, f := range m { - if err = os.RemoveAll(f); err != nil { - log.Println(err) + if err = removeAll(Location(c), f); err != nil { + logError.Println(err) continue } } diff --git a/cmd/geneos/webserver.go b/cmd/geneos/webserver.go index 11240f2..f26b5ec 100644 --- a/cmd/geneos/webserver.go +++ b/cmd/geneos/webserver.go @@ -40,11 +40,12 @@ func init() { } func webserverInstance(name string) interface{} { - // Bootstrap + local, remote := splitInstanceName(name) c := &Webservers{} - c.Root = RunningConfig.ITRSHome + c.Root = remoteRoot(remote) c.Type = Webserver.String() - c.Name = name + c.Name = local + c.Location = remote setDefaults(&c) return c } @@ -129,7 +130,7 @@ func webserverAdd(name string, username string, params []string) (c Instance, er return } - if err = os.MkdirAll(filepath.Join(Home(c), "webapps"), 0777); err != nil { + if err = mkdirAll(Location(c), filepath.Join(Home(c), "webapps"), 0777); err != nil { return } diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index e62ceb6..1f82956 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -79,7 +79,16 @@ func (g GeneosLogger) Write(p []byte) (n int, err error) { io.WriteString(g.Writer, line) os.Exit(1) case ERROR: - line = fmt.Sprintf("%s%s", prefix, p) + var fnName string = "UNKNOWN" + pc, _, _, ok := runtime.Caller(3) + if ok { + fn := runtime.FuncForPC(pc) + if fn != nil { + fnName = fn.Name() + } + } + + line = fmt.Sprintf("%s%s() %s", prefix, fnName, p) case DEBUG: var fnName string = "UNKNOWN" pc, f, ln, ok := runtime.Caller(3)