From 44f6d7040925a885e8ae3aa49e5cb8e674ca604e Mon Sep 17 00:00:00 2001 From: TJ Moore Date: Mon, 16 Dec 2024 14:37:57 -0500 Subject: [PATCH] Update the support export tool to gather Patroni logs Ensures that the on volume Patroni log file is exported, if that file exists. If the PostgresCluster is not configured to create this file, note in the debug logs that this is acceptable for some configurations. Issue: PGO-1701 --- internal/cmd/exec.go | 11 ++++ internal/cmd/exec_test.go | 19 +++++++ internal/cmd/export.go | 113 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 143 insertions(+) diff --git a/internal/cmd/exec.go b/internal/cmd/exec.go index 3e675b9..5eab7ef 100644 --- a/internal/cmd/exec.go +++ b/internal/cmd/exec.go @@ -79,6 +79,17 @@ func (exec Executor) listBackrestLogFiles() (string, string, error) { return stdout.String(), stderr.String(), err } +// listPatroniLogFiles returns the full path of Patroni log file. +// These are the Patroni logs stored on the Postgres instance. +func (exec Executor) listPatroniLogFiles() (string, string, error) { + var stdout, stderr bytes.Buffer + + command := "ls -1dt pgdata/patroni/log/*" + err := exec(nil, &stdout, &stderr, "bash", "-ceu", "--", command) + + return stdout.String(), stderr.String(), err +} + // listBackrestRepoHostLogFiles returns the full path of pgBackRest log files. // These are the pgBackRest logs stored on the repo host func (exec Executor) listBackrestRepoHostLogFiles() (string, string, error) { diff --git a/internal/cmd/exec_test.go b/internal/cmd/exec_test.go index de26c8b..e66e51b 100644 --- a/internal/cmd/exec_test.go +++ b/internal/cmd/exec_test.go @@ -79,6 +79,25 @@ func TestListPGLogFiles(t *testing.T) { } +func TestListPatroniLogFiles(t *testing.T) { + + t.Run("default", func(t *testing.T) { + expected := errors.New("pass-through") + exec := func( + stdin io.Reader, stdout, stderr io.Writer, command ...string, + ) error { + assert.DeepEqual(t, command, []string{"bash", "-ceu", "--", "ls -1dt pgdata/patroni/log/*"}) + assert.Assert(t, stdout != nil, "should capture stdout") + assert.Assert(t, stderr != nil, "should capture stderr") + return expected + } + _, _, err := Executor(exec).listPatroniLogFiles() + assert.ErrorContains(t, err, "pass-through") + + }) + +} + func TestCatFile(t *testing.T) { t.Run("default", func(t *testing.T) { diff --git a/internal/cmd/export.go b/internal/cmd/export.go index bb85efd..5a0b2a9 100644 --- a/internal/cmd/export.go +++ b/internal/cmd/export.go @@ -471,6 +471,12 @@ Collecting PGO CLI logs... writeInfo(cmd, fmt.Sprintf("Error gathering pgBackRest DB Hosts Logs: %s", err)) } + // Patroni Logs that are stored on the Postgres Instances + err = gatherPatroniLogs(ctx, clientset, restConfig, namespace, clusterName, tw, cmd) + if err != nil { + writeInfo(cmd, fmt.Sprintf("Error gathering Patroni Logs from Instance Pods: %s", err)) + } + // All pgBackRest Logs on the Repo Host err = gatherRepoHostLogs(ctx, clientset, restConfig, namespace, clusterName, tw, cmd) if err != nil { @@ -1308,6 +1314,113 @@ func gatherDbBackrestLogs(ctx context.Context, return nil } +// gatherPatroniLogs gathers all the file-based Patroni logs on the DB instance, +// if configured. By default, these logs will be sent to stdout and captured as +// Pod logs instead. +func gatherPatroniLogs(ctx context.Context, + clientset *kubernetes.Clientset, + config *rest.Config, + namespace string, + clusterName string, + tw *tar.Writer, + cmd *cobra.Command, +) error { + writeInfo(cmd, "Collecting Patroni logs...") + + dbPods, err := clientset.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: util.DBInstanceLabels(clusterName), + }) + + if err != nil { + if apierrors.IsForbidden(err) { + writeInfo(cmd, err.Error()) + return nil + } + return err + } + + if len(dbPods.Items) == 0 { + writeInfo(cmd, "No database instance pod found for gathering logs") + return nil + } + + writeDebug(cmd, fmt.Sprintf("Found %d Pods\n", len(dbPods.Items))) + + podExec, err := util.NewPodExecutor(config) + if err != nil { + return err + } + + for _, pod := range dbPods.Items { + writeDebug(cmd, fmt.Sprintf("Pod Name is %s\n", pod.Name)) + + exec := func(stdin io.Reader, stdout, stderr io.Writer, command ...string, + ) error { + return podExec(namespace, pod.Name, util.ContainerDatabase, + stdin, stdout, stderr, command...) + } + + // Get Patroni Log Files + stdout, stderr, err := Executor(exec).listPatroniLogFiles() + + // Depending upon the list* function above: + // An error may happen when err is non-nil or stderr is non-empty. + // In both cases, we want to print helpful information and continue to the + // next iteration. + if err != nil || stderr != "" { + + if apierrors.IsForbidden(err) { + writeInfo(cmd, err.Error()) + return nil + } + + writeDebug(cmd, "Error getting Patroni logs\n") + + if err != nil { + writeDebug(cmd, fmt.Sprintf("%s\n", err.Error())) + } + if stderr != "" { + writeDebug(cmd, stderr) + } + + if strings.Contains(stderr, "No such file or directory") { + writeDebug(cmd, "Cannot find any Patroni log files. This is acceptable in some configurations.\n") + } + continue + } + + logFiles := strings.Split(strings.TrimSpace(stdout), "\n") + for _, logFile := range logFiles { + writeDebug(cmd, fmt.Sprintf("LOG FILE: %s\n", logFile)) + var buf bytes.Buffer + + stdout, stderr, err := Executor(exec).catFile(logFile) + if err != nil { + if apierrors.IsForbidden(err) { + writeInfo(cmd, err.Error()) + // Continue and output errors for each log file + // Allow the user to see and address all issues at once + continue + } + return err + } + + buf.Write([]byte(stdout)) + if stderr != "" { + str := fmt.Sprintf("\nError returned: %s\n", stderr) + buf.Write([]byte(str)) + } + + path := clusterName + fmt.Sprintf("/pods/%s/", pod.Name) + logFile + if err := writeTar(tw, buf.Bytes(), path, cmd); err != nil { + return err + } + } + + } + return nil +} + // gatherRepoHostLogs gathers all the file-based pgBackRest logs on the repo host. // There may not be any logs depending upon pgBackRest's log-level-file. func gatherRepoHostLogs(ctx context.Context,