From b90c68ccdf07e3835c42db0e44e1dcb8468dfbbb Mon Sep 17 00:00:00 2001 From: Vladimir Nadvornik Date: Wed, 30 Oct 2024 13:27:57 +0100 Subject: [PATCH 1/2] Dedicated container for database --- mgradm/cmd/install/podman/utils.go | 18 ++ mgradm/cmd/install/shared/flags.go | 12 +- mgradm/cmd/uninstall/podman.go | 5 + mgradm/shared/pgsql/pgsql.go | 180 ++++++++++++++++++ .../templates/mgrSetupScriptTemplate.go | 10 +- .../shared/templates/pgsqlConfigTemplate.go | 54 ++++++ .../shared/templates/pgsqlServiceTemplate.go | 84 ++++++++ mgradm/shared/utils/cmd_utils.go | 18 ++ mgradm/shared/utils/types.go | 7 + shared/connection.go | 36 ++++ shared/podman/systemd.go | 3 + shared/podman/utils.go | 3 + shared/utils/ports.go | 6 +- shared/utils/volumes.go | 26 ++- 14 files changed, 445 insertions(+), 17 deletions(-) create mode 100644 mgradm/shared/pgsql/pgsql.go create mode 100644 mgradm/shared/templates/pgsqlConfigTemplate.go create mode 100644 mgradm/shared/templates/pgsqlServiceTemplate.go diff --git a/mgradm/cmd/install/podman/utils.go b/mgradm/cmd/install/podman/utils.go index 039856eab..92170a539 100644 --- a/mgradm/cmd/install/podman/utils.go +++ b/mgradm/cmd/install/podman/utils.go @@ -15,6 +15,7 @@ import ( install_shared "github.com/uyuni-project/uyuni-tools/mgradm/cmd/install/shared" "github.com/uyuni-project/uyuni-tools/mgradm/shared/coco" "github.com/uyuni-project/uyuni-tools/mgradm/shared/hub" + "github.com/uyuni-project/uyuni-tools/mgradm/shared/pgsql" "github.com/uyuni-project/uyuni-tools/mgradm/shared/podman" "github.com/uyuni-project/uyuni-tools/shared" . "github.com/uyuni-project/uyuni-tools/shared/l10n" @@ -110,6 +111,18 @@ func installForPodman( "CERT_PASS": caPassword, } + if flags.ReportDB.Host == "" { + flags.ReportDB.Host = "uyuni-pgsql-server.mgr.internal" + } + + if flags.DB.Host == "" { + flags.DB.Host = "uyuni-pgsql-server.mgr.internal" + + if err := pgsql.SetupPgsql(systemd, authFile, flags.Pgsql, flags.DB.Admin.User, flags.DB.Admin.Password); err != nil { + return err + } + } + log.Info().Msg(L("Run setup command in the container")) if err := install_shared.RunSetup(cnx, &flags.InstallFlags, fqdn, env); err != nil { @@ -119,6 +132,11 @@ func installForPodman( return err } + log.Info().Msg(L("Enabling SSL in the postgres container")) + if err := pgsql.EnableSSL(systemd); err != nil { + return err + } + if path, err := exec.LookPath("uyuni-payg-extract-data"); err == nil { // the binary is installed err = utils.RunCmdStdMapping(zerolog.DebugLevel, path) diff --git a/mgradm/cmd/install/shared/flags.go b/mgradm/cmd/install/shared/flags.go index 40f5746eb..7e2b2ca54 100644 --- a/mgradm/cmd/install/shared/flags.go +++ b/mgradm/cmd/install/shared/flags.go @@ -52,6 +52,7 @@ type InstallFlags struct { Scc types.SCCCredentials Debug DebugFlags Image types.ImageFlags `mapstructure:",squash"` + Pgsql cmd_utils.PgsqlFlags Coco cmd_utils.CocoFlags HubXmlrpc cmd_utils.HubXmlrpcFlags Admin apiTypes.User @@ -100,6 +101,7 @@ func (flags *InstallFlags) CheckParameters(cmd *cobra.Command, command string) { if flags.TZ == "" { flags.TZ = utils.GetLocalTimezone() } + utils.AskPasswordIfMissing(&flags.DB.Admin.Password, cmd.Flag("db-admin-password").Usage, 5, 48) utils.AskIfMissing(&flags.Email, cmd.Flag("email").Usage, 1, 128, emailChecker) utils.AskIfMissing(&flags.EmailFrom, cmd.Flag("emailfrom").Usage, 0, 0, emailChecker) @@ -129,11 +131,11 @@ func AddInstallFlags(cmd *cobra.Command) { cmd.Flags().String("db-user", "spacewalk", L("Database user")) cmd.Flags().String("db-password", "", L("Database password. Randomly generated by default")) cmd.Flags().String("db-name", "susemanager", L("Database name")) - cmd.Flags().String("db-host", "localhost", L("Database host")) + cmd.Flags().String("db-host", "", L("Database host")) cmd.Flags().Int("db-port", 5432, L("Database port")) cmd.Flags().String("db-protocol", "tcp", L("Database protocol")) - cmd.Flags().String("db-admin-user", "", L("External database admin user name")) - cmd.Flags().String("db-admin-password", "", L("External database admin password")) + cmd.Flags().String("db-admin-user", "postgres", L("Database admin user name")) + cmd.Flags().String("db-admin-password", "", L("Database admin password")) cmd.Flags().String("db-provider", "", L("External database provider. Possible values 'aws'")) _ = utils.AddFlagHelpGroup(cmd, &utils.Group{ID: "db", Title: L("Database Flags")}) @@ -149,7 +151,7 @@ func AddInstallFlags(cmd *cobra.Command) { cmd.Flags().Bool("tftp", true, L("Enable TFTP")) cmd.Flags().String("reportdb-name", "reportdb", L("Report database name")) - cmd.Flags().String("reportdb-host", "localhost", L("Report database host")) + cmd.Flags().String("reportdb-host", "", L("Report database host")) cmd.Flags().Int("reportdb-port", 5432, L("Report database port")) cmd.Flags().String("reportdb-user", "pythia_susemanager", L("Report Database username")) cmd.Flags().String("reportdb-password", "", L("Report database password. Randomly generated by default")) @@ -200,6 +202,8 @@ func AddInstallFlags(cmd *cobra.Command) { cmd_utils.AddHubXmlrpcFlags(cmd) + cmd_utils.AddPgsqlFlags(cmd) + cmd.Flags().String("admin-login", "admin", L("Administrator user name")) cmd.Flags().String("admin-password", "", L("Administrator password")) cmd.Flags().String("admin-firstName", "Administrator", L("First name of the administrator")) diff --git a/mgradm/cmd/uninstall/podman.go b/mgradm/cmd/uninstall/podman.go index e71cdee6c..014221221 100644 --- a/mgradm/cmd/uninstall/podman.go +++ b/mgradm/cmd/uninstall/podman.go @@ -26,6 +26,7 @@ func uninstallForPodman( podman.GetServiceImage(podman.ServerService), podman.GetServiceImage(podman.ServerAttestationService + "@"), podman.GetServiceImage(podman.HubXmlrpcService), + podman.GetServiceImage(podman.PgsqlService), } // Uninstall the service @@ -35,6 +36,7 @@ func uninstallForPodman( systemd.UninstallInstantiatedService(podman.ServerAttestationService, !flags.Force) systemd.UninstallInstantiatedService(podman.HubXmlrpcService, !flags.Force) + systemd.UninstallInstantiatedService(podman.PgsqlService, !flags.Force) // Remove the volumes if flags.Purge.Volumes { @@ -43,6 +45,9 @@ func uninstallForPodman( for _, volume := range utils.ServerVolumeMounts { volumes = append(volumes, volume.Name) } + for _, volume := range utils.PgsqlRequiredVolumeMounts { + volumes = append(volumes, volume.Name) + } for _, volume := range volumes { if err := podman.DeleteVolume(volume, !flags.Force); err != nil { log.Warn().Err(err).Msgf(L("Failed to remove volume %s"), volume) diff --git a/mgradm/shared/pgsql/pgsql.go b/mgradm/shared/pgsql/pgsql.go new file mode 100644 index 000000000..66da6c76e --- /dev/null +++ b/mgradm/shared/pgsql/pgsql.go @@ -0,0 +1,180 @@ +// SPDX-FileCopyrightText: 2024 SUSE LLC +// +// SPDX-License-Identifier: Apache-2.0 + +package pgsql + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/rs/zerolog/log" + "github.com/uyuni-project/uyuni-tools/mgradm/shared/templates" + cmd_utils "github.com/uyuni-project/uyuni-tools/mgradm/shared/utils" + "github.com/uyuni-project/uyuni-tools/shared" + . "github.com/uyuni-project/uyuni-tools/shared/l10n" + "github.com/uyuni-project/uyuni-tools/shared/podman" + "github.com/uyuni-project/uyuni-tools/shared/utils" +) + +// SetupPgsql prepares the systemd service and starts it if needed. +func SetupPgsql( + systemd podman.Systemd, + authFile string, + pgsqlFlags cmd_utils.PgsqlFlags, + admin string, + password string, +) error { + image := pgsqlFlags.Image + currentReplicas := systemd.CurrentReplicaCount(podman.PgsqlService) + log.Debug().Msgf("Current HUB replicas running are %d.", currentReplicas) + + if pgsqlFlags.Replicas == 0 { + log.Debug().Msg("No pgsql requested.") + } + if !pgsqlFlags.IsChanged { + log.Info().Msgf(L("No changes requested for hub. Keep %d replicas."), currentReplicas) + } + + pullEnabled := (pgsqlFlags.Replicas > 0 && pgsqlFlags.IsChanged) || (currentReplicas > 0 && !pgsqlFlags.IsChanged) + + pgsqlImage, err := utils.ComputeImage(pgsqlFlags.Image.Registry, pgsqlFlags.Image.Tag, image) + + if err != nil { + return utils.Errorf(err, L("failed to compute image URL")) + } + + preparedImage, err := podman.PrepareImage(authFile, pgsqlImage, pgsqlFlags.Image.PullPolicy, pullEnabled) + if err != nil { + return err + } + + initdbDir, _, err := utils.TempDir() + if err != nil { + return err + } + + // fixme: for now, we need the script outside of this func, in EnableSSL + // defer cleaner() + + _ = os.Chmod(initdbDir, 0555) + + data := templates.PgsqlConfigTemplateData{} + + scriptName := "pgsqlConfig.sh" + scriptPath := filepath.Join(initdbDir, scriptName) + if err := utils.WriteTemplateToFile(data, scriptPath, 0555, true); err != nil { + return fmt.Errorf(L("failed to generate %s"), scriptName) + } + + if err := generatePgsqlSystemdService(systemd, preparedImage, initdbDir, admin, password); err != nil { + return utils.Errorf(err, L("cannot generate systemd service")) + } + + if err := EnablePgsql(systemd, 0); err != nil { + return err + } + if err := EnablePgsql(systemd, pgsqlFlags.Replicas); err != nil { + return err + } + cnx := shared.NewConnection("podman", podman.PgsqlContainerName, "") + if err := cnx.WaitForHealthcheck(); err != nil { + return err + } + + // Now the servisce is up and ready, the admin credentials are no longer needed + if err := generatePgsqlSystemdService(systemd, preparedImage, "", "", ""); err != nil { + return utils.Errorf(err, L("cannot generate systemd service")) + } + return nil +} + +// EnableSSL enables ssl in postgres container, as long as the certs are mounted. +func EnableSSL(systemd podman.Systemd) error { + cnx := shared.NewConnection("podman", podman.PgsqlContainerName, "") + if _, err := cnx.Exec("/docker-entrypoint-initdb.d/pgsqlConfig.sh"); err != nil { + return err + } + + if err := systemd.RestartInstantiated(podman.PgsqlService); err != nil { + return utils.Errorf(err, L("cannot restart service")) + } + + if err := cnx.WaitForHealthcheck(); err != nil { + return err + } + return nil +} + +// EnablePgsql enables the hub xmlrpc service if the number of replicas is 1. +// This function is meant for installation or migration, to enable or disable the service after, use ScaleService. +func EnablePgsql(systemd podman.Systemd, replicas int) error { + if replicas > 1 { + log.Warn().Msg(L("Multiple Hub XML-RPC container replicas are not currently supported, setting up only one.")) + replicas = 1 + } + + if err := systemd.ScaleService(replicas, podman.PgsqlService); err != nil { + return utils.Errorf(err, L("cannot enable service")) + } + return nil +} + +// Upgrade updates the systemd service files and restarts the containers if needed. +func Upgrade( + systemd podman.Systemd, + authFile string, + pgsqlFlags cmd_utils.PgsqlFlags, + admin string, + password string, +) error { + if err := SetupPgsql(systemd, authFile, pgsqlFlags, admin, password); err != nil { + return err + } + + if err := systemd.ReloadDaemon(false); err != nil { + return err + } + + return systemd.RestartInstantiated(podman.PgsqlService) +} + +// generatePgsqlSystemdService creates the Hub XMLRPC systemd files. +func generatePgsqlSystemdService( + systemd podman.Systemd, + image string, + initdb string, + admin string, + password string, +) error { + pgsqlData := templates.PgsqlServiceTemplateData{ + Volumes: utils.PgsqlRequiredVolumeMounts, + Ports: utils.PgsqlPorts, + NamePrefix: "uyuni", + Network: podman.UyuniNetwork, + Image: image, + } + if err := utils.WriteTemplateToFile( + pgsqlData, podman.GetServicePath(podman.PgsqlService+"@"), 0555, true, + ); err != nil { + return utils.Errorf(err, L("failed to generate systemd service unit file")) + } + + environment := fmt.Sprintf("Environment=UYUNI_IMAGE=%s\n", image) + if initdb != "" { + environment += fmt.Sprintf("Environment=PODMAN_EXTRA_ARGS=\"-v %s:/docker-entrypoint-initdb.d:z\"\n", initdb) + } + if admin != "" { + environment += fmt.Sprintf("Environment=POSTGRES_USER=\"%s\"\n", admin) + } + if password != "" { + environment += fmt.Sprintf("Environment=POSTGRES_PASSWORD=\"%s\"\n", password) + } + + if err := podman.GenerateSystemdConfFile(podman.PgsqlService+"@", "generated.conf", environment, true); err != nil { + return utils.Errorf(err, L("cannot generate systemd conf file")) + } + + return systemd.ReloadDaemon(false) +} diff --git a/mgradm/shared/templates/mgrSetupScriptTemplate.go b/mgradm/shared/templates/mgrSetupScriptTemplate.go index 2c17b5707..f104f5a70 100644 --- a/mgradm/shared/templates/mgrSetupScriptTemplate.go +++ b/mgradm/shared/templates/mgrSetupScriptTemplate.go @@ -10,10 +10,15 @@ import ( ) //nolint:lll -const mgrSetupScriptTemplate = `#!/bin/sh +const mgrSetupScriptTemplate = `#!/bin/sh -x {{- range $name, $value := .Env }} export {{ $name }}='{{ $value }}' {{- end }} +sed -i -e "s|/bin/bash|/bin/bash -x|" /usr/bin/uyuni-setup-reportdb +sed -i -e "s|verify-full|disable|" /usr/lib/perl5/vendor_perl/5.26.1/Spacewalk/Setup.pm +sed -i -e "s|CREATE EXTENSION pgcrypto;||" /usr/share/susemanager/db/postgres/main.sql +sed -i -e "s|create extension dblink;||" /usr/share/susemanager/db/postgres/main.sql +sed -i -e "s|create extension dblink;||" /usr/share/susemanager/db/reportdb/main.sql {{- if .DebugJava }} echo 'JAVA_OPTS=" $JAVA_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,address=*:8003,server=y,suspend=n" ' >> /etc/tomcat/conf.d/remote_debug.conf @@ -21,7 +26,8 @@ echo 'JAVA_OPTS=" $JAVA_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,address=*:800 echo 'JAVA_OPTS=" $JAVA_OPTS -Xdebug -Xrunjdwp:transport=dt_socket,address=*:8002,server=y,suspend=n" ' >> /usr/share/rhn/config-defaults/rhn_search_daemon.conf {{- end }} -/usr/lib/susemanager/bin/mgr-setup -s -n +#sed -i -e "s|EXTERNALDB=0|EXTERNALDB=1|" /usr/lib/susemanager/bin/mgr-setup +/bin/bash -x /usr/lib/susemanager/bin/mgr-setup -s -n RESULT=$? # clean before leaving diff --git a/mgradm/shared/templates/pgsqlConfigTemplate.go b/mgradm/shared/templates/pgsqlConfigTemplate.go new file mode 100644 index 000000000..e4855c76c --- /dev/null +++ b/mgradm/shared/templates/pgsqlConfigTemplate.go @@ -0,0 +1,54 @@ +// SPDX-FileCopyrightText: 2024 SUSE LLC +// +// SPDX-License-Identifier: Apache-2.0 + +package templates + +import ( + "io" + "text/template" + // "github.com/uyuni-project/uyuni-tools/shared/types" +) + +const pgsqlConfigTemplate = `#!/bin/sh -x +POSTGRESQL=/var/lib/pgsql/data/postgresql.conf +SSL_CERT=/etc/pki/tls/certs/spacewalk.crt +SSL_KEY=/etc/pki/tls/private/pg-spacewalk.key + +postgres_reconfig() { + if grep -E "^$1[[:space:]]*=" $POSTGRESQL >/dev/null; then + sed -i "s|^$1[[:space:]]*=.*|$1 = $2|" $POSTGRESQL + else + echo "$1 = $2" >> $POSTGRESQL + fi +} + + +postgres_reconfig effective_cache_size 1152MB +postgres_reconfig maintenance_work_mem 96MB +postgres_reconfig max_connections 600 +postgres_reconfig shared_buffers 384MB +postgres_reconfig wal_buffers 4MB +postgres_reconfig work_mem 2560kB +postgres_reconfig jit off + +if [ -f $SSL_KEY ] ; then + chown postgres $SSL_KEY + chmod 400 $SSL_KEY + postgres_reconfig "ssl" "on" + postgres_reconfig "ssl_cert_file" "'$SSL_CERT'" + postgres_reconfig "ssl_key_file" "'$SSL_KEY'" +fi + +echo "postgresql.conf updated" +` + +// PgsqlConfigTemplateData POD information to create systemd file. +type PgsqlConfigTemplateData struct { +} + +// Render will create the systemd configuration file. +func (data PgsqlConfigTemplateData) Render(wr io.Writer) error { + t := template.Must(template.New("script").Parse(pgsqlConfigTemplate)) + return t.Execute(wr, data) +} diff --git a/mgradm/shared/templates/pgsqlServiceTemplate.go b/mgradm/shared/templates/pgsqlServiceTemplate.go new file mode 100644 index 000000000..391649cb5 --- /dev/null +++ b/mgradm/shared/templates/pgsqlServiceTemplate.go @@ -0,0 +1,84 @@ +// SPDX-FileCopyrightText: 2024 SUSE LLC +// +// SPDX-License-Identifier: Apache-2.0 + +package templates + +import ( + "io" + "text/template" + + "github.com/uyuni-project/uyuni-tools/shared/types" +) + +const pgsqlServiceTemplate = `# uyuni-pgsql-server.service, generated by mgradm +# Use an uyuni-pgsql-server.service.d/local.conf file to override + +[Unit] +Description=Uyuni postgres server image container service +Wants=network.target +After=network-online.target +RequiresMountsFor=%t/containers + +[Service] +Environment=PODMAN_SYSTEMD_UNIT=%n +Restart=on-failure +ExecStartPre=/bin/rm -f %t/uyuni-pgsql-server.pid %t/%n.ctr-id +ExecStartPre=/usr/bin/podman rm --ignore --force -t 10 {{ .NamePrefix }}-pgsql-server +ExecStart=/bin/sh -c '/usr/bin/podman run \ + --conmon-pidfile %t/uyuni-pgsql-server.pid \ + --cidfile=%t/%n.ctr-id \ + --cgroups=no-conmon \ + --shm-size=0 \ + --shm-size-systemd=0 \ + --sdnotify=conmon \ + -d \ + --name {{ .NamePrefix }}-pgsql-server \ + --hostname {{ .NamePrefix }}-pgsql-server.mgr.internal \ + {{ .Args }} \ + {{- range .Ports }} + -p {{ .Exposed }}:{{ .Port }}{{if .Protocol}}/{{ .Protocol }}{{end}} \ + {{- if $.IPV6Enabled }} + -p [::]:{{ .Exposed }}:{{ .Port }}{{if .Protocol}}/{{ .Protocol }}{{end}} \ + {{- end }} + {{- end }} + {{- range .Volumes }} + -v {{ .Name }}:{{ .MountPath }}:z \ + {{- end }} + -e TZ=${TZ} \ + -e POSTGRES_PASSWORD \ + --network {{ .Network }} \ + ${PODMAN_EXTRA_ARGS} ${UYUNI_IMAGE}' +ExecStop=/usr/bin/podman stop \ + --ignore -t 10 \ + --cidfile=%t/%n.ctr-id +ExecStopPost=/usr/bin/podman rm \ + -f \ + --ignore -t 10 \ + --cidfile=%t/%n.ctr-id + +PIDFile=%t/uyuni-pgsql-server.pid +TimeoutStopSec=180 +TimeoutStartSec=900 +Type=forking + +[Install] +WantedBy=multi-user.target default.target +` + +// PostgresServiceTemplateData POD information to create systemd file. +type PgsqlServiceTemplateData struct { + Volumes []types.VolumeMount + NamePrefix string + Args string + Ports []types.PortMap + Image string + Network string + IPV6Enabled bool +} + +// Render will create the systemd configuration file. +func (data PgsqlServiceTemplateData) Render(wr io.Writer) error { + t := template.Must(template.New("service").Parse(pgsqlServiceTemplate)) + return t.Execute(wr, data) +} diff --git a/mgradm/shared/utils/cmd_utils.go b/mgradm/shared/utils/cmd_utils.go index 168260d33..16fc065fe 100644 --- a/mgradm/shared/utils/cmd_utils.go +++ b/mgradm/shared/utils/cmd_utils.go @@ -154,3 +154,21 @@ func AddUpgradeHubXmlrpcFlags(cmd *cobra.Command) { Leave it unset if you want to keep the previous number of replicas.`)) _ = utils.AddFlagToHelpGroupID(cmd, "hubxmlrpc-replicas", "hubxmlrpc-container") } + +// AddPgsqlFlags adds hub XML-RPC related parameters to cmd. +func AddPgsqlFlags(cmd *cobra.Command) { + _ = utils.AddFlagHelpGroup(cmd, &utils.Group{ID: "pgsql-container", Title: L("Postgresql Database Container Flags")}) + AddContainerImageFlags(cmd, "pgsql", L("Postgresql Database"), "pgsql-container", "suse/postgres") + cmd.Flags().Int("pgsql-replicas", 1, L("How many replicas of the Postgresql service container should be started.")) + _ = utils.AddFlagToHelpGroupID(cmd, "pgsql-replicas", "pgsql-container") +} + +// AddUpgradePgsqlFlags adds hub XML-RPC related parameters to cmd upgrade. +func AddUpgradePgsqlFlags(cmd *cobra.Command) { + _ = utils.AddFlagHelpGroup(cmd, &utils.Group{ID: "pgsql-container", Title: L("Postgresql Database Container Flags")}) + AddContainerImageFlags(cmd, "pgsql", L("Postgresql Database"), "pgsql-container", "pgsql-container") + cmd.Flags().Int("pgsql-replicas", 0, + L(`How many replicas of the Postgresql service container should be started. +Leave it unset if you want to keep the previous number of replicas.`)) + _ = utils.AddFlagToHelpGroupID(cmd, "pgsql-replicas", "pgsql-container") +} diff --git a/mgradm/shared/utils/types.go b/mgradm/shared/utils/types.go index 2c2265737..cc3cb95ab 100644 --- a/mgradm/shared/utils/types.go +++ b/mgradm/shared/utils/types.go @@ -29,6 +29,13 @@ type SslCertFlags struct { Server ssl.SslPair } +// PgsqlFlags contains settings for Pgsql container. +type PgsqlFlags struct { + Replicas int + Image types.ImageFlags `mapstructure:",squash"` + IsChanged bool +} + // HubXmlrpcFlags contains settings for Hub XMLRPC container. type HubXmlrpcFlags struct { Replicas int diff --git a/shared/connection.go b/shared/connection.go index d62ff4e26..7a6b5f1aa 100644 --- a/shared/connection.go +++ b/shared/connection.go @@ -242,6 +242,27 @@ func (c *Connection) Exec(command string, args ...string) ([]byte, error) { return utils.RunCmdOutput(zerolog.DebugLevel, cmd, cmdArgs...) } +// Healthcheck runs healthcheck command inside the container. +func (c *Connection) Healthcheck() ([]byte, error) { + if c.podName == "" { + if _, err := c.GetPodName(); c.podName == "" { + return nil, utils.Errorf(err, L("Healthcheck not executed")) + } + } + + cmd, cmdErr := c.GetCommand() + if cmdErr != nil { + return nil, cmdErr + } + + cmdArgs := []string{"healthcheck", "run", c.podName} + // if cmd == "kubectl" { + // fixme + // } + + return utils.RunCmdOutput(zerolog.DebugLevel, cmd, cmdArgs...) +} + // WaitForContainer waits up to 10 sec for the container to appear. func (c *Connection) WaitForContainer() error { for i := 0; i < 10; i++ { @@ -307,6 +328,21 @@ func (c *Connection) WaitForServer() error { return errors.New(L("server didn't start within 60s. Check for the service status")) } +// WaitForHealthcheck waits at most 60s for healtcheck to succeed. +func (c *Connection) WaitForHealthcheck() error { + // Wait for the system to be up + for i := 0; i < 60; i++ { + _, err := c.Healthcheck() + if err != nil { + log.Debug().Err(err) + time.Sleep(1 * time.Second) + continue + } + return nil + } + return errors.New(L("container didn't start within 60s. Check for the service status")) +} + // Copy transfers a file to or from the container. // Prefix one of src or dst parameters with `server:` to designate the path is in the container // user and group parameters are used to set the owner of a file transferred in the container. diff --git a/shared/podman/systemd.go b/shared/podman/systemd.go index 5d5fbb203..80a5330de 100644 --- a/shared/podman/systemd.go +++ b/shared/podman/systemd.go @@ -22,6 +22,9 @@ var servicesPath = "/etc/systemd/system/" // ServerService is the name of the systemd service for the server. const ServerService = "uyuni-server" +// PgsqlService is the name of the systemd service for the Pgsql container. +const PgsqlService = "uyuni-pgsql-server" + // ServerAttestationService is the name of the systemd service for the coco attestation container. const ServerAttestationService = "uyuni-server-attestation" diff --git a/shared/podman/utils.go b/shared/podman/utils.go index 1fefb93c1..c9fe0172a 100644 --- a/shared/podman/utils.go +++ b/shared/podman/utils.go @@ -28,6 +28,9 @@ const ServerContainerName = "uyuni-server" // HubXmlrpcContainerName is the container name for the Hub XML-RPC API. const HubXmlrpcContainerName = "uyuni-hub-xmlrpc" +// PgsqlContainerName represents the postgres container name. +const PgsqlContainerName = "uyuni-pgsql-server" + // ProxyContainerNames represents all the proxy container names. var ProxyContainerNames = []string{ "uyuni-proxy-httpd", diff --git a/shared/utils/ports.go b/shared/utils/ports.go index 6cd7e7402..1558e0fab 100644 --- a/shared/utils/ports.go +++ b/shared/utils/ports.go @@ -18,7 +18,6 @@ func NewPortMap(name string, exposed int, port int) types.PortMap { // TCPPorts are the tcp ports required by the server // The port names should be less than 15 characters long and lowercased for traefik to eat them. var TCPPorts = []types.PortMap{ - NewPortMap("postgres", 5432, 5432), NewPortMap("salt-publish", 4505, 4505), NewPortMap("salt-request", 4506, 4506), NewPortMap("cobbler", 25151, 25151), @@ -28,6 +27,11 @@ var TCPPorts = []types.PortMap{ NewPortMap("tasko-mtrx", 9800, 9800), } +// PgsqlPorts are the tcp ports required by the postgres server. +var PgsqlPorts = []types.PortMap{ + NewPortMap("postgres", 5432, 5432), +} + // TCPPodmanPorts are the tcp ports required by the server on podman. var TCPPodmanPorts = []types.PortMap{ // TODO: Replace Node exporter with cAdvisor diff --git a/shared/utils/volumes.go b/shared/utils/volumes.go index 4181a8bda..350a384db 100644 --- a/shared/utils/volumes.go +++ b/shared/utils/volumes.go @@ -6,19 +6,20 @@ package utils import "github.com/uyuni-project/uyuni-tools/shared/types" -// PgsqlRequiredVolumeMounts represents volumes mount used by PostgreSQL. -var PgsqlRequiredVolumeMounts = []types.VolumeMount{ +// PgsqlRequiredSharedVolumeMounts represents shared volumes mount used by PostgreSQL. +var PgsqlRequiredSharedVolumeMounts = []types.VolumeMount{ {MountPath: "/etc/pki/tls", Name: "etc-tls"}, - {MountPath: "/var/lib/pgsql", Name: "var-pgsql"}, - {MountPath: "/etc/rhn", Name: "etc-rhn"}, {MountPath: "/etc/pki/spacewalk-tls", Name: "tls-key"}, } -// PgsqlRequiredVolumes represents volumes used by PostgreSQL. -var PgsqlRequiredVolumes = []types.Volume{ +// PgsqlRequiredVolumeMounts represents volumes mount used by PostgreSQL. +var PgsqlRequiredVolumeMounts = append([]types.VolumeMount{ + {MountPath: "/var/lib/pgsql/data", Name: "var-pgsql"}, +}, PgsqlRequiredSharedVolumeMounts[:]...) + +// PgsqlRequiredSharedVolumes represents volumes used by PostgreSQL. +var PgsqlRequiredSharedVolumes = []types.Volume{ {Name: "etc-tls", PersistentVolumeClaim: &types.PersistentVolumeClaim{ClaimName: "etc-tls"}}, - {Name: "var-pgsql", PersistentVolumeClaim: &types.PersistentVolumeClaim{ClaimName: "var-pgsql"}}, - {Name: "etc-rhn", PersistentVolumeClaim: &types.PersistentVolumeClaim{ClaimName: "etc-rhn"}}, {Name: "tls-key", Secret: &types.Secret{ SecretName: "uyuni-cert", Items: []types.SecretItem{ @@ -29,6 +30,11 @@ var PgsqlRequiredVolumes = []types.Volume{ }, } +// PgsqlRequiredVolumes represents volumes used by PostgreSQL. +var PgsqlRequiredVolumes = append([]types.Volume{ + {Name: "var-pgsql", PersistentVolumeClaim: &types.PersistentVolumeClaim{ClaimName: "var-pgsql"}}, +}, PgsqlRequiredSharedVolumes[:]...) + // EtcServerVolumeMounts represents volumes mounted in /etc folder. var EtcServerVolumeMounts = []types.VolumeMount{ {MountPath: "/etc/apache2", Name: "etc-apache2"}, @@ -57,8 +63,8 @@ var EtcServerVolumes = []types.Volume{ {Name: "etc-sssd", PersistentVolumeClaim: &types.PersistentVolumeClaim{ClaimName: "etc-sssd"}}, } -var etcAndPgsqlVolumeMounts = append(PgsqlRequiredVolumeMounts, EtcServerVolumeMounts[:]...) -var etcAndPgsqlVolumes = append(PgsqlRequiredVolumes, EtcServerVolumes[:]...) +var etcAndPgsqlVolumeMounts = append(PgsqlRequiredSharedVolumeMounts, EtcServerVolumeMounts[:]...) +var etcAndPgsqlVolumes = append(PgsqlRequiredSharedVolumes, EtcServerVolumes[:]...) // ServerVolumeMounts should match the volumes mapping from the container definition in both // the helm chart and the systemctl services definitions. From a80fa6a51e4370457bde16f932075d35445d5891 Mon Sep 17 00:00:00 2001 From: Vladimir Nadvornik Date: Mon, 25 Nov 2024 16:13:44 +0100 Subject: [PATCH 2/2] Create dedicated postgres container in upgrade and migration --- mgradm/cmd/migrate/kubernetes/utils.go | 2 +- mgradm/cmd/migrate/podman/utils.go | 11 +++ mgradm/cmd/migrate/shared/flags.go | 2 + mgradm/cmd/support/ptf/podman/utils.go | 3 +- mgradm/cmd/upgrade/podman/utils.go | 2 +- mgradm/cmd/upgrade/shared/flags.go | 2 + mgradm/shared/kubernetes/k3s.go | 2 +- mgradm/shared/pgsql/pgsql.go | 75 +++++++++++-------- mgradm/shared/podman/podman.go | 70 +++++++++++++++-- .../shared/templates/migrateScriptTemplate.go | 15 ++-- .../shared/templates/pgsqlConfigTemplate.go | 54 ------------- .../templates/pgsqlFinalizeScriptTemplate.go | 19 ----- .../pgsqlVersionUpgradeScriptTemplate.go | 22 +++--- mgradm/shared/utils/cmd_utils.go | 4 +- mgradm/shared/utils/exec.go | 34 ++++----- shared/utils/serverinspector.go | 21 ++++-- shared/utils/volumes.go | 24 ++++-- 17 files changed, 201 insertions(+), 161 deletions(-) delete mode 100644 mgradm/shared/templates/pgsqlConfigTemplate.go diff --git a/mgradm/cmd/migrate/kubernetes/utils.go b/mgradm/cmd/migrate/kubernetes/utils.go index 78750a86f..d81170e8e 100644 --- a/mgradm/cmd/migrate/kubernetes/utils.go +++ b/mgradm/cmd/migrate/kubernetes/utils.go @@ -58,7 +58,7 @@ func migrateToKubernetes( sshConfigPath, sshKnownhostsPath := migration_shared.GetSSHPaths() // Prepare the migration script and folder - scriptDir, cleaner, err := adm_utils.GenerateMigrationScript(fqdn, flags.User, true, flags.Prepare) + scriptDir, cleaner, err := adm_utils.GenerateMigrationScript(fqdn, flags.User, true, flags.Prepare, "fixme", "fixme") if err != nil { return utils.Errorf(err, L("failed to generate migration script")) } diff --git a/mgradm/cmd/migrate/podman/utils.go b/mgradm/cmd/migrate/podman/utils.go index 07d615f65..4749e5139 100644 --- a/mgradm/cmd/migrate/podman/utils.go +++ b/mgradm/cmd/migrate/podman/utils.go @@ -14,6 +14,7 @@ import ( migration_shared "github.com/uyuni-project/uyuni-tools/mgradm/cmd/migrate/shared" "github.com/uyuni-project/uyuni-tools/mgradm/shared/coco" "github.com/uyuni-project/uyuni-tools/mgradm/shared/hub" + "github.com/uyuni-project/uyuni-tools/mgradm/shared/pgsql" "github.com/uyuni-project/uyuni-tools/mgradm/shared/podman" "github.com/uyuni-project/uyuni-tools/shared" podman_utils "github.com/uyuni-project/uyuni-tools/shared/podman" @@ -86,6 +87,16 @@ func migrateToPodman( } } + if err := podman_utils.SetupNetwork(false); err != nil { + return err + } + + if err := pgsql.Upgrade( + systemd, authFile, flags.Pgsql, + ); err != nil { + return err + } + schemaUpdateRequired := oldPgVersion != newPgVersion if err := podman.RunPgsqlFinalizeScript(preparedImage, schemaUpdateRequired, true); err != nil { return utils.Errorf(err, L("cannot run PostgreSQL finalize script")) diff --git a/mgradm/cmd/migrate/shared/flags.go b/mgradm/cmd/migrate/shared/flags.go index e86a2e2e0..5141c59e5 100644 --- a/mgradm/cmd/migrate/shared/flags.go +++ b/mgradm/cmd/migrate/shared/flags.go @@ -16,6 +16,7 @@ type MigrateFlags struct { Prepare bool Image types.ImageFlags `mapstructure:",squash"` DBUpgradeImage types.ImageFlags `mapstructure:"dbupgrade"` + Pgsql utils.PgsqlFlags Coco utils.CocoFlags User string Mirror string @@ -30,6 +31,7 @@ func AddMigrateFlags(cmd *cobra.Command) { utils.AddSCCFlag(cmd) utils.AddImageFlag(cmd) utils.AddDBUpgradeImageFlag(cmd) + utils.AddUpgradePgsqlFlags(cmd) utils.AddUpgradeCocoFlag(cmd) utils.AddUpgradeHubXmlrpcFlags(cmd) cmd.Flags().String("user", "root", diff --git a/mgradm/cmd/support/ptf/podman/utils.go b/mgradm/cmd/support/ptf/podman/utils.go index caf1ab1b0..0b07a17f3 100644 --- a/mgradm/cmd/support/ptf/podman/utils.go +++ b/mgradm/cmd/support/ptf/podman/utils.go @@ -31,6 +31,7 @@ func ptfForPodman( dummyImage := types.ImageFlags{} dummyCoco := adm_utils.CocoFlags{} dummyHubXmlrpc := adm_utils.HubXmlrpcFlags{} + dummyPgsql := adm_utils.PgsqlFlags{} if err := flags.checkParameters(); err != nil { return err } @@ -46,7 +47,7 @@ func ptfForPodman( } defer cleaner() - return podman.Upgrade(systemd, authFile, "", flags.Image, dummyImage, dummyCoco, dummyHubXmlrpc) + return podman.Upgrade(systemd, authFile, "", flags.Image, dummyImage, dummyCoco, dummyHubXmlrpc, dummyPgsql) } func (flags *podmanPTFFlags) checkParameters() error { diff --git a/mgradm/cmd/upgrade/podman/utils.go b/mgradm/cmd/upgrade/podman/utils.go index 82eb46de8..6a9efd95e 100644 --- a/mgradm/cmd/upgrade/podman/utils.go +++ b/mgradm/cmd/upgrade/podman/utils.go @@ -28,6 +28,6 @@ func upgradePodman(globalFlags *types.GlobalFlags, flags *podmanUpgradeFlags, cm defer cleaner() return podman.Upgrade( - systemd, authFile, flags.Image.Registry, flags.Image, flags.DBUpgradeImage, flags.Coco, flags.HubXmlrpc, + systemd, authFile, flags.Image.Registry, flags.Image, flags.DBUpgradeImage, flags.Coco, flags.HubXmlrpc, flags.Pgsql, ) } diff --git a/mgradm/cmd/upgrade/shared/flags.go b/mgradm/cmd/upgrade/shared/flags.go index d11046cde..32ae5a182 100644 --- a/mgradm/cmd/upgrade/shared/flags.go +++ b/mgradm/cmd/upgrade/shared/flags.go @@ -16,6 +16,7 @@ type UpgradeFlags struct { DBUpgradeImage types.ImageFlags `mapstructure:"dbupgrade"` Coco utils.CocoFlags HubXmlrpc utils.HubXmlrpcFlags + Pgsql utils.PgsqlFlags } // AddUpgradeFlags add upgrade flags to a command. @@ -26,6 +27,7 @@ func AddUpgradeFlags(cmd *cobra.Command) { utils.AddUpgradeCocoFlag(cmd) utils.AddUpgradeHubXmlrpcFlags(cmd) + utils.AddUpgradePgsqlFlags(cmd) } // AddUpgradeListFlags add upgrade list flags to a command. diff --git a/mgradm/shared/kubernetes/k3s.go b/mgradm/shared/kubernetes/k3s.go index 2aceb20e7..10c478785 100644 --- a/mgradm/shared/kubernetes/k3s.go +++ b/mgradm/shared/kubernetes/k3s.go @@ -121,7 +121,7 @@ func RunPgsqlFinalizeScript( defer cleaner() pgsqlFinalizeContainer := "uyuni-finalize-pgsql" pgsqlFinalizeScriptName, err := adm_utils.GenerateFinalizePostgresScript( - scriptDir, true, schemaUpdateRequired, true, migration, true, + scriptDir, true, schemaUpdateRequired, migration, true, ) if err != nil { return utils.Errorf(err, L("cannot generate PostgreSQL finalization script")) diff --git a/mgradm/shared/pgsql/pgsql.go b/mgradm/shared/pgsql/pgsql.go index 66da6c76e..14bb18a36 100644 --- a/mgradm/shared/pgsql/pgsql.go +++ b/mgradm/shared/pgsql/pgsql.go @@ -6,8 +6,6 @@ package pgsql import ( "fmt" - "os" - "path/filepath" "github.com/rs/zerolog/log" "github.com/uyuni-project/uyuni-tools/mgradm/shared/templates" @@ -50,25 +48,7 @@ func SetupPgsql( return err } - initdbDir, _, err := utils.TempDir() - if err != nil { - return err - } - - // fixme: for now, we need the script outside of this func, in EnableSSL - // defer cleaner() - - _ = os.Chmod(initdbDir, 0555) - - data := templates.PgsqlConfigTemplateData{} - - scriptName := "pgsqlConfig.sh" - scriptPath := filepath.Join(initdbDir, scriptName) - if err := utils.WriteTemplateToFile(data, scriptPath, 0555, true); err != nil { - return fmt.Errorf(L("failed to generate %s"), scriptName) - } - - if err := generatePgsqlSystemdService(systemd, preparedImage, initdbDir, admin, password); err != nil { + if err := generatePgsqlSystemdService(systemd, preparedImage, admin, password); err != nil { return utils.Errorf(err, L("cannot generate systemd service")) } @@ -82,18 +62,18 @@ func SetupPgsql( if err := cnx.WaitForHealthcheck(); err != nil { return err } - // Now the servisce is up and ready, the admin credentials are no longer needed - if err := generatePgsqlSystemdService(systemd, preparedImage, "", "", ""); err != nil { + if err := generatePgsqlSystemdService(systemd, preparedImage, "", ""); err != nil { return utils.Errorf(err, L("cannot generate systemd service")) } + return nil } // EnableSSL enables ssl in postgres container, as long as the certs are mounted. func EnableSSL(systemd podman.Systemd) error { cnx := shared.NewConnection("podman", podman.PgsqlContainerName, "") - if _, err := cnx.Exec("/docker-entrypoint-initdb.d/pgsqlConfig.sh"); err != nil { + if _, err := cnx.Exec("/docker-entrypoint-initdb.d/uyuni-postgres-config.sh"); err != nil { return err } @@ -126,25 +106,59 @@ func Upgrade( systemd podman.Systemd, authFile string, pgsqlFlags cmd_utils.PgsqlFlags, - admin string, - password string, ) error { - if err := SetupPgsql(systemd, authFile, pgsqlFlags, admin, password); err != nil { + image := pgsqlFlags.Image + currentReplicas := systemd.CurrentReplicaCount(podman.PgsqlService) + log.Debug().Msgf("Current HUB replicas running are %d.", currentReplicas) + + if pgsqlFlags.Replicas == 0 { + log.Debug().Msg("No pgsql requested.") + } + if !pgsqlFlags.IsChanged { + log.Info().Msgf(L("No changes requested for hub. Keep %d replicas."), currentReplicas) + } + + pullEnabled := (pgsqlFlags.Replicas > 0 && pgsqlFlags.IsChanged) || (currentReplicas > 0 && !pgsqlFlags.IsChanged) + + pgsqlImage, err := utils.ComputeImage(pgsqlFlags.Image.Registry, pgsqlFlags.Image.Tag, image) + + if err != nil { + return utils.Errorf(err, L("failed to compute image URL")) + } + + preparedImage, err := podman.PrepareImage(authFile, pgsqlImage, pgsqlFlags.Image.PullPolicy, pullEnabled) + if err != nil { + return err + } + err = podman.RunContainer("uyuni-db-migrate", pgsqlImage, utils.PgsqlRequiredVolumeMounts, []string{}, + []string{"chown", "postgres", "/etc/pki/tls/private/pg-spacewalk.key"}) + if err != nil { return err } + if err := generatePgsqlSystemdService(systemd, preparedImage, "", ""); err != nil { + return utils.Errorf(err, L("cannot generate systemd service")) + } + if err := systemd.ReloadDaemon(false); err != nil { return err } - return systemd.RestartInstantiated(podman.PgsqlService) + if err := EnablePgsql(systemd, 0); err != nil { + return err + } + if err := EnablePgsql(systemd, pgsqlFlags.Replicas); err != nil { + return err + } + + cnx := shared.NewConnection("podman", podman.PgsqlContainerName, "") + return cnx.WaitForHealthcheck() } // generatePgsqlSystemdService creates the Hub XMLRPC systemd files. func generatePgsqlSystemdService( systemd podman.Systemd, image string, - initdb string, admin string, password string, ) error { @@ -162,9 +176,6 @@ func generatePgsqlSystemdService( } environment := fmt.Sprintf("Environment=UYUNI_IMAGE=%s\n", image) - if initdb != "" { - environment += fmt.Sprintf("Environment=PODMAN_EXTRA_ARGS=\"-v %s:/docker-entrypoint-initdb.d:z\"\n", initdb) - } if admin != "" { environment += fmt.Sprintf("Environment=POSTGRES_USER=\"%s\"\n", admin) } diff --git a/mgradm/shared/podman/podman.go b/mgradm/shared/podman/podman.go index 8f7f2640e..b9a9b377a 100644 --- a/mgradm/shared/podman/podman.go +++ b/mgradm/shared/podman/podman.go @@ -17,6 +17,7 @@ import ( "github.com/rs/zerolog/log" "github.com/uyuni-project/uyuni-tools/mgradm/shared/coco" "github.com/uyuni-project/uyuni-tools/mgradm/shared/hub" + "github.com/uyuni-project/uyuni-tools/mgradm/shared/pgsql" "github.com/uyuni-project/uyuni-tools/mgradm/shared/ssl" "github.com/uyuni-project/uyuni-tools/mgradm/shared/templates" adm_utils "github.com/uyuni-project/uyuni-tools/mgradm/shared/utils" @@ -194,7 +195,14 @@ func RunMigration( user string, prepare bool, ) (*utils.InspectResult, error) { - scriptDir, cleaner, err := adm_utils.GenerateMigrationScript(sourceFqdn, user, false, prepare) + scriptDir, cleaner, err := adm_utils.GenerateMigrationScript( + sourceFqdn, + user, + false, + prepare, + "uyuni-pgsql-server.mgr.internal", + "uyuni-pgsql-server.mgr.internal", + ) if err != nil { return nil, utils.Errorf(err, L("cannot generate migration script")) } @@ -216,7 +224,7 @@ func RunMigration( } log.Info().Msg(L("Migrating server")) - if err := podman.RunContainer("uyuni-migration", preparedImage, utils.ServerVolumeMounts, extraArgs, + if err := podman.RunContainer("uyuni-migration", preparedImage, utils.ServerMigrationVolumeMounts, extraArgs, []string{"/var/lib/uyuni-tools/migrate.sh"}); err != nil { return nil, utils.Errorf(err, L("cannot run uyuni migration container")) } @@ -294,7 +302,7 @@ func RunPgsqlVersionUpgrade( return utils.Errorf(err, L("cannot generate PostgreSQL database version upgrade script")) } - err = podman.RunContainer(pgsqlVersionUpgradeContainer, preparedImage, utils.ServerVolumeMounts, extraArgs, + err = podman.RunContainer(pgsqlVersionUpgradeContainer, preparedImage, utils.PgsqlRequiredVolumeMounts, extraArgs, []string{"/var/lib/uyuni-tools/" + pgsqlVersionUpgradeScriptName}) if err != nil { return err @@ -314,10 +322,11 @@ func RunPgsqlFinalizeScript(serverImage string, schemaUpdateRequired bool, migra extraArgs := []string{ "-v", scriptDir + ":/var/lib/uyuni-tools/", "--security-opt", "label=disable", + "--network", podman.UyuniNetwork, } pgsqlFinalizeContainer := "uyuni-finalize-pgsql" pgsqlFinalizeScriptName, err := adm_utils.GenerateFinalizePostgresScript( - scriptDir, true, schemaUpdateRequired, true, migration, false, + scriptDir, true, schemaUpdateRequired, migration, false, ) if err != nil { return utils.Errorf(err, L("cannot generate PostgreSQL finalization script")) @@ -363,6 +372,7 @@ func Upgrade( upgradeImage types.ImageFlags, cocoFlags adm_utils.CocoFlags, hubXmlrpcFlags adm_utils.HubXmlrpcFlags, + pgsqlFlags adm_utils.PgsqlFlags, ) error { if err := CallCloudGuestRegistryAuth(); err != nil { return err @@ -396,6 +406,22 @@ func Upgrade( defer func() { err = systemd.StartService(podman.ServerService) }() + + if inspectedValues.CurrentPgVersionNotMigrated != "" || + inspectedValues.DBHost == "localhost" || + inspectedValues.ReportDBHost == "localhost" { + log.Info().Msgf(L("Configuring external postgresql %s %s"), + inspectedValues.CurrentPgVersion, inspectedValues.CurrentPgVersionNotMigrated) + + if err := RunPgsqlContainerMigration( + preparedImage, "uyuni-pgsql-server.mgr.internal", "uyuni-pgsql-server.mgr.internal", + ); err != nil { + return utils.Errorf(err, L("cannot run PostgreSQL version upgrade script")) + } + inspectedValues.CurrentPgVersion = inspectedValues.CurrentPgVersionNotMigrated + pgsqlFlags.Replicas = 1 // migrating pgsql to separate container + } + if inspectedValues.ImagePgVersion > inspectedValues.CurrentPgVersion { log.Info().Msgf( L("Previous postgresql is %[1]s, instead new one is %[2]s. Performing a DB version upgrade…"), @@ -415,6 +441,12 @@ func Upgrade( ) } + if err := pgsql.Upgrade( + systemd, authFile, pgsqlFlags, + ); err != nil { + return err + } + schemaUpdateRequired := inspectedValues.CurrentPgVersion != inspectedValues.ImagePgVersion if err := RunPgsqlFinalizeScript(preparedImage, schemaUpdateRequired, false); err != nil { return utils.Errorf(err, L("cannot run PostgreSQL finalize script")) @@ -497,7 +529,7 @@ func Inspect(preparedImage string) (*utils.ServerInspectData, error) { "--security-opt", "label=disable", } - err = podman.RunContainer("uyuni-inspect", preparedImage, utils.ServerVolumeMounts, podmanArgs, + err = podman.RunContainer("uyuni-inspect", preparedImage, utils.ServerMigrationVolumeMounts, podmanArgs, []string{utils.InspectContainerDirectory + "/" + utils.InspectScriptFilename}) if err != nil { return nil, err @@ -511,6 +543,34 @@ func Inspect(preparedImage string) (*utils.ServerInspectData, error) { return inspectResult, err } +// RunPgsqlContainerMigration migrate to separate postgres container. +func RunPgsqlContainerMigration(serverImage string, dbHost string, reportDBHost string) error { + scriptDir, cleaner, err := utils.TempDir() + if err != nil { + return err + } + defer cleaner() + + data := templates.PgsqlMigrateScriptTemplateData{ + DBHost: dbHost, + ReportDBHost: reportDBHost, + } + + scriptPath := filepath.Join(scriptDir, "pgmigrate.sh") + if err = utils.WriteTemplateToFile(data, scriptPath, 0555, true); err != nil { + return utils.Errorf(err, L("failed to generate postgresql migration script")) + } + + podmanArgs := []string{ + "-v", scriptDir + ":" + scriptDir, + "--security-opt", "label=disable", + } + err = podman.RunContainer("uyuni-db-migrate", serverImage, utils.ServerMigrationVolumeMounts, podmanArgs, + []string{scriptPath}) + + return err +} + // CallCloudGuestRegistryAuth calls cloudguestregistryauth if it is available. func CallCloudGuestRegistryAuth() error { cloudguestregistryauth := "cloudguestregistryauth" diff --git a/mgradm/shared/templates/migrateScriptTemplate.go b/mgradm/shared/templates/migrateScriptTemplate.go index 7be270584..67eb16788 100644 --- a/mgradm/shared/templates/migrateScriptTemplate.go +++ b/mgradm/shared/templates/migrateScriptTemplate.go @@ -134,7 +134,8 @@ grep '^db_port' /etc/rhn/rhn.conf | sed 's/[ \t]//g' >>/var/lib/uyuni-tools/data $SSH {{ .SourceFqdn }} sh -c "systemctl list-unit-files | grep hub-xmlrpc-api | grep -q active && echo has_hubxmlrpc=true || echo has_hubxmlrpc=false" >>/var/lib/uyuni-tools/data echo "Altering configuration for domain resolution..." -sed 's/report_db_host = {{ .SourceFqdn }}/report_db_host = localhost/' -i /etc/rhn/rhn.conf; +sed 's/^db_host = .*/db_host = {{ .DBHost }}/' -i /etc/rhn/rhn.conf; +sed 's/^report_db_host = .*/report_db_host = {{ .ReportDBHost }}/' -i /etc/rhn/rhn.conf; sed 's/server\.jabber_server/java\.hostname/' -i /etc/rhn/rhn.conf; sed 's/client_use_localhost: false/client_use_localhost: true/' -i /etc/cobbler/settings.yaml; @@ -189,11 +190,13 @@ echo "DONE"` // MigrateScriptTemplateData represents migration information used to create migration script. type MigrateScriptTemplateData struct { - Volumes []types.VolumeMount - SourceFqdn string - User string - Kubernetes bool - Prepare bool + Volumes []types.VolumeMount + SourceFqdn string + User string + Kubernetes bool + Prepare bool + DBHost string + ReportDBHost string } // Render will create migration script. diff --git a/mgradm/shared/templates/pgsqlConfigTemplate.go b/mgradm/shared/templates/pgsqlConfigTemplate.go deleted file mode 100644 index e4855c76c..000000000 --- a/mgradm/shared/templates/pgsqlConfigTemplate.go +++ /dev/null @@ -1,54 +0,0 @@ -// SPDX-FileCopyrightText: 2024 SUSE LLC -// -// SPDX-License-Identifier: Apache-2.0 - -package templates - -import ( - "io" - "text/template" - // "github.com/uyuni-project/uyuni-tools/shared/types" -) - -const pgsqlConfigTemplate = `#!/bin/sh -x -POSTGRESQL=/var/lib/pgsql/data/postgresql.conf -SSL_CERT=/etc/pki/tls/certs/spacewalk.crt -SSL_KEY=/etc/pki/tls/private/pg-spacewalk.key - -postgres_reconfig() { - if grep -E "^$1[[:space:]]*=" $POSTGRESQL >/dev/null; then - sed -i "s|^$1[[:space:]]*=.*|$1 = $2|" $POSTGRESQL - else - echo "$1 = $2" >> $POSTGRESQL - fi -} - - -postgres_reconfig effective_cache_size 1152MB -postgres_reconfig maintenance_work_mem 96MB -postgres_reconfig max_connections 600 -postgres_reconfig shared_buffers 384MB -postgres_reconfig wal_buffers 4MB -postgres_reconfig work_mem 2560kB -postgres_reconfig jit off - -if [ -f $SSL_KEY ] ; then - chown postgres $SSL_KEY - chmod 400 $SSL_KEY - postgres_reconfig "ssl" "on" - postgres_reconfig "ssl_cert_file" "'$SSL_CERT'" - postgres_reconfig "ssl_key_file" "'$SSL_KEY'" -fi - -echo "postgresql.conf updated" -` - -// PgsqlConfigTemplateData POD information to create systemd file. -type PgsqlConfigTemplateData struct { -} - -// Render will create the systemd configuration file. -func (data PgsqlConfigTemplateData) Render(wr io.Writer) error { - t := template.Must(template.New("script").Parse(pgsqlConfigTemplate)) - return t.Execute(wr, data) -} diff --git a/mgradm/shared/templates/pgsqlFinalizeScriptTemplate.go b/mgradm/shared/templates/pgsqlFinalizeScriptTemplate.go index 39c63e740..e845ef3b6 100644 --- a/mgradm/shared/templates/pgsqlFinalizeScriptTemplate.go +++ b/mgradm/shared/templates/pgsqlFinalizeScriptTemplate.go @@ -13,20 +13,6 @@ import ( const postgresFinalizeScriptTemplate = `#!/bin/bash set -e -{{ if .Migration }} -echo "Adding database access for other containers..." -db_user=$(sed -n '/^db_user/{s/^.*=[ \t]\+\(.*\)$/\1/ ; p}' /etc/rhn/rhn.conf) -db_name=$(sed -n '/^db_name/{s/^.*=[ \t]\+\(.*\)$/\1/ ; p}' /etc/rhn/rhn.conf) -ip=$(ip -o -4 addr show up scope global | head -1 | awk '{print $4}' || true) -echo "host $db_name $db_user $ip scram-sha-256" >> /var/lib/pgsql/data/pg_hba.conf -{{ end }} - -{{ if .RunAutotune }} -echo "Running smdba system-check autotuning..." -smdba system-check autotuning -{{ end }} -echo "Starting Postgresql..." -su -s /bin/bash - postgres -c "/usr/share/postgresql/postgresql-script start" {{ if .RunReindex }} echo "Reindexing database. This may take a while, please do not cancel it!" database=$(sed -n "s/^\s*db_name\s*=\s*\([^ ]*\)\s*$/\1/p" /etc/rhn/rhn.conf) @@ -57,15 +43,10 @@ where not exists (select 1 from rhntaskorun r join rhntaskotemplate t on r.templ join rhntaskobunch b on t.bunch_id = b.id where b.name='update-system-overview-bunch' limit 1); EOT - -echo "Stopping Postgresql..." -su -s /bin/bash - postgres -c "/usr/share/postgresql/postgresql-script stop" -echo "DONE" ` // FinalizePostgresTemplateData represents information used to create PostgreSQL migration script. type FinalizePostgresTemplateData struct { - RunAutotune bool RunReindex bool RunSchemaUpdate bool Migration bool diff --git a/mgradm/shared/templates/pgsqlVersionUpgradeScriptTemplate.go b/mgradm/shared/templates/pgsqlVersionUpgradeScriptTemplate.go index 7758820a3..69da43513 100644 --- a/mgradm/shared/templates/pgsqlVersionUpgradeScriptTemplate.go +++ b/mgradm/shared/templates/pgsqlVersionUpgradeScriptTemplate.go @@ -14,19 +14,23 @@ const postgreSQLVersionUpgradeScriptTemplate = `#!/bin/bash set -e echo "PostgreSQL version upgrade" +ls -la /var/lib/pgsql/data/ OLD_VERSION={{ .OldVersion }} NEW_VERSION={{ .NewVersion }} -FAST_UPGRADE=--link +FAST_UPGRADE= #--link echo "Testing presence of postgresql$NEW_VERSION..." test -d /usr/lib/postgresql$NEW_VERSION/bin echo "Testing presence of postgresql$OLD_VERSION..." test -d /usr/lib/postgresql$OLD_VERSION/bin -echo "Create a backup at /var/lib/pgsql/data-pg$OLD_VERSION..." -mv /var/lib/pgsql/data /var/lib/pgsql/data-pg$OLD_VERSION +echo "Create a backup at /var/lib/pgsql/data-backup..." +mkdir -p /var/lib/pgsql/data-backup +chown postgres:postgres /var/lib/pgsql/data-backup +chmod 700 /var/lib/pgsql/data-backup +shopt -s dotglob +mv /var/lib/pgsql/data/* /var/lib/pgsql/data-backup echo "Create new database directory..." -mkdir -p /var/lib/pgsql/data chown -R postgres:postgres /var/lib/pgsql echo "Enforce key permission" chown -R postgres:postgres /etc/pki/tls/private/pg-spacewalk.key @@ -45,12 +49,12 @@ echo "Running initdb using postgres user" echo "Any suggested command from the console should be run using postgres user" su -s /bin/bash - postgres -c "initdb -D /var/lib/pgsql/data --locale=$POSTGRES_LANG" echo "Successfully initialized new postgresql $NEW_VERSION database." -su -s /bin/bash - postgres -c "pg_upgrade --old-bindir=/usr/lib/postgresql$OLD_VERSION/bin --new-bindir=/usr/lib/postgresql$NEW_VERSION/bin --old-datadir=/var/lib/pgsql/data-pg$OLD_VERSION --new-datadir=/var/lib/pgsql/data $FAST_UPGRADE" +su -s /bin/bash - postgres -c "pg_upgrade --old-bindir=/usr/lib/postgresql$OLD_VERSION/bin --new-bindir=/usr/lib/postgresql$NEW_VERSION/bin --old-datadir=/var/lib/pgsql/data-backup --new-datadir=/var/lib/pgsql/data $FAST_UPGRADE" -cp /var/lib/pgsql/data-pg$OLD_VERSION/pg_hba.conf /var/lib/pgsql/data -mv /var/lib/pgsql/data-pg$OLD_VERSION/pg_hba.conf /var/lib/pgsql/data-pg$OLD_VERSION/pg_hba.conf.migrated -cp /var/lib/pgsql/data-pg$OLD_VERSION/postgresql.conf /var/lib/pgsql/data/ -mv /var/lib/pgsql/data-pg$OLD_VERSION/postgresql.conf /var/lib/pgsql/data-pg$OLD_VERSION/postgresql.conf.migrated +cp /var/lib/pgsql/data-backup/pg_hba.conf /var/lib/pgsql/data +mv /var/lib/pgsql/data-backup/pg_hba.conf /var/lib/pgsql/data-backup/pg_hba.conf.migrated +cp /var/lib/pgsql/data-backup/postgresql.conf /var/lib/pgsql/data/ +mv /var/lib/pgsql/data-backup/postgresql.conf /var/lib/pgsql/data-backup/postgresql.conf.migrated echo "DONE"` diff --git a/mgradm/shared/utils/cmd_utils.go b/mgradm/shared/utils/cmd_utils.go index 16fc065fe..481ee5163 100644 --- a/mgradm/shared/utils/cmd_utils.go +++ b/mgradm/shared/utils/cmd_utils.go @@ -158,7 +158,7 @@ Leave it unset if you want to keep the previous number of replicas.`)) // AddPgsqlFlags adds hub XML-RPC related parameters to cmd. func AddPgsqlFlags(cmd *cobra.Command) { _ = utils.AddFlagHelpGroup(cmd, &utils.Group{ID: "pgsql-container", Title: L("Postgresql Database Container Flags")}) - AddContainerImageFlags(cmd, "pgsql", L("Postgresql Database"), "pgsql-container", "suse/postgres") + AddContainerImageFlags(cmd, "pgsql", L("Postgresql Database"), "pgsql-container", "server-postgres") cmd.Flags().Int("pgsql-replicas", 1, L("How many replicas of the Postgresql service container should be started.")) _ = utils.AddFlagToHelpGroupID(cmd, "pgsql-replicas", "pgsql-container") } @@ -166,7 +166,7 @@ func AddPgsqlFlags(cmd *cobra.Command) { // AddUpgradePgsqlFlags adds hub XML-RPC related parameters to cmd upgrade. func AddUpgradePgsqlFlags(cmd *cobra.Command) { _ = utils.AddFlagHelpGroup(cmd, &utils.Group{ID: "pgsql-container", Title: L("Postgresql Database Container Flags")}) - AddContainerImageFlags(cmd, "pgsql", L("Postgresql Database"), "pgsql-container", "pgsql-container") + AddContainerImageFlags(cmd, "pgsql", L("Postgresql Database"), "pgsql-container", "server-postgres") cmd.Flags().Int("pgsql-replicas", 0, L(`How many replicas of the Postgresql service container should be started. Leave it unset if you want to keep the previous number of replicas.`)) diff --git a/mgradm/shared/utils/exec.go b/mgradm/shared/utils/exec.go index c505eff85..0ee488a23 100644 --- a/mgradm/shared/utils/exec.go +++ b/mgradm/shared/utils/exec.go @@ -74,10 +74,9 @@ func GeneratePgsqlVersionUpgradeScript( // GenerateFinalizePostgresScript generates the script to finalize PostgreSQL upgrade. func GenerateFinalizePostgresScript( - scriptDir string, runAutotune bool, runReindex bool, runSchemaUpdate bool, migration bool, kubernetes bool, + scriptDir string, runReindex bool, runSchemaUpdate bool, migration bool, kubernetes bool, ) (string, error) { data := templates.FinalizePostgresTemplateData{ - RunAutotune: runAutotune, RunReindex: runReindex, RunSchemaUpdate: runSchemaUpdate, Migration: migration, @@ -115,18 +114,27 @@ func RunMigration(cnx *shared.Connection, tmpPath string, scriptName string) err } // GenerateMigrationScript generates the script that perform migration. -func GenerateMigrationScript(sourceFqdn string, user string, kubernetes bool, prepare bool) (string, func(), error) { +func GenerateMigrationScript( + sourceFqdn string, + user string, + kubernetes bool, + prepare bool, + dbHost string, + reportDBHost string, +) (string, func(), error) { scriptDir, cleaner, err := utils.TempDir() if err != nil { return "", nil, err } data := templates.MigrateScriptTemplateData{ - Volumes: utils.ServerVolumeMounts, - SourceFqdn: sourceFqdn, - User: user, - Kubernetes: kubernetes, - Prepare: prepare, + Volumes: append(utils.ServerVolumeMounts, utils.PgsqlDataVolumeMounts[:]...), + SourceFqdn: sourceFqdn, + User: user, + Kubernetes: kubernetes, + Prepare: prepare, + DBHost: dbHost, + ReportDBHost: reportDBHost, } scriptPath := filepath.Join(scriptDir, "migrate.sh") @@ -226,16 +234,6 @@ func SanityCheck(cnx *shared.Connection, inspectedValues *utils.ServerInspectDat ) } } - - if inspectedValues.ImagePgVersion == "" { - return fmt.Errorf(L("cannot fetch postgresql version from %s"), serverImage) - } - log.Debug().Msgf("Image %s has PostgreSQL %s", serverImage, inspectedValues.ImagePgVersion) - if inspectedValues.CurrentPgVersion == "" { - return errors.New(L("posgresql is not installed in the current deployment")) - } - log.Debug().Msgf("Current deployment has PostgreSQL %s", inspectedValues.CurrentPgVersion) - return nil } diff --git a/shared/utils/serverinspector.go b/shared/utils/serverinspector.go index 3c90f88f8..d4822bbd6 100644 --- a/shared/utils/serverinspector.go +++ b/shared/utils/serverinspector.go @@ -33,6 +33,8 @@ func NewServerInspector(scriptDir string) ServerInspector { "rpm -qa --qf '%{VERSION}\\n' 'name=postgresql[0-8][0-9]-server' | cut -d. -f1 | sort -n | tail -1 || true"), types.NewInspectData("current_pg_version", "(test -e /var/lib/pgsql/data/PG_VERSION && cat /var/lib/pgsql/data/PG_VERSION) || true"), + types.NewInspectData("current_pg_version_not_migrated", + "(test -e /var/lib/pgsql/data/data/PG_VERSION && cat /var/lib/pgsql/data/data/PG_VERSION) || true"), types.NewInspectData("db_user", "cat /etc/rhn/rhn.conf 2>/dev/null | grep '^db_user' | cut -d' ' -f3 || true"), types.NewInspectData("db_password", @@ -41,6 +43,10 @@ func NewServerInspector(scriptDir string) ServerInspector { "cat /etc/rhn/rhn.conf 2>/dev/null | grep '^db_name' | cut -d' ' -f3 || true"), types.NewInspectData("db_port", "cat /etc/rhn/rhn.conf 2>/dev/null | grep '^db_port' | cut -d' ' -f3 || true"), + types.NewInspectData("db_host", + "cat /etc/rhn/rhn.conf 2>/dev/null | grep '^db_host' | cut -d' ' -f3 || true"), + types.NewInspectData("report_db_host", + "cat /etc/rhn/rhn.conf 2>/dev/null | grep '^report_db_host' | cut -d' ' -f3 || true"), }, ScriptDir: scriptDir, DataPath: path.Join(InspectContainerDirectory, inspectDataFile), @@ -52,12 +58,15 @@ func NewServerInspector(scriptDir string) ServerInspector { // CommonInspectData are data common between the migration source inspect and server inspector results. type CommonInspectData struct { - CurrentPgVersion string `mapstructure:"current_pg_version"` - ImagePgVersion string `mapstructure:"image_pg_version"` - DBUser string `mapstructure:"db_user"` - DBPassword string `mapstructure:"db_password"` - DBName string `mapstructure:"db_name"` - DBPort int `mapstructure:"db_port"` + CurrentPgVersion string `mapstructure:"current_pg_version"` + CurrentPgVersionNotMigrated string `mapstructure:"current_pg_version_not_migrated"` + ImagePgVersion string `mapstructure:"image_pg_version"` + DBUser string `mapstructure:"db_user"` + DBPassword string `mapstructure:"db_password"` + DBName string `mapstructure:"db_name"` + DBPort int `mapstructure:"db_port"` + DBHost string `mapstructure:"db_host"` + ReportDBHost string `mapstructure:"report_db_host"` } // ServerInspectData are the data extracted by a server inspector. diff --git a/shared/utils/volumes.go b/shared/utils/volumes.go index 350a384db..f83857684 100644 --- a/shared/utils/volumes.go +++ b/shared/utils/volumes.go @@ -12,10 +12,13 @@ var PgsqlRequiredSharedVolumeMounts = []types.VolumeMount{ {MountPath: "/etc/pki/spacewalk-tls", Name: "tls-key"}, } -// PgsqlRequiredVolumeMounts represents volumes mount used by PostgreSQL. -var PgsqlRequiredVolumeMounts = append([]types.VolumeMount{ +// PgsqlDataVolumeMounts represents volumes mount used by PostgreSQL. +var PgsqlDataVolumeMounts = []types.VolumeMount{ {MountPath: "/var/lib/pgsql/data", Name: "var-pgsql"}, -}, PgsqlRequiredSharedVolumeMounts[:]...) +} + +// PgsqlRequiredVolumeMounts represents volumes mount used by PostgreSQL. +var PgsqlRequiredVolumeMounts = append(PgsqlDataVolumeMounts, PgsqlRequiredSharedVolumeMounts[:]...) // PgsqlRequiredSharedVolumes represents volumes used by PostgreSQL. var PgsqlRequiredSharedVolumes = []types.Volume{ @@ -30,10 +33,13 @@ var PgsqlRequiredSharedVolumes = []types.Volume{ }, } -// PgsqlRequiredVolumes represents volumes used by PostgreSQL. -var PgsqlRequiredVolumes = append([]types.Volume{ +// PgsqlDataVolumes represents volumes used by PostgreSQL. +var PgsqlDataVolumes = []types.Volume{ {Name: "var-pgsql", PersistentVolumeClaim: &types.PersistentVolumeClaim{ClaimName: "var-pgsql"}}, -}, PgsqlRequiredSharedVolumes[:]...) +} + +// PgsqlRequiredVolumes represents volumes used by PostgreSQL. +var PgsqlRequiredVolumes = append(PgsqlDataVolumes, PgsqlRequiredSharedVolumes[:]...) // EtcServerVolumeMounts represents volumes mounted in /etc folder. var EtcServerVolumeMounts = []types.VolumeMount{ @@ -104,6 +110,12 @@ var ServerVolumes = append([]types.Volume{ {Name: "ca-cert", PersistentVolumeClaim: &types.PersistentVolumeClaim{ClaimName: "ca-cert"}}, }, etcAndPgsqlVolumes[:]...) +// ServerMigrationVolumeMounts match server + postgres volume mounts, used for migration. +var ServerMigrationVolumeMounts = append(ServerVolumeMounts, PgsqlDataVolumeMounts[:]...) + +// ServerMigrationVolumes match server + postgres volumes, used for migration. +var ServerMigrationVolumes = append(ServerVolumes, PgsqlDataVolumes[:]...) + // HubXmlrpcVolumeMounts represents volumes used by Hub Xmlrpc container. var HubXmlrpcVolumeMounts = []types.VolumeMount{ {MountPath: "/etc/pki/trust/anchors", Name: "ca-cert"},