From 6b98073c53495d13f8bbafa06e293c21e6fef5d7 Mon Sep 17 00:00:00 2001 From: Kjartan Einarsson Date: Wed, 7 Aug 2024 11:57:19 -0700 Subject: [PATCH 01/12] initial setup for backup containers --- containers/backup/.gitattributes | 12 + containers/backup/.gitignore | 20 + containers/backup/LICENSE | 201 ++++++ containers/backup/README.md | 624 ++++++++++++++++++ containers/backup/backup.conf | 51 ++ containers/backup/config/backup.conf | 51 ++ containers/backup/docker/Dockerfile | 43 ++ containers/backup/docker/Dockerfile_MSSQL | 46 ++ containers/backup/docker/Dockerfile_MariaDB | 37 ++ containers/backup/docker/Dockerfile_Mongo | 45 ++ containers/backup/docker/backup.config.utils | 501 ++++++++++++++ .../backup/docker/backup.container.utils | 82 +++ containers/backup/docker/backup.file.utils | 245 +++++++ containers/backup/docker/backup.ftp | 23 + containers/backup/docker/backup.logging | 128 ++++ .../backup/docker/backup.mariadb.plugin | 218 ++++++ containers/backup/docker/backup.misc.utils | 53 ++ containers/backup/docker/backup.mongo.plugin | 299 +++++++++ containers/backup/docker/backup.mssql.plugin | 229 +++++++ containers/backup/docker/backup.null.plugin | 208 ++++++ .../backup/docker/backup.postgres.plugin | 276 ++++++++ containers/backup/docker/backup.s3 | 21 + containers/backup/docker/backup.server.utils | 39 ++ containers/backup/docker/backup.settings | 55 ++ containers/backup/docker/backup.sh | 144 ++++ containers/backup/docker/backup.usage | 138 ++++ containers/backup/docker/backup.utils | 289 ++++++++ containers/backup/docker/uid_entrypoint | 7 + .../backup-cronjob/backup-cronjob.yaml | 253 +++++++ .../backup/templates/backup/backup-build.yaml | 74 +++ .../templates/backup/backup-deploy.yaml | 447 +++++++++++++ containers/backup/templates/nsp/build.yaml | 50 ++ containers/backup/templates/nsp/deploy.yaml | 50 ++ 33 files changed, 4959 insertions(+) create mode 100644 containers/backup/.gitattributes create mode 100644 containers/backup/.gitignore create mode 100644 containers/backup/LICENSE create mode 100644 containers/backup/README.md create mode 100644 containers/backup/backup.conf create mode 100644 containers/backup/config/backup.conf create mode 100644 containers/backup/docker/Dockerfile create mode 100644 containers/backup/docker/Dockerfile_MSSQL create mode 100644 containers/backup/docker/Dockerfile_MariaDB create mode 100644 containers/backup/docker/Dockerfile_Mongo create mode 100644 containers/backup/docker/backup.config.utils create mode 100644 containers/backup/docker/backup.container.utils create mode 100644 containers/backup/docker/backup.file.utils create mode 100644 containers/backup/docker/backup.ftp create mode 100644 containers/backup/docker/backup.logging create mode 100644 containers/backup/docker/backup.mariadb.plugin create mode 100644 containers/backup/docker/backup.misc.utils create mode 100644 containers/backup/docker/backup.mongo.plugin create mode 100644 containers/backup/docker/backup.mssql.plugin create mode 100644 containers/backup/docker/backup.null.plugin create mode 100644 containers/backup/docker/backup.postgres.plugin create mode 100644 containers/backup/docker/backup.s3 create mode 100644 containers/backup/docker/backup.server.utils create mode 100644 containers/backup/docker/backup.settings create mode 100755 containers/backup/docker/backup.sh create mode 100644 containers/backup/docker/backup.usage create mode 100644 containers/backup/docker/backup.utils create mode 100644 containers/backup/docker/uid_entrypoint create mode 100644 containers/backup/templates/backup-cronjob/backup-cronjob.yaml create mode 100644 containers/backup/templates/backup/backup-build.yaml create mode 100644 containers/backup/templates/backup/backup-deploy.yaml create mode 100644 containers/backup/templates/nsp/build.yaml create mode 100644 containers/backup/templates/nsp/deploy.yaml diff --git a/containers/backup/.gitattributes b/containers/backup/.gitattributes new file mode 100644 index 00000000..a295ec35 --- /dev/null +++ b/containers/backup/.gitattributes @@ -0,0 +1,12 @@ +# Set the default behavior, in case people don't have core.autocrlf set. +* text=auto + +# Declare files that will always have LF line endings on checkout. +backup.* text eol=lf +*.sh text eol=lf +*.md text eol=lf +*.json text eol=lf +*.conf text eol=lf +**/s2i/bin/* text eol=lf +**/root/**/* text eol=lf +**/.scripts/* text eol=lf \ No newline at end of file diff --git a/containers/backup/.gitignore b/containers/backup/.gitignore new file mode 100644 index 00000000..24669f53 --- /dev/null +++ b/containers/backup/.gitignore @@ -0,0 +1,20 @@ +# See http://help.github.com/ignore-files/ for more about ignoring files. +.DS_Store + +# Files created by the scripts from; https://github.com/BCDevOps/openshift-project-tools +*_DeploymentConfig.json +*_BuildConfig.json +*.local.* +*.overrides.* +*.param +settings*.sh + +# Visual Studio Code +.vscode + +# Local config +.env +docker/backup.conf +backups +minio-data +pg-data diff --git a/containers/backup/LICENSE b/containers/backup/LICENSE new file mode 100644 index 00000000..261eeb9e --- /dev/null +++ b/containers/backup/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/containers/backup/README.md b/containers/backup/README.md new file mode 100644 index 00000000..3743699a --- /dev/null +++ b/containers/backup/README.md @@ -0,0 +1,624 @@ +--- +title: Backup Container +description: A simple containerized backup solution for backing up one or more supported databases to a secondary location. +author: WadeBarnes +resourceType: Components +personas: + - Developer + - Product Owner + - Designer +labels: + - backup + - backups + - postgres + - mongo + - mssql + - database +--- + +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](LICENSE) + +_Table of Contents_ + + + +- [Backup Container](#backup-container) + - [Supported Databases](#supported-databases) +- [Backup Container Options](#backup-container-options) + - [Backups in OpenShift](#backups-in-openshift) + - [Storage](#storage) + - [Backup Storage Volume](#backup-storage-volume) + - [NFS Storage Backup and Retention Policy](#nfs-storage-backup-and-retention-policy) + - [Restore/Verification Storage Volume](#restoreverification-storage-volume) + - [Storage Performance](#storage-performance) + - [Deployment / Configuration](#deployment--configuration) + - [backup.conf](#backupconf) + - [Cron Mode](#cron-mode) + - [Cronjob Deployment / Configuration / Constraints](#cronjob-deployment--configuration--constraints) + - [Resources](#resources) + - [Multiple Databases](#multiple-databases) + - [Backup Strategies](#backup-strategies) + - [Daily](#daily) + - [Rolling](#rolling) + - [Using the Backup Script](#using-the-backup-script) + - [Using Backup Verification](#using-backup-verification) + - [Using the FTP backup](#using-the-ftp-backup) + - [Using the Webhook Integration](#using-the-webhook-integration) + - [Database Plugin Support](#database-plugin-support) + - [Backup](#backup) + - [Immediate Backup:](#immediate-backup) + - [Execute a single backup cycle with the pod deployment](#execute-a-single-backup-cycle-with-the-pod-deployment) + - [Execute an on demand backup using the scheduled job](#execute-an-on-demand-backup-using-the-scheduled-job) + - [Restore](#restore) + - [Network Policies](#network-policies) +- [Example Deployments](#example-deployments) + - [Deploy with Helm Chart](#deploy-with-helm-chart) +- [Prebuilt Container Images](#prebuilt-container-images) +- [Postgres Base Version](#postgres-base-version) +- [Tip and Tricks](#tip-and-tricks) +- [Getting Help or Reporting an Issue](#getting-help-or-reporting-an-issue) +- [How to Contribute](#how-to-contribute) + + + +# Introduction + +This backup system is a straightforward containerized solution designed to back up one or more supported databases to a secondary location. +## Supported Databases & Secondary Locations + +### Databases + +- PostgresSQL +- MongoDB +- MariaDB +- MSSQL - Currently MSSQL requires that the nfs db volume be shared with the database for backups to function correctly. + +### Secondary Locations + +- OCIO backup infrastructure +- Amazon S3 (S3 Compatible / OCIO Object Store) +- FTP Server + +# Backup Container Options + +You have the option to run the Backup Container for supported databases either separately or in a mixed environment. If you choose the mixed environment, please follow these guidelines: + +1. It is required to use the recommended `backup.conf` configuration. +2. Within the `backup.conf` file, make sure to specify the `DatabaseType` for each listed database. +3. For each type of supported backup container being used, you will need to create a build and deployment configuration. +4. Make sure to mount the same `backup.conf` file (ConfigMap) to each deployed container. + +These steps will help ensure the smooth operation of the backup system. + +## Backups in OpenShift + +This project provides you with a starting point for integrating backups into your OpenShift projects. The scripts and templates provided in the [openshift](./openshift) directory are compatible with the [openshift-developer-tools](https://github.com/BCDevOps/openshift-developer-tools) scripts. They help you create an OpenShift deployment or cronjob called `backup` in your projects that runs backups on databases within the project environment. You only need to integrate the scripts and templates into your project(s), the builds can be done with this repository as the source. + +As an alternative to using the command line interface `oc` ([OpenShift CLI](https://access.redhat.com/documentation/en-us/openshift_container_platform/4.12/html-single/cli_tools/index)), you can integrate the backup configurations (Build and Deployment templates, override script, and config) directly into your project configuration and manage the publishing and updating of the Build and Deployment configurations using the [BCDevOps/openshift-developer-tools](https://github.com/BCDevOps/openshift-developer-tools/tree/master/bin) scripts. An example can be found in the [bcgov/orgbook-configurations](https://github.com/bcgov/orgbook-configurations) repository under the [backup templates folder](https://github.com/bcgov/orgbook-configurations/tree/master/openshift/templates/backup). + +Simplified documentation on how to use the tools can be found [here](https://github.com/bcgov/jag-cullencommission/tree/master/openshift). All scripts support a `-c` option that allows you to perform operations on a single component of your application such as the backup container. In the orgbook-configurations example above, note the `-c backup` argument supplied. + +Following are the instructions for running the backups and a restore. + +## Storage + +The backup container utilizes two volumes: one for storing the backups and another for restore/verification testing. The deployment template deliberately separates these volumes. + +The upcoming sections on storage will provide you with recommendations and limitations regarding the storage classes. + +### Backup Storage Volume + +We recommend using the `netapp-file-backup` storage class for the backup Persistent Volume Claim (PVC). This storage class is supported by the standard OCIO backup infrastructure and has a default quota of 25Gi. If you require additional storage, please submit an iStore request to adjust the quota accordingly. The backup retention policy for the backup infrastructure is as follows: + +- Backup: + - Daily: Incremental + - Monthly: Full +- Retention: 90 days + +If you are utilizing S3 storage or the corporate S3 compatible storage, you may not need to use the `netapp-file-backup` storage class. These systems are already replicated and highly redundant. In such cases, we recommend using the following storage classes: +- `netapp-file-standard` for backup storage +- `netapp-file-standard` for restore/verification storage + +To implement this, create a PVC using the the appropriate storage class and mount it to your pod at the `/backups` mount point. Or, if you're using the provided deployment template, update or override the `BACKUP_VOLUME_STORAGE_CLASS` parameter. + +For more detailed information, please visit the [DevHub](https://developer.gov.bc.ca/OCP4-Backup-and-Restore) page. + +### Restore / Verification Storage Volume + +The restore/verification volume should use the default storage class `netapp-file-standard`. Please avoid using `netapp-file-backup` as it is not suitable for transient workloads. The provided deployment template will automatically provision this volume when it is published. + +Ensure that the volume is large enough to accommodate your largest database. You can set the size by updating or overriding the `VERIFICATION_VOLUME_SIZE` parameter in the provided OpenShift template. + +### Storage Performance + +Our PVC are supported by NetApp storage. It's important to note that the performance of the storage is not affected by the storage class chosen. +## Deployment / Configuration + +Together, the scripts and templates provided in the [openshift](./openshift) directory will automatically deploy the `backup` app as described below. The [backup-deploy.overrides.sh](./openshift/backup-deploy.overrides.sh) script generates the deployment configuration necessary for the [backup.conf](config/backup.conf) file to be mounted as a ConfigMap by the `backup` container. + +The following environment variables are defaults used by the `backup` app. + +**NOTE**: These environment variables MUST MATCH those used by the database container(s) you are planning to backup. + +| Name | Default (if not set) | Purpose | +| -------------------------- | -------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| BACKUP_STRATEGY | rolling | To control the backup strategy used for backups. This is explained more below. | +| BACKUP_DIR | /backups/ | The directory under which backups will be stored. The deployment configuration mounts the persistent volume claim to this location when first deployed. | +| NUM_BACKUPS | 31 | Used for backward compatibility only, this value is used with the daily backup strategy to set the number of backups to retain before pruning. | +| DAILY_BACKUPS | 6 | When using the rolling backup strategy this value is used to determine the number of daily (Mon-Sat) backups to retain before pruning. | +| WEEKLY_BACKUPS | 4 | When using the rolling backup strategy this value is used to determine the number of weekly (Sun) backups to retain before pruning. | +| MONTHLY_BACKUPS | 1 | When using the rolling backup strategy this value is used to determine the number of monthly (last day of the month) backups to retain before pruning. | +| BACKUP_PERIOD | 1d | Only used for Legacy Mode. Ignored when running in Cron Mode. The schedule on which to run the backups. The value is used by a sleep command and can be defined in d, h, m, or s. | +| DATABASE_SERVICE_NAME | postgresql | Used for backward compatibility only. The name of the service/host for the _default_ database target. | +| DATABASE_USER_KEY_NAME | database-user | The database user key name stored in database deployment resources specified by DATABASE_DEPLOYMENT_NAME. | +| DATABASE_PASSWORD_KEY_NAME | database-password | The database password key name stored in database deployment resources specified by DATABASE_DEPLOYMENT_NAME. | +| DATABASE_NAME | my_postgres_db | Used for backward compatibility only. The name of the _default_ database target; the name of the database you want to backup. | +| DATABASE_USER | _wired to a secret_ | The username for the database(s) hosted by the database server. The deployment configuration makes the assumption you have your database credentials stored in secrets (which you should), and the key for the username is `database-user`. The name of the secret must be provided as the `DATABASE_DEPLOYMENT_NAME` parameter to the deployment configuration template. | +| DATABASE_PASSWORD | _wired to a secret_ | The password for the database(s) hosted by the database server. The deployment configuration makes the assumption you have your database credentials stored in secrets (which you should), and the key for the username is `database-password`. The name of the secret must be provided as the `DATABASE_DEPLOYMENT_NAME` parameter to the deployment configuration template. | +| FTP_URL | | The FTP server URL. If not specified, the FTP backup feature is disabled. The default value in the deployment configuration is an empty value - not specified. | +| FTP_USER | _wired to a secret_ | The username for the FTP server. The deployment configuration creates a secret with the name specified in the FTP_SECRET_KEY parameter (default: `ftp-secret`). The key for the username is `ftp-user` and the value is an empty value by default. | +| FTP_PASSWORD | _wired to a secret_ | The password for the FTP server. The deployment configuration creates a secret with the name specified in the FTP_SECRET_KEY parameter (default: `ftp-secret`). The key for the password is `ftp-password` and the value is an empty value by default. | +| S3_USER | No Default | The username for the S3 compatible object store. This may also be referred to as the "Access key" in AWS S3. | +| S3_PASSWORD | No Default | The password for the S3 compatible object store. This may also be referred to as the "Secret key" in AWS. | +| S3_ENDPOINT | None | The AWS endpoint to use for S3 compatible object storage. For OpenShift minio use `http://minio-service:9000` | +| S3_BUCKET | None | The bucket where you backups will be transferd to. | +| PGDUTY_SVC_KEY | | PagerDuty service integration key. | +| PGDUTY_URL | | PagerDuty events API url, the default url (the default url is https://events.pagerduty.com/generic/2010-04-15/create_event.json) | +| WEBHOOK_URL | | The URL of the webhook endpoint to use for notifications. If not specified, the webhook integration feature is disabled. The default value in the deployment configuration is an empty value - not specified. | +| ENVIRONMENT_FRIENDLY_NAME | | A friendly (human readable) name of the environment. This variable is used by the webhook integration to identify the environment from which the backup notifications originate. The default value in the deployment configuration is an empty value - not specified. | +| ENVIRONMENT_NAME | | A name or ID of the environment. This variable is used by the webhook integration to identify the environment from which the backup notifications originate. The default value in the deployment configuration is an empty value - not specified. | + +### backup.conf + +Using this default configuration you can easily back up a single postgres database, however we recommend you extend the configuration and use the `backup.conf` file to list a number of databases for backup and even set a cron schedule for the backups. + +When using the `backup.conf` file the following environment variables are ignored, since you list all of your `host`/`database` pairs in the file; `DATABASE_SERVICE_NAME`, `DATABASE_NAME`. To provide the credentials needed for the listed databases you extend the deployment configuration to include `hostname_USER` and `hostname_PASSWORD` credential pairs which are wired to the appropriate secrets (where hostname matches the hostname/servicename, in all caps and underscores, of the database). For example, if you are backing up a database named `wallet-db/my_wallet`, you would have to extend the deployment configuration to include a `WALLET_DB_USER` and `WALLET_DB_PASSWORD` credential pair, wired to the appropriate secrets, to access the database(s) on the `wallet-db` server. + +### Cron Mode + +The `backup` container supports running the backups on a cron schedule. The schedule is specified in the `backup.conf` file. Refer to the [backup.conf](./config/backup.conf) file for additional details and examples. + +### Cronjob Deployment / Configuration / Constraints + +_This section describes the configuration of an OpenShift CronJob this is different than the Cron Mode supported by the container when deployed in "long running" mode._ + +The cronjob object can be deployed in the same manner as the application, and will also have a dependency on the image built by the build config. The main constraint for the cronjob objects is that they will require a configmap in place of environment variables and does not support the `backup.conf` for multiple database backups in the same job. In order to backup multiple databases, create multiple cronjob objects with their associated configmaps and secrets. + +The following variables are supported in the first iteration of the backup cronjob: + +| Name | Default (if not set) | Purpose | +| -------------------------- | -------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| BACKUP_STRATEGY | daily | To control the backup strategy used for backups. This is explained more below. | +| BACKUP_DIR | /backups/ | The directory under which backups will be stored. The deployment configuration mounts the persistent volume claim to this location when first deployed. | +| SCHEDULE | 0 1 \* \* \* | Cron Schedule to Execute the Job (using local cluster system TZ). | +| NUM_BACKUPS | 31 | For backward compatibility this value is used with the daily backup strategy to set the number of backups to retain before pruning. | +| DAILY_BACKUPS | 6 | When using the rolling backup strategy this value is used to determine the number of daily (Mon-Sat) backups to retain before pruning. | +| WEEKLY_BACKUPS | 4 | When using the rolling backup strategy this value is used to determine the number of weekly (Sun) backups to retain before pruning. | +| MONTHLY_BACKUPS | 1 | When using the rolling backup strategy this value is used to determine the number of monthly (last day of the month) backups to retain before pruning. | +| DATABASE_SERVICE_NAME | postgresql | The name of the service/host for the _default_ database target. | +| DATABASE_USER_KEY_NAME | database-user | The database user key name stored in database deployment resources specified by DATABASE_DEPLOYMENT_NAME. | +| DATABASE_PASSWORD_KEY_NAME | database-password | The database password key name stored in database deployment resources specified by DATABASE_DEPLOYMENT_NAME. | +| POSTGRESQL_DATABASE | my_postgres_db | The name of the _default_ database target; the name of the database you want to backup. | +| POSTGRESQL_USER | _wired to a secret_ | The username for the database(s) hosted by the `postgresql` Postgres server. The deployment configuration makes the assumption you have your database credentials stored in secrets (which you should), and the key for the username is `database-user`. The name of the secret must be provided as the `DATABASE_DEPLOYMENT_NAME` parameter to the deployment configuration template. | +| POSTGRESQL_PASSWORD | _wired to a secret_ | The password for the database(s) hosted by the `postgresql` Postgres server. The deployment configuration makes the assumption you have your database credentials stored in secrets (which you should), and the key for the username is `database-password`. The name of the secret must be provided as the `DATABASE_DEPLOYMENT_NAME` parameter to the deployment configuration template. | + +The following variables are NOT supported: + +| Name | Default (if not set) | Purpose | +| ------------- | -------------------- | -------------------------------------------------------------------------------------------------------- | +| BACKUP_PERIOD | 1d | The schedule on which to run the backups. The value is replaced by the cron schedule variable (SCHEDULE) | + +The scheduled job does not yet support the FTP environment variables. + +| Name | +| ------------ | +| FTP_URL | +| FTP_USER | +| FTP_PASSWORD | + +### Resources + +The backup-container is assigned with `Best-effort` resource type (setting zero for request and limit), which allows the resources to scale up and down without an explicit limit as resource on the node allow. It benefits from large bursts of recourses for short periods of time to get things more quickly. After some time of running the backup-container, you could then set the request and limit according to the average resource consumption. + +## Multiple Databases + +When backing up multiple databases, the retention settings apply to each database individually. For instance if you use the `daily` strategy and set the retention number(s) to 5, you will retain 5 copies of each database. So plan your backup storage accordingly. + +An example of the backup container in action can be found here; [example log output](./docs/ExampleLog.md) + +## Backup Strategies + +The `backup` app supports two backup strategies, each are explained below. Regardless of the strategy backups are identified using a core name derived from the `host/database` specification and a timestamp. All backups are compressed using gzip. + +### Daily + +The daily backup strategy is very simple. Backups are created in dated folders under the top level `/backups/` folder. When the maximum number of backups (`NUM_BACKUPS`) is exceeded, the oldest ones are pruned from disk. + +For example (faked): + +``` +================================================================================================================================ +Current Backups: +-------------------------------------------------------------------------------------------------------------------------------- +1.0K 2018-10-03 22:16 ./backups/2018-10-03/postgresql-TheOrgBook_Database_2018-10-03_22-16-11.sql.gz +1.0K 2018-10-03 22:16 ./backups/2018-10-03/postgresql-TheOrgBook_Database_2018-10-03_22-16-28.sql.gz +1.0K 2018-10-03 22:16 ./backups/2018-10-03/postgresql-TheOrgBook_Database_2018-10-03_22-16-46.sql.gz +1.0K 2018-10-03 22:16 ./backups/2018-10-03/wallet-db-tob_holder_2018-10-03_22-16-13.sql.gz +1.0K 2018-10-03 22:16 ./backups/2018-10-03/wallet-db-tob_holder_2018-10-03_22-16-31.sql.gz +1.0K 2018-10-03 22:16 ./backups/2018-10-03/wallet-db-tob_holder_2018-10-03_22-16-48.sql.gz +1.0K 2018-10-03 22:16 ./backups/2018-10-03/wallet-db-tob_verifier_2018-10-03_22-16-08.sql.gz +1.0K 2018-10-03 22:16 ./backups/2018-10-03/wallet-db-tob_verifier_2018-10-03_22-16-25.sql.gz +1.0K 2018-10-03 22:16 ./backups/2018-10-03/wallet-db-tob_verifier_2018-10-03_22-16-43.sql.gz +13K 2018-10-03 22:16 ./backups/2018-10-03 +... +61K 2018-10-04 10:43 ./backups/ +================================================================================================================================ +``` + +### Rolling + +The rolling backup strategy provides a bit more flexibility. It allows you to keep a number of recent `daily` backups, a number of `weekly` backups, and a number of `monthly` backups. + +- Daily backups are any backups done Monday through Saturday. +- Weekly backups are any backups done at the end of the week, which we're calling Sunday. +- Monthly backups are any backups done on the last day of a month. + +There are retention settings you can set for each. The defaults provide you with a week's worth of `daily` backups, a month's worth of `weekly` backups, and a single backup for the previous month. + +Although the example does not show any `weekly` or `monthly` backups, you can see from the example that the folders are further broken down into the backup type. + +For example (faked): + +``` +================================================================================================================================ +Current Backups: +-------------------------------------------------------------------------------------------------------------------------------- +0 2018-10-03 22:16 ./backups/daily/2018-10-03 +1.0K 2018-10-04 09:29 ./backups/daily/2018-10-04/postgresql-TheOrgBook_Database_2018-10-04_09-29-52.sql.gz +1.0K 2018-10-04 10:37 ./backups/daily/2018-10-04/postgresql-TheOrgBook_Database_2018-10-04_10-37-15.sql.gz +1.0K 2018-10-04 09:29 ./backups/daily/2018-10-04/wallet-db-tob_holder_2018-10-04_09-29-55.sql.gz +1.0K 2018-10-04 10:37 ./backups/daily/2018-10-04/wallet-db-tob_holder_2018-10-04_10-37-18.sql.gz +1.0K 2018-10-04 09:29 ./backups/daily/2018-10-04/wallet-db-tob_verifier_2018-10-04_09-29-49.sql.gz +1.0K 2018-10-04 10:37 ./backups/daily/2018-10-04/wallet-db-tob_verifier_2018-10-04_10-37-12.sql.gz +22K 2018-10-04 10:43 ./backups/daily/2018-10-04 +22K 2018-10-04 10:43 ./backups/daily +4.0K 2018-10-03 22:16 ./backups/monthly/2018-10-03 +4.0K 2018-10-03 22:16 ./backups/monthly +4.0K 2018-10-03 22:16 ./backups/weekly/2018-10-03 +4.0K 2018-10-03 22:16 ./backups/weekly +61K 2018-10-04 10:43 ./backups/ +================================================================================================================================ +``` + +## Using the Backup Script + +The [backup script](./docker/backup.sh) has a few utility features built into it. For a full list of features and documentation run `backup.sh -h`. + +Features include: + +- The ability to list the existing backups, `backup.sh -l` +- Listing the current configuration, `backup.sh -c` +- Running a single backup cycle, `backup.sh -1` +- Restoring a database from backup, `backup.sh -r [-f ]` + - Restore mode will allow you to restore a database to a different location (host, and/or database name) provided it can contact the host and you can provide the appropriate credentials. +- Verifying backups, `backup.sh [-s] -v [-f ]` + - Verify mode will restore a backup to the local server to ensure it can be restored without error. Once restored a table query is performed to ensure there was at least one table restored and queries against the database succeed without error. All database files and configuration are destroyed following the tests. + +## Using Backup Verification + +The [backup script](./docker/backup.sh) supports running manual or scheduled verifications on your backups; `backup.sh [-s] -v [-f ]`. Refer to the script documentation `backup.sh -h`, and the configuration documentation, [backup.conf](config/backup.conf), for additional details on how to use this feature. + +## Using the FTP backup + +- The FTP backup feature is enabled by specifying the FTP server URL `FTP_URL`. +- The FTP server must support FTPS. +- Path can be added to the URL. For example, the URL can be `ftp://ftp.gov.bc.ca/schoolbus-db-backup/`. Note that when adding path, the URL must be ended with `/` as the example. +- The username and password must be populated in the secret key. Refer to the deployment configuration section. +- There is a known issue for FTPS with Windows 2012 FTP. http://redoubtsolutions.com/fix-the-supplied-message-is-incomplete-error-when-you-use-an-ftps-client-to-upload-a-file-in-windows/ + +## Using the Webhook Integration + +The Webhook integration feature is enabled by specifying the webhook URL, `WEBHOOK_URL`, in your configuration. It's recommended that you also provide values for `ENVIRONMENT_FRIENDLY_NAME` and `ENVIRONMENT_NAME`, so you can better identify the environment from which the messages originate and do things like produce links to the environment. + +The Webhook integration feature was built with Rocket.Chat in mind and an integration script for Rocket.Chat can be found in [rocket.chat.integration.js](./scripts/rocket.chat.integration.js). This script was developed to support the BC OpenShift Pathfinder environment and will format the notifications from the backup script into Rocket.Chat messages (examples below). If you provide values for the environment name (`ENVIRONMENT_FRIENDLY_NAME` and `ENVIRONMENT_NAME`) hyperlinks will be added to the messages to link you to the pathfinder project console. + +Sample Message: + +![Sample Message](./docs/SampleRocketChatMessage.png) + +Sample Error Message: + +![Sample Erros Message](./docs/SampleRocketChatErrorMessage.png) + +For information on how setup a webhook in Rocket.Chat refer to [Incoming WebHook Scripting](https://rocket.chat/docs/administrator-guides/integrations/). The **Webhook URL** created during this process is the URL you use for `WEBHOOK_URL` to enable the Webhook integration feature. + +## Database Plugin Support + +The backup container uses a plugin architecture to perform the database specific operations needed to support various database types. + +The plugins are loaded dynamically based on the container type. By default the `backup.null.plugin` will be loaded when the container type is not recognized. + +To add support for a new database type: + +1. Update the `getContainerType` function in [backup.container.utils](./docker/backup.container.utils) to detect the new type of database. +2. Using the existing plugins as reference, implement the database specific scripts for the new database type. +3. Using the existing docker files as reference, create a new one to build the new container type. +4. Update the build and deployment templates and their documentation as needed. +5. Update the project documentation as needed. +6. Test, test, test. +7. Submit a PR. + +Plugin Examples: + +- [backup.postgres.plugin](./docker/backup.postgres.plugin) + + - Postgres backup implementation. + +- [backup.mongo.plugin](./docker/backup.mongo.plugin) + + - Mongo backup implementation. + +- [backup.mssql.plugin](./docker/backup.mssql.plugin) + + - MSSQL backup implementation. + +- [backup.mariadb.plugin](./docker/backup.mariadb.plugin) + + - MariaDB backup implementation. This plugin should also work with mysql, but is currently untested. + +- [backup.null.plugin](./docker/backup.null.plugin) + - Sample/Template backup implementation that simply outputs log messages for the various operations. + +## Backup + +_The following sections describes (some) postgres specific implementation, however the steps are generally the same between database implementations._ + +The purpose of the backup app is to do automatic backups. Deploy the Backup app to do daily backups. Viewing the Logs for the Backup App will show a record of backups that have been completed. + +The Backup app performs the following sequence of operations: + +1. Create a directory that will be used to store the backup. +2. Use the `pg_dump` and `gzip` commands to make a backup. +3. Cull backups more than $NUM_BACKUPS (default 31 - configured in deployment script) +4. Wait/Sleep for a period of time and repeat + +Note that with the pod deployment, we support cron schedule(s) or the legacy mode (which uses a simple "sleep") to run the backup periodically. With the OpenShift Scheduled Job deployment, use the backup-cronjob.yaml template and set the schedule via the OpenShift cronjob object SCHEDULE template parameter. + +A separate pod is used vs. having the backups run from the Postgres Pod for fault tolerant purposes - to keep the backups separate from the database storage. We don't want to, for example, lose the storage of the database, or have the database and backups storage fill up, and lose both the database and the backups. + +### Immediate Backup: + +#### Execute a single backup cycle with the pod deployment + +- Check the logs of the Backup pod to make sure a backup isn't run right now (pretty unlikely...) +- Open a terminal window to the pod +- Run `backup.sh -1` + - This will run a single backup cycle and exit. + +#### Execute an on demand backup using the scheduled job + +- Run the following: `oc create job ${SOMEJOBNAME} --from=cronjob/${BACKUP_CRONJOB_NAME}` + - example: `oc create job my-backup-1 --from=cronjob/backup-postgresql` + - this will run a single backup job and exit. + - note: the jobs created in this manner are NOT cleaned up by the scheduler like the automated jobs are. + +### Restore + +The `backup.sh` script's restore mode makes it very simple to restore the most recent backup of a particular database. It's as simple as running a the following command, for example (run `backup.sh -h` for full details on additional options); + + backup.sh -r postgresql/TheOrgBook_Database + +Following are more detailed steps to perform a restore of a backup. + +1. Log into the OpenShift Console and log into OpenShift on the command shell window. + 1. The instructions here use a mix of the console and command line, but all could be done from a command shell using "oc" commands. +1. Scale to 0 all Apps that use the database connection. + 1. This is necessary as the Apps will need to restart to pull data from the restored backup. + 1. It is recommended that you also scale down to 0 your client application so that users know the application is unavailable while the database restore is underway. + 1. A nice addition to this would be a user-friendly "This application is offline" message - not yet implemented. +1. Restart the database pod as a quick way of closing any other database connections from users using port forward or that have rsh'd to directly connect to the database. +1. Open an rsh into the backup pod: + 1. Open a command prompt connection to OpenShift using `oc login` with parameters appropriate for your OpenShift host. + 1. Change to the OpenShift project containing the Backup App `oc project ` + 1. List pods using `oc get pods` + 1. Open a remote shell connection to the **backup** pod. `oc rsh ` +1. In the rsh run the backup script in restore mode, `./backup.sh -r `, to restore the desired backup file. For full information on how to use restore mode, refer to the script documentation, `./backup.sh -h`. Have the Admin password for the database handy, the script will ask for it during the restore process. + 1. The restore script will automatically grant the database user access to the restored database. If there are other users needing access to the database, such as the DBA group, you will need to additionally run the following commands on the database pod itself using `psql`: + 1. Get a list of the users by running the command `\du` + 1. For each user that is not "postgres" and $POSTGRESQL_USER, execute the command `GRANT SELECT ON ALL TABLES IN SCHEMA public TO "";` + 1. If users have been set up with other grants, set them up as well. +1. Verify that the database restore worked + 1. On the database pod, query a table - e.g the USER table: `SELECT * FROM "SBI_USER";` - you can look at other tables if you want. + 1. Verify the expected data is shown. +1. Exit remote shells back to your local command line +1. From the Openshift Console restart the app: + 1. Scale up any pods you scaled down and wait for them to finish starting up. View the logs to verify there were no startup issues. +1. Verify full application functionality. + +Done! + +## Network Policies + +The default `backup-container` template contains a basic Network Policy that is designed to be functioning out-of-the-box for most standard deployments. It provides: +- Internal traffic authorization towards target databases: for this to work, the target database deployments must be in the same namespace/environment AND must be labelled with `backup=true`. + +The default Network Policy is meant to be a "one size fits all" starter policy to facilitate standing up the `backup-container` in a new environment. Please consider updating/tweaking it to better fit your needs, depending on your setup. + +# Example Deployments + +
Example of a Postgres deployment + +The following outlines the deployment of a simple backup of three PostgreSQL databases in the same project namespace, on OCP v4.x. + +1. As per OCP4 [docs](https://developer.gov.bc.ca/OCP4-Backup-and-Restore), 25G of the storage class `netapp-file-backup` is the default quota. If this is insufficient, you may [request](https://github.com/BCDevOps/devops-requests/issues/new/choose) more. + +2. `git clone https://github.com/BCDevOps/backup-container.git && cd backup-container`. + +Create the image. + +```bash +oc -n 599f0a-tools process -f ./openshift/templates/backup/backup-build.yaml \ + -p NAME=nrmsurveys-bkup OUTPUT_IMAGE_TAG=v1 | oc -n 599f0a-tools create -f - +``` + +3. Configure (./config/backup.conf) (listing your database(s), and setting your cron schedule). + +```bash +postgres=eaofider-postgresql:5432/eaofider +postgres=pawslimesurvey-postgresql:5432/pawslimesurvey + +0 1 * * * default ./backup.sh -s +0 4 * * * default ./backup.sh -s -v all +``` + +4. Configure references to your DB credentials in [backup-deploy.yaml](./openshift/templates/backup/backup-deploy.yaml), replacing the boilerplate `DATABASE_USER` and `DATABASE_PASSWORD` environment variables. + +```yaml +- name: EAOFIDER_POSTGRESQL_USER + valueFrom: + secretKeyRef: + name: eaofider-postgresql + key: "${DATABASE_USER_KEY_NAME}" +- name: EAOFIDER_POSTGRESQL_PASSWORD + valueFrom: + secretKeyRef: + name: eaofider-postgresql + key: "${DATABASE_PASSWORD_KEY_NAME}" +``` + +Note that underscores should be used in the environment variable names. + +5. Create your customized `./openshift/backup-deploy.overrides.param` parameter file, if required. + +6. Deploy the app; here the example namespace is `599f0a-dev` and the app name is `nrmsurveys-bkup`: + +```bash +oc -n 599f0a-dev create configmap backup-conf --from-file=./config/backup.conf +oc -n 599f0a-dev label configmap backup-conf app=nrmsurveys-bkup + +oc -n 599f0a-dev process -f ./openshift/templates/backup/backup-deploy.yaml \ + -p NAME=nrmsurveys-bkup \ + -p IMAGE_NAMESPACE=599f0a-tools \ + -p SOURCE_IMAGE_NAME=nrmsurveys-bkup \ + -p TAG_NAME=v1 \ + -p BACKUP_VOLUME_NAME=nrmsurveys-bkup-pvc -p BACKUP_VOLUME_SIZE=20Gi \ + -p VERIFICATION_VOLUME_SIZE=5Gi \ + -p ENVIRONMENT_FRIENDLY_NAME='NRM Survey DB Backups' | oc -n 599f0a-dev create -f - +``` + +To clean up the deployment + +```bash +oc -n 599f0a-dev delete pvc/nrmsurveys-bkup-pvc pvc/backup-verification secret/nrmsurveys-bkup secret/ftp-secret dc/nrmsurveys-bkup networkpolicy/nrmsurveys-bkup configmap/backup-conf +``` + +To clean up the image stream and build configuration + +```bash +oc -n 599f0a-dev delete buildconfig/nrmsurveys-bkup imagestream/nrmsurveys-bkup +``` + +
+ +
Example of a MongoDB deployment + +The following outlines the deployment of a simple backup of a single MongoDB database with backup validation. + +1. Decide on amount of backup storage required. While 25Gi is the default quota limit in BC Gov OCP4 provisioned namespaces for `netapp-file-backup`-class storage, teams are able to request more. If you are backing up a non-production environment or an environment outside of BC Gov OCP, you can use a different storage class and thus, different default storage quota. This example assumes that you're using 5Gi of `netapp-file-backup`-class storage. +2. `git clone https://github.com/BCDevOps/backup-container.git && cd backup-container`. +3. Determine the OpenShift namespace for the image (e.g. `abc123-dev`), the app name (e.g. `myapp-backup`), and the image tag (e.g. `v1`). Then build the image in your `-tools` namespace. + +```bash +oc -n abc123-tools process -f ./openshift/templates/backup/backup-build.yaml \ + -p DOCKER_FILE_PATH=Dockerfile_Mongo + -p NAME=myapp-backup -p OUTPUT_IMAGE_TAG=v1 -p BASE_IMAGE_FOR_BUILD=registry.access.redhat.com/rhscl/mongodb-36-rhel7 | oc -n abc123-tools create -f - +``` + +4. Configure `./config/backup.conf`. This defines the database(s) to backup and the schedule that backups are to follow. Additionally, this sets up backup validation (identified by `-v all` flag). + +```bash +# Database(s) +mongo=myapp-mongodb:27017/mydb + +# Cron Schedule(s) +0 1 * * * default ./backup.sh -s +0 4 * * * default ./backup.sh -s -v all +``` + +5. Configure references to your DB credentials in [backup-deploy.yaml](./openshift/templates/backup/backup-deploy.yaml), replacing the boilerplate `DATABASE_USER` and `DATABASE_PASSWORD` environment variable names. Note the hostname of the database to be backed up. This example uses a hostname of `myapp-mongodb` which maps to environement variables named `MYAPP_MONGODB_USER` and `MYAPP_MONGODB_PASSWORD`. See the [backup.conf](#backupconf) section above for more in depth instructions. This example also assumes that the name of the secret containing your database username and password is the same as the provided `DATABASE_DEPLOYMENT_NAME` parameter. If that's not the case for your service, the secret name can be overridden. + +```yaml +- name: MYAPP_MONGODB_USER + valueFrom: + secretKeyRef: + name: "${DATABASE_DEPLOYMENT_NAME}" + key: "${DATABASE_USER_KEY_NAME}" +- name: MYAPP_MONGODB_PASSWORD + valueFrom: + secretKeyRef: + name: "${DATABASE_DEPLOYMENT_NAME}" + key: "${DATABASE_PASSWORD_KEY_NAME}" +``` + +6. Deploy the app. In this example, the namespace is `abc123-dev` and the app name is `myapp-backup`. Note that the key names within the database secret referencing database username and password are `username` and `password`, respectively. If this is not the case for your deployment, specify the correct key names as parameters `DATABASE_USER_KEY_NAME` and `DATABASE_PASSWORD_KEY_NAME`. Also note that `BACKUP_VOLUME_NAME` is from Step 2 above. + +```bash +oc -n abc123-dev create configmap backup-conf --from-file=./config/backup.conf +oc -n abc123-dev label configmap backup-conf app=myapp-backup + +oc -n abc123-dev process -f ./openshift/templates/backup/backup-deploy.yaml \ + -p NAME=myapp-backup \ + -p IMAGE_NAMESPACE=abc123-tools \ + -p SOURCE_IMAGE_NAME=myapp-backup \ + -p TAG_NAME=v1 \ + -p BACKUP_VOLUME_NAME=bk-abc123-dev-v9k7xgyvwdxm \ + -p BACKUP_VOLUME_SIZE=5Gi \ + -p VERIFICATION_VOLUME_SIZE=10Gi \ + -p VERIFICATION_VOLUME_CLASS=netapp-file-standard \ + -p DATABASE_DEPLOYMENT_NAME=myapp-mongodb \ + -p DATABASE_USER_KEY_NAME=username \ + -p DATABASE_PASSWORD_KEY_NAME=password \ + -p ENVIRONMENT_FRIENDLY_NAME='My App MongoDB Backups' | oc -n abc123-dev create -f - + +``` + +
+ +## Deploy with Helm Chart + +``` +helm repo add bcgov http://bcgov.github.io/helm-charts +helm upgrade --install db-backup-storage bcgov/backup-storage +``` + +For customizing the configuration, go to: https://github.com/bcgov/helm-charts/tree/master/charts/backup-storage + +# Prebuilt Container Images + +Starting with v2.3.3, prebuilt container images are built and published with each release: + +- [bcgovimages/backup-container](https://hub.docker.com/r/bcgovimages/backup-container) (`PostgreSQL`) +- [bcgovimages/backup-container-mongo](https://hub.docker.com/r/bcgovimages/backup-container-mongo) +- [bcgovimages/backup-container-mssql](https://hub.docker.com/r/bcgovimages/backup-container-mssql) +- [bcgovimages/backup-container-mariadb](https://hub.docker.com/r/bcgovimages/backup-container-mariadb) + +# Postgres Base Version + +The backup container works on top of the base postgres image [here](./docker/Dockerfile) + +To use previous supported versions of postgres - V9 to V12, use the images from the [Prebuilt Container Images](#prebuilt-container-images) + +# Tip and Tricks + +Please refer to the [Tips and Tricks](./docs/TipsAndTricks.md) document for solutions to known issues. + +# Getting Help or Reporting an Issue + +To report bugs/issues/feature requests, please file an [issue](../../issues). + +# How to Contribute + +If you would like to contribute, please see our [CONTRIBUTING](./CONTRIBUTING.md) guidelines. + +Please note that this project is released with a [Contributor Code of Conduct](./CODE_OF_CONDUCT.md). +By participating in this project you agree to abide by its terms. diff --git a/containers/backup/backup.conf b/containers/backup/backup.conf new file mode 100644 index 00000000..27f9bf29 --- /dev/null +++ b/containers/backup/backup.conf @@ -0,0 +1,51 @@ +# ============================================================ +# Databases: +# ------------------------------------------------------------ +# List the databases you want backed up here. +# Databases will be backed up in the order they are listed. +# +# The entries must be in one of the following forms: +# - / +# - :/ +# - =/ +# - =:/ +# can be postgres, mongo or mssql +# MUST be specified when you are sharing a +# single backup.conf file between postgres, mongo and mssql +# backup containers. If you do not specify +# the listed databases are assumed to be valid for the +# backup container in which the configuration is mounted. +# +# Examples: +# - postgres=postgresql/my_database +# - postgres=postgresql:5432/my_database +# - mongo=mongodb/my_database +# - mongo=mongodb:27017/my_database +# - mssql=mssql_server:1433/my_database +# ----------------------------------------------------------- +# Cron Scheduling: +# ----------------------------------------------------------- +# List your backup and verification schedule(s) here as well. +# The schedule(s) must be listed as cron tabs that +# execute the script in 'scheduled' mode: +# - ./backup.sh -s +# +# Examples (assuming system's TZ is set to PST): +# - 0 1 * * * default ./backup.sh -s +# - Run a backup at 1am Pacific every day. +# +# - 0 4 * * * default ./backup.sh -s -v all +# - Verify the most recent backups for all datbases +# at 4am Pacific every day. +# ----------------------------------------------------------- +# Full Example: +# ----------------------------------------------------------- +# postgres=postgresql:5432/TheOrgBook_Database +# mongo=mender-mongodb:27017/useradm +# postgres=wallet-db/tob_issuer +# mssql=pims-db-dev:1433/pims +# mariadb=matomo-db:3306/matomo +# +# 0 1 * * * default ./backup.sh -s +# 0 4 * * * default ./backup.sh -s -v all +# ============================================================ \ No newline at end of file diff --git a/containers/backup/config/backup.conf b/containers/backup/config/backup.conf new file mode 100644 index 00000000..f44e692d --- /dev/null +++ b/containers/backup/config/backup.conf @@ -0,0 +1,51 @@ +# ============================================================ +# Databases: +# ------------------------------------------------------------ +# List the databases you want backed up here. +# Databases will be backed up in the order they are listed. +# +# The entries must be in one of the following forms: +# - / +# - :/ +# - =/ +# - =:/ +# can be postgres, mongo or mssql +# MUST be specified when you are sharing a +# single backup.conf file between postgres, mongo and mssql +# backup containers. If you do not specify +# the listed databases are assumed to be valid for the +# backup container in which the configuration is mounted. +# +# Examples: +# - postgres=postgresql/my_database +- postgres=postgresql:5432/restoration-db-all-container +# - mongo=mongodb/my_database +# - mongo=mongodb:27017/my_database +# - mssql=mssql_server:1433/my_database +# ----------------------------------------------------------- +# Cron Scheduling: +# ----------------------------------------------------------- +# List your backup and verification schedule(s) here as well. +# The schedule(s) must be listed as cron tabs that +# execute the script in 'scheduled' mode: +# - ./backup.sh -s +# +# Examples (assuming system's TZ is set to PST): +- 0 1 * * * default ./backup.sh -s +# - Run a backup at 1am Pacific every day. +# +- 0 4 * * * default ./backup.sh -s -v all +# - Verify the most recent backups for all datbases +# at 4am Pacific every day. +# ----------------------------------------------------------- +# Full Example: +# ----------------------------------------------------------- +# postgres=postgresql:5432/TheOrgBook_Database +# mongo=mender-mongodb:27017/useradm +# postgres=wallet-db/tob_issuer +# mssql=pims-db-dev:1433/pims +# mariadb=matomo-db:3306/matomo +# +# 0 1 * * * default ./backup.sh -s +# 0 4 * * * default ./backup.sh -s -v all +# ============================================================ \ No newline at end of file diff --git a/containers/backup/docker/Dockerfile b/containers/backup/docker/Dockerfile new file mode 100644 index 00000000..a7e8debf --- /dev/null +++ b/containers/backup/docker/Dockerfile @@ -0,0 +1,43 @@ +# This image provides a postgres installation from which to run backups +FROM --platform=linux/amd64 quay.io/fedora/postgresql-15:15 + +# Change timezone to PST for convenience +ENV TZ=PST8PDT + +# Set the workdir to be root +WORKDIR / + +# Load the backup scripts into the container (must be executable). +COPY backup.* / + +# ======================================================================================================== +# Install go-crond (from https://github.com/webdevops/go-crond) +# +# CRON Jobs in OpenShift: +# - https://blog.danman.eu/cron-jobs-in-openshift/ +# -------------------------------------------------------------------------------------------------------- +ARG SOURCE_REPO=webdevops +ARG GOCROND_VERSION=23.2.0 +ADD https://github.com/$SOURCE_REPO/go-crond/releases/download/$GOCROND_VERSION/go-crond.linux.amd64 /usr/bin/go-crond + +USER root + +RUN curl https://dl.min.io/client/mc/release/linux-amd64/mc -o /usr/bin/mc +RUN chmod +x /usr/bin/mc +RUN chmod +x /usr/bin/go-crond +# ======================================================================================================== + +# ======================================================================================================== +# Perform operations that require root privilages here ... +# -------------------------------------------------------------------------------------------------------- +RUN echo $TZ > /etc/timezone +# ======================================================================================================== + +# The column command is missing from quay.io/fedora/postgresql-14:14 +RUN dnf install -y util-linux + +# Important - Reset to the base image's user account. +USER 26 + +# Set the default CMD. +CMD sh /backup.sh diff --git a/containers/backup/docker/Dockerfile_MSSQL b/containers/backup/docker/Dockerfile_MSSQL new file mode 100644 index 00000000..30dd72c8 --- /dev/null +++ b/containers/backup/docker/Dockerfile_MSSQL @@ -0,0 +1,46 @@ +FROM mcr.microsoft.com/mssql/rhel/server:2019-CU1-rhel-8 + +# Change timezone to PST for convenience +ENV TZ=PST8PDT + +# Set the workdir to be root +WORKDIR / + +# Load the backup scripts into the container (must be executable). +COPY backup.* / + +# ======================================================================================================== +# Install go-crond (from https://github.com/webdevops/go-crond) +# +# CRON Jobs in OpenShift: +# - https://blog.danman.eu/cron-jobs-in-openshift/ +# -------------------------------------------------------------------------------------------------------- +ARG SOURCE_REPO=webdevops +ARG GOCROND_VERSION=23.2.0 +ADD https://github.com/$SOURCE_REPO/go-crond/releases/download/$GOCROND_VERSION/go-crond.linux.amd64 /usr/bin/go-crond + +USER root + +RUN chmod +x /usr/bin/go-crond +# ======================================================================================================== + +# ======================================================================================================== +# Perform operations that require root privilages here ... +# -------------------------------------------------------------------------------------------------------- +RUN echo $TZ > /etc/timezone +# ======================================================================================================== +COPY uid_entrypoint /opt/mssql-tools/bin/ +RUN chmod -R a+rwx /opt/mssql-tools/bin/uid_entrypoint + +ENV PATH=${PATH}:/opt/mssql/bin:/opt/mssql-tools/bin +RUN mkdir -p /var/opt/mssql/data && \ + chmod -R g=u /var/opt/mssql /etc/passwd + +# Important - Reset to the base image's user account. +USER 10001 + +# Set the default CMD. +CMD sh /backup.sh + +### user name recognition at runtime w/ an arbitrary uid - for OpenShift deployments +ENTRYPOINT [ "/opt/mssql-tools/bin/uid_entrypoint" ] \ No newline at end of file diff --git a/containers/backup/docker/Dockerfile_MariaDB b/containers/backup/docker/Dockerfile_MariaDB new file mode 100644 index 00000000..ce0da9cd --- /dev/null +++ b/containers/backup/docker/Dockerfile_MariaDB @@ -0,0 +1,37 @@ +from registry.fedoraproject.org/f31/mariadb + +# Change timezone to PST for convenience +ENV TZ=PST8PDT + +# Set the workdir to be root +WORKDIR / + +# Load the backup scripts into the container (must be executable). +COPY backup.* / + +# ======================================================================================================== +# Install go-crond (from https://github.com/webdevops/go-crond) +# +# CRON Jobs in OpenShift: +# - https://blog.danman.eu/cron-jobs-in-openshift/ +# -------------------------------------------------------------------------------------------------------- +ARG SOURCE_REPO=webdevops +ARG GOCROND_VERSION=23.2.0 +ADD https://github.com/$SOURCE_REPO/go-crond/releases/download/$GOCROND_VERSION/go-crond.linux.amd64 /usr/bin/go-crond + +USER root + +RUN chmod +x /usr/bin/go-crond +# ======================================================================================================== + +# ======================================================================================================== +# Perform operations that require root privilages here ... +# -------------------------------------------------------------------------------------------------------- +RUN echo $TZ > /etc/timezone +# ======================================================================================================== + +# Important - Reset to the base image's user account. +USER 26 + +# Set the default CMD. +CMD sh /backup.sh \ No newline at end of file diff --git a/containers/backup/docker/Dockerfile_Mongo b/containers/backup/docker/Dockerfile_Mongo new file mode 100644 index 00000000..6995e436 --- /dev/null +++ b/containers/backup/docker/Dockerfile_Mongo @@ -0,0 +1,45 @@ +# This image provides a mongo installation from which to run backups +FROM mongodb/mongodb-community-server:6.0.6-ubi8 + +ARG uid=998 +ARG user=mongod + +# Change timezone to PST for convenience +ENV TZ=PST8PDT + +# Set the workdir to be root +WORKDIR / + +# Load the backup scripts into the container (must be executable). +COPY backup.* / + +# ======================================================================================================== +# Install go-crond (from https://github.com/webdevops/go-crond) +# +# CRON Jobs in OpenShift: +# - https://blog.danman.eu/cron-jobs-in-openshift/ +# -------------------------------------------------------------------------------------------------------- +ARG SOURCE_REPO=webdevops +ARG GOCROND_VERSION=23.2.0 +ADD https://github.com/$SOURCE_REPO/go-crond/releases/download/$GOCROND_VERSION/go-crond.linux.amd64 /usr/bin/go-crond + +USER root + +RUN chmod +x /usr/bin/go-crond +RUN chown -R $user:root /data/db && \ + chmod -R ug+rw /data/db + +RUN usermod -a -G 0 $user +# ======================================================================================================== + +# ======================================================================================================== +# Perform operations that require root privilages here ... +# -------------------------------------------------------------------------------------------------------- +RUN echo $TZ > /etc/timezone +# ======================================================================================================== + +# Important - Reset to the base image's user account. +USER $uid + +# Set the default CMD. +CMD bash /backup.sh diff --git a/containers/backup/docker/backup.config.utils b/containers/backup/docker/backup.config.utils new file mode 100644 index 00000000..46652737 --- /dev/null +++ b/containers/backup/docker/backup.config.utils @@ -0,0 +1,501 @@ +#!/bin/bash +# ================================================================================================================= +# Configuration Utility Functions: +# ----------------------------------------------------------------------------------------------------------------- +function getDatabaseName(){ + ( + _databaseSpec=${1} + _databaseName=$(echo ${_databaseSpec} | sed -n 's~^.*/\(.*$\)~\1~p') + echo "${_databaseName}" + ) +} + +function getDatabaseType(){ + ( + _databaseSpec=${1} + _databaseType=$(echo ${_databaseSpec} | sed -n 's~^\(.*\)=.*$~\1~p' | tr '[:upper:]' '[:lower:]') + echo "${_databaseType}" + ) +} + +function getPort(){ + ( + local OPTIND + local localhost + unset localhost + while getopts :l FLAG; do + case $FLAG in + l ) localhost=1 ;; + esac + done + shift $((OPTIND-1)) + + _databaseSpec=${1} + + if [ -z "${localhost}" ]; then + portsed="s~^.*:\([[:digit:]]\+\)/.*$~\1~p" + _port=$(echo ${_databaseSpec} | sed -n "${portsed}") + fi + + echo "${_port}" + ) +} + +function getHostname(){ + ( + local OPTIND + local localhost + unset localhost + while getopts :l FLAG; do + case $FLAG in + l ) localhost=1 ;; + esac + done + shift $((OPTIND-1)) + + _databaseSpec=${1} + + if [ -z "${localhost}" ]; then + _hostname=$(echo ${_databaseSpec} | sed 's~^.\+[=]~~;s~[:/].*~~') + else + _hostname="127.0.0.1" + fi + + echo "${_hostname}" + ) +} + +function getHostPrefix(){ + ( + _hostname=${1} + _hostPrefix=$(echo ${_hostname} | tr '[:lower:]' '[:upper:]' | sed "s~-~_~g") + echo "${_hostPrefix}" + ) +} + +function getHostUserParam(){ + ( + _hostname=${1} + _hostUser=$(getHostPrefix ${_hostname})_USER + echo "${_hostUser}" + ) +} + +function getHostPasswordParam(){ + ( + _hostname=${1} + _hostPassword=$(getHostPrefix ${_hostname})_PASSWORD + echo "${_hostPassword}" + ) +} + +function readConf(){ + ( + local OPTIND + local readCron + local quiet + local all + unset readCron + unset quiet + while getopts cqa FLAG; do + case $FLAG in + c ) readCron=1 ;; + q ) quiet=1 ;; + a ) all=1 ;; + esac + done + shift $((OPTIND-1)) + + # Remove all comments and any blank lines + filters="/^[[:blank:]]*$/d;/^[[:blank:]]*#/d;/#.*/d;" + + if [ -z "${readCron}" ]; then + # Read in the database config ... + # - Remove any lines that do not match the expected database spec format(s) + # - [=]/ + # - [=]:/ + filters+="/^[a-zA-Z0-9=_/-]*\(:[0-9]*\)\?\/[a-zA-Z0-9_/-]*$/!d;" + if [ -z "${all}" ]; then + # Remove any database configs that are not for the current container type + # Database configs that do not define the database type are assumed to be for the current container type + filters+="/\(^[a-zA-Z0-9_/-]*\(:[0-9]*\)\?\/[a-zA-Z0-9_/-]*$\)\|\(^${CONTAINER_TYPE}=\)/!d;" + fi + else + # Read in the cron config ... + # - Remove any lines that MATCH expected database spec format(s), + # leaving, what should be, cron tabs. + filters+="/^[a-zA-Z0-9=_/-]*\(:[0-9]*\)\?\/[a-zA-Z0-9_/-]*$/d;" + fi + + if [ -f ${BACKUP_CONF} ]; then + if [ -z "${quiet}" ]; then + echo "Reading backup config from ${BACKUP_CONF} ..." >&2 + fi + _value=$(sed "${filters}" ${BACKUP_CONF}) + fi + + if [ -z "${_value}" ] && [ -z "${readCron}" ]; then + # Backward compatibility + if [ -z "${quiet}" ]; then + echo "Reading backup config from environment variables ..." >&2 + fi + _value="${DATABASE_SERVICE_NAME}${DEFAULT_PORT:+:${DEFAULT_PORT}}${POSTGRESQL_DATABASE:+/${POSTGRESQL_DATABASE}}" + fi + + echo "${_value}" + ) +} + +function getNumBackupsToRetain(){ + ( + _count=0 + _backupType=${1:-$(getBackupType)} + + case "${_backupType}" in + daily) + _count=${DAILY_BACKUPS} + if (( ${_count} <= 0 )) && (( ${WEEKLY_BACKUPS} <= 0 )) && (( ${MONTHLY_BACKUPS} <= 0 )); then + _count=1 + fi + ;; + weekly) + _count=${WEEKLY_BACKUPS} + ;; + monthly) + _count=${MONTHLY_BACKUPS} + ;; + *) + _count=${NUM_BACKUPS} + ;; + esac + + echo "${_count}" + ) +} + +function getUsername(){ + ( + _databaseSpec=${1} + _hostname=$(getHostname ${_databaseSpec}) + _paramName=$(getHostUserParam ${_hostname}) + # Backward compatibility ... + _username="${!_paramName:-${DATABASE_USER}}" + echo ${_username} + ) +} + +function getPassword(){ + ( + _databaseSpec=${1} + _hostname=$(getHostname ${_databaseSpec}) + _paramName=$(getHostPasswordParam ${_hostname}) + # Backward compatibility ... + _password="${!_paramName:-${DATABASE_PASSWORD}}" + echo ${_password} + ) +} + +function isLastDayOfMonth(){ + ( + _date=${1:-$(date)} + _day=$(date -d "${_date}" +%-d) + _month=$(date -d "${_date}" +%-m) + _lastDayOfMonth=$(date -d "${_month}/1 + 1 month - 1 day" "+%-d") + + if (( ${_day} == ${_lastDayOfMonth} )); then + return 0 + else + return 1 + fi + ) +} + +function isLastDayOfWeek(){ + ( + # We're calling Sunday the last dayt of the week in this case. + _date=${1:-$(date)} + _dayOfWeek=$(date -d "${_date}" +%u) + + if (( ${_dayOfWeek} == 7 )); then + return 0 + else + return 1 + fi + ) +} + +function getBackupType(){ + ( + _backupType="" + if rollingStrategy; then + if isLastDayOfMonth && (( "${MONTHLY_BACKUPS}" > 0 )); then + _backupType="monthly" + elif isLastDayOfWeek; then + _backupType="weekly" + else + _backupType="daily" + fi + fi + echo "${_backupType}" + ) +} + +function rollingStrategy(){ + if [[ "${BACKUP_STRATEGY}" == "rolling" ]] && (( "${WEEKLY_BACKUPS}" >= 0 )) && (( "${MONTHLY_BACKUPS}" >= 0 )); then + return 0 + else + return 1 + fi +} + +function dailyStrategy(){ + if [[ "${BACKUP_STRATEGY}" == "daily" ]] || (( "${WEEKLY_BACKUPS}" < 0 )); then + return 0 + else + return 1 + fi +} + +function listSettings(){ + _backupDirectory=${1:-$(createBackupFolder -g)} + _databaseList=${2:-$(readConf -q)} + _yellow='\e[33m' + _nc='\e[0m' # No Color + _notConfigured="${_yellow}not configured${_nc}" + + echo -e \\n"Settings:" + _mode=$(getMode 2>/dev/null) + echo -e "- Run mode: ${_mode}"\\n + + if rollingStrategy; then + echo "- Backup strategy: rolling" + fi + if dailyStrategy; then + echo "- Backup strategy: daily" + fi + if ! rollingStrategy && ! dailyStrategy; then + echoYellow "- Backup strategy: Unknown backup strategy; ${BACKUP_STRATEGY}" + _configurationError=1 + fi + backupType=$(getBackupType) + if [ -z "${backupType}" ]; then + echo "- Current backup type: flat daily" + else + echo "- Current backup type: ${backupType}" + fi + echo "- Backups to retain:" + if rollingStrategy; then + echo " - Daily: $(getNumBackupsToRetain daily)" + echo " - Weekly: $(getNumBackupsToRetain weekly)" + echo " - Monthly: $(getNumBackupsToRetain monthly)" + else + echo " - Total: $(getNumBackupsToRetain)" + fi + echo "- Current backup folder: ${_backupDirectory}" + + if [[ "${_mode}" != ${ONCE} ]]; then + if [[ "${_mode}" == ${CRON} ]] || [[ "${_mode}" == ${SCHEDULED} ]]; then + _backupSchedule=$(readConf -cq) + echo "- Time Zone: $(date +"%Z %z")" + fi + _backupSchedule=$(formatList "${_backupSchedule:-${BACKUP_PERIOD}}") + echo -e \\n"- Schedule:" + echo "${_backupSchedule}" + fi + + if [[ "${CONTAINER_TYPE}" == "${UNKNOWN_DB}" ]] && [ -z "${_allowNullPlugin}" ]; then + echoRed "\n- Container Type: ${CONTAINER_TYPE}" + _configurationError=1 + else + echo -e "\n- Container Type: ${CONTAINER_TYPE}" + fi + + _databaseList=$(formatList "${_databaseList}") + echo "- Databases (filtered by container type):" + echo "${_databaseList}" + echo + + if [ -z "${FTP_URL}" ]; then + echo -e "- FTP server: ${_notConfigured}" + else + echo "- FTP server: ${FTP_URL}" + fi + + if [ -z "${S3_ENDPOINT}" ]; then + echo -e "- S3 endpoint: ${_notConfigured}" + else + echo "- S3 endpoint: ${S3_ENDPOINT}" + fi + + if [ -z "${WEBHOOK_URL}" ]; then + echo -e "- Webhook Endpoint: ${_notConfigured}" + else + echo "- Webhook Endpoint: ${WEBHOOK_URL}" + fi + + if [ -z "${ENVIRONMENT_FRIENDLY_NAME}" ]; then + echo -e "- Environment Friendly Name: ${_notConfigured}" + else + echo -e "- Environment Friendly Name: ${ENVIRONMENT_FRIENDLY_NAME}" + fi + if [ -z "${ENVIRONMENT_NAME}" ]; then + echo -e "- Environment Name (Id): ${_notConfigured}" + else + echo "- Environment Name (Id): ${ENVIRONMENT_NAME}" + fi + + if [ ! -z "${_configurationError}" ]; then + echo + logError "Configuration error! The script will exit." + sleep 5 + exit 1 + fi + echo +} + +function isScheduled(){ + ( + if [ ! -z "${SCHEDULED_RUN}" ]; then + return 0 + else + return 1 + fi + ) +} + +function isScripted(){ + ( + if [ ! -z "${SCHEDULED_RUN}" ]; then + return 0 + else + return 1 + fi + ) +} + +function restoreMode(){ + ( + if [ ! -z "${_restoreDatabase}" ]; then + return 0 + else + return 1 + fi + ) +} + +function verifyMode(){ + ( + if [ ! -z "${_verifyBackup}" ]; then + return 0 + else + return 1 + fi + ) +} + +function pruneMode(){ + ( + if [ ! -z "${RUN_PRUNE}" ]; then + return 0 + else + return 1 + fi + ) +} + +function cronMode(){ + ( + cronTabs=$(readConf -cq) + if isInstalled "go-crond" && [ ! -z "${cronTabs}" ]; then + return 0 + else + return 1 + fi + ) +} + +function runOnce() { + if [ ! -z "${RUN_ONCE}" ]; then + return 0 + else + return 1 + fi +} + +function getMode(){ + ( + unset _mode + + if pruneMode; then + _mode="${PRUNE}" + fi + + if [ -z "${_mode}" ] && restoreMode; then + _mode="${RESTORE}" + fi + + if [ -z "${_mode}" ] && verifyMode; then + # Determine if this is a scheduled verification or a manual one. + if isScheduled; then + if cronMode; then + _mode="${SCHEDULED_VERIFY}" + else + _mode="${ERROR}" + logError "Scheduled mode cannot be used without cron being installed and at least one cron tab being defined in ${BACKUP_CONF}." + fi + else + _mode="${VERIFY}" + fi + fi + + if [ -z "${_mode}" ] && runOnce; then + _mode="${ONCE}" + fi + + if [ -z "${_mode}" ] && isScheduled; then + if cronMode; then + _mode="${SCHEDULED}" + else + _mode="${ERROR}" + logError "Scheduled mode cannot be used without cron being installed and at least one cron tab being defined in ${BACKUP_CONF}." + fi + fi + + if [ -z "${_mode}" ] && cronMode; then + _mode="${CRON}" + fi + + if [ -z "${_mode}" ]; then + _mode="${LEGACY}" + fi + + echo "${_mode}" + ) +} + +function validateOperation(){ + ( + _databaseSpec=${1} + _mode=${2} + _rtnCd=0 + + if [[ "${_mode}" == ${RESTORE} ]] && ! isForContainerType ${_databaseSpec}; then + echoRed "\nYou are attempting to restore database '${_databaseSpec}' from a ${CONTAINER_TYPE} container." + echoRed "Cannot continue with the restore. It must be initiated from the matching container type." + _rtnCd=1 + fi + + return ${_rtnCd} + ) +} + +function ignoreErrors(){ + ( + if [ ! -z "${IGNORE_ERRORS}" ]; then + return 0 + else + return 1 + fi + ) +} +# ====================================================================================== \ No newline at end of file diff --git a/containers/backup/docker/backup.container.utils b/containers/backup/docker/backup.container.utils new file mode 100644 index 00000000..2d44960e --- /dev/null +++ b/containers/backup/docker/backup.container.utils @@ -0,0 +1,82 @@ +#!/bin/bash +# ================================================================================================================= +# Container Utility Functions: +# ----------------------------------------------------------------------------------------------------------------- +function isPostgres(){ + ( + if isInstalled "psql"; then + return 0 + else + return 1 + fi + ) +} + +function isMongo(){ + ( + if isInstalled "mongosh"; then + return 0 + else + return 1 + fi + ) +} + +function isMsSql(){ + ( + if isInstalled "sqlcmd"; then + return 0 + else + return 1 + fi + ) +} + +function isMariaDb(){ + ( + # If a seperate mysql plugin is added, this check may be insufficient to establish the container type. + if isInstalled "mysql"; then + return 0 + else + return 1 + fi + ) +} + +function getContainerType(){ + ( + local _containerType=${UNKNOWN_DB} + _rtnCd=0 + + if isPostgres; then + _containerType=${POSTGRE_DB} + elif isMongo; then + _containerType=${MONGO_DB} + elif isMsSql; then + _containerType=${MSSQL_DB} + elif isMariaDb; then + _containerType=${MARIA_DB} + else + _containerType=${UNKNOWN_DB} + _rtnCd=1 + fi + + echo "${_containerType}" + return ${_rtnCd} + ) +} + +function isForContainerType(){ + ( + _databaseSpec=${1} + _databaseType=$(getDatabaseType ${_databaseSpec}) + + # If the database type has not been defined, assume the database spec is valid for the current databse container type. + if [ -z "${_databaseType}" ] || [[ "${_databaseType}" == "${CONTAINER_TYPE}" ]]; then + return 0 + else + return 1 + fi + ) +} +# ====================================================================================== diff --git a/containers/backup/docker/backup.file.utils b/containers/backup/docker/backup.file.utils new file mode 100644 index 00000000..06ef069a --- /dev/null +++ b/containers/backup/docker/backup.file.utils @@ -0,0 +1,245 @@ +#!/bin/bash +# ================================================================================================================= +# File Utility Functions +# ----------------------------------------------------------------------------------------------------------------- +function makeDirectory() +{ + ( + # Creates directories with permissions reclusively. + # ${1} is the directory to be created + # Inspired by https://unix.stackexchange.com/questions/49263/recursive-mkdir + directory="${1}" + test $# -eq 1 || { echo "Function 'makeDirectory' can create only one directory (with it's parent directories)."; exit 1; } + test -d "${directory}" && return 0 + test -d "$(dirname "${directory}")" || { makeDirectory "$(dirname "${directory}")" || return 1; } + test -d "${directory}" || { mkdir --mode=g+w "${directory}" || return 1; } + return 0 + ) +} + +function finalizeBackup(){ + ( + _filename=${1} + _inProgressFilename="${_filename}${IN_PROGRESS_BACKUP_FILE_EXTENSION}" + _finalFilename="${_filename}${BACKUP_FILE_EXTENSION}" + + if [ -f ${_inProgressFilename} ]; then + mv "${_inProgressFilename}" "${_finalFilename}" + echo "${_finalFilename}" + fi + ) +} + +function listExistingBackups(){ + ( + local _backupDir=${1:-${ROOT_BACKUP_DIR}} + local database + local databases=$(readConf -q) + local output="\nDatabase,Current Size" + + for database in ${databases}; do + if isForContainerType ${database}; then + output+="\n${database},$(getDbSize "${database}")" + fi + done + + echoMagenta "\n================================================================================================================================" + echoMagenta "Current Backups:" + echoMagenta "\n$(echo -ne "${output}" | column -t -s ,)" + echoMagenta "\n$(df -h ${_backupDir})" + echoMagenta "--------------------------------------------------------------------------------------------------------------------------------" + du -ah --time ${_backupDir} + echoMagenta "================================================================================================================================\n" + ) +} + +function getDirectoryName(){ + ( + local path=${1} + path="${path%"${path##*[!/]}"}" + local name="${path##*/}" + echo "${name}" + ) +} + +function getBackupTypeFromPath(){ + ( + local path=${1} + path="${path%"${path##*[!/]}"}" + path="$(dirname "${path}")" + local backupType=$(getDirectoryName "${path}") + echo "${backupType}" + ) +} + +function prune(){ + ( + local database + local backupDirs + local backupDir + local backupType + local backupTypes + local pruneBackup + unset backupTypes + unset backupDirs + unset pruneBackup + + local databases=$(readConf -q) + if rollingStrategy; then + backupTypes="daily weekly monthly" + for backupType in ${backupTypes}; do + backupDirs="${backupDirs} $(createBackupFolder -g ${backupType})" + done + else + backupDirs=$(createBackupFolder -g) + fi + + if [ ! -z "${_fromBackup}" ]; then + pruneBackup="$(findBackup "" "${_fromBackup}")" + while [ ! -z "${pruneBackup}" ]; do + echoYellow "\nAbout to delete backup file: ${pruneBackup}" + waitForAnyKey + rm -rfvd "${pruneBackup}" + + # Quietly delete any empty directories that are left behind ... + find ${ROOT_BACKUP_DIR} -type d -empty -delete > /dev/null 2>&1 + pruneBackup="$(findBackup "" "${_fromBackup}")" + done + else + for backupDir in ${backupDirs}; do + for database in ${databases}; do + unset backupType + if rollingStrategy; then + backupType=$(getBackupTypeFromPath "${backupDir}") + fi + pruneBackups "${backupDir}" "${database}" "${backupType}" + done + done + fi + ) +} + +function pruneBackups(){ + ( + _backupDir=${1} + _databaseSpec=${2} + _backupType=${3:-''} + _pruneDir="$(dirname "${_backupDir}")" + _numBackupsToRetain=$(getNumBackupsToRetain "${_backupType}") + _coreFilename=$(generateCoreFilename ${_databaseSpec}) + + if [ -d ${_pruneDir} ]; then + let _index=${_numBackupsToRetain}+1 + _filesToPrune=$(find ${_pruneDir}* -type f -printf '%T@ %p\n' | grep ${_coreFilename} | sort -r | tail -n +${_index} | sed 's~^.* \(.*$\)~\1~') + + if [ ! -z "${_filesToPrune}" ]; then + echoYellow "\nPruning ${_coreFilename} backups from ${_pruneDir} ..." + echo "${_filesToPrune}" | xargs rm -rfvd + + # Quietly delete any empty directories that are left behind ... + find ${ROOT_BACKUP_DIR} -type d -empty -delete > /dev/null 2>&1 + fi + fi + ) +} + +function touchBackupFile() { + ( + # For safety, make absolutely certain the directory and file exist. + # The pruning process removes empty directories, so if there is an error + # during a backup the backup directory could be deleted. + _backupFile=${1} + _backupDir="${_backupFile%/*}" + makeDirectory ${_backupDir} && touch ${_backupFile} + ) +} + +function findBackup(){ + ( + _databaseSpec=${1} + _fileName=${2} + + # If no backup file was specified, find the most recent for the database. + # Otherwise treat the value provided as a filter to find the most recent backup file matching the filter. + if [ -z "${_fileName}" ]; then + _coreFilename=$(generateCoreFilename ${_databaseSpec}) + _fileName=$(find ${ROOT_BACKUP_DIR}* -type f -printf '%T@ %p\n' | grep ${_coreFilename} | sort | tail -n 1 | sed 's~^.* \(.*$\)~\1~') + else + _fileName=$(find ${ROOT_BACKUP_DIR}* -type f -printf '%T@ %p\n' | grep ${_fileName} | sort | tail -n 1 | sed 's~^.* \(.*$\)~\1~') + fi + + echo "${_fileName}" + ) +} + +function createBackupFolder(){ + ( + local OPTIND + local genOnly + unset genOnly + while getopts g FLAG; do + case $FLAG in + g ) genOnly=1 ;; + esac + done + shift $((OPTIND-1)) + + _backupTypeDir="${1:-$(getBackupType)}" + if [ ! -z "${_backupTypeDir}" ]; then + _backupTypeDir=${_backupTypeDir}/ + fi + + _backupDir="${ROOT_BACKUP_DIR}${_backupTypeDir}`date +\%Y-\%m-\%d`/" + + # Don't actually create the folder if we're just generating it for printing the configuation. + if [ -z "${genOnly}" ]; then + echo "Making backup directory ${_backupDir} ..." >&2 + if ! makeDirectory ${_backupDir}; then + logError "Failed to create backup directory ${_backupDir}." + exit 1; + fi; + fi + + echo ${_backupDir} + ) +} + +function generateFilename(){ + ( + _backupDir=${1} + _databaseSpec=${2} + _coreFilename=$(generateCoreFilename ${_databaseSpec}) + _filename="${_backupDir}${_coreFilename}_`date +\%Y-\%m-\%d_%H-%M-%S`" + echo ${_filename} + ) +} + +function generateCoreFilename(){ + ( + _databaseSpec=${1} + _hostname=$(getHostname ${_databaseSpec}) + _database=$(getDatabaseName ${_databaseSpec}) + _coreFilename="${_hostname}-${_database}" + echo ${_coreFilename} + ) +} + +function getFileSize(){ + ( + _filename=${1} + echo $(du -h "${_filename}" | awk '{print $1}') + ) +} + +function dirIsEmpty(){ + ( + dir="${@}" + rtnVal=$(find ${dir} -maxdepth 0 -empty) + if [ -z "${rtnVal}" ] || [ "${dir}" != "${rtnVal}" ]; then + return 1 + else + return 0 + fi + ) +} +# ================================================================================================================= \ No newline at end of file diff --git a/containers/backup/docker/backup.ftp b/containers/backup/docker/backup.ftp new file mode 100644 index 00000000..d0a935cf --- /dev/null +++ b/containers/backup/docker/backup.ftp @@ -0,0 +1,23 @@ +#!/bin/bash +# ================================================================================================================= +# FTP Support Functions: +# ----------------------------------------------------------------------------------------------------------------- +function ftpBackup(){ + ( + if [ -z "${FTP_URL}" ] ; then + return 0 + fi + + _filename=${1} + _filenameWithExtension="${_filename}${BACKUP_FILE_EXTENSION}" + echo "Transferring ${_filenameWithExtension} to ${FTP_URL}" + curl --ftp-ssl -T ${_filenameWithExtension} --user ${FTP_USER}:${FTP_PASSWORD} ${FTP_URL} + + if [ ${?} -eq 0 ]; then + logInfo "Successfully transferred ${_filenameWithExtension} to the FTP server" + else + logError "Failed to transfer ${_filenameWithExtension} with the exit code ${?}" + fi + ) +} +# ================================================================================================================= diff --git a/containers/backup/docker/backup.logging b/containers/backup/docker/backup.logging new file mode 100644 index 00000000..6c5e05df --- /dev/null +++ b/containers/backup/docker/backup.logging @@ -0,0 +1,128 @@ +#!/bin/bash +# ================================================================================================================= +# Logging Functions: +# ----------------------------------------------------------------------------------------------------------------- +function debugMsg (){ + _msg="${@}" + if [ "${BACKUP_LOG_LEVEL}" == "debug" ]; then + echoGreen "$(date) - [DEBUG] - ${@}" >&2 + fi +} + +function echoRed (){ + _msg="${@}" + _red='\e[31m' + _nc='\e[0m' # No Color + echo -e "${_red}${_msg}${_nc}" +} + +function echoYellow (){ + _msg="${@}" + _yellow='\e[33m' + _nc='\e[0m' # No Color + echo -e "${_yellow}${_msg}${_nc}" +} + +function echoBlue (){ + _msg="${@}" + _blue='\e[34m' + _nc='\e[0m' # No Color + echo -e "${_blue}${_msg}${_nc}" +} + +function echoGreen (){ + _msg="${@}" + _green='\e[32m' + _nc='\e[0m' # No Color + echo -e "${_green}${_msg}${_nc}" +} + +function echoMagenta (){ + _msg="${@}" + _magenta='\e[35m' + _nc='\e[0m' # No Color + echo -e "${_magenta}${_msg}${_nc}" +} + +function logInfo(){ + ( + infoMsg="${1}" + echo -e "${infoMsg}" + postMsgToWebhook "${ENVIRONMENT_FRIENDLY_NAME}" \ + "${ENVIRONMENT_NAME}" \ + "INFO" \ + "${infoMsg}" + ) +} + +function logWarn(){ + ( + warnMsg="${1}" + echoYellow "${warnMsg}" + postMsgToWebhook "${ENVIRONMENT_FRIENDLY_NAME}" \ + "${ENVIRONMENT_NAME}" \ + "WARN" \ + "${warnMsg}" + ) +} + +function logError(){ + ( + errorMsg="${1}" + echoRed "[!!ERROR!!] - ${errorMsg}" >&2 + postMsgToWebhook "${ENVIRONMENT_FRIENDLY_NAME}" \ + "${ENVIRONMENT_NAME}" \ + "ERROR" \ + "${errorMsg}" + postMsgToPagerDuty "${errorMsg}" + ) +} + +function postMsgToWebhook(){ + ( + if [ -z "${WEBHOOK_URL}" ]; then + return 0 + fi + + projectFriendlyName=${1} + projectName=${2} + statusCode=${3} + message=$(echo -e "${4}") + + curl -s \ + -X POST \ + -H "Content-Type: application/x-www-form-urlencoded" \ + --data-urlencode "projectFriendlyName=${projectFriendlyName}" \ + --data-urlencode "projectName=${projectName}" \ + --data-urlencode "statusCode=${statusCode}" \ + --data-urlencode "message=${message}" \ + "${WEBHOOK_URL}" > /dev/null + ) +} + +function postMsgToPagerDuty(){ + ( + if [ -z "${PGDUTY_SVC_KEY}" ]; then + echo "Missing PagerDuty service key" + return 0 + fi + + if [ -z "${PGDUTY_URL}" ]; then + echo "Missing PagerDuty API url" + return 0 + fi + + message=$(echo -e "${1}" | tr '\n' ' ') + + curl -s \ + -X POST \ + -H "Content-type: application/json" \ + -d "{ + \"service_key\": \"${PGDUTY_SVC_KEY}\", + \"event_type\": \"trigger\", + \"description\": \"${message}\" + }" \ + "${PGDUTY_URL}" > /dev/null + ) +} +# ================================================================================================================= diff --git a/containers/backup/docker/backup.mariadb.plugin b/containers/backup/docker/backup.mariadb.plugin new file mode 100644 index 00000000..76024c52 --- /dev/null +++ b/containers/backup/docker/backup.mariadb.plugin @@ -0,0 +1,218 @@ +#!/bin/bash +# ================================================================================================================= +# MariaDB Backup and Restore Functions: +# - Dynamically loaded as a plug-in +# - Refer to existing plug-ins for implementation examples. +# ----------------------------------------------------------------------------------------------------------------- +export serverDataDirectory="/var/lib/mysql/data" + +function onBackupDatabase(){ + ( + local OPTIND + local unset flags + while getopts : FLAG; do + case $FLAG in + ? ) flags+="-${OPTARG} ";; + esac + done + shift $((OPTIND-1)) + + _databaseSpec=${1} + _backupFile=${2} + + _hostname=$(getHostname ${_databaseSpec}) + _database=$(getDatabaseName ${_databaseSpec}) + _port=$(getPort ${_databaseSpec}) + _portArg=${_port:+"-P ${_port}"} + _username=$(getUsername ${_databaseSpec}) + _password=$(getPassword ${_databaseSpec}) + echoGreen "Backing up '${_hostname}${_port:+:${_port}}${_database:+/${_database}}' to '${_backupFile}' ..." + + MYSQL_PWD=${_password} mysqldump -h "${_hostname}" -u "${_username}" ${_portArg} "${_database}" | gzip > ${_backupFile} + return ${?} + ) +} + +function onRestoreDatabase(){ + ( + local OPTIND + local unset flags + while getopts : FLAG; do + case $FLAG in + ? ) flags+="-${OPTARG} ";; + esac + done + shift $((OPTIND-1)) + + _databaseSpec=${1} + _fileName=${2} + _adminPassword=${3} + + _hostname=$(getHostname ${flags} ${_databaseSpec}) + _database=$(getDatabaseName ${_databaseSpec}) + _port=$(getPort ${flags} ${_databaseSpec}) + _portArg=${_port:+"-P ${_port}"} + _username=$(getUsername ${_databaseSpec}) + _password=$(getPassword ${_databaseSpec}) + echo -e "Restoring '${_fileName}' to '${_hostname}${_port:+:${_port}}${_database:+/${_database}}' ...\n" >&2 + + + # Restore + gunzip -c "${_fileName}" | MYSQL_PWD=${_password} mysql -h ${_hostname} -u ${_username} ${_portArg} ${_database} + _rtnCd=${PIPESTATUS[1]} + + return ${_rtnCd} + ) +} + +function onStartServer(){ + ( + local OPTIND + local unset flags + while getopts : FLAG; do + case $FLAG in + ? ) flags+="-${OPTARG} ";; + esac + done + shift $((OPTIND-1)) + + _databaseSpec=${1} + + # Start a local MariaDB instance + MYSQL_USER=$(getUsername "${_databaseSpec}") \ + MYSQL_PASSWORD=$(getPassword "${_databaseSpec}") \ + MYSQL_DATABASE=$(getDatabaseName "${_databaseSpec}") \ + MYSQL_ROOT_PASSWORD=$(getPassword "${_databaseSpec}") \ + run-mysqld >/dev/null 2>&1 & + ) +} + +function onStopServer(){ + ( + local OPTIND + local unset flags + while getopts : FLAG; do + case $FLAG in + ? ) flags+="-${OPTARG} ";; + esac + done + shift $((OPTIND-1)) + + mysqladmin --defaults-file=${MYSQL_DEFAULTS_FILE:-/etc/my.cnf} -u root --socket=/var/lib/mysql/mysql.sock flush-privileges shutdown + ) +} + +function onCleanup(){ + ( + if ! dirIsEmpty ${serverDataDirectory}; then + # Delete the database files and configuration + echo -e "Cleaning up ...\n" >&2 + rm -rf ${serverDataDirectory}/* + else + echo -e "Already clean ...\n" >&2 + fi + ) +} + +function onPingDbServer(){ + ( + local OPTIND + local unset flags + while getopts : FLAG; do + case $FLAG in + ? ) flags+="-${OPTARG} ";; + esac + done + shift $((OPTIND-1)) + + _databaseSpec=${1} + + _hostname=$(getHostname ${flags} ${_databaseSpec}) + _database=$(getDatabaseName ${_databaseSpec}) + _port=$(getPort ${flags} ${_databaseSpec}) + _portArg=${_port:+"-P ${_port}"} + _username=$(getUsername ${_databaseSpec}) + _password=$(getPassword ${_databaseSpec}) + + if MYSQL_PWD=${_password} mysql -h ${_hostname} -u ${_username} ${_portArg} ${_database} -e "SELECT 1;" >/dev/null 2>&1; then + return 0 + else + return 1 + fi + ) +} + +function onVerifyBackup(){ + ( + local OPTIND + local unset flags + while getopts : FLAG; do + case $FLAG in + ? ) flags+="-${OPTARG} ";; + esac + done + shift $((OPTIND-1)) + + _databaseSpec=${1} + + _hostname=$(getHostname -l ${_databaseSpec}) + _database=$(getDatabaseName ${_databaseSpec}) + _port=$(getPort -l ${_databaseSpec}) + _portArg=${_port:+"-P ${_port}"} + _username=$(getUsername ${_databaseSpec}) + _password=$(getPassword ${_databaseSpec}) + + debugMsg "backup.mariadb.plugin - onVerifyBackup" + tables=$(MYSQL_PWD=${_password} mysql -h "${_hostname}" -u "${_username}" ${_portArg} "${_database}" -e "SHOW TABLES;") + rtnCd=${?} + + # Get the size of the restored database + if (( ${rtnCd} == 0 )); then + size=$(getDbSize -l "${_databaseSpec}") + rtnCd=${?} + fi + + if (( ${rtnCd} == 0 )); then + numResults=$(echo "${tables}"| wc -l) + if [[ ! -z "${tables}" ]] && (( numResults >= 1 )); then + # All good + verificationLog="\nThe restored database contained ${numResults} tables, and is ${size} in size." + else + # Not so good + verificationLog="\nNo tables were found in the restored database." + rtnCd="3" + fi + fi + + echo ${verificationLog} + return ${rtnCd} + ) +} + +function onGetDbSize(){ + ( + local OPTIND + local unset flags + while getopts : FLAG; do + case $FLAG in + ? ) flags+="-${OPTARG} ";; + esac + done + shift $((OPTIND-1)) + + _databaseSpec=${1} + + _hostname=$(getHostname ${flags} ${_databaseSpec}) + _database=$(getDatabaseName ${_databaseSpec}) + _port=$(getPort ${flags} ${_databaseSpec}) + _portArg=${_port:+"-P ${_port}"} + _username=$(getUsername ${_databaseSpec}) + _password=$(getPassword ${_databaseSpec}) + + MYSQL_PWD=${_password} mysql -h "${_hostname}" -u "${_username}" ${_portArg} "${_database}" -e "SELECT ROUND(SUM(data_length + index_length) / 1024 / 1024, 1) AS \"size in mb\" FROM information_schema.tables WHERE table_schema=\"${_database}\" GROUP BY table_schema;" + + echo ${size} + return ${rtnCd} + ) +} +# ================================================================================================================= diff --git a/containers/backup/docker/backup.misc.utils b/containers/backup/docker/backup.misc.utils new file mode 100644 index 00000000..d636cd88 --- /dev/null +++ b/containers/backup/docker/backup.misc.utils @@ -0,0 +1,53 @@ +#!/bin/bash +# ================================================================================================================= +# General Utility Functions: +# ----------------------------------------------------------------------------------------------------------------- +function waitForAnyKey() { + read -n1 -s -r -p $'\e[33mWould you like to continue?\e[0m Press Ctrl-C to exit, or any other key to continue ...' key + echo -e \\n + + # If we get here the user did NOT press Ctrl-C ... + return 0 +} + +function formatList(){ + ( + filters='s~^~ - ~;' + _value=$(echo "${1}" | sed "${filters}") + echo "${_value}" + ) +} + +function isInstalled(){ + rtnVal=$(type "$1" >/dev/null 2>&1) + rtnCd=$? + if [ ${rtnCd} -ne 0 ]; then + return 1 + else + return 0 + fi +} + +function getElapsedTime(){ + ( + local startTime=${1} + local endTime=${2} + local duration=$(($endTime - $startTime)) + echo $(getElapsedTimeFromDuration "${duration}") + ) +} + +function getElapsedTimeFromDuration(){ + local duration_ns=${1} + + local hours=$((duration_ns / 3600000000000)) + local minutes=$(( (duration_ns % 3600000000000) / 60000000000 )) + local seconds=$(( (duration_ns % 60000000000) / 1000000000 )) + local milliseconds=$(( (duration_ns % 1000000000) / 1000000 )) + local microseconds=$(( (duration_ns % 1000000) / 1000 )) + local nanoseconds=$(( duration_ns % 1000 )) + + local elapsedTime="${hours}h:${minutes}m:${seconds}s:${milliseconds}ms:${microseconds}µs:${nanoseconds}ns" + echo ${elapsedTime} +} +# ====================================================================================== diff --git a/containers/backup/docker/backup.mongo.plugin b/containers/backup/docker/backup.mongo.plugin new file mode 100644 index 00000000..34837212 --- /dev/null +++ b/containers/backup/docker/backup.mongo.plugin @@ -0,0 +1,299 @@ +#!/bin/bash +# ================================================================================================================= +# Mongo Backup and Restore Functions: +# - Dynamically loaded as a plug-in +# ----------------------------------------------------------------------------------------------------------------- +export serverDataDirectory="/data/db" + +function onBackupDatabase(){ + ( + local OPTIND + local unset flags + while getopts : FLAG; do + case $FLAG in + ? ) flags+="-${OPTARG} ";; + esac + done + shift $((OPTIND-1)) + + _databaseSpec=${1} + _backupFile=${2} + + _hostname=$(getHostname ${_databaseSpec}) + _database=$(getDatabaseName ${_databaseSpec}) + _port=$(getPort ${_databaseSpec}) + _portArg=${_port:+"--port=${_port}"} + _username=$(getUsername ${_databaseSpec}) + _password=$(getPassword ${_databaseSpec}) + echoGreen "Backing up '${_hostname}${_port:+:${_port}}${_database:+/${_database}}' to '${_backupFile}' ..." + + _authDbArg=${MONGODB_AUTHENTICATION_DATABASE:+"--authenticationDatabase ${MONGODB_AUTHENTICATION_DATABASE}"} + mongodump -h "${_hostname}" -d "${_database}" ${_authDbArg} ${_portArg} -u "${_username}" -p "${_password}" --quiet --gzip --archive=${_backupFile} + return ${?} + ) +} + +function onRestoreDatabase(){ + ( + local OPTIND + local unset flags + while getopts : FLAG; do + case $FLAG in + ? ) flags+="-${OPTARG} ";; + esac + done + shift $((OPTIND-1)) + + _databaseSpec=${1} + _fileName=${2} + _adminPassword=${3} + + _hostname=$(getHostname ${flags} ${_databaseSpec}) + _database=$(getDatabaseName ${_databaseSpec}) + _port=$(getPort ${flags} ${_databaseSpec}) + _portArg=${_port:+"--port=${_port}"} + _username=$(getUsername ${_databaseSpec}) + _password=$(getPassword ${_databaseSpec}) + echo -e "Restoring '${_fileName}' to '${_hostname}${_port:+:${_port}}${_database:+/${_database}}' ...\n" >&2 + + # ToDo: + # - Add support for restoring to a different database. + # The following implementation only supports restoring to a database of the same name, + # unlike the postgres implementation that allows the database to be restored to a database of a different + # name for testing. + # Ref: https://stackoverflow.com/questions/36321899/mongorestore-to-a-different-database + + _authDbArg=${MONGODB_AUTHENTICATION_DATABASE:+"--authenticationDatabase ${MONGODB_AUTHENTICATION_DATABASE}"} + mongorestore --drop -h ${_hostname} -d "${_database}" ${_authDbArg} ${_portArg} -u "${_username}" -p "${_password}" --gzip --archive=${_fileName} --nsInclude="*" + return ${?} + ) +} + +function onDatabaseInit() { + local OPTIND + local unset flags + while getopts : FLAG; do + case $FLAG in + ? ) flags+="-${OPTARG} ";; + esac + done + shift $((OPTIND-1)) + + # Retrieve database details + local _databaseSpec=${1} + local _database=$(getDatabaseName "${_databaseSpec}") + local _username=$(getUsername "${_databaseSpec}") + local _password=$(getPassword "${_databaseSpec}") + + # Check if the database already exists + if mongosh --quiet --eval "db.getMongo().getDBNames().indexOf('$_database') >= 0"; then + echoYellow "Database '$_database' already exists, skipping initialization." + return 0 + fi + + # Initialize the database by creating the user with the roles + mongosh "$_database" --quiet --eval " + db.createUser({ + user: '$_username', + pwd: '$_password', + roles: [ + { role: 'dbOwner', db: '$_database' }, + { role: 'readWrite', db: '$_database' }, + { role: 'clusterAdmin', db: 'admin' } + ] + }); + " + + # Check the exit status of the createUser command + if [ $? -eq 0 ]; then + echoGreen "Database '$_database' initialized successfully." + return 0 + else + echoRed "Failed to initialize database '$_database'." + return 1 + fi +} + + +function onStartServer(){ + ( + local OPTIND + local unset flags + while getopts : FLAG; do + case $FLAG in + ? ) flags+="-${OPTARG} ";; + esac + done + shift $((OPTIND-1)) + + _databaseSpec=${1} + + _hostname=$(getHostname ${flags} ${_databaseSpec}) + + # Start a local MongoDb instance + mongod --bind_ip $_hostname > /dev/null 2>&1 & + + # Initialize database if necessary + onDatabaseInit "${_databaseSpec}" + ) +} + +function onStopServer(){ + ( + local OPTIND + local unset flags + while getopts : FLAG; do + case $FLAG in + ? ) flags+="-${OPTARG} ";; + esac + done + shift $((OPTIND-1)) + + _databaseSpec=${1} + + _hostname=$(getHostname ${flags} ${_databaseSpec}) + + mongosh ${_hostname}/admin --quiet --eval "db.shutdownServer()" > /dev/null 2>&1 + + # Wait for server to stop ... + local startTime=$(date +%s%N) + + printf "waiting for server to stop" + while onPingDbServer -l ${@}; do + printf "." + local duration_ns=$(($(date +%s%N) - $startTime)) + if (( ${duration_ns} >= ${DATABASE_SERVER_TIMEOUT_NS} )); then + echoRed "\nThe server failed to stop within $(getElapsedTimeFromDuration ${duration})." + echoRed "Killing the mongod process ...\n" + pkill -INT mongod + break + fi + sleep 1 + done + ) +} + +function onCleanup(){ + ( + if ! dirIsEmpty ${serverDataDirectory}; then + # Delete the database files and configuration + echo -e "Cleaning up ...\n" >&2 + rm -rf ${serverDataDirectory}/* + else + echo -e "Already clean ...\n" >&2 + fi + ) +} + +function onPingDbServer(){ + ( + local OPTIND + local unset flags + while getopts : FLAG; do + case $FLAG in + ? ) flags+="-${OPTARG} ";; + esac + done + shift $((OPTIND-1)) + + _databaseSpec=${1} + + _hostname=$(getHostname ${flags} ${_databaseSpec}) + _database=$(getDatabaseName ${_databaseSpec}) + _port=$(getPort ${flags} ${_databaseSpec}) + _portArg=${_port:+"--port ${_port}"} + _username=$(getUsername ${_databaseSpec}) + _password=$(getPassword ${_databaseSpec}) + + _dbAddressArg=${_hostname}${_port:+:${_port}}${_database:+/${_database}} + _authDbArg=${MONGODB_AUTHENTICATION_DATABASE:+"--authenticationDatabase ${MONGODB_AUTHENTICATION_DATABASE}"} + + mongosh ${_dbAddressArg} ${_authDbArg} -u "${_username}" -p "${_password}" --quiet --eval='db.runCommand("ping").ok' > /dev/null 2>&1 + local mongoshExitCode=$? + + if (( ${mongoshExitCode} != 0 )); then + return 1 + else + return 0 + fi + ) +} + +function onVerifyBackup(){ + ( + local OPTIND + local unset flags + while getopts : FLAG; do + case $FLAG in + ? ) flags+="-${OPTARG} ";; + esac + done + shift $((OPTIND-1)) + + _databaseSpec=${1} + + _hostname=$(getHostname -l ${_databaseSpec}) + _database=$(getDatabaseName ${_databaseSpec}) + _port=$(getPort -l ${_databaseSpec}) + _portArg=${_port:+"--port ${_port}"} + _username=$(getUsername ${_databaseSpec}) + _password=$(getPassword ${_databaseSpec}) + + _dbAddressArg=${_hostname}${_port:+:${_port}}${_database:+/${_database}} + _authDbArg=${MONGODB_AUTHENTICATION_DATABASE:+"--authenticationDatabase ${MONGODB_AUTHENTICATION_DATABASE}"} + collections=$(mongosh ${_dbAddressArg} ${_authDbArg} -u "${_username}" -p "${_password}" --quiet --eval 'var dbs = [];dbs = db.getCollectionNames();for (i in dbs){ print(db.dbs[i]);}';) + rtnCd=${?} + + # Get the size of the restored database + if (( ${rtnCd} == 0 )); then + size=$(getDbSize -l "${_databaseSpec}") + rtnCd=${?} + fi + + if (( ${rtnCd} == 0 )); then + numResults=$(echo "${collections}"| wc -l) + if [[ ! -z "${collections}" ]] && (( numResults >= 1 )); then + # All good + verificationLog="\nThe restored database contained ${numResults} collections, and is ${size} in size." + else + # Not so good + verificationLog="\nNo collections were found in the restored database ${_database}." + rtnCd="3" + fi + fi + + echo ${verificationLog} + return ${rtnCd} + ) +} + +function onGetDbSize(){ + ( + local OPTIND + local unset flags + while getopts : FLAG; do + case $FLAG in + ? ) flags+="-${OPTARG} ";; + esac + done + shift $((OPTIND-1)) + + _databaseSpec=${1} + + _hostname=$(getHostname ${flags} ${_databaseSpec}) + _database=$(getDatabaseName ${_databaseSpec}) + _port=$(getPort ${flags} ${_databaseSpec}) + _portArg=${_port:+"--port ${_port}"} + _username=$(getUsername ${_databaseSpec}) + _password=$(getPassword ${_databaseSpec}) + + _dbAddressArg=${_hostname}${_port:+:${_port}}${_database:+/${_database}} + _authDbArg=${MONGODB_AUTHENTICATION_DATABASE:+"--authenticationDatabase ${MONGODB_AUTHENTICATION_DATABASE}"} + size=$(mongosh ${_dbAddressArg} ${_authDbArg} -u "${_username}" -p "${_password}" --quiet --eval 'printjson(db.stats().fsTotalSize)') + rtnCd=${?} + + echo ${size} + return ${rtnCd} + ) +} +# ================================================================================================================= diff --git a/containers/backup/docker/backup.mssql.plugin b/containers/backup/docker/backup.mssql.plugin new file mode 100644 index 00000000..e9bbc3a1 --- /dev/null +++ b/containers/backup/docker/backup.mssql.plugin @@ -0,0 +1,229 @@ +#!/bin/bash +# ================================================================================================================= +# MSSQL Backup and Restore Functions: +# - Dynamically loaded as a plug-in +# ----------------------------------------------------------------------------------------------------------------- +export serverDataDirectory="/var/opt/mssql/data" + +function onBackupDatabase(){ + ( + local OPTIND + local unset flags + while getopts : FLAG; do + case $FLAG in + ? ) flags+="-${OPTARG} ";; + esac + done + shift $((OPTIND-1)) + + _databaseSpec=${1} + _backupFile=${2} + + _hostname=$(getHostname ${_databaseSpec}) + _database=$(getDatabaseName ${_databaseSpec}) + _port=$(getPort ${_databaseSpec}) + _portArg=${_port:+",${_port}"} + _username=$(getLocalOrDBUsername ${_databaseSpec} ${_hostname}) + _password=$(getPassword ${_databaseSpec}) + echoGreen "Backing up '${_hostname}${_port:+:${_port}}${_database:+/${_database}}' to '${_backupFile}' ..." + + #TODO: add support for backing up transaction log as well. + sqlcmd -S ${_hostname}${_portArg} -U ${_username} -P ${_password} -Q "BACKUP DATABASE ${_database} TO DISK = N'${_fileName}' WITH NOFORMAT, NOINIT, SKIP, NOREWIND, NOUNLOAD, STATS = 10" + return ${?} + ) +} + +function onRestoreDatabase(){ + ( + local OPTIND + local unset flags + while getopts : FLAG; do + case $FLAG in + ? ) flags+="-${OPTARG} ";; + esac + done + shift $((OPTIND-1)) + + _databaseSpec=${1} + _fileName=${2} + _adminPassword=${3} + + _hostname=$(getHostname ${flags} ${_databaseSpec}) + _database=$(getDatabaseName ${_databaseSpec}) + _port=$(getPort ${flags} ${_databaseSpec}) + _portArg=${_port:+",${_port}"} + _username=$(getLocalOrDBUsername ${_databaseSpec} ${_hostname}) + _password=$(getPassword ${_databaseSpec}) + echo -e "Restoring '${_fileName}' to '${_hostname}${_port:+:${_port}}${_database:+/${_database}}' ...\n" >&2 + + #force single user mode on database to ensure restore works properly + sqlcmd -S ${_hostname}${_portArg} -U ${_username} -P ${_password} -Q "ALTER DATABASE ${_database} SET SINGLE_USER WITH ROLLBACK AFTER 30;RESTORE DATABASE ${_database} FROM DISK = N'${_fileName}' WITH FILE = 1, NOUNLOAD, REPLACE, STATS=5;ALTER DATABASE ${_database} SET MULTI_USER" + return ${?} + ) +} + +function onStartServer(){ + ( + local OPTIND + local unset flags + while getopts : FLAG; do + case $FLAG in + ? ) flags+="-${OPTARG} ";; + esac + done + shift $((OPTIND-1)) + + _databaseSpec=${1} + + /opt/mssql/bin/sqlservr --accept-eula & + ) +} + +function onStopServer(){ + ( + local OPTIND + local unset flags + while getopts : FLAG; do + case $FLAG in + ? ) flags+="-${OPTARG} ";; + esac + done + shift $((OPTIND-1)) + + _hostname=$(getHostname ${flags} ${_databaseSpec}) + _port=$(getPort ${flags} ${_databaseSpec}) + _portArg=${_port:+",${_port}"} + _username=$(getLocalOrDBUsername ${_databaseSpec} ${_hostname}) + _password=$(getPassword ${_databaseSpec}) + + sqlcmd -S ${_hostname}${_portArg} -U ${_username} -P ${_password} -Q "SHUTDOWN" + ) +} + +function onCleanup(){ + ( + if ! dirIsEmpty ${serverDataDirectory}; then + # Delete the database files and configuration + echo -e "Cleaning up ...\n" >&2 + rm -rf ${serverDataDirectory}/* + else + echo -e "Already clean ...\n" >&2 + fi + ) +} + +function onPingDbServer(){ + ( + local OPTIND + local unset flags + while getopts : FLAG; do + case $FLAG in + ? ) flags+="-${OPTARG} ";; + esac + done + shift $((OPTIND-1)) + + _databaseSpec=${1} + + _hostname=$(getHostname ${flags} ${_databaseSpec}) + _database=$(getDatabaseName ${_databaseSpec}) + _port=$(getPort ${flags} ${_databaseSpec}) + _portArg=${_port:+",${_port}"} + _username=$(getLocalOrDBUsername ${_databaseSpec} ${_hostname}) + _password=$(getPassword ${_databaseSpec}) + + if sqlcmd -S ${_hostname}${_portArg} -U ${_username} -P ${_password} -Q "SELECT 1" >/dev/null 2>&1; then + return 0 + else + return 1 + fi + ) +} + +function onVerifyBackup(){ + ( + local OPTIND + local unset flags + while getopts : FLAG; do + case $FLAG in + ? ) flags+="-${OPTARG} ";; + esac + done + shift $((OPTIND-1)) + + _databaseSpec=${1} + _hostname=$(getHostname -l ${_databaseSpec}) + _database=$(getDatabaseName ${_databaseSpec}) + _port=$(getPort -l ${_databaseSpec}) + _portArg=${_port:+",${_port}"} + _username=$(getLocalOrDBUsername ${_databaseSpec} ${_hostname}) + _password=$(getPassword ${_databaseSpec}) + + tables=$(sqlcmd -S ${_hostname}${_portArg} -U ${_username} -P ${_password} -d ${_database} -Q "SELECT table_name FROM information_schema.tables WHERE TABLE_CATALOG = '${_database}' AND table_type='BASE TABLE';") + rtnCd=${?} + + # Get the size of the restored database + if (( ${rtnCd} == 0 )); then + size=$(getDbSize -l "${_databaseSpec}") + rtnCd=${?} + fi + + if (( ${rtnCd} == 0 )); then + numResults=$(echo "${tables}"| wc -l) + if [[ ! -z "${tables}" ]] && (( numResults >= 1 )); then + # All good + verificationLog="\nThe restored database contained ${numResults} tables, and is ${size} in size." + else + # Not so good + verificationLog="\nNo tables were found in the restored database ${_database}." + rtnCd="3" + fi + fi + + echo ${verificationLog} + return ${rtnCd} + ) +} + +function onGetDbSize(){ + ( + local OPTIND + local unset flags + while getopts : FLAG; do + case $FLAG in + ? ) flags+="-${OPTARG} ";; + esac + done + shift $((OPTIND-1)) + + _databaseSpec=${1} + + _hostname=$(getHostname ${flags} ${_databaseSpec}) + _database=$(getDatabaseName ${_databaseSpec}) + _port=$(getPort ${flags} ${_databaseSpec}) + _portArg=${_port:+",${_port}"} + _username=$(getLocalOrDBUsername ${_databaseSpec} ${_hostname}) + _password=$(getPassword ${_databaseSpec}) + + size=$(sqlcmd -S ${_hostname}${_portArg} -U ${_username} -P ${_password} -d ${_database} -Q "SELECT CONVERT(VARCHAR,SUM(size)*8/1024)+' MB' AS 'size' FROM sys.master_files m INNER JOIN sys.databases d ON d.database_id = m.database_id WHERE d.name = '${_database}' AND m.type_desc = 'ROWS' GROUP BY d.name") + rtnCd=${?} + + echo ${size} + return ${rtnCd} + ) +} + +function getLocalOrDBUsername(){ + ( + _databaseSpec=${1} + _localhost="127.0.0.1" + _hostname=${2} + if [ "$_hostname" == "$_localhost" ]; then + _username=sa + else + _username=$(getUsername ${_databaseSpec}) + fi + echo ${_username} + ) +} +# ================================================================================================================= \ No newline at end of file diff --git a/containers/backup/docker/backup.null.plugin b/containers/backup/docker/backup.null.plugin new file mode 100644 index 00000000..653bb06a --- /dev/null +++ b/containers/backup/docker/backup.null.plugin @@ -0,0 +1,208 @@ +#!/bin/bash +# ================================================================================================================= +# Null Backup and Restore Functions: +# - Dynamically loaded as a plug-in +# - Refer to existing plug-ins for implementation examples. +# ----------------------------------------------------------------------------------------------------------------- +export serverDataDirectory="/var/lib/pgsql/data" + +function onBackupDatabase(){ + ( + local OPTIND + local unset flags + while getopts : FLAG; do + case $FLAG in + ? ) flags+="-${OPTARG} ";; + esac + done + shift $((OPTIND-1)) + + _databaseSpec=${1} + _backupFile=${2} + + _hostname=$(getHostname ${_databaseSpec}) + _database=$(getDatabaseName ${_databaseSpec}) + _port=$(getPort ${_databaseSpec}) + _portArg=${_port:+"--port ${_port}"} + _username=$(getUsername ${_databaseSpec}) + _password=$(getPassword ${_databaseSpec}) + echoGreen "Backing up '${_hostname}${_port:+:${_port}}${_database:+/${_database}}' to '${_backupFile}' ..." + + echoRed "[backup.null.plugin] onBackupDatabase - Not Implemented" + # echoGreen "Starting database backup ..." + # Add your database specific backup operation(s) here. + return ${?} + ) +} + +function onRestoreDatabase(){ + ( + local OPTIND + local unset flags + while getopts : FLAG; do + case $FLAG in + ? ) flags+="-${OPTARG} ";; + esac + done + shift $((OPTIND-1)) + + _databaseSpec=${1} + _fileName=${2} + _adminPassword=${3} + + _hostname=$(getHostname ${flags} ${_databaseSpec}) + _database=$(getDatabaseName ${_databaseSpec}) + _port=$(getPort ${flags} ${_databaseSpec}) + _portArg=${_port:+"--port ${_port}"} + _username=$(getUsername ${_databaseSpec}) + _password=$(getPassword ${_databaseSpec}) + echo -e "Restoring '${_fileName}' to '${_hostname}${_port:+:${_port}}${_database:+/${_database}}' ...\n" >&2 + + echoRed "[backup.null.plugin] onRestoreDatabase - Not Implemented" + # Add your database specific restore operation(s) here. + return ${?} + ) +} + +function onStartServer(){ + ( + local OPTIND + local unset flags + while getopts : FLAG; do + case $FLAG in + ? ) flags+="-${OPTARG} ";; + esac + done + shift $((OPTIND-1)) + + _databaseSpec=${1} + + echoRed "[backup.null.plugin] onStartServer - Not Implemented" + # Add your NON-BLOCKING database specific startup operation(s) here. + # - Start the database server as a background job. + ) +} + +function onStopServer(){ + ( + local OPTIND + local unset flags + while getopts : FLAG; do + case $FLAG in + ? ) flags+="-${OPTARG} ";; + esac + done + shift $((OPTIND-1)) + + _databaseSpec=${1} + _username=$(getUsername ${_databaseSpec}) + _password=$(getPassword ${_databaseSpec}) + + echoRed "[backup.null.plugin] onStopServer - Not Implemented" + + # echo "Shutting down..." + # Add your database specific shutdown operation(s) here. + ) +} + +function onCleanup(){ + ( + # Add your database specific cleanup operation(s) here. + echoRed "[backup.null.plugin] onCleanup - Not Implemented" + + # if ! dirIsEmpty ${serverDataDirectory}; then + # # Delete the database files and configuration + # echo -e "Cleaning up ...\n" >&2 + # rm -rf ${serverDataDirectory}/* + # else + # echo -e "Already clean ...\n" >&2 + # fi + ) +} + +function onPingDbServer(){ + ( + local OPTIND + local unset flags + while getopts : FLAG; do + case $FLAG in + ? ) flags+="-${OPTARG} ";; + esac + done + shift $((OPTIND-1)) + + _databaseSpec=${1} + + _hostname=$(getHostname ${flags} ${_databaseSpec}) + _database=$(getDatabaseName ${_databaseSpec}) + _port=$(getPort ${flags} ${_databaseSpec}) + _portArg=${_port:+"--port ${_port}"} + _username=$(getUsername ${_databaseSpec}) + _password=$(getPassword ${_databaseSpec}) + + echoRed "[backup.null.plugin] onPingDbServer - Not Implemented" + # Add your database specific ping operation(s) here. + # if ; then + # return 0 + # else + # return 1 + # fi + ) +} + +function onVerifyBackup(){ + ( + local OPTIND + local unset flags + while getopts : FLAG; do + case $FLAG in + ? ) flags+="-${OPTARG} ";; + esac + done + shift $((OPTIND-1)) + + _databaseSpec=${1} + + _hostname=$(getHostname -l ${_databaseSpec}) + _database=$(getDatabaseName ${_databaseSpec}) + _port=$(getPort -l ${_databaseSpec}) + _portArg=${_port:+"--port ${_port}"} + _username=$(getUsername ${_databaseSpec}) + _password=$(getPassword ${_databaseSpec}) + + echoRed "[backup.null.plugin] onVerifyBackup - Not Implemented" + # Add your database specific verification operation(s) here. + + # echo ${verificationLog} + # return ${rtnCd} + ) +} + +function onGetDbSize(){ + ( + local OPTIND + local unset flags + while getopts : FLAG; do + case $FLAG in + ? ) flags+="-${OPTARG} ";; + esac + done + shift $((OPTIND-1)) + + _databaseSpec=${1} + + _hostname=$(getHostname ${flags} ${_databaseSpec}) + _database=$(getDatabaseName ${_databaseSpec}) + _port=$(getPort ${flags} ${_databaseSpec}) + _portArg=${_port:+"--port ${_port}"} + _username=$(getUsername ${_databaseSpec}) + _password=$(getPassword ${_databaseSpec}) + + echoRed "[backup.null.plugin] onGetDbSize - Not Implemented" + # Add your database specific get size operation(s) here. + + # echo ${size} + # return ${rtnCd} + ) +} +# ================================================================================================================= diff --git a/containers/backup/docker/backup.postgres.plugin b/containers/backup/docker/backup.postgres.plugin new file mode 100644 index 00000000..d4710f67 --- /dev/null +++ b/containers/backup/docker/backup.postgres.plugin @@ -0,0 +1,276 @@ +#!/bin/bash +# ================================================================================================================= +# Postgres Backup and Restore Functions: +# - Dynamically loaded as a plug-in +# ----------------------------------------------------------------------------------------------------------------- +export serverDataDirectory="/var/lib/pgsql/data" + +function onBackupDatabase(){ + ( + local OPTIND + local unset flags + while getopts : FLAG; do + case $FLAG in + ? ) flags+="-${OPTARG} ";; + esac + done + shift $((OPTIND-1)) + + _databaseSpec=${1} + _backupFile=${2} + + _hostname=$(getHostname ${_databaseSpec}) + _database=$(getDatabaseName ${_databaseSpec}) + _port=$(getPort ${_databaseSpec}) + _portArg=${_port:+"-p ${_port}"} + _username=$(getUsername ${_databaseSpec}) + _password=$(getPassword ${_databaseSpec}) + echoGreen "Backing up '${_hostname}${_port:+:${_port}}${_database:+/${_database}}' to '${_backupFile}' ..." + + export PGPASSWORD=${_password} + pg_dump -Fp -h "${_hostname}" ${_portArg} -U "${_username}" "${_database}" > "${BACKUP_DIR}backup.sql" + pg_dumpall -h "${_hostname}" ${_portArg} -U "${_username}" --roles-only --no-role-passwords > "${BACKUP_DIR}roles.sql" + cat "${BACKUP_DIR}roles.sql" "${BACKUP_DIR}backup.sql" | gzip > ${_backupFile} + rm "${BACKUP_DIR}roles.sql" && rm "${BACKUP_DIR}backup.sql" + return ${PIPESTATUS[0]} + ) +} + +function onRestoreDatabase(){ + ( + local OPTIND + local unset quiet + local unset flags + while getopts :q FLAG; do + case $FLAG in + q ) + quiet=1 + flags+="-${FLAG} " + ;; + ? ) flags+="-${OPTARG} ";; + esac + done + shift $((OPTIND-1)) + + _databaseSpec=${1} + _fileName=${2} + _adminPassword=${3} + + _hostname=$(getHostname ${flags} ${_databaseSpec}) + _database=$(getDatabaseName ${_databaseSpec}) + _port=$(getPort ${flags} ${_databaseSpec}) + _portArg=${_port:+"-p ${_port}"} + _username=$(getUsername ${_databaseSpec}) + _password=$(getPassword ${_databaseSpec}) + echo -e "Restoring '${_fileName}' to '${_hostname}${_port:+:${_port}}${_database:+/${_database}}' ...\n" >&2 + + export PGPASSWORD=${_adminPassword} + _stopOnErrors="-v ON_ERROR_STOP=1" + if ignoreErrors; then + _stopOnErrors="-v ON_ERROR_STOP=0" + fi + _rtnCd=0 + + # Drop + if (( ${_rtnCd} == 0 )); then + psql -h "${_hostname}" ${_portArg} -ac "DROP DATABASE \"${_database}\";" + _rtnCd=${?} + echo + fi + + # Create + if (( ${_rtnCd} == 0 )); then + psql -h "${_hostname}" ${_portArg} -ac "CREATE DATABASE \"${_database}\";" + _rtnCd=${?} + echo + fi + + # Drop Patroni-specific schemas + if (( ${_rtnCd} == 0 )); then + psql -h "${_hostname}" ${_portArg} -a -d ${_database} </dev/null 2>&1 & + ) +} + +function onStopServer(){ + ( + local OPTIND + local unset flags + while getopts : FLAG; do + case $FLAG in + ? ) flags+="-${OPTARG} ";; + esac + done + shift $((OPTIND-1)) + + _databaseSpec=${1} + + # Stop the local PostgreSql instance + pg_ctl stop -D ${serverDataDirectory}/userdata + ) +} + +function onCleanup(){ + ( + if ! dirIsEmpty ${serverDataDirectory}; then + # Delete the database files and configuration + echo -e "Cleaning up ...\n" >&2 + rm -rf ${serverDataDirectory}/* + else + echo -e "Already clean ...\n" >&2 + fi + ) +} + +function onPingDbServer(){ + ( + local OPTIND + local unset flags + while getopts : FLAG; do + case $FLAG in + ? ) flags+="-${OPTARG} ";; + esac + done + shift $((OPTIND-1)) + + _databaseSpec=${1} + + _hostname=$(getHostname ${flags} ${_databaseSpec}) + _database=$(getDatabaseName ${_databaseSpec}) + _port=$(getPort ${flags} ${_databaseSpec}) + _portArg=${_port:+"-p ${_port}"} + _username=$(getUsername ${_databaseSpec}) + _password=$(getPassword ${_databaseSpec}) + + if PGPASSWORD=${_password} psql -h ${_hostname} ${_portArg} -U ${_username} -q -d ${_database} -c 'SELECT 1' >/dev/null 2>&1; then + return 0 + else + return 1 + fi + ) +} + +function onVerifyBackup(){ + ( + local OPTIND + local unset flags + while getopts : FLAG; do + case $FLAG in + ? ) flags+="-${OPTARG} ";; + esac + done + shift $((OPTIND-1)) + + _databaseSpec=${1} + + _hostname=$(getHostname -l ${_databaseSpec}) + _database=$(getDatabaseName ${_databaseSpec}) + _port=$(getPort -l ${_databaseSpec}) + _portArg=${_port:+"-p ${_port}"} + _username=$(getUsername ${_databaseSpec}) + _password=$(getPassword ${_databaseSpec}) + + debugMsg "backup.postgres.plugin - onVerifyBackup" + tables=$(psql -h "${_hostname}" ${_portArg} -d "${_database}" -t -c "SELECT table_name FROM information_schema.tables WHERE table_schema='${TABLE_SCHEMA}' AND table_type='BASE TABLE';") + rtnCd=${?} + + # Get the size of the restored database + if (( ${rtnCd} == 0 )); then + size=$(getDbSize -l "${_databaseSpec}") + rtnCd=${?} + fi + + if (( ${rtnCd} == 0 )); then + numResults=$(echo "${tables}"| wc -l) + if [[ ! -z "${tables}" ]] && (( numResults >= 1 )); then + # All good + verificationLog="\nThe restored database contained ${numResults} tables, and is ${size} in size." + else + # Not so good + verificationLog="\nNo tables were found in the restored database." + rtnCd="3" + fi + fi + + echo ${verificationLog} + return ${rtnCd} + ) +} + +function onGetDbSize(){ + ( + local OPTIND + local unset flags + while getopts : FLAG; do + case $FLAG in + ? ) flags+="-${OPTARG} ";; + esac + done + shift $((OPTIND-1)) + + _databaseSpec=${1} + + _hostname=$(getHostname ${flags} ${_databaseSpec}) + _database=$(getDatabaseName ${_databaseSpec}) + _port=$(getPort ${flags} ${_databaseSpec}) + _portArg=${_port:+"-p ${_port}"} + _username=$(getUsername ${_databaseSpec}) + _password=$(getPassword ${_databaseSpec}) + + size=$(PGPASSWORD=${_password} psql -h "${_hostname}" ${_portArg} -U "${_username}" -d "${_database}" -t -c "SELECT pg_size_pretty(pg_database_size(current_database())) as size;") + rtnCd=${?} + + echo ${size} + return ${rtnCd} + ) +} +# ================================================================================================================= diff --git a/containers/backup/docker/backup.s3 b/containers/backup/docker/backup.s3 new file mode 100644 index 00000000..ec73cc4f --- /dev/null +++ b/containers/backup/docker/backup.s3 @@ -0,0 +1,21 @@ +#!/bin/bash + +function s3Backup() { + local db_backup="$1" + local minio_alias=minio_s3 + local minio_container=backup-container-minio-1 + + if [[ $S3_ENDPOINT ]]; then + if ! mc alias ls | grep -o "^$minio_alias" > /dev/null; then + echo "Creating $minio_alias.." + mc alias set $minio_alias $S3_ENDPOINT $S3_USER $S3_PASSWORD + fi + + if ! mc ls $minio_alias/$S3_BUCKET &> /dev/null; then + echo "Creating $S3_BUCKET bucket.." + mc mb $minio_alias/$S3_BUCKET + fi + + mc cp "$db_backup" $minio_alias/$S3_BUCKET + fi +} diff --git a/containers/backup/docker/backup.server.utils b/containers/backup/docker/backup.server.utils new file mode 100644 index 00000000..9e938a15 --- /dev/null +++ b/containers/backup/docker/backup.server.utils @@ -0,0 +1,39 @@ +#!/bin/bash +# ================================================================================================================= +# Backup Server Utility Functions: +# ----------------------------------------------------------------------------------------------------------------- +function startCron(){ + logInfo "Starting backup server in cron mode ..." + listSettings + echoBlue "Starting go-crond as a background task ...\n" + CRON_CMD="go-crond -v --default-user=${UID} --allow-unprivileged ${BACKUP_CONF}" + exec ${CRON_CMD} & + wait +} + +function startLegacy(){ + ( + while true; do + runBackups + + echoYellow "Sleeping for ${BACKUP_PERIOD} ...\n" + sleep ${BACKUP_PERIOD} + done + ) +} + +function shutDown(){ + jobIds=$(jobs | awk -F '[][]' '{print $2}' ) + for jobId in ${jobIds} ; do + echo "Shutting down background job '${jobId}' ..." + kill %${jobId} + done + + if [ ! -z "${jobIds}" ]; then + echo "Waiting for any background jobs to complete ..." + fi + wait + + exit 0 +} +# ====================================================================================== \ No newline at end of file diff --git a/containers/backup/docker/backup.settings b/containers/backup/docker/backup.settings new file mode 100644 index 00000000..47ad2035 --- /dev/null +++ b/containers/backup/docker/backup.settings @@ -0,0 +1,55 @@ +#!/bin/bash +# ====================================================================================== +# Default Settings +# -------------------------------------------------------------------------------------- +export BACKUP_FILE_EXTENSION=".sql.gz" +export IN_PROGRESS_BACKUP_FILE_EXTENSION=".sql.gz.in_progress" +export DEFAULT_PORT=${POSTGRESQL_PORT_NUM:-5432} +export DATABASE_SERVICE_NAME=${DATABASE_SERVICE_NAME:-postgresql} +export POSTGRESQL_DATABASE=${POSTGRESQL_DATABASE:-my_postgres_db} +export TABLE_SCHEMA=${TABLE_SCHEMA:-public} + +# Supports: +# - daily +# - rolling +export BACKUP_STRATEGY=$(echo "${BACKUP_STRATEGY:-rolling}" | tr '[:upper:]' '[:lower:]') +export BACKUP_PERIOD=${BACKUP_PERIOD:-1d} +export ROOT_BACKUP_DIR=${ROOT_BACKUP_DIR:-${BACKUP_DIR:-/backups/}} +export BACKUP_CONF=${BACKUP_CONF:-backup.conf} + +# Used to prune the total number of backup when using the daily backup strategy. +# Default provides for one full month of backups +export NUM_BACKUPS=${NUM_BACKUPS:-31} + +# Used to prune the total number of backup when using the rolling backup strategy. +# Defaults provide for: +# - A week's worth of daily backups +# - A month's worth of weekly backups +# - The previous month's backup +export DAILY_BACKUPS=${DAILY_BACKUPS:-6} +export WEEKLY_BACKUPS=${WEEKLY_BACKUPS:-4} +export MONTHLY_BACKUPS=${MONTHLY_BACKUPS:-1} + +# Modes: +export ONCE="once" +export SCHEDULED="scheduled" +export RESTORE="restore" +export VERIFY="verify" +export CRON="cron" +export LEGACY="legacy" +export ERROR="error" +export SCHEDULED_VERIFY="scheduled-verify" +export PRUNE="prune" + +# Supported Database Containers +export UNKNOWN_DB="null" +export MONGO_DB="mongo" +export POSTGRE_DB="postgres" +export MSSQL_DB="mssql" +export MARIA_DB="mariadb" +export CONTAINER_TYPE="$(getContainerType)" + +# Other: +export DATABASE_SERVER_TIMEOUT=${DATABASE_SERVER_TIMEOUT:-120} +export DATABASE_SERVER_TIMEOUT_NS=$((DATABASE_SERVER_TIMEOUT * 1000000000)) +# ====================================================================================== diff --git a/containers/backup/docker/backup.sh b/containers/backup/docker/backup.sh new file mode 100755 index 00000000..1d41f210 --- /dev/null +++ b/containers/backup/docker/backup.sh @@ -0,0 +1,144 @@ +#!/bin/bash + +# ====================================================================================== +# Imports +# -------------------------------------------------------------------------------------- +. ./backup.usage # Usage information +. ./backup.logging # Logging functions +. ./backup.config.utils # Configuration functions +. ./backup.container.utils # Container Utility Functions +. ./backup.s3 # S3 Support functions +. ./backup.ftp # FTP Support functions +. ./backup.misc.utils # General Utility Functions +. ./backup.file.utils # File Utility Functions +. ./backup.utils # Primary Database Backup and Restore Functions +. ./backup.server.utils # Backup Server Utility Functions +. ./backup.settings # Default Settings +# ====================================================================================== + +# ====================================================================================== +# Initialization: +# -------------------------------------------------------------------------------------- +trap shutDown EXIT TERM + +# Load database plug-in based on the container type ... +. ./backup.${CONTAINER_TYPE}.plugin > /dev/null 2>&1 +if [[ ${?} != 0 ]]; then + echoRed "backup.${CONTAINER_TYPE}.plugin not found." + + # Default to null plugin. + export CONTAINER_TYPE=${UNKNOWN_DB} + . ./backup.${CONTAINER_TYPE}.plugin > /dev/null 2>&1 +fi + +while getopts nclr:v:f:1spha:I FLAG; do + case $FLAG in + n) + # Allow null database plugin ... + # Without this flag loading the null plugin is considered a configuration error. + # The null plugin can be used for testing. + export _allowNullPlugin=1 + ;; + c) + echoBlue "\nListing configuration settings ..." + listSettings + exit 0 + ;; + l) + listExistingBackups ${ROOT_BACKUP_DIR} + exit 0 + ;; + r) + # Trigger restore mode ... + export _restoreDatabase=${OPTARG} + ;; + v) + # Trigger verify mode ... + export _verifyBackup=${OPTARG} + ;; + f) + # Optionally specify the backup file to verify or restore from ... + export _fromBackup=${OPTARG} + ;; + 1) + export RUN_ONCE=1 + ;; + s) + export SCHEDULED_RUN=1 + ;; + p) + export RUN_PRUNE=1 + ;; + a) + export _adminPassword=${OPTARG} + ;; + I) + export IGNORE_ERRORS=1 + ;; + h) + usage + ;; + \?) + echo -e \\n"Invalid option: -${OPTARG}"\\n + usage + ;; + esac +done +shift $((OPTIND-1)) +# ====================================================================================== + +# ====================================================================================== +# Main Script +# -------------------------------------------------------------------------------------- +case $(getMode) in + ${ONCE}) + runBackups + echoGreen "Single backup run complete.\n" + ;; + + ${SCHEDULED}) + runBackups + echoGreen "Scheduled backup run complete.\n" + ;; + + ${RESTORE}) + unset restoreFlags + if isScripted; then + restoreFlags="-q" + fi + + if validateOperation "${_restoreDatabase}" "${RESTORE}"; then + restoreDatabase ${restoreFlags} "${_restoreDatabase}" "${_fromBackup}" + fi + ;; + + ${VERIFY}) + verifyBackups "${_verifyBackup}" "${_fromBackup}" + ;; + + ${SCHEDULED_VERIFY}) + verifyBackups -q "${_verifyBackup}" "${_fromBackup}" + ;; + + ${CRON}) + startCron + ;; + + ${LEGACY}) + startLegacy + ;; + + ${PRUNE}) + prune + ;; + + ${ERROR}) + echoRed "A configuration error has occurred, review the details above." + usage + ;; + *) + echoYellow "Unrecognized operational mode; ${_mode}" + usage + ;; +esac +# ====================================================================================== diff --git a/containers/backup/docker/backup.usage b/containers/backup/docker/backup.usage new file mode 100644 index 00000000..70ada874 --- /dev/null +++ b/containers/backup/docker/backup.usage @@ -0,0 +1,138 @@ +#!/bin/bash +# ================================================================================================================= +# Usage: +# ----------------------------------------------------------------------------------------------------------------- +function usage () { + cat <<-EOF + + Automated backup script for PostgreSQL, MongoDB and MSSQL databases. + + There are two modes of scheduling backups: + - Cron Mode: + - Allows one or more schedules to be defined as cron tabs in ${BACKUP_CONF}. + - If cron (go-crond) is installed (which is handled by the Docker file) and at least one cron tab is defined, the script will startup in Cron Mode, + otherwise it will default to Legacy Mode. + - Refer to ${BACKUP_CONF} for additional details and exples of using cron scheduling. + + - Legacy Mode: + - Uses a simple sleep command to set the schedule based on the setting of BACKUP_PERIOD; defaults to ${BACKUP_PERIOD} + + Refer to the project documentation for additional details on how to use this script. + - https://github.com/BCDevOps/backup-container + + Usage: + $0 [options] + + Standard Options: + ================= + -h prints this usage documentation. + + -1 run once. + Performs a single set of backups and exits. + + -s run in scheduled/silent (no questions asked) mode. + A flag to be used by cron scheduled backups to indicate they are being run on a schedule. + Requires cron (go-crond) to be installed and at least one cron tab to be defined in ${BACKUP_CONF} + Refer to ${BACKUP_CONF} for additional details and examples of using cron scheduling. + + -l lists existing backups. + Great for listing the available backups for a restore. + + -c lists the current configuration settings and exits. + Great for confirming the current settings, and listing the databases included in the backup schedule. + + -p prune backups + Used to manually prune backups. + This can be used with the '-f' option, see below, to prune specific backups or sets of backups. + Use caution when using the '-f' option. + + -I ignore errors + This flag can be used with the Restore Options, when restoring a postgres database, to continue the + database restoration process when errors are encountered. By default the postgres restoration script + stops on the first error. + + Verify Options: + ================ + The verify process performs the following basic operations: + - Start a local database server instance. + - Restore the selected backup locally, watching for errors. + - Run a table query on the restored database as a simple test to ensure tables were restored + and queries against the database succeed without error. + - Stop the local database server instance. + - Delete the local database and configuration. + + -v ; in the form =/, or =:/ + where defaults to container database type if omitted + must be one of "postgres" or "mongo" + must be specified in a mixed database container project + + Triggers verify mode and starts verify mode on the specified database. + + Example: + $0 -v postgresql=postgresql:5432/TheOrgBook_Database + - Would start the verification process on the database using the most recent backup for the database. + + $0 -v all + - Verify the most recent backup of all databases. + + -f ; an OPTIONAL filter to use to find/identify the backup file to restore. + Refer to the same option under 'Restore Options' for details. + + Restore Options: + ================ + The restore process performs the following basic operations: + - Drop and recreate the selected database. + - Grant the database user access to the recreated database + - Restore the database from the selected backup file + + Have the 'Admin' (postgres or mongo) password handy, the script will ask you for it during the restore. + + When in restore mode, the script will list the settings it will use and wait for your confirmation to continue. + This provides you with an opportunity to ensure you have selected the correct database and backup file + for the job. + + Restore mode will allow you to restore a database to a different location (host, and/or database name) provided + it can contact the host and you can provide the appropriate credentials. If you choose to do this, you will need + to provide a file filter using the '-f' option, since the script will likely not be able to determine which backup + file you would want to use. This functionality provides a convenient way to test your backups or migrate your + database/data without affecting the original database. + + -r ; in the form =/, or =:/ + where defaults to container database type if omitted + must be one of "postgres" or "mongo" + must be specified in a mixed database container project + + Triggers restore mode and starts restore mode on the specified database. + + Example: + $0 -r postgresql:5432/TheOrgBook_Database/postgres + - Would start the restore process on the database using the most recent backup for the database. + + -f ; an OPTIONAL filter to use to find/identify the backup file to restore. + This can be a full or partial file specification. When only part of a filename is specified the restore process + attempts to find the most recent backup matching the filter. + If not specified, the restore process attempts to locate the most recent backup file for the specified database. + + Examples: + $0 -r postgresql=wallet-db/test_db/postgres -f wallet-db-tob_holder + - Would try to find the latest backup matching on the partial file name provided. + + $0 -r wallet-db/test_db/postgres -f /backups/daily/2018-11-07/wallet-db-tob_holder_2018-11-07_23-59-35.sql.gz + - Would use the specific backup file. + + $0 -r wallet-db/test_db/postgres -f wallet-db-tob_holder_2018-11-07_23-59-35.sql.gz + - Would use the specific backup file regardless of its location in the root backup folder. + + -s OPTIONAL flag. Use with caution. Could cause unintentional data loss. + Run the restore in scripted/scheduled mode. In this mode the restore will not ask you to confirm the settings, + nor will ask you for the 'Admin' password. It will simply attempt to restore a database from a backup. + It's up to you to ensure it's targeting the correct database and using the correct backup file. + + -a ; an OPTIONAL flag used to specify the 'Admin' password. + Use with the '-s' flag to specify the 'Admin' password. Under normal usage conditions it's better to supply the + password when prompted so it is not visible on the console. + +EOF +exit 1 +} +# ================================================================================================================= \ No newline at end of file diff --git a/containers/backup/docker/backup.utils b/containers/backup/docker/backup.utils new file mode 100644 index 00000000..4c562c1d --- /dev/null +++ b/containers/backup/docker/backup.utils @@ -0,0 +1,289 @@ +#!/bin/bash +# ================================================================================================================= +# Primary Database Backup and Restore Functions: +# ----------------------------------------------------------------------------------------------------------------- +function backupDatabase(){ + ( + _databaseSpec=${1} + _fileName=${2} + + _backupFile="${_fileName}${IN_PROGRESS_BACKUP_FILE_EXTENSION}" + + touchBackupFile "${_backupFile}" + onBackupDatabase "${_databaseSpec}" "${_backupFile}" + _rtnCd=${?} + + if (( ${_rtnCd} != 0 )); then + rm -rfvd ${_backupFile} + fi + + return ${_rtnCd} + ) +} + +function restoreDatabase(){ + ( + local OPTIND + local quiet + local localhost + unset quiet + unset localhost + unset flags + while getopts ql FLAG; do + case $FLAG in + q ) + quiet=1 + flags+="-${FLAG} " + ;; + * ) flags+="-${FLAG} ";; + esac + done + shift $((OPTIND-1)) + + _databaseSpec=${1} + _fileName=${2} + _fileName=$(findBackup "${_databaseSpec}" "${_fileName}") + + if [ -z "${quiet}" ]; then + echoBlue "\nRestoring database ..." + echo -e "\nSettings:" + echo "- Database: ${_databaseSpec}" + + if [ ! -z "${_fileName}" ]; then + echo -e "- Backup file: ${_fileName}\n" + else + echoRed "- Backup file: No backup file found or specified. Cannot continue with the restore.\n" + exit 1 + fi + waitForAnyKey + fi + + if [ -z "${quiet}" ] && [ -z "${_adminPassword}" ]; then + # Ask for the Admin Password for the database, if it has not already been provided. + _msg="Admin password (${_databaseSpec}):" + _yellow='\033[1;33m' + _nc='\033[0m' # No Color + _message=$(echo -e "${_yellow}${_msg}${_nc}") + read -r -s -p $"${_message}" _adminPassword + echo -e "\n" + fi + + local startTime=$(date +%s%N) + onRestoreDatabase ${flags} "${_databaseSpec}" "${_fileName}" "${_adminPassword}" + _rtnCd=${?} + + local endTime=$(date +%s%N) + if (( ${_rtnCd} == 0 )); then + echoGreen "\nRestore complete - Elapsed time: $(getElapsedTime ${startTime} ${endTime})\n" + else + echoRed "\nRestore failed.\n" >&2 + fi + + return ${_rtnCd} + ) +} + +function runBackups(){ + ( + echoBlue "\nStarting backup process ..." + databases=$(readConf) + backupDir=$(createBackupFolder) + listSettings "${backupDir}" "${databases}" + + for database in ${databases}; do + if isForContainerType ${database}; then + local startTime=$(date +%s%N) + filename=$(generateFilename "${backupDir}" "${database}") + backupDatabase "${database}" "${filename}" + rtnCd=${?} + local endTime=$(date +%s%N) + local elapsedTime="\n\nElapsed time: $(getElapsedTime ${startTime} ${endTime}) - Status Code: ${rtnCd}" + + if (( ${rtnCd} == 0 )); then + backupPath=$(finalizeBackup "${filename}") + dbSize=$(getDbSize "${database}") + backupSize=$(getFileSize "${backupPath}") + logInfo "Successfully backed up ${database}.\nBackup written to ${backupPath}.\nDatabase Size: ${dbSize}\nBackup Size: ${backupSize}${elapsedTime}" + + s3Backup "${backupPath}" + ftpBackup "${filename}" + pruneBackups "${backupDir}" "${database}" + else + logError "Failed to backup ${database}.${elapsedTime}" + fi + fi + done + + listExistingBackups ${ROOT_BACKUP_DIR} + ) +} + +function startServer(){ + ( + # Start a local server instance ... + onStartServer ${@} + + # Wait for server to start ... + local startTime=$(date +%s%N) + rtnCd=0 + printf "waiting for server to start" + while ! pingDbServer ${@}; do + printf "." + local duration_ns=$(($(date +%s%N) - $startTime)) + if (( ${duration_ns} >= ${DATABASE_SERVER_TIMEOUT_NS} )); then + echoRed "\nThe server failed to start within $(getElapsedTimeFromDuration ${duration}).\n" + rtnCd=1 + break + fi + sleep 1 + done + + echoBlue "\nThe server started in $(getElapsedTimeFromDuration ${duration}).\n" + echo + return ${rtnCd} + ) +} + +function stopServer(){ + ( + onStopServer ${@} + ) +} + +function cleanUp(){ + ( + onCleanup + ) +} + +function pingDbServer(){ + ( + onPingDbServer ${@} + return ${?} + ) +} + +function verifyBackups(){ + ( + local OPTIND + local flags + unset flags + while getopts q FLAG; do + case $FLAG in + * ) flags+="-${FLAG} " ;; + esac + done + shift $((OPTIND-1)) + + _databaseSpec=${1} + _fileName=${2} + if [[ "${_databaseSpec}" == "all" ]]; then + databases=$(readConf -q) + else + databases=${_databaseSpec} + fi + + for database in ${databases}; do + if isForContainerType ${database}; then + verifyBackup ${flags} "${database}" "${_fileName}" + fi + done + ) +} + +function verifyBackup(){ + ( + local OPTIND + local quiet + unset quiet + while getopts q FLAG; do + case $FLAG in + q ) quiet=1 ;; + esac + done + shift $((OPTIND-1)) + + _databaseSpec=${1} + _fileName=${2} + _fileName=$(findBackup "${_databaseSpec}" "${_fileName}") + + echoBlue "\nVerifying backup ..." + echo -e "\nSettings:" + echo "- Database: ${_databaseSpec}" + + if [ ! -z "${_fileName}" ]; then + echo -e "- Backup file: ${_fileName}\n" + else + echoRed "- Backup file: No backup file found or specified. Cannot continue with the backup verification.\n" + exit 0 + fi + + if [ -z "${quiet}" ]; then + waitForAnyKey + fi + + # Make sure the server is not running ... + if pingDbServer -l "${_databaseSpec}"; then + logError "Backup verification failed: ${_fileName}\n\nThe verification server is still running." + exit 1 + fi + + # Make sure things have been cleaned up before we start ... + cleanUp + + local startTime=$(date +%s%N) + startServer -l "${_databaseSpec}" + rtnCd=${?} + + # Restore the database + if (( ${rtnCd} == 0 )); then + if [ -z "${quiet}" ]; then + restoreDatabase -ql "${_databaseSpec}" "${_fileName}" + rtnCd=${?} + else + # Filter out stdout, keep stderr + echo "Restoring from backup ..." + restoreLog=$(restoreDatabase -ql "${_databaseSpec}" "${_fileName}" 2>&1 >/dev/null) + rtnCd=${?} + + if [ ! -z "${restoreLog}" ] && (( ${rtnCd} == 0 )); then + echo ${restoreLog} + unset restoreLog + elif [ ! -z "${restoreLog}" ] && (( ${rtnCd} != 0 )); then + restoreLog="\n\nThe following issues were encountered during backup verification;\n${restoreLog}" + fi + fi + fi + + # Ensure there are tables in the databse and general queries work + if (( ${rtnCd} == 0 )); then + verificationLog=$(onVerifyBackup "${_databaseSpec}") + rtnCd=${?} + fi + + # Stop the database server and clean up ... + stopServer -l "${_databaseSpec}" + cleanUp + + local endTime=$(date +%s%N) + local elapsedTime="\n\nElapsed time: $(getElapsedTime ${startTime} ${endTime}) - Status Code: ${rtnCd}" + + if (( ${rtnCd} == 0 )); then + logInfo "Successfully verified backup: ${_fileName}${verificationLog}${restoreLog}${elapsedTime}" + else + logError "Backup verification failed: ${_fileName}${verificationLog}${restoreLog}${elapsedTime}" + fi + return ${rtnCd} + ) +} + +function getDbSize(){ + ( + size=$(onGetDbSize ${@}) + rtnCd=${?} + + echo ${size} + return ${rtnCd} + ) +} +# ================================================================================================================= diff --git a/containers/backup/docker/uid_entrypoint b/containers/backup/docker/uid_entrypoint new file mode 100644 index 00000000..ba474c91 --- /dev/null +++ b/containers/backup/docker/uid_entrypoint @@ -0,0 +1,7 @@ +#!/bin/sh +if ! whoami &> /dev/null; then + if [ -w /etc/passwd ]; then + echo "${USER_NAME:-sqlservr}:x:$(id -u):0:${USER_NAME:-sqlservr} user:${HOME}:/sbin/nologin" >> /etc/passwd + fi +fi +exec "$@" \ No newline at end of file diff --git a/containers/backup/templates/backup-cronjob/backup-cronjob.yaml b/containers/backup/templates/backup-cronjob/backup-cronjob.yaml new file mode 100644 index 00000000..b19239f1 --- /dev/null +++ b/containers/backup/templates/backup-cronjob/backup-cronjob.yaml @@ -0,0 +1,253 @@ +--- +kind: "Template" +apiVersion: "template.openshift.io/v1" +metadata: + name: "{$JOB_NAME}-cronjob-template" + annotations: + description: "Scheduled Task to perform a Database Backup" + tags: "cronjob,backup" +parameters: + - name: "JOB_NAME" + displayName: "Job Name" + description: "Name of the Scheduled Job to Create." + value: "backup-postgres" + required: true + - name: "JOB_PERSISTENT_STORAGE_NAME" + displayName: "Backup Persistent Storage Name" + description: "Pre-Created PVC to use for backup target" + value: "bk-devex-von-tools-a9vlgd1jpsg1" + required: true + - name: "SCHEDULE" + displayName: "Cron Schedule" + description: "Cron Schedule to Execute the Job (using local cluster system TZ)" + # Currently targeting 1:00 AM Daily + value: "0 1 * * *" + required: true + - name: "SOURCE_IMAGE_NAME" + displayName: "Source Image Name" + description: "The name of the image to use for this resource." + required: true + value: "backup-container" + - name: "IMAGE_REGISTRY" + description: "The base OpenShift docker registry" + displayName: "Docker Image Registry" + required: true + # Set value to "docker-registry.default.svc:5000" if using OCP3 + value: "docker.io" + - name: "IMAGE_NAMESPACE" + displayName: "Image Namespace" + description: "The namespace of the OpenShift project containing the imagestream for the application." + required: true + value: "bcgovimages" + - name: "TAG_NAME" + displayName: "Environment TAG name" + description: "The TAG name for this environment, e.g., dev, test, prod" + required: true + value: "dev" + - name: "DATABASE_SERVICE_NAME" + displayName: "Database Service Name" + description: "The name of the database service." + required: true + value: "postgresql" + - name: "DATABASE_DEFAULT_PORT" + displayName: "Database Service Port" + description: "The configured port for the database service" + required: true + value: "5432" + - name: "DATABASE_NAME" + displayName: "Database Name" + description: "The name of the database." + required: true + value: "MyDatabase" + - name: "DATABASE_DEPLOYMENT_NAME" + displayName: "Database Deployment Name" + description: "The name associated to the database deployment resources. In particular, this is used to wire up the credentials associated to the database." + required: true + value: "postgresql" + - name: DATABASE_USER_KEY_NAME + displayName: Database User Key Name + description: + The database user key name stored in database deployment resources specified + by DATABASE_DEPLOYMENT_NAME. + required: true + value: database-user + - name: DATABASE_PASSWORD_KEY_NAME + displayName: Database Password Key Name + description: + The database password key name stored in database deployment resources + specified by DATABASE_DEPLOYMENT_NAME. + required: true + value: database-password + - name: "BACKUP_STRATEGY" + displayName: "Backup Strategy" + description: "The strategy to use for backups; for example daily, or rolling." + required: true + value: "rolling" + - name: "BACKUP_DIR" + displayName: "The root backup directory" + description: "The name of the root backup directory" + required: true + value: "/backups/" + - name: "NUM_BACKUPS" + displayName: "The number of backup files to be retained" + description: "The number of backup files to be retained. Used for the `daily` backup strategy. Ignored when using the `rolling` backup strategy." + required: false + value: "5" + - name: "DAILY_BACKUPS" + displayName: "Number of Daily Backups to Retain" + description: "The number of daily backup files to be retained. Used for the `rolling` backup strategy." + required: false + value: "7" + - name: "WEEKLY_BACKUPS" + displayName: "Number of Weekly Backups to Retain" + description: "The number of weekly backup files to be retained. Used for the `rolling` backup strategy." + required: false + value: "4" + - name: "MONTHLY_BACKUPS" + displayName: "Number of Monthly Backups to Retain" + description: "The number of monthly backup files to be retained. Used for the `rolling` backup strategy." + required: false + value: "1" + - name: "JOB_SERVICE_ACCOUNT" + displayName: "Service Account Name" + description: "Name of the Service Account To Exeucte the Job As." + value: "default" + required: true + - name: "SUCCESS_JOBS_HISTORY_LIMIT" + displayName: "Successful Job History Limit" + description: "The number of successful jobs that will be retained" + value: "5" + required: true + - name: "FAILED_JOBS_HISTORY_LIMIT" + displayName: "Failed Job History Limit" + description: "The number of failed jobs that will be retained" + value: "2" + required: true + - name: "JOB_BACKOFF_LIMIT" + displayName: "Job Backoff Limit" + description: "The number of attempts to try for a successful job outcome" + value: "0" + required: false +objects: +- kind: ConfigMap + apiVersion: v1 + metadata: + name: "${JOB_NAME}-config" + labels: + template: "${JOB_NAME}-config-template" + cronjob: "${JOB_NAME}" + data: + DATABASE_SERVICE_NAME: "${DATABASE_SERVICE_NAME}" + DEFAULT_PORT: "${DATABASE_DEFAULT_PORT}" + POSTGRESQL_DATABASE: "${DATABASE_NAME}" + # BACKUP_STRATEGY: "daily" + BACKUP_STRATEGY: "rolling" + RETENTION.NUM_BACKUPS: "${NUM_BACKUPS}" + RETENTION.DAILY_BACKUPS: "${DAILY_BACKUPS}" + RETENTION.WEEKLY_BACKUPS: "${WEEKLY_BACKUPS}" + RETENTION.MONTHLY_BACKUPS: "${MONTHLY_BACKUPS}" + +- kind: "CronJob" + apiVersion: "batch/v1" + metadata: + name: "${JOB_NAME}" + labels: + template: "${JOB_NAME}-cronjob" + cronjob: "${JOB_NAME}" + spec: + schedule: "${SCHEDULE}" + concurrencyPolicy: "Forbid" + successfulJobsHistoryLimit: "${{SUCCESS_JOBS_HISTORY_LIMIT}}" + failedJobsHistoryLimit: "${{FAILED_JOBS_HISTORY_LIMIT}}" + jobTemplate: + metadata: + labels: + template: "${JOB_NAME}-job" + cronjob: "${JOB_NAME}" + spec: + backoffLimit: ${{JOB_BACKOFF_LIMIT}} + template: + metadata: + labels: + template: "${JOB_NAME}-job" + cronjob: "${JOB_NAME}" + spec: + containers: + - name: "${JOB_NAME}-cronjob" + image: "${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/${SOURCE_IMAGE_NAME}:${TAG_NAME}" + # image: backup + command: + - "/bin/bash" + - "-c" + - "/backup.sh -1" + volumeMounts: + - mountPath: "${BACKUP_DIR}" + name: "backup" + env: + - name: BACKUP_DIR + value: "${BACKUP_DIR}" + - name: BACKUP_STRATEGY + valueFrom: + configMapKeyRef: + name: "${JOB_NAME}-config" + key: BACKUP_STRATEGY + - name: NUM_BACKUPS + valueFrom: + configMapKeyRef: + name: "${JOB_NAME}-config" + key: RETENTION.NUM_BACKUPS + optional: true + - name: DAILY_BACKUPS + valueFrom: + configMapKeyRef: + name: "${JOB_NAME}-config" + key: RETENTION.DAILY_BACKUPS + optional: true + - name: WEEKLY_BACKUPS + valueFrom: + configMapKeyRef: + name: "${JOB_NAME}-config" + key: RETENTION.WEEKLY_BACKUPS + optional: true + - name: MONTHLY_BACKUPS + valueFrom: + configMapKeyRef: + name: "${JOB_NAME}-config" + key: RETENTION.MONTHLY_BACKUPS + optional: true + - name: DATABASE_SERVICE_NAME + valueFrom: + configMapKeyRef: + name: "${JOB_NAME}-config" + key: DATABASE_SERVICE_NAME + - name: DEFAULT_PORT + valueFrom: + configMapKeyRef: + name: "${JOB_NAME}-config" + key: DEFAULT_PORT + optional: true + - name: POSTGRESQL_DATABASE + valueFrom: + configMapKeyRef: + name: "${JOB_NAME}-config" + key: POSTGRESQL_DATABASE + - name: DATABASE_USER + valueFrom: + secretKeyRef: + name: "${DATABASE_DEPLOYMENT_NAME}" + key: "${DATABASE_USER_KEY_NAME}" + - name: DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: "${DATABASE_DEPLOYMENT_NAME}" + key: "${DATABASE_PASSWORD_KEY_NAME}" + volumes: + - name: backup + persistentVolumeClaim: + claimName: "${JOB_PERSISTENT_STORAGE_NAME}" + restartPolicy: "Never" + terminationGracePeriodSeconds: 30 + activeDeadlineSeconds: 1600 + dnsPolicy: "ClusterFirst" + serviceAccountName: "${JOB_SERVICE_ACCOUNT}" + serviceAccount: "${JOB_SERVICE_ACCOUNT}" diff --git a/containers/backup/templates/backup/backup-build.yaml b/containers/backup/templates/backup/backup-build.yaml new file mode 100644 index 00000000..dd404bec --- /dev/null +++ b/containers/backup/templates/backup/backup-build.yaml @@ -0,0 +1,74 @@ +kind: Template +apiVersion: "template.openshift.io/v1" +metadata: + name: ${NAME}-build-template + creationTimestamp: null +objects: + - kind: ImageStream + apiVersion: v1 + metadata: + name: ${NAME} + - kind: BuildConfig + apiVersion: v1 + metadata: + name: ${NAME} + labels: + app: ${NAME} + spec: + triggers: + - type: ImageChange + - type: ConfigChange + runPolicy: Serial + source: + type: Git + git: + uri: ${GIT_REPO_URL} + ref: ${GIT_REF} + contextDir: ${SOURCE_CONTEXT_DIR} + strategy: + type: Docker + dockerStrategy: + from: + kind: DockerImage + name: ${BASE_IMAGE_FOR_BUILD} + dockerfilePath: ${DOCKER_FILE_PATH} + output: + to: + kind: ImageStreamTag + name: ${NAME}:${OUTPUT_IMAGE_TAG} +parameters: + - name: NAME + displayName: Name + description: The name assigned to all of the resources. Use 'backup-{database name}' depending on your database provider + required: true + value: backup-postgres + - name: GIT_REPO_URL + displayName: Git Repo URL + description: The URL to your GIT repo. + required: true + value: https://github.com/BCDevOps/backup-container.git + - name: GIT_REF + displayName: Git Reference + description: The git reference or branch. + required: true + value: master + - name: SOURCE_CONTEXT_DIR + displayName: Source Context Directory + description: The source context directory. + required: false + value: /docker + - name: DOCKER_FILE_PATH + displayName: Docker File + description: The path and file of the docker file defining the build. Choose either 'Dockerfile' for Postgres builds or 'Dockerfile_Mongo' for MongoDB builds or 'Dockerfile_MSSQL' for MSSQL builds. + required: false + value: Dockerfile + - name: OUTPUT_IMAGE_TAG + displayName: Output Image Tag + description: The tag given to the built image. + required: true + value: latest + - name: BASE_IMAGE_FOR_BUILD + displayName: FROM Image Tag + description: Base image to build from. Docker creds or Artificatory setup may be needed to alleviate docker rate-limiting + required: true + value: quay.io/fedora/postgresql-14:14 diff --git a/containers/backup/templates/backup/backup-deploy.yaml b/containers/backup/templates/backup/backup-deploy.yaml new file mode 100644 index 00000000..94efdb3d --- /dev/null +++ b/containers/backup/templates/backup/backup-deploy.yaml @@ -0,0 +1,447 @@ +kind: Template +apiVersion: "template.openshift.io/v1" +metadata: + name: ${NAME}-deployment-template +objects: + - kind: NetworkPolicy + apiVersion: networking.k8s.io/v1 + metadata: + name: ${NAME} + labels: + name: ${NAME} + app: ${APP_NAME} + env: ${TAG_NAME} + spec: + description: | + Allow the ${NAME} to access databases in this namespace. + ingress: + - from: + - podSelector: + matchLabels: + name: ${NAME} + app: ${APP_NAME} + role: ${ROLE} + env: ${TAG_NAME} + namespaceSelector: + matchLabels: + name: ${NAMESPACE_NAME} + environment: ${TAG_NAME} + podSelector: + matchLabels: + backup: "true" + env: ${TAG_NAME} + + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: ${BACKUP_VOLUME_NAME} + labels: + name: ${NAME} + app: ${APP_NAME} + role: ${ROLE} + env: ${TAG_NAME} + spec: + storageClassName: ${BACKUP_VOLUME_CLASS} + accessModes: + - ReadWriteOnce + resources: + requests: + storage: ${BACKUP_VOLUME_SIZE} + + - kind: PersistentVolumeClaim + apiVersion: v1 + metadata: + name: ${VERIFICATION_VOLUME_NAME} + labels: + name: ${NAME} + app: ${APP_NAME} + role: ${ROLE} + env: ${TAG_NAME} + spec: + storageClassName: ${VERIFICATION_VOLUME_CLASS} + accessModes: + - ReadWriteOnce + resources: + requests: + storage: ${VERIFICATION_VOLUME_SIZE} + + - kind: Secret + apiVersion: v1 + metadata: + name: ${NAME} + labels: + name: ${NAME} + app: ${APP_NAME} + role: ${ROLE} + env: ${TAG_NAME} + type: Opaque + stringData: + webhook-url: ${WEBHOOK_URL} + webhook-url-host: ${WEBHOOK_URL_HOST} + + - kind: Secret + apiVersion: v1 + metadata: + name: ${FTP_SECRET_KEY} + labels: + name: ${NAME} + app: ${APP_NAME} + role: ${ROLE} + env: ${TAG_NAME} + type: Opaque + stringData: + ftp-url: ${FTP_URL} + ftp-user: ${FTP_USER} + ftp-password: ${FTP_PASSWORD} + ftp-url-host: ${FTP_URL_HOST} + + - kind: DeploymentConfig + apiVersion: v1 + metadata: + name: ${NAME} + labels: + name: ${NAME} + app: ${APP_NAME} + role: ${ROLE} + env: ${TAG_NAME} + spec: + strategy: + type: Recreate + triggers: + - type: ConfigChange + - type: ImageChange + imageChangeParams: + automatic: true + containerNames: + - ${NAME} + from: + kind: ImageStreamTag + namespace: ${IMAGE_NAMESPACE} + name: ${SOURCE_IMAGE_NAME}:${TAG_NAME} + replicas: 1 + selector: + name: ${NAME} + template: + metadata: + name: ${NAME} + labels: + name: ${NAME} + app: ${APP_NAME} + role: ${ROLE} + env: ${TAG_NAME} + spec: + volumes: + - name: ${BACKUP_VOLUME_NAME} + persistentVolumeClaim: + claimName: ${BACKUP_VOLUME_NAME} + - name: ${VERIFICATION_VOLUME_NAME} + persistentVolumeClaim: + claimName: ${VERIFICATION_VOLUME_NAME} + - name: ${NAME}-config-volume + configMap: + name: ${CONFIG_MAP_NAME} + items: + - key: ${CONFIG_FILE_NAME} + path: ${CONFIG_FILE_NAME} + containers: + - name: ${NAME} + image: "" + ports: [] + env: + - name: BACKUP_STRATEGY + value: ${BACKUP_STRATEGY} + - name: BACKUP_DIR + value: ${BACKUP_DIR} + - name: NUM_BACKUPS + value: ${NUM_BACKUPS} + - name: DAILY_BACKUPS + value: ${DAILY_BACKUPS} + - name: WEEKLY_BACKUPS + value: ${WEEKLY_BACKUPS} + - name: MONTHLY_BACKUPS + value: ${MONTHLY_BACKUPS} + - name: BACKUP_PERIOD + value: ${BACKUP_PERIOD} + - name: DATABASE_SERVICE_NAME + value: ${DATABASE_SERVICE_NAME} + - name: DATABASE_NAME + value: ${DATABASE_NAME} + - name: MONGODB_AUTHENTICATION_DATABASE + value: ${MONGODB_AUTHENTICATION_DATABASE} + - name: TABLE_SCHEMA + value: ${TABLE_SCHEMA} + - name: DATABASE_USER + valueFrom: + secretKeyRef: + name: ${DATABASE_DEPLOYMENT_NAME} + key: ${DATABASE_USER_KEY_NAME} + - name: DATABASE_PASSWORD + valueFrom: + secretKeyRef: + name: ${DATABASE_DEPLOYMENT_NAME} + key: ${DATABASE_PASSWORD_KEY_NAME} + - name: FTP_URL + valueFrom: + secretKeyRef: + name: ${FTP_SECRET_KEY} + key: ftp-url + - name: FTP_USER + valueFrom: + secretKeyRef: + name: ${FTP_SECRET_KEY} + key: ftp-user + - name: FTP_PASSWORD + valueFrom: + secretKeyRef: + name: ${FTP_SECRET_KEY} + key: ftp-password + - name: WEBHOOK_URL + valueFrom: + secretKeyRef: + name: ${NAME} + key: webhook-url + - name: ENVIRONMENT_FRIENDLY_NAME + value: ${ENVIRONMENT_FRIENDLY_NAME} + - name: ENVIRONMENT_NAME + value: ${ENVIRONMENT_NAME} + resources: + requests: + cpu: ${CPU_REQUEST} + memory: ${MEMORY_REQUEST} + limits: + cpu: ${CPU_LIMIT} + memory: ${MEMORY_LIMIT} + volumeMounts: + - name: ${BACKUP_VOLUME_NAME} + mountPath: ${BACKUP_DIR} + - name: ${VERIFICATION_VOLUME_NAME} + mountPath: ${VERIFICATION_VOLUME_MOUNT_PATH} + - name: ${NAME}-config-volume + mountPath: ${CONFIG_MOUNT_PATH}${CONFIG_FILE_NAME} + subPath: ${CONFIG_FILE_NAME} + +parameters: + - name: NAME + displayName: Name + description: The name assigned to all of the resources. Use 'backup-{database name}' depending on your database provider + required: true + value: backup-postgres + - name: SOURCE_IMAGE_NAME + displayName: Source Image Name + description: The name of the image to use for this resource. Use 'backup-{database name}' depending on your database provider + required: true + value: backup-postgres + - name: APP_NAME + displayName: App Name + description: Used to group resources together. Defaults to backup-container + required: true + value: backup-container + - name: ROLE + displayName: ROLE + description: The role assigned to all of the resources. Defaults to backup-container + required: true + value: backup-container + - name: NAMESPACE_NAME + displayName: Namespace Name + description: The name of the namespace being deployed to.. + required: true + value: devex-von-image + - name: IMAGE_NAMESPACE + displayName: Image Namespace + description: The namespace of the OpenShift project containing the imagestream for the application. + required: true + value: + - name: TAG_NAME + displayName: Environment TAG name + description: The TAG name for this environment, e.g., dev, test, prod + required: true + value: dev + - name: DATABASE_SERVICE_NAME + displayName: Database Service Name + description: Used for backward compatibility only. Not needed when using the recommended 'backup.conf' configuration. The name of the database service. + required: false + value: "" + - name: DATABASE_NAME + displayName: Database Name + description: Used for backward compatibility only. Not needed when using the recommended 'backup.conf' configuration. The name of the database. + required: false + value: "" + - name: MONGODB_AUTHENTICATION_DATABASE + displayName: MongoDB Authentication Database + description: This is only required if you are backing up mongo database with a separate authentication database. + required: false + value: "" + - name: DATABASE_DEPLOYMENT_NAME + displayName: Database Deployment Name + description: The name associated to the database deployment resources. In particular, this is used to wire up the credentials associated to the database. + required: true + value: postgresql + - name: DATABASE_USER_KEY_NAME + displayName: Database User Key Name + description: The database user key name stored in database deployment resources specified by DATABASE_DEPLOYMENT_NAME. + required: true + value: database-user + - name: DATABASE_PASSWORD_KEY_NAME + displayName: Database Password Key Name + description: The database password key name stored in database deployment resources specified by DATABASE_DEPLOYMENT_NAME. + required: true + value: database-password + - name: MSSQL_SA_PASSWORD + displayName: MSSQL SA Password + description: The database password to use for the local backup database. + required: false + - name: TABLE_SCHEMA + displayName: Table Schema + description: The table schema for your database. Used for Postgres backups. + required: true + value: public + - name: BACKUP_STRATEGY + displayName: Backup Strategy + description: The strategy to use for backups; for example daily, or rolling. + required: true + value: rolling + - name: FTP_SECRET_KEY + displayName: FTP Secret Key + description: The FTP secret key is used to wire up the credentials associated to the FTP. + required: false + value: ftp-secret + - name: FTP_URL + displayName: FTP Server URL + description: The URL of the backup FTP server + required: false + value: "" + - name: FTP_URL_HOST + displayName: Ftp URL Hostname + description: Ftp URL Hostname. The backup-deploy.overrides.sh will parse this from the supplied FTP_URL, and fetch it from a secret for updates. + required: false + value: + - name: FTP_USER + displayName: FTP user name + description: FTP user name + required: false + value: "" + - name: FTP_PASSWORD + displayName: FTP password + description: FTP password + required: false + value: "" + - name: WEBHOOK_URL + displayName: Webhook URL + description: The URL of the webhook to use for notifications. If not specified, the webhook integration feature is disabled. + required: false + value: "" + - name: WEBHOOK_URL_HOST + displayName: Webhook URL Hostname + description: Webhook URL Hostname. The backup-deploy.overrides.sh will parse this from the supplied WEBHOOK_URL, and fetch it from a secret for updates. + required: false + value: + - name: ENVIRONMENT_FRIENDLY_NAME + displayName: Friendly Environment Name + description: The human readable name of the environment. This variable is used by the webhook integration to identify the environment in which the backup notifications originate. + required: false + value: "" + - name: ENVIRONMENT_NAME + displayName: Environment Name (Environment Id) + description: The name or Id of the environment. This variable is used by the webhook integration and by the NetworkSecurityPolicies to identify the environment in which the backup notifications originate. + required: true + value: "" + - name: BACKUP_DIR + displayName: The root backup directory + description: The name of the root backup directory. The backup volume will be mounted to this directory. + required: true + value: /backups/ + - name: NUM_BACKUPS + displayName: The number of backup files to be retained + description: Used for backward compatibility only. Ignored when using the recommended `rolling` backup strategy. The number of backup files to be retained. Used for the `daily` backup strategy. + required: false + value: "" + - name: DAILY_BACKUPS + displayName: Number of Daily Backups to Retain + description: The number of daily backup files to be retained. Used for the `rolling` backup strategy. + required: false + value: "" + - name: WEEKLY_BACKUPS + displayName: Number of Weekly Backups to Retain + description: The number of weekly backup files to be retained. Used for the `rolling` backup strategy. + required: false + value: "" + - name: MONTHLY_BACKUPS + displayName: Number of Monthly Backups to Retain + description: The number of monthly backup files to be retained. Used for the `rolling` backup strategy. + required: false + value: "" + - name: BACKUP_PERIOD + displayName: Period (d,m,s) between backups in a format used by the sleep command + description: Used for backward compatibility only. Ignored when using the recommended `backup.conf` and cron backup strategy. Period (d,m,s) between backups in a format used by the sleep command + required: false + value: "" + - name: CONFIG_FILE_NAME + displayName: Config File Name + description: The name of the configuration file. + required: true + value: backup.conf + - name: CONFIG_MAP_NAME + displayName: Config Map Name + description: The name of the configuration map. + required: true + value: backup-conf + - name: CONFIG_MOUNT_PATH + displayName: Config Mount Path + description: The path to use to mount the config file. + required: true + value: / + - name: BACKUP_VOLUME_NAME + displayName: Backup Volume Name + description: The name of the persistent volume used to store the backups. + required: true + value: backup + - name: BACKUP_VOLUME_SIZE + displayName: Backup Volume Size + description: The size of the persistent volume used to store the backups, e.g. 512Mi, 1Gi, 2Gi. Ensure this is sized correctly. Refer to the container documentation for details. + required: true + value: 5Gi + - name: BACKUP_VOLUME_CLASS + displayName: Backup Volume Class + description: The class of the persistent volume used to store the backups; netapp-file-standard is the recommended default. + required: true + value: netapp-file-backup + - name: VERIFICATION_VOLUME_NAME + displayName: Verification Volume Name + description: The name for the verification volume, used for restoring and verifying backups. + required: false + value: backup-verification + - name: VERIFICATION_VOLUME_SIZE + displayName: Backup Volume Size + description: The size of the persistent volume used for restoring and verifying backups, e.g. 512Mi, 1Gi, 2Gi. Ensure this is sized correctly. It should be large enough to contain your largest database. + required: true + value: 1Gi + - name: VERIFICATION_VOLUME_CLASS + displayName: Backup Volume Class + description: The class of the persistent volume used for restoring and verifying backups; netapp-file-standard, netapp-block-standard. + required: true + value: netapp-file-standard + - name: VERIFICATION_VOLUME_MOUNT_PATH + displayName: Verification Volume Mount Path + description: The path on which to mount the verification volume. This is used by the database server to contain the database configuration and data files. For Mongo, please use /var/lib/mongodb/data . For MSSQL, please use /var/opt/mssql/data. For MariaDB, please use /var/lib/mysql/data + required: true + value: /var/lib/pgsql/data + - name: CPU_REQUEST + displayName: Resources CPU Request + description: The resources CPU request (in cores) for this build. + required: true + value: "0" + - name: CPU_LIMIT + displayName: Resources CPU Limit + description: The resources CPU limit (in cores) for this build. + required: true + value: "0" + - name: MEMORY_REQUEST + displayName: Resources Memory Request + description: The resources Memory request (in Mi, Gi, etc) for this build. + required: true + value: 0Mi + - name: MEMORY_LIMIT + displayName: Resources Memory Limit + description: The resources Memory limit (in Mi, Gi, etc) for this build. + required: true + value: 0Mi diff --git a/containers/backup/templates/nsp/build.yaml b/containers/backup/templates/nsp/build.yaml new file mode 100644 index 00000000..031e6bc9 --- /dev/null +++ b/containers/backup/templates/nsp/build.yaml @@ -0,0 +1,50 @@ +--- +kind: Template +apiVersion: "template.openshift.io/v1" +metadata: + name: global-nsp-build-template +objects: + - kind: NetworkPolicy + apiVersion: networking.k8s.io/v1 + metadata: + name: deny-by-default + spec: + description: | + Deny all traffic by default. + podSelector: {} + ingress: [] + + - kind: NetworkSecurityPolicy + apiVersion: security.devops.gov.bc.ca/v1alpha1 + metadata: + name: any-to-any + spec: + description: | + Disable Aporeto policies - Allow all pods within the namespace to communicate. + source: + - - $namespace=${NAMESPACE_NAME}-${ENV_NAME} + destination: + - - $namespace=${NAMESPACE_NAME}-${ENV_NAME} + + - kind: NetworkSecurityPolicy + apiVersion: security.devops.gov.bc.ca/v1alpha1 + metadata: + name: any-to-external + spec: + description: | + Disable Aporeto policies - Allow all pods within the namespace full access to external systems. + source: + - - $namespace=${NAMESPACE_NAME}-${ENV_NAME} + destination: + - - ext:network=any + +parameters: + - name: NAMESPACE_NAME + displayName: The target namespace for the resources. + required: true + value: + - name: ENV_NAME + displayName: Environment Name + description: Environment name. For the build environment this will typically be 'tools' + required: true + value: tools diff --git a/containers/backup/templates/nsp/deploy.yaml b/containers/backup/templates/nsp/deploy.yaml new file mode 100644 index 00000000..6dab41e6 --- /dev/null +++ b/containers/backup/templates/nsp/deploy.yaml @@ -0,0 +1,50 @@ +--- +kind: Template +apiVersion: "template.openshift.io/v1" +metadata: + name: global-nsp-build-template +objects: + - kind: NetworkPolicy + apiVersion: networking.k8s.io/v1 + metadata: + name: deny-by-default + spec: + description: | + Deny all traffic by default. + podSelector: {} + ingress: [] + + - kind: NetworkSecurityPolicy + apiVersion: security.devops.gov.bc.ca/v1alpha1 + metadata: + name: any-to-any + spec: + description: | + Disable Aporeto policies - Allow all pods within the namespace to communicate. + source: + - - $namespace=${NAMESPACE_NAME}-${TAG_NAME} + destination: + - - $namespace=${NAMESPACE_NAME}-${TAG_NAME} + + - kind: NetworkSecurityPolicy + apiVersion: security.devops.gov.bc.ca/v1alpha1 + metadata: + name: any-to-external + spec: + description: | + Disable Aporeto policies - Allow all pods within the namespace full access to external systems. + source: + - - $namespace=${NAMESPACE_NAME}-${TAG_NAME} + destination: + - - ext:network=any + +parameters: + - name: NAMESPACE_NAME + displayName: The target namespace for the resources. + required: true + value: + - name: TAG_NAME + displayName: Environment Name + description: Environment name. For the build environment this will typically be 'tools' + required: true + value: dev From 8e95f4d61c9d1b93fa2d70c20c9c8318604002e8 Mon Sep 17 00:00:00 2001 From: Kjartan Einarsson Date: Wed, 7 Aug 2024 16:17:41 -0700 Subject: [PATCH 02/12] remove extra docker scripts, update yamls and MD --- containers/backup/DEVELOPER.md | 75 +++ containers/backup/backup.conf | 51 -- containers/backup/config/backup.conf | 2 +- containers/backup/docker/Dockerfile | 43 -- containers/backup/docker/Dockerfile_MSSQL | 46 -- containers/backup/docker/Dockerfile_MariaDB | 37 -- containers/backup/docker/Dockerfile_Mongo | 45 -- containers/backup/docker/backup.config.utils | 501 ------------------ .../backup/docker/backup.container.utils | 82 --- containers/backup/docker/backup.file.utils | 245 --------- containers/backup/docker/backup.ftp | 23 - containers/backup/docker/backup.logging | 128 ----- .../backup/docker/backup.mariadb.plugin | 218 -------- containers/backup/docker/backup.misc.utils | 53 -- containers/backup/docker/backup.mongo.plugin | 299 ----------- containers/backup/docker/backup.mssql.plugin | 229 -------- containers/backup/docker/backup.null.plugin | 208 -------- .../backup/docker/backup.postgres.plugin | 276 ---------- containers/backup/docker/backup.s3 | 21 - containers/backup/docker/backup.server.utils | 39 -- containers/backup/docker/backup.settings | 55 -- containers/backup/docker/backup.sh | 144 ----- containers/backup/docker/backup.usage | 138 ----- containers/backup/docker/backup.utils | 289 ---------- containers/backup/docker/uid_entrypoint | 7 - .../backup-cronjob/backup-cronjob.yaml | 32 ++ .../templates/backup/backup-deploy.yaml | 39 +- 27 files changed, 141 insertions(+), 3184 deletions(-) create mode 100644 containers/backup/DEVELOPER.md delete mode 100644 containers/backup/backup.conf delete mode 100644 containers/backup/docker/Dockerfile delete mode 100644 containers/backup/docker/Dockerfile_MSSQL delete mode 100644 containers/backup/docker/Dockerfile_MariaDB delete mode 100644 containers/backup/docker/Dockerfile_Mongo delete mode 100644 containers/backup/docker/backup.config.utils delete mode 100644 containers/backup/docker/backup.container.utils delete mode 100644 containers/backup/docker/backup.file.utils delete mode 100644 containers/backup/docker/backup.ftp delete mode 100644 containers/backup/docker/backup.logging delete mode 100644 containers/backup/docker/backup.mariadb.plugin delete mode 100644 containers/backup/docker/backup.misc.utils delete mode 100644 containers/backup/docker/backup.mongo.plugin delete mode 100644 containers/backup/docker/backup.mssql.plugin delete mode 100644 containers/backup/docker/backup.null.plugin delete mode 100644 containers/backup/docker/backup.postgres.plugin delete mode 100644 containers/backup/docker/backup.s3 delete mode 100644 containers/backup/docker/backup.server.utils delete mode 100644 containers/backup/docker/backup.settings delete mode 100755 containers/backup/docker/backup.sh delete mode 100644 containers/backup/docker/backup.usage delete mode 100644 containers/backup/docker/backup.utils delete mode 100644 containers/backup/docker/uid_entrypoint diff --git a/containers/backup/DEVELOPER.md b/containers/backup/DEVELOPER.md new file mode 100644 index 00000000..d433d3bd --- /dev/null +++ b/containers/backup/DEVELOPER.md @@ -0,0 +1,75 @@ +# Openshift Commands to setup Backup Container + +## Example of a Postgres deployment + +The following outlines the deployment of a simple backup of three PostgreSQL databases in the same project namespace, on OCP v4.x. + +1. As per OCP4 [docs](https://developer.gov.bc.ca/OCP4-Backup-and-Restore), 25G of the storage class `netapp-file-backup` is the default quota. If this is insufficient, you may [request](https://github.com/BCDevOps/devops-requests/issues/new/choose) more. + +2. `git clone https://github.com/BCDevOps/backup-container.git && cd backup-container`. + +Create the image. + +```bash +oc -n d83219-tools process -f ./openshift/templates/backup/backup-build.yaml \ + -p NAME=nert-bkup OUTPUT_IMAGE_TAG=v1 | oc -n d83219-tools create -f - +``` + +3. Configure (./config/backup.conf) (listing your database(s), and setting your cron schedule). + +```bash +postgres=restoration-tracker-db-postgresql:5432/restoration-tracker +# postgres=pawslimesurvey-postgresql:5432/pawslimesurvey + +0 1 * * * default ./backup.sh -s +0 4 * * * default ./backup.sh -s -v all +``` + +4. Configure references to your DB credentials in [backup-deploy.yaml](./openshift/templates/backup/backup-deploy.yaml), replacing the boilerplate `DATABASE_USER` and `DATABASE_PASSWORD` environment variables. + +```yaml +- name: EAOFIDER_POSTGRESQL_USER + valueFrom: + secretKeyRef: + name: eaofider-postgresql + key: "${DATABASE_USER_KEY_NAME}" +- name: EAOFIDER_POSTGRESQL_PASSWORD + valueFrom: + secretKeyRef: + name: eaofider-postgresql + key: "${DATABASE_PASSWORD_KEY_NAME}" +``` + +Note that underscores should be used in the environment variable names. + +5. Create your customized `./openshift/backup-deploy.overrides.param` parameter file, if required. + +6. Deploy the app; here the example namespace is `d83219-dev` and the app name is `nert-bkup`: + +```bash +oc -n d83219-dev create configmap backup-conf --from-file=./config/backup.conf +oc -n d83219-dev label configmap backup-conf app=nert-bkup + +oc -n d83219-dev process -f ./templates/backup/backup-deploy.yaml \ + -p NAME=nert-bkup \ + -p IMAGE_NAMESPACE=d83219-tools \ + -p SOURCE_IMAGE_NAME=nert-bkup \ + -p TAG_NAME=v1 \ + -p BACKUP_VOLUME_NAME=nert-bkup-pvc -p BACKUP_VOLUME_SIZE=2Gi \ + -p VERIFICATION_VOLUME_SIZE=1Gi \ + -p ENVIRONMENT_NAME=dev \ + -p ENVIRONMENT_FRIENDLY_NAME='NERT DB Backups' | oc -n d83219-dev create -f - +``` + +To clean up the deployment + +```bash +oc -n d83219-dev delete pvc/nert-bkup-pvc pvc/backup-verification secret/nert-bkup secret/ftp-secret dc/nert-bkup networkpolicy/nert-bkup configmap/backup-conf +``` + +To clean up the image stream and build configuration + +```bash +oc -n d83219-dev delete buildconfig/nert-bkup imagestream/nert-bkup +``` + diff --git a/containers/backup/backup.conf b/containers/backup/backup.conf deleted file mode 100644 index 27f9bf29..00000000 --- a/containers/backup/backup.conf +++ /dev/null @@ -1,51 +0,0 @@ -# ============================================================ -# Databases: -# ------------------------------------------------------------ -# List the databases you want backed up here. -# Databases will be backed up in the order they are listed. -# -# The entries must be in one of the following forms: -# - / -# - :/ -# - =/ -# - =:/ -# can be postgres, mongo or mssql -# MUST be specified when you are sharing a -# single backup.conf file between postgres, mongo and mssql -# backup containers. If you do not specify -# the listed databases are assumed to be valid for the -# backup container in which the configuration is mounted. -# -# Examples: -# - postgres=postgresql/my_database -# - postgres=postgresql:5432/my_database -# - mongo=mongodb/my_database -# - mongo=mongodb:27017/my_database -# - mssql=mssql_server:1433/my_database -# ----------------------------------------------------------- -# Cron Scheduling: -# ----------------------------------------------------------- -# List your backup and verification schedule(s) here as well. -# The schedule(s) must be listed as cron tabs that -# execute the script in 'scheduled' mode: -# - ./backup.sh -s -# -# Examples (assuming system's TZ is set to PST): -# - 0 1 * * * default ./backup.sh -s -# - Run a backup at 1am Pacific every day. -# -# - 0 4 * * * default ./backup.sh -s -v all -# - Verify the most recent backups for all datbases -# at 4am Pacific every day. -# ----------------------------------------------------------- -# Full Example: -# ----------------------------------------------------------- -# postgres=postgresql:5432/TheOrgBook_Database -# mongo=mender-mongodb:27017/useradm -# postgres=wallet-db/tob_issuer -# mssql=pims-db-dev:1433/pims -# mariadb=matomo-db:3306/matomo -# -# 0 1 * * * default ./backup.sh -s -# 0 4 * * * default ./backup.sh -s -v all -# ============================================================ \ No newline at end of file diff --git a/containers/backup/config/backup.conf b/containers/backup/config/backup.conf index f44e692d..62eb5e17 100644 --- a/containers/backup/config/backup.conf +++ b/containers/backup/config/backup.conf @@ -18,7 +18,7 @@ # # Examples: # - postgres=postgresql/my_database -- postgres=postgresql:5432/restoration-db-all-container +- postgres=restoration-tracker-db-postgresql-dev-deploy:5432/restoration-tracker # - mongo=mongodb/my_database # - mongo=mongodb:27017/my_database # - mssql=mssql_server:1433/my_database diff --git a/containers/backup/docker/Dockerfile b/containers/backup/docker/Dockerfile deleted file mode 100644 index a7e8debf..00000000 --- a/containers/backup/docker/Dockerfile +++ /dev/null @@ -1,43 +0,0 @@ -# This image provides a postgres installation from which to run backups -FROM --platform=linux/amd64 quay.io/fedora/postgresql-15:15 - -# Change timezone to PST for convenience -ENV TZ=PST8PDT - -# Set the workdir to be root -WORKDIR / - -# Load the backup scripts into the container (must be executable). -COPY backup.* / - -# ======================================================================================================== -# Install go-crond (from https://github.com/webdevops/go-crond) -# -# CRON Jobs in OpenShift: -# - https://blog.danman.eu/cron-jobs-in-openshift/ -# -------------------------------------------------------------------------------------------------------- -ARG SOURCE_REPO=webdevops -ARG GOCROND_VERSION=23.2.0 -ADD https://github.com/$SOURCE_REPO/go-crond/releases/download/$GOCROND_VERSION/go-crond.linux.amd64 /usr/bin/go-crond - -USER root - -RUN curl https://dl.min.io/client/mc/release/linux-amd64/mc -o /usr/bin/mc -RUN chmod +x /usr/bin/mc -RUN chmod +x /usr/bin/go-crond -# ======================================================================================================== - -# ======================================================================================================== -# Perform operations that require root privilages here ... -# -------------------------------------------------------------------------------------------------------- -RUN echo $TZ > /etc/timezone -# ======================================================================================================== - -# The column command is missing from quay.io/fedora/postgresql-14:14 -RUN dnf install -y util-linux - -# Important - Reset to the base image's user account. -USER 26 - -# Set the default CMD. -CMD sh /backup.sh diff --git a/containers/backup/docker/Dockerfile_MSSQL b/containers/backup/docker/Dockerfile_MSSQL deleted file mode 100644 index 30dd72c8..00000000 --- a/containers/backup/docker/Dockerfile_MSSQL +++ /dev/null @@ -1,46 +0,0 @@ -FROM mcr.microsoft.com/mssql/rhel/server:2019-CU1-rhel-8 - -# Change timezone to PST for convenience -ENV TZ=PST8PDT - -# Set the workdir to be root -WORKDIR / - -# Load the backup scripts into the container (must be executable). -COPY backup.* / - -# ======================================================================================================== -# Install go-crond (from https://github.com/webdevops/go-crond) -# -# CRON Jobs in OpenShift: -# - https://blog.danman.eu/cron-jobs-in-openshift/ -# -------------------------------------------------------------------------------------------------------- -ARG SOURCE_REPO=webdevops -ARG GOCROND_VERSION=23.2.0 -ADD https://github.com/$SOURCE_REPO/go-crond/releases/download/$GOCROND_VERSION/go-crond.linux.amd64 /usr/bin/go-crond - -USER root - -RUN chmod +x /usr/bin/go-crond -# ======================================================================================================== - -# ======================================================================================================== -# Perform operations that require root privilages here ... -# -------------------------------------------------------------------------------------------------------- -RUN echo $TZ > /etc/timezone -# ======================================================================================================== -COPY uid_entrypoint /opt/mssql-tools/bin/ -RUN chmod -R a+rwx /opt/mssql-tools/bin/uid_entrypoint - -ENV PATH=${PATH}:/opt/mssql/bin:/opt/mssql-tools/bin -RUN mkdir -p /var/opt/mssql/data && \ - chmod -R g=u /var/opt/mssql /etc/passwd - -# Important - Reset to the base image's user account. -USER 10001 - -# Set the default CMD. -CMD sh /backup.sh - -### user name recognition at runtime w/ an arbitrary uid - for OpenShift deployments -ENTRYPOINT [ "/opt/mssql-tools/bin/uid_entrypoint" ] \ No newline at end of file diff --git a/containers/backup/docker/Dockerfile_MariaDB b/containers/backup/docker/Dockerfile_MariaDB deleted file mode 100644 index ce0da9cd..00000000 --- a/containers/backup/docker/Dockerfile_MariaDB +++ /dev/null @@ -1,37 +0,0 @@ -from registry.fedoraproject.org/f31/mariadb - -# Change timezone to PST for convenience -ENV TZ=PST8PDT - -# Set the workdir to be root -WORKDIR / - -# Load the backup scripts into the container (must be executable). -COPY backup.* / - -# ======================================================================================================== -# Install go-crond (from https://github.com/webdevops/go-crond) -# -# CRON Jobs in OpenShift: -# - https://blog.danman.eu/cron-jobs-in-openshift/ -# -------------------------------------------------------------------------------------------------------- -ARG SOURCE_REPO=webdevops -ARG GOCROND_VERSION=23.2.0 -ADD https://github.com/$SOURCE_REPO/go-crond/releases/download/$GOCROND_VERSION/go-crond.linux.amd64 /usr/bin/go-crond - -USER root - -RUN chmod +x /usr/bin/go-crond -# ======================================================================================================== - -# ======================================================================================================== -# Perform operations that require root privilages here ... -# -------------------------------------------------------------------------------------------------------- -RUN echo $TZ > /etc/timezone -# ======================================================================================================== - -# Important - Reset to the base image's user account. -USER 26 - -# Set the default CMD. -CMD sh /backup.sh \ No newline at end of file diff --git a/containers/backup/docker/Dockerfile_Mongo b/containers/backup/docker/Dockerfile_Mongo deleted file mode 100644 index 6995e436..00000000 --- a/containers/backup/docker/Dockerfile_Mongo +++ /dev/null @@ -1,45 +0,0 @@ -# This image provides a mongo installation from which to run backups -FROM mongodb/mongodb-community-server:6.0.6-ubi8 - -ARG uid=998 -ARG user=mongod - -# Change timezone to PST for convenience -ENV TZ=PST8PDT - -# Set the workdir to be root -WORKDIR / - -# Load the backup scripts into the container (must be executable). -COPY backup.* / - -# ======================================================================================================== -# Install go-crond (from https://github.com/webdevops/go-crond) -# -# CRON Jobs in OpenShift: -# - https://blog.danman.eu/cron-jobs-in-openshift/ -# -------------------------------------------------------------------------------------------------------- -ARG SOURCE_REPO=webdevops -ARG GOCROND_VERSION=23.2.0 -ADD https://github.com/$SOURCE_REPO/go-crond/releases/download/$GOCROND_VERSION/go-crond.linux.amd64 /usr/bin/go-crond - -USER root - -RUN chmod +x /usr/bin/go-crond -RUN chown -R $user:root /data/db && \ - chmod -R ug+rw /data/db - -RUN usermod -a -G 0 $user -# ======================================================================================================== - -# ======================================================================================================== -# Perform operations that require root privilages here ... -# -------------------------------------------------------------------------------------------------------- -RUN echo $TZ > /etc/timezone -# ======================================================================================================== - -# Important - Reset to the base image's user account. -USER $uid - -# Set the default CMD. -CMD bash /backup.sh diff --git a/containers/backup/docker/backup.config.utils b/containers/backup/docker/backup.config.utils deleted file mode 100644 index 46652737..00000000 --- a/containers/backup/docker/backup.config.utils +++ /dev/null @@ -1,501 +0,0 @@ -#!/bin/bash -# ================================================================================================================= -# Configuration Utility Functions: -# ----------------------------------------------------------------------------------------------------------------- -function getDatabaseName(){ - ( - _databaseSpec=${1} - _databaseName=$(echo ${_databaseSpec} | sed -n 's~^.*/\(.*$\)~\1~p') - echo "${_databaseName}" - ) -} - -function getDatabaseType(){ - ( - _databaseSpec=${1} - _databaseType=$(echo ${_databaseSpec} | sed -n 's~^\(.*\)=.*$~\1~p' | tr '[:upper:]' '[:lower:]') - echo "${_databaseType}" - ) -} - -function getPort(){ - ( - local OPTIND - local localhost - unset localhost - while getopts :l FLAG; do - case $FLAG in - l ) localhost=1 ;; - esac - done - shift $((OPTIND-1)) - - _databaseSpec=${1} - - if [ -z "${localhost}" ]; then - portsed="s~^.*:\([[:digit:]]\+\)/.*$~\1~p" - _port=$(echo ${_databaseSpec} | sed -n "${portsed}") - fi - - echo "${_port}" - ) -} - -function getHostname(){ - ( - local OPTIND - local localhost - unset localhost - while getopts :l FLAG; do - case $FLAG in - l ) localhost=1 ;; - esac - done - shift $((OPTIND-1)) - - _databaseSpec=${1} - - if [ -z "${localhost}" ]; then - _hostname=$(echo ${_databaseSpec} | sed 's~^.\+[=]~~;s~[:/].*~~') - else - _hostname="127.0.0.1" - fi - - echo "${_hostname}" - ) -} - -function getHostPrefix(){ - ( - _hostname=${1} - _hostPrefix=$(echo ${_hostname} | tr '[:lower:]' '[:upper:]' | sed "s~-~_~g") - echo "${_hostPrefix}" - ) -} - -function getHostUserParam(){ - ( - _hostname=${1} - _hostUser=$(getHostPrefix ${_hostname})_USER - echo "${_hostUser}" - ) -} - -function getHostPasswordParam(){ - ( - _hostname=${1} - _hostPassword=$(getHostPrefix ${_hostname})_PASSWORD - echo "${_hostPassword}" - ) -} - -function readConf(){ - ( - local OPTIND - local readCron - local quiet - local all - unset readCron - unset quiet - while getopts cqa FLAG; do - case $FLAG in - c ) readCron=1 ;; - q ) quiet=1 ;; - a ) all=1 ;; - esac - done - shift $((OPTIND-1)) - - # Remove all comments and any blank lines - filters="/^[[:blank:]]*$/d;/^[[:blank:]]*#/d;/#.*/d;" - - if [ -z "${readCron}" ]; then - # Read in the database config ... - # - Remove any lines that do not match the expected database spec format(s) - # - [=]/ - # - [=]:/ - filters+="/^[a-zA-Z0-9=_/-]*\(:[0-9]*\)\?\/[a-zA-Z0-9_/-]*$/!d;" - if [ -z "${all}" ]; then - # Remove any database configs that are not for the current container type - # Database configs that do not define the database type are assumed to be for the current container type - filters+="/\(^[a-zA-Z0-9_/-]*\(:[0-9]*\)\?\/[a-zA-Z0-9_/-]*$\)\|\(^${CONTAINER_TYPE}=\)/!d;" - fi - else - # Read in the cron config ... - # - Remove any lines that MATCH expected database spec format(s), - # leaving, what should be, cron tabs. - filters+="/^[a-zA-Z0-9=_/-]*\(:[0-9]*\)\?\/[a-zA-Z0-9_/-]*$/d;" - fi - - if [ -f ${BACKUP_CONF} ]; then - if [ -z "${quiet}" ]; then - echo "Reading backup config from ${BACKUP_CONF} ..." >&2 - fi - _value=$(sed "${filters}" ${BACKUP_CONF}) - fi - - if [ -z "${_value}" ] && [ -z "${readCron}" ]; then - # Backward compatibility - if [ -z "${quiet}" ]; then - echo "Reading backup config from environment variables ..." >&2 - fi - _value="${DATABASE_SERVICE_NAME}${DEFAULT_PORT:+:${DEFAULT_PORT}}${POSTGRESQL_DATABASE:+/${POSTGRESQL_DATABASE}}" - fi - - echo "${_value}" - ) -} - -function getNumBackupsToRetain(){ - ( - _count=0 - _backupType=${1:-$(getBackupType)} - - case "${_backupType}" in - daily) - _count=${DAILY_BACKUPS} - if (( ${_count} <= 0 )) && (( ${WEEKLY_BACKUPS} <= 0 )) && (( ${MONTHLY_BACKUPS} <= 0 )); then - _count=1 - fi - ;; - weekly) - _count=${WEEKLY_BACKUPS} - ;; - monthly) - _count=${MONTHLY_BACKUPS} - ;; - *) - _count=${NUM_BACKUPS} - ;; - esac - - echo "${_count}" - ) -} - -function getUsername(){ - ( - _databaseSpec=${1} - _hostname=$(getHostname ${_databaseSpec}) - _paramName=$(getHostUserParam ${_hostname}) - # Backward compatibility ... - _username="${!_paramName:-${DATABASE_USER}}" - echo ${_username} - ) -} - -function getPassword(){ - ( - _databaseSpec=${1} - _hostname=$(getHostname ${_databaseSpec}) - _paramName=$(getHostPasswordParam ${_hostname}) - # Backward compatibility ... - _password="${!_paramName:-${DATABASE_PASSWORD}}" - echo ${_password} - ) -} - -function isLastDayOfMonth(){ - ( - _date=${1:-$(date)} - _day=$(date -d "${_date}" +%-d) - _month=$(date -d "${_date}" +%-m) - _lastDayOfMonth=$(date -d "${_month}/1 + 1 month - 1 day" "+%-d") - - if (( ${_day} == ${_lastDayOfMonth} )); then - return 0 - else - return 1 - fi - ) -} - -function isLastDayOfWeek(){ - ( - # We're calling Sunday the last dayt of the week in this case. - _date=${1:-$(date)} - _dayOfWeek=$(date -d "${_date}" +%u) - - if (( ${_dayOfWeek} == 7 )); then - return 0 - else - return 1 - fi - ) -} - -function getBackupType(){ - ( - _backupType="" - if rollingStrategy; then - if isLastDayOfMonth && (( "${MONTHLY_BACKUPS}" > 0 )); then - _backupType="monthly" - elif isLastDayOfWeek; then - _backupType="weekly" - else - _backupType="daily" - fi - fi - echo "${_backupType}" - ) -} - -function rollingStrategy(){ - if [[ "${BACKUP_STRATEGY}" == "rolling" ]] && (( "${WEEKLY_BACKUPS}" >= 0 )) && (( "${MONTHLY_BACKUPS}" >= 0 )); then - return 0 - else - return 1 - fi -} - -function dailyStrategy(){ - if [[ "${BACKUP_STRATEGY}" == "daily" ]] || (( "${WEEKLY_BACKUPS}" < 0 )); then - return 0 - else - return 1 - fi -} - -function listSettings(){ - _backupDirectory=${1:-$(createBackupFolder -g)} - _databaseList=${2:-$(readConf -q)} - _yellow='\e[33m' - _nc='\e[0m' # No Color - _notConfigured="${_yellow}not configured${_nc}" - - echo -e \\n"Settings:" - _mode=$(getMode 2>/dev/null) - echo -e "- Run mode: ${_mode}"\\n - - if rollingStrategy; then - echo "- Backup strategy: rolling" - fi - if dailyStrategy; then - echo "- Backup strategy: daily" - fi - if ! rollingStrategy && ! dailyStrategy; then - echoYellow "- Backup strategy: Unknown backup strategy; ${BACKUP_STRATEGY}" - _configurationError=1 - fi - backupType=$(getBackupType) - if [ -z "${backupType}" ]; then - echo "- Current backup type: flat daily" - else - echo "- Current backup type: ${backupType}" - fi - echo "- Backups to retain:" - if rollingStrategy; then - echo " - Daily: $(getNumBackupsToRetain daily)" - echo " - Weekly: $(getNumBackupsToRetain weekly)" - echo " - Monthly: $(getNumBackupsToRetain monthly)" - else - echo " - Total: $(getNumBackupsToRetain)" - fi - echo "- Current backup folder: ${_backupDirectory}" - - if [[ "${_mode}" != ${ONCE} ]]; then - if [[ "${_mode}" == ${CRON} ]] || [[ "${_mode}" == ${SCHEDULED} ]]; then - _backupSchedule=$(readConf -cq) - echo "- Time Zone: $(date +"%Z %z")" - fi - _backupSchedule=$(formatList "${_backupSchedule:-${BACKUP_PERIOD}}") - echo -e \\n"- Schedule:" - echo "${_backupSchedule}" - fi - - if [[ "${CONTAINER_TYPE}" == "${UNKNOWN_DB}" ]] && [ -z "${_allowNullPlugin}" ]; then - echoRed "\n- Container Type: ${CONTAINER_TYPE}" - _configurationError=1 - else - echo -e "\n- Container Type: ${CONTAINER_TYPE}" - fi - - _databaseList=$(formatList "${_databaseList}") - echo "- Databases (filtered by container type):" - echo "${_databaseList}" - echo - - if [ -z "${FTP_URL}" ]; then - echo -e "- FTP server: ${_notConfigured}" - else - echo "- FTP server: ${FTP_URL}" - fi - - if [ -z "${S3_ENDPOINT}" ]; then - echo -e "- S3 endpoint: ${_notConfigured}" - else - echo "- S3 endpoint: ${S3_ENDPOINT}" - fi - - if [ -z "${WEBHOOK_URL}" ]; then - echo -e "- Webhook Endpoint: ${_notConfigured}" - else - echo "- Webhook Endpoint: ${WEBHOOK_URL}" - fi - - if [ -z "${ENVIRONMENT_FRIENDLY_NAME}" ]; then - echo -e "- Environment Friendly Name: ${_notConfigured}" - else - echo -e "- Environment Friendly Name: ${ENVIRONMENT_FRIENDLY_NAME}" - fi - if [ -z "${ENVIRONMENT_NAME}" ]; then - echo -e "- Environment Name (Id): ${_notConfigured}" - else - echo "- Environment Name (Id): ${ENVIRONMENT_NAME}" - fi - - if [ ! -z "${_configurationError}" ]; then - echo - logError "Configuration error! The script will exit." - sleep 5 - exit 1 - fi - echo -} - -function isScheduled(){ - ( - if [ ! -z "${SCHEDULED_RUN}" ]; then - return 0 - else - return 1 - fi - ) -} - -function isScripted(){ - ( - if [ ! -z "${SCHEDULED_RUN}" ]; then - return 0 - else - return 1 - fi - ) -} - -function restoreMode(){ - ( - if [ ! -z "${_restoreDatabase}" ]; then - return 0 - else - return 1 - fi - ) -} - -function verifyMode(){ - ( - if [ ! -z "${_verifyBackup}" ]; then - return 0 - else - return 1 - fi - ) -} - -function pruneMode(){ - ( - if [ ! -z "${RUN_PRUNE}" ]; then - return 0 - else - return 1 - fi - ) -} - -function cronMode(){ - ( - cronTabs=$(readConf -cq) - if isInstalled "go-crond" && [ ! -z "${cronTabs}" ]; then - return 0 - else - return 1 - fi - ) -} - -function runOnce() { - if [ ! -z "${RUN_ONCE}" ]; then - return 0 - else - return 1 - fi -} - -function getMode(){ - ( - unset _mode - - if pruneMode; then - _mode="${PRUNE}" - fi - - if [ -z "${_mode}" ] && restoreMode; then - _mode="${RESTORE}" - fi - - if [ -z "${_mode}" ] && verifyMode; then - # Determine if this is a scheduled verification or a manual one. - if isScheduled; then - if cronMode; then - _mode="${SCHEDULED_VERIFY}" - else - _mode="${ERROR}" - logError "Scheduled mode cannot be used without cron being installed and at least one cron tab being defined in ${BACKUP_CONF}." - fi - else - _mode="${VERIFY}" - fi - fi - - if [ -z "${_mode}" ] && runOnce; then - _mode="${ONCE}" - fi - - if [ -z "${_mode}" ] && isScheduled; then - if cronMode; then - _mode="${SCHEDULED}" - else - _mode="${ERROR}" - logError "Scheduled mode cannot be used without cron being installed and at least one cron tab being defined in ${BACKUP_CONF}." - fi - fi - - if [ -z "${_mode}" ] && cronMode; then - _mode="${CRON}" - fi - - if [ -z "${_mode}" ]; then - _mode="${LEGACY}" - fi - - echo "${_mode}" - ) -} - -function validateOperation(){ - ( - _databaseSpec=${1} - _mode=${2} - _rtnCd=0 - - if [[ "${_mode}" == ${RESTORE} ]] && ! isForContainerType ${_databaseSpec}; then - echoRed "\nYou are attempting to restore database '${_databaseSpec}' from a ${CONTAINER_TYPE} container." - echoRed "Cannot continue with the restore. It must be initiated from the matching container type." - _rtnCd=1 - fi - - return ${_rtnCd} - ) -} - -function ignoreErrors(){ - ( - if [ ! -z "${IGNORE_ERRORS}" ]; then - return 0 - else - return 1 - fi - ) -} -# ====================================================================================== \ No newline at end of file diff --git a/containers/backup/docker/backup.container.utils b/containers/backup/docker/backup.container.utils deleted file mode 100644 index 2d44960e..00000000 --- a/containers/backup/docker/backup.container.utils +++ /dev/null @@ -1,82 +0,0 @@ -#!/bin/bash -# ================================================================================================================= -# Container Utility Functions: -# ----------------------------------------------------------------------------------------------------------------- -function isPostgres(){ - ( - if isInstalled "psql"; then - return 0 - else - return 1 - fi - ) -} - -function isMongo(){ - ( - if isInstalled "mongosh"; then - return 0 - else - return 1 - fi - ) -} - -function isMsSql(){ - ( - if isInstalled "sqlcmd"; then - return 0 - else - return 1 - fi - ) -} - -function isMariaDb(){ - ( - # If a seperate mysql plugin is added, this check may be insufficient to establish the container type. - if isInstalled "mysql"; then - return 0 - else - return 1 - fi - ) -} - -function getContainerType(){ - ( - local _containerType=${UNKNOWN_DB} - _rtnCd=0 - - if isPostgres; then - _containerType=${POSTGRE_DB} - elif isMongo; then - _containerType=${MONGO_DB} - elif isMsSql; then - _containerType=${MSSQL_DB} - elif isMariaDb; then - _containerType=${MARIA_DB} - else - _containerType=${UNKNOWN_DB} - _rtnCd=1 - fi - - echo "${_containerType}" - return ${_rtnCd} - ) -} - -function isForContainerType(){ - ( - _databaseSpec=${1} - _databaseType=$(getDatabaseType ${_databaseSpec}) - - # If the database type has not been defined, assume the database spec is valid for the current databse container type. - if [ -z "${_databaseType}" ] || [[ "${_databaseType}" == "${CONTAINER_TYPE}" ]]; then - return 0 - else - return 1 - fi - ) -} -# ====================================================================================== diff --git a/containers/backup/docker/backup.file.utils b/containers/backup/docker/backup.file.utils deleted file mode 100644 index 06ef069a..00000000 --- a/containers/backup/docker/backup.file.utils +++ /dev/null @@ -1,245 +0,0 @@ -#!/bin/bash -# ================================================================================================================= -# File Utility Functions -# ----------------------------------------------------------------------------------------------------------------- -function makeDirectory() -{ - ( - # Creates directories with permissions reclusively. - # ${1} is the directory to be created - # Inspired by https://unix.stackexchange.com/questions/49263/recursive-mkdir - directory="${1}" - test $# -eq 1 || { echo "Function 'makeDirectory' can create only one directory (with it's parent directories)."; exit 1; } - test -d "${directory}" && return 0 - test -d "$(dirname "${directory}")" || { makeDirectory "$(dirname "${directory}")" || return 1; } - test -d "${directory}" || { mkdir --mode=g+w "${directory}" || return 1; } - return 0 - ) -} - -function finalizeBackup(){ - ( - _filename=${1} - _inProgressFilename="${_filename}${IN_PROGRESS_BACKUP_FILE_EXTENSION}" - _finalFilename="${_filename}${BACKUP_FILE_EXTENSION}" - - if [ -f ${_inProgressFilename} ]; then - mv "${_inProgressFilename}" "${_finalFilename}" - echo "${_finalFilename}" - fi - ) -} - -function listExistingBackups(){ - ( - local _backupDir=${1:-${ROOT_BACKUP_DIR}} - local database - local databases=$(readConf -q) - local output="\nDatabase,Current Size" - - for database in ${databases}; do - if isForContainerType ${database}; then - output+="\n${database},$(getDbSize "${database}")" - fi - done - - echoMagenta "\n================================================================================================================================" - echoMagenta "Current Backups:" - echoMagenta "\n$(echo -ne "${output}" | column -t -s ,)" - echoMagenta "\n$(df -h ${_backupDir})" - echoMagenta "--------------------------------------------------------------------------------------------------------------------------------" - du -ah --time ${_backupDir} - echoMagenta "================================================================================================================================\n" - ) -} - -function getDirectoryName(){ - ( - local path=${1} - path="${path%"${path##*[!/]}"}" - local name="${path##*/}" - echo "${name}" - ) -} - -function getBackupTypeFromPath(){ - ( - local path=${1} - path="${path%"${path##*[!/]}"}" - path="$(dirname "${path}")" - local backupType=$(getDirectoryName "${path}") - echo "${backupType}" - ) -} - -function prune(){ - ( - local database - local backupDirs - local backupDir - local backupType - local backupTypes - local pruneBackup - unset backupTypes - unset backupDirs - unset pruneBackup - - local databases=$(readConf -q) - if rollingStrategy; then - backupTypes="daily weekly monthly" - for backupType in ${backupTypes}; do - backupDirs="${backupDirs} $(createBackupFolder -g ${backupType})" - done - else - backupDirs=$(createBackupFolder -g) - fi - - if [ ! -z "${_fromBackup}" ]; then - pruneBackup="$(findBackup "" "${_fromBackup}")" - while [ ! -z "${pruneBackup}" ]; do - echoYellow "\nAbout to delete backup file: ${pruneBackup}" - waitForAnyKey - rm -rfvd "${pruneBackup}" - - # Quietly delete any empty directories that are left behind ... - find ${ROOT_BACKUP_DIR} -type d -empty -delete > /dev/null 2>&1 - pruneBackup="$(findBackup "" "${_fromBackup}")" - done - else - for backupDir in ${backupDirs}; do - for database in ${databases}; do - unset backupType - if rollingStrategy; then - backupType=$(getBackupTypeFromPath "${backupDir}") - fi - pruneBackups "${backupDir}" "${database}" "${backupType}" - done - done - fi - ) -} - -function pruneBackups(){ - ( - _backupDir=${1} - _databaseSpec=${2} - _backupType=${3:-''} - _pruneDir="$(dirname "${_backupDir}")" - _numBackupsToRetain=$(getNumBackupsToRetain "${_backupType}") - _coreFilename=$(generateCoreFilename ${_databaseSpec}) - - if [ -d ${_pruneDir} ]; then - let _index=${_numBackupsToRetain}+1 - _filesToPrune=$(find ${_pruneDir}* -type f -printf '%T@ %p\n' | grep ${_coreFilename} | sort -r | tail -n +${_index} | sed 's~^.* \(.*$\)~\1~') - - if [ ! -z "${_filesToPrune}" ]; then - echoYellow "\nPruning ${_coreFilename} backups from ${_pruneDir} ..." - echo "${_filesToPrune}" | xargs rm -rfvd - - # Quietly delete any empty directories that are left behind ... - find ${ROOT_BACKUP_DIR} -type d -empty -delete > /dev/null 2>&1 - fi - fi - ) -} - -function touchBackupFile() { - ( - # For safety, make absolutely certain the directory and file exist. - # The pruning process removes empty directories, so if there is an error - # during a backup the backup directory could be deleted. - _backupFile=${1} - _backupDir="${_backupFile%/*}" - makeDirectory ${_backupDir} && touch ${_backupFile} - ) -} - -function findBackup(){ - ( - _databaseSpec=${1} - _fileName=${2} - - # If no backup file was specified, find the most recent for the database. - # Otherwise treat the value provided as a filter to find the most recent backup file matching the filter. - if [ -z "${_fileName}" ]; then - _coreFilename=$(generateCoreFilename ${_databaseSpec}) - _fileName=$(find ${ROOT_BACKUP_DIR}* -type f -printf '%T@ %p\n' | grep ${_coreFilename} | sort | tail -n 1 | sed 's~^.* \(.*$\)~\1~') - else - _fileName=$(find ${ROOT_BACKUP_DIR}* -type f -printf '%T@ %p\n' | grep ${_fileName} | sort | tail -n 1 | sed 's~^.* \(.*$\)~\1~') - fi - - echo "${_fileName}" - ) -} - -function createBackupFolder(){ - ( - local OPTIND - local genOnly - unset genOnly - while getopts g FLAG; do - case $FLAG in - g ) genOnly=1 ;; - esac - done - shift $((OPTIND-1)) - - _backupTypeDir="${1:-$(getBackupType)}" - if [ ! -z "${_backupTypeDir}" ]; then - _backupTypeDir=${_backupTypeDir}/ - fi - - _backupDir="${ROOT_BACKUP_DIR}${_backupTypeDir}`date +\%Y-\%m-\%d`/" - - # Don't actually create the folder if we're just generating it for printing the configuation. - if [ -z "${genOnly}" ]; then - echo "Making backup directory ${_backupDir} ..." >&2 - if ! makeDirectory ${_backupDir}; then - logError "Failed to create backup directory ${_backupDir}." - exit 1; - fi; - fi - - echo ${_backupDir} - ) -} - -function generateFilename(){ - ( - _backupDir=${1} - _databaseSpec=${2} - _coreFilename=$(generateCoreFilename ${_databaseSpec}) - _filename="${_backupDir}${_coreFilename}_`date +\%Y-\%m-\%d_%H-%M-%S`" - echo ${_filename} - ) -} - -function generateCoreFilename(){ - ( - _databaseSpec=${1} - _hostname=$(getHostname ${_databaseSpec}) - _database=$(getDatabaseName ${_databaseSpec}) - _coreFilename="${_hostname}-${_database}" - echo ${_coreFilename} - ) -} - -function getFileSize(){ - ( - _filename=${1} - echo $(du -h "${_filename}" | awk '{print $1}') - ) -} - -function dirIsEmpty(){ - ( - dir="${@}" - rtnVal=$(find ${dir} -maxdepth 0 -empty) - if [ -z "${rtnVal}" ] || [ "${dir}" != "${rtnVal}" ]; then - return 1 - else - return 0 - fi - ) -} -# ================================================================================================================= \ No newline at end of file diff --git a/containers/backup/docker/backup.ftp b/containers/backup/docker/backup.ftp deleted file mode 100644 index d0a935cf..00000000 --- a/containers/backup/docker/backup.ftp +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -# ================================================================================================================= -# FTP Support Functions: -# ----------------------------------------------------------------------------------------------------------------- -function ftpBackup(){ - ( - if [ -z "${FTP_URL}" ] ; then - return 0 - fi - - _filename=${1} - _filenameWithExtension="${_filename}${BACKUP_FILE_EXTENSION}" - echo "Transferring ${_filenameWithExtension} to ${FTP_URL}" - curl --ftp-ssl -T ${_filenameWithExtension} --user ${FTP_USER}:${FTP_PASSWORD} ${FTP_URL} - - if [ ${?} -eq 0 ]; then - logInfo "Successfully transferred ${_filenameWithExtension} to the FTP server" - else - logError "Failed to transfer ${_filenameWithExtension} with the exit code ${?}" - fi - ) -} -# ================================================================================================================= diff --git a/containers/backup/docker/backup.logging b/containers/backup/docker/backup.logging deleted file mode 100644 index 6c5e05df..00000000 --- a/containers/backup/docker/backup.logging +++ /dev/null @@ -1,128 +0,0 @@ -#!/bin/bash -# ================================================================================================================= -# Logging Functions: -# ----------------------------------------------------------------------------------------------------------------- -function debugMsg (){ - _msg="${@}" - if [ "${BACKUP_LOG_LEVEL}" == "debug" ]; then - echoGreen "$(date) - [DEBUG] - ${@}" >&2 - fi -} - -function echoRed (){ - _msg="${@}" - _red='\e[31m' - _nc='\e[0m' # No Color - echo -e "${_red}${_msg}${_nc}" -} - -function echoYellow (){ - _msg="${@}" - _yellow='\e[33m' - _nc='\e[0m' # No Color - echo -e "${_yellow}${_msg}${_nc}" -} - -function echoBlue (){ - _msg="${@}" - _blue='\e[34m' - _nc='\e[0m' # No Color - echo -e "${_blue}${_msg}${_nc}" -} - -function echoGreen (){ - _msg="${@}" - _green='\e[32m' - _nc='\e[0m' # No Color - echo -e "${_green}${_msg}${_nc}" -} - -function echoMagenta (){ - _msg="${@}" - _magenta='\e[35m' - _nc='\e[0m' # No Color - echo -e "${_magenta}${_msg}${_nc}" -} - -function logInfo(){ - ( - infoMsg="${1}" - echo -e "${infoMsg}" - postMsgToWebhook "${ENVIRONMENT_FRIENDLY_NAME}" \ - "${ENVIRONMENT_NAME}" \ - "INFO" \ - "${infoMsg}" - ) -} - -function logWarn(){ - ( - warnMsg="${1}" - echoYellow "${warnMsg}" - postMsgToWebhook "${ENVIRONMENT_FRIENDLY_NAME}" \ - "${ENVIRONMENT_NAME}" \ - "WARN" \ - "${warnMsg}" - ) -} - -function logError(){ - ( - errorMsg="${1}" - echoRed "[!!ERROR!!] - ${errorMsg}" >&2 - postMsgToWebhook "${ENVIRONMENT_FRIENDLY_NAME}" \ - "${ENVIRONMENT_NAME}" \ - "ERROR" \ - "${errorMsg}" - postMsgToPagerDuty "${errorMsg}" - ) -} - -function postMsgToWebhook(){ - ( - if [ -z "${WEBHOOK_URL}" ]; then - return 0 - fi - - projectFriendlyName=${1} - projectName=${2} - statusCode=${3} - message=$(echo -e "${4}") - - curl -s \ - -X POST \ - -H "Content-Type: application/x-www-form-urlencoded" \ - --data-urlencode "projectFriendlyName=${projectFriendlyName}" \ - --data-urlencode "projectName=${projectName}" \ - --data-urlencode "statusCode=${statusCode}" \ - --data-urlencode "message=${message}" \ - "${WEBHOOK_URL}" > /dev/null - ) -} - -function postMsgToPagerDuty(){ - ( - if [ -z "${PGDUTY_SVC_KEY}" ]; then - echo "Missing PagerDuty service key" - return 0 - fi - - if [ -z "${PGDUTY_URL}" ]; then - echo "Missing PagerDuty API url" - return 0 - fi - - message=$(echo -e "${1}" | tr '\n' ' ') - - curl -s \ - -X POST \ - -H "Content-type: application/json" \ - -d "{ - \"service_key\": \"${PGDUTY_SVC_KEY}\", - \"event_type\": \"trigger\", - \"description\": \"${message}\" - }" \ - "${PGDUTY_URL}" > /dev/null - ) -} -# ================================================================================================================= diff --git a/containers/backup/docker/backup.mariadb.plugin b/containers/backup/docker/backup.mariadb.plugin deleted file mode 100644 index 76024c52..00000000 --- a/containers/backup/docker/backup.mariadb.plugin +++ /dev/null @@ -1,218 +0,0 @@ -#!/bin/bash -# ================================================================================================================= -# MariaDB Backup and Restore Functions: -# - Dynamically loaded as a plug-in -# - Refer to existing plug-ins for implementation examples. -# ----------------------------------------------------------------------------------------------------------------- -export serverDataDirectory="/var/lib/mysql/data" - -function onBackupDatabase(){ - ( - local OPTIND - local unset flags - while getopts : FLAG; do - case $FLAG in - ? ) flags+="-${OPTARG} ";; - esac - done - shift $((OPTIND-1)) - - _databaseSpec=${1} - _backupFile=${2} - - _hostname=$(getHostname ${_databaseSpec}) - _database=$(getDatabaseName ${_databaseSpec}) - _port=$(getPort ${_databaseSpec}) - _portArg=${_port:+"-P ${_port}"} - _username=$(getUsername ${_databaseSpec}) - _password=$(getPassword ${_databaseSpec}) - echoGreen "Backing up '${_hostname}${_port:+:${_port}}${_database:+/${_database}}' to '${_backupFile}' ..." - - MYSQL_PWD=${_password} mysqldump -h "${_hostname}" -u "${_username}" ${_portArg} "${_database}" | gzip > ${_backupFile} - return ${?} - ) -} - -function onRestoreDatabase(){ - ( - local OPTIND - local unset flags - while getopts : FLAG; do - case $FLAG in - ? ) flags+="-${OPTARG} ";; - esac - done - shift $((OPTIND-1)) - - _databaseSpec=${1} - _fileName=${2} - _adminPassword=${3} - - _hostname=$(getHostname ${flags} ${_databaseSpec}) - _database=$(getDatabaseName ${_databaseSpec}) - _port=$(getPort ${flags} ${_databaseSpec}) - _portArg=${_port:+"-P ${_port}"} - _username=$(getUsername ${_databaseSpec}) - _password=$(getPassword ${_databaseSpec}) - echo -e "Restoring '${_fileName}' to '${_hostname}${_port:+:${_port}}${_database:+/${_database}}' ...\n" >&2 - - - # Restore - gunzip -c "${_fileName}" | MYSQL_PWD=${_password} mysql -h ${_hostname} -u ${_username} ${_portArg} ${_database} - _rtnCd=${PIPESTATUS[1]} - - return ${_rtnCd} - ) -} - -function onStartServer(){ - ( - local OPTIND - local unset flags - while getopts : FLAG; do - case $FLAG in - ? ) flags+="-${OPTARG} ";; - esac - done - shift $((OPTIND-1)) - - _databaseSpec=${1} - - # Start a local MariaDB instance - MYSQL_USER=$(getUsername "${_databaseSpec}") \ - MYSQL_PASSWORD=$(getPassword "${_databaseSpec}") \ - MYSQL_DATABASE=$(getDatabaseName "${_databaseSpec}") \ - MYSQL_ROOT_PASSWORD=$(getPassword "${_databaseSpec}") \ - run-mysqld >/dev/null 2>&1 & - ) -} - -function onStopServer(){ - ( - local OPTIND - local unset flags - while getopts : FLAG; do - case $FLAG in - ? ) flags+="-${OPTARG} ";; - esac - done - shift $((OPTIND-1)) - - mysqladmin --defaults-file=${MYSQL_DEFAULTS_FILE:-/etc/my.cnf} -u root --socket=/var/lib/mysql/mysql.sock flush-privileges shutdown - ) -} - -function onCleanup(){ - ( - if ! dirIsEmpty ${serverDataDirectory}; then - # Delete the database files and configuration - echo -e "Cleaning up ...\n" >&2 - rm -rf ${serverDataDirectory}/* - else - echo -e "Already clean ...\n" >&2 - fi - ) -} - -function onPingDbServer(){ - ( - local OPTIND - local unset flags - while getopts : FLAG; do - case $FLAG in - ? ) flags+="-${OPTARG} ";; - esac - done - shift $((OPTIND-1)) - - _databaseSpec=${1} - - _hostname=$(getHostname ${flags} ${_databaseSpec}) - _database=$(getDatabaseName ${_databaseSpec}) - _port=$(getPort ${flags} ${_databaseSpec}) - _portArg=${_port:+"-P ${_port}"} - _username=$(getUsername ${_databaseSpec}) - _password=$(getPassword ${_databaseSpec}) - - if MYSQL_PWD=${_password} mysql -h ${_hostname} -u ${_username} ${_portArg} ${_database} -e "SELECT 1;" >/dev/null 2>&1; then - return 0 - else - return 1 - fi - ) -} - -function onVerifyBackup(){ - ( - local OPTIND - local unset flags - while getopts : FLAG; do - case $FLAG in - ? ) flags+="-${OPTARG} ";; - esac - done - shift $((OPTIND-1)) - - _databaseSpec=${1} - - _hostname=$(getHostname -l ${_databaseSpec}) - _database=$(getDatabaseName ${_databaseSpec}) - _port=$(getPort -l ${_databaseSpec}) - _portArg=${_port:+"-P ${_port}"} - _username=$(getUsername ${_databaseSpec}) - _password=$(getPassword ${_databaseSpec}) - - debugMsg "backup.mariadb.plugin - onVerifyBackup" - tables=$(MYSQL_PWD=${_password} mysql -h "${_hostname}" -u "${_username}" ${_portArg} "${_database}" -e "SHOW TABLES;") - rtnCd=${?} - - # Get the size of the restored database - if (( ${rtnCd} == 0 )); then - size=$(getDbSize -l "${_databaseSpec}") - rtnCd=${?} - fi - - if (( ${rtnCd} == 0 )); then - numResults=$(echo "${tables}"| wc -l) - if [[ ! -z "${tables}" ]] && (( numResults >= 1 )); then - # All good - verificationLog="\nThe restored database contained ${numResults} tables, and is ${size} in size." - else - # Not so good - verificationLog="\nNo tables were found in the restored database." - rtnCd="3" - fi - fi - - echo ${verificationLog} - return ${rtnCd} - ) -} - -function onGetDbSize(){ - ( - local OPTIND - local unset flags - while getopts : FLAG; do - case $FLAG in - ? ) flags+="-${OPTARG} ";; - esac - done - shift $((OPTIND-1)) - - _databaseSpec=${1} - - _hostname=$(getHostname ${flags} ${_databaseSpec}) - _database=$(getDatabaseName ${_databaseSpec}) - _port=$(getPort ${flags} ${_databaseSpec}) - _portArg=${_port:+"-P ${_port}"} - _username=$(getUsername ${_databaseSpec}) - _password=$(getPassword ${_databaseSpec}) - - MYSQL_PWD=${_password} mysql -h "${_hostname}" -u "${_username}" ${_portArg} "${_database}" -e "SELECT ROUND(SUM(data_length + index_length) / 1024 / 1024, 1) AS \"size in mb\" FROM information_schema.tables WHERE table_schema=\"${_database}\" GROUP BY table_schema;" - - echo ${size} - return ${rtnCd} - ) -} -# ================================================================================================================= diff --git a/containers/backup/docker/backup.misc.utils b/containers/backup/docker/backup.misc.utils deleted file mode 100644 index d636cd88..00000000 --- a/containers/backup/docker/backup.misc.utils +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash -# ================================================================================================================= -# General Utility Functions: -# ----------------------------------------------------------------------------------------------------------------- -function waitForAnyKey() { - read -n1 -s -r -p $'\e[33mWould you like to continue?\e[0m Press Ctrl-C to exit, or any other key to continue ...' key - echo -e \\n - - # If we get here the user did NOT press Ctrl-C ... - return 0 -} - -function formatList(){ - ( - filters='s~^~ - ~;' - _value=$(echo "${1}" | sed "${filters}") - echo "${_value}" - ) -} - -function isInstalled(){ - rtnVal=$(type "$1" >/dev/null 2>&1) - rtnCd=$? - if [ ${rtnCd} -ne 0 ]; then - return 1 - else - return 0 - fi -} - -function getElapsedTime(){ - ( - local startTime=${1} - local endTime=${2} - local duration=$(($endTime - $startTime)) - echo $(getElapsedTimeFromDuration "${duration}") - ) -} - -function getElapsedTimeFromDuration(){ - local duration_ns=${1} - - local hours=$((duration_ns / 3600000000000)) - local minutes=$(( (duration_ns % 3600000000000) / 60000000000 )) - local seconds=$(( (duration_ns % 60000000000) / 1000000000 )) - local milliseconds=$(( (duration_ns % 1000000000) / 1000000 )) - local microseconds=$(( (duration_ns % 1000000) / 1000 )) - local nanoseconds=$(( duration_ns % 1000 )) - - local elapsedTime="${hours}h:${minutes}m:${seconds}s:${milliseconds}ms:${microseconds}µs:${nanoseconds}ns" - echo ${elapsedTime} -} -# ====================================================================================== diff --git a/containers/backup/docker/backup.mongo.plugin b/containers/backup/docker/backup.mongo.plugin deleted file mode 100644 index 34837212..00000000 --- a/containers/backup/docker/backup.mongo.plugin +++ /dev/null @@ -1,299 +0,0 @@ -#!/bin/bash -# ================================================================================================================= -# Mongo Backup and Restore Functions: -# - Dynamically loaded as a plug-in -# ----------------------------------------------------------------------------------------------------------------- -export serverDataDirectory="/data/db" - -function onBackupDatabase(){ - ( - local OPTIND - local unset flags - while getopts : FLAG; do - case $FLAG in - ? ) flags+="-${OPTARG} ";; - esac - done - shift $((OPTIND-1)) - - _databaseSpec=${1} - _backupFile=${2} - - _hostname=$(getHostname ${_databaseSpec}) - _database=$(getDatabaseName ${_databaseSpec}) - _port=$(getPort ${_databaseSpec}) - _portArg=${_port:+"--port=${_port}"} - _username=$(getUsername ${_databaseSpec}) - _password=$(getPassword ${_databaseSpec}) - echoGreen "Backing up '${_hostname}${_port:+:${_port}}${_database:+/${_database}}' to '${_backupFile}' ..." - - _authDbArg=${MONGODB_AUTHENTICATION_DATABASE:+"--authenticationDatabase ${MONGODB_AUTHENTICATION_DATABASE}"} - mongodump -h "${_hostname}" -d "${_database}" ${_authDbArg} ${_portArg} -u "${_username}" -p "${_password}" --quiet --gzip --archive=${_backupFile} - return ${?} - ) -} - -function onRestoreDatabase(){ - ( - local OPTIND - local unset flags - while getopts : FLAG; do - case $FLAG in - ? ) flags+="-${OPTARG} ";; - esac - done - shift $((OPTIND-1)) - - _databaseSpec=${1} - _fileName=${2} - _adminPassword=${3} - - _hostname=$(getHostname ${flags} ${_databaseSpec}) - _database=$(getDatabaseName ${_databaseSpec}) - _port=$(getPort ${flags} ${_databaseSpec}) - _portArg=${_port:+"--port=${_port}"} - _username=$(getUsername ${_databaseSpec}) - _password=$(getPassword ${_databaseSpec}) - echo -e "Restoring '${_fileName}' to '${_hostname}${_port:+:${_port}}${_database:+/${_database}}' ...\n" >&2 - - # ToDo: - # - Add support for restoring to a different database. - # The following implementation only supports restoring to a database of the same name, - # unlike the postgres implementation that allows the database to be restored to a database of a different - # name for testing. - # Ref: https://stackoverflow.com/questions/36321899/mongorestore-to-a-different-database - - _authDbArg=${MONGODB_AUTHENTICATION_DATABASE:+"--authenticationDatabase ${MONGODB_AUTHENTICATION_DATABASE}"} - mongorestore --drop -h ${_hostname} -d "${_database}" ${_authDbArg} ${_portArg} -u "${_username}" -p "${_password}" --gzip --archive=${_fileName} --nsInclude="*" - return ${?} - ) -} - -function onDatabaseInit() { - local OPTIND - local unset flags - while getopts : FLAG; do - case $FLAG in - ? ) flags+="-${OPTARG} ";; - esac - done - shift $((OPTIND-1)) - - # Retrieve database details - local _databaseSpec=${1} - local _database=$(getDatabaseName "${_databaseSpec}") - local _username=$(getUsername "${_databaseSpec}") - local _password=$(getPassword "${_databaseSpec}") - - # Check if the database already exists - if mongosh --quiet --eval "db.getMongo().getDBNames().indexOf('$_database') >= 0"; then - echoYellow "Database '$_database' already exists, skipping initialization." - return 0 - fi - - # Initialize the database by creating the user with the roles - mongosh "$_database" --quiet --eval " - db.createUser({ - user: '$_username', - pwd: '$_password', - roles: [ - { role: 'dbOwner', db: '$_database' }, - { role: 'readWrite', db: '$_database' }, - { role: 'clusterAdmin', db: 'admin' } - ] - }); - " - - # Check the exit status of the createUser command - if [ $? -eq 0 ]; then - echoGreen "Database '$_database' initialized successfully." - return 0 - else - echoRed "Failed to initialize database '$_database'." - return 1 - fi -} - - -function onStartServer(){ - ( - local OPTIND - local unset flags - while getopts : FLAG; do - case $FLAG in - ? ) flags+="-${OPTARG} ";; - esac - done - shift $((OPTIND-1)) - - _databaseSpec=${1} - - _hostname=$(getHostname ${flags} ${_databaseSpec}) - - # Start a local MongoDb instance - mongod --bind_ip $_hostname > /dev/null 2>&1 & - - # Initialize database if necessary - onDatabaseInit "${_databaseSpec}" - ) -} - -function onStopServer(){ - ( - local OPTIND - local unset flags - while getopts : FLAG; do - case $FLAG in - ? ) flags+="-${OPTARG} ";; - esac - done - shift $((OPTIND-1)) - - _databaseSpec=${1} - - _hostname=$(getHostname ${flags} ${_databaseSpec}) - - mongosh ${_hostname}/admin --quiet --eval "db.shutdownServer()" > /dev/null 2>&1 - - # Wait for server to stop ... - local startTime=$(date +%s%N) - - printf "waiting for server to stop" - while onPingDbServer -l ${@}; do - printf "." - local duration_ns=$(($(date +%s%N) - $startTime)) - if (( ${duration_ns} >= ${DATABASE_SERVER_TIMEOUT_NS} )); then - echoRed "\nThe server failed to stop within $(getElapsedTimeFromDuration ${duration})." - echoRed "Killing the mongod process ...\n" - pkill -INT mongod - break - fi - sleep 1 - done - ) -} - -function onCleanup(){ - ( - if ! dirIsEmpty ${serverDataDirectory}; then - # Delete the database files and configuration - echo -e "Cleaning up ...\n" >&2 - rm -rf ${serverDataDirectory}/* - else - echo -e "Already clean ...\n" >&2 - fi - ) -} - -function onPingDbServer(){ - ( - local OPTIND - local unset flags - while getopts : FLAG; do - case $FLAG in - ? ) flags+="-${OPTARG} ";; - esac - done - shift $((OPTIND-1)) - - _databaseSpec=${1} - - _hostname=$(getHostname ${flags} ${_databaseSpec}) - _database=$(getDatabaseName ${_databaseSpec}) - _port=$(getPort ${flags} ${_databaseSpec}) - _portArg=${_port:+"--port ${_port}"} - _username=$(getUsername ${_databaseSpec}) - _password=$(getPassword ${_databaseSpec}) - - _dbAddressArg=${_hostname}${_port:+:${_port}}${_database:+/${_database}} - _authDbArg=${MONGODB_AUTHENTICATION_DATABASE:+"--authenticationDatabase ${MONGODB_AUTHENTICATION_DATABASE}"} - - mongosh ${_dbAddressArg} ${_authDbArg} -u "${_username}" -p "${_password}" --quiet --eval='db.runCommand("ping").ok' > /dev/null 2>&1 - local mongoshExitCode=$? - - if (( ${mongoshExitCode} != 0 )); then - return 1 - else - return 0 - fi - ) -} - -function onVerifyBackup(){ - ( - local OPTIND - local unset flags - while getopts : FLAG; do - case $FLAG in - ? ) flags+="-${OPTARG} ";; - esac - done - shift $((OPTIND-1)) - - _databaseSpec=${1} - - _hostname=$(getHostname -l ${_databaseSpec}) - _database=$(getDatabaseName ${_databaseSpec}) - _port=$(getPort -l ${_databaseSpec}) - _portArg=${_port:+"--port ${_port}"} - _username=$(getUsername ${_databaseSpec}) - _password=$(getPassword ${_databaseSpec}) - - _dbAddressArg=${_hostname}${_port:+:${_port}}${_database:+/${_database}} - _authDbArg=${MONGODB_AUTHENTICATION_DATABASE:+"--authenticationDatabase ${MONGODB_AUTHENTICATION_DATABASE}"} - collections=$(mongosh ${_dbAddressArg} ${_authDbArg} -u "${_username}" -p "${_password}" --quiet --eval 'var dbs = [];dbs = db.getCollectionNames();for (i in dbs){ print(db.dbs[i]);}';) - rtnCd=${?} - - # Get the size of the restored database - if (( ${rtnCd} == 0 )); then - size=$(getDbSize -l "${_databaseSpec}") - rtnCd=${?} - fi - - if (( ${rtnCd} == 0 )); then - numResults=$(echo "${collections}"| wc -l) - if [[ ! -z "${collections}" ]] && (( numResults >= 1 )); then - # All good - verificationLog="\nThe restored database contained ${numResults} collections, and is ${size} in size." - else - # Not so good - verificationLog="\nNo collections were found in the restored database ${_database}." - rtnCd="3" - fi - fi - - echo ${verificationLog} - return ${rtnCd} - ) -} - -function onGetDbSize(){ - ( - local OPTIND - local unset flags - while getopts : FLAG; do - case $FLAG in - ? ) flags+="-${OPTARG} ";; - esac - done - shift $((OPTIND-1)) - - _databaseSpec=${1} - - _hostname=$(getHostname ${flags} ${_databaseSpec}) - _database=$(getDatabaseName ${_databaseSpec}) - _port=$(getPort ${flags} ${_databaseSpec}) - _portArg=${_port:+"--port ${_port}"} - _username=$(getUsername ${_databaseSpec}) - _password=$(getPassword ${_databaseSpec}) - - _dbAddressArg=${_hostname}${_port:+:${_port}}${_database:+/${_database}} - _authDbArg=${MONGODB_AUTHENTICATION_DATABASE:+"--authenticationDatabase ${MONGODB_AUTHENTICATION_DATABASE}"} - size=$(mongosh ${_dbAddressArg} ${_authDbArg} -u "${_username}" -p "${_password}" --quiet --eval 'printjson(db.stats().fsTotalSize)') - rtnCd=${?} - - echo ${size} - return ${rtnCd} - ) -} -# ================================================================================================================= diff --git a/containers/backup/docker/backup.mssql.plugin b/containers/backup/docker/backup.mssql.plugin deleted file mode 100644 index e9bbc3a1..00000000 --- a/containers/backup/docker/backup.mssql.plugin +++ /dev/null @@ -1,229 +0,0 @@ -#!/bin/bash -# ================================================================================================================= -# MSSQL Backup and Restore Functions: -# - Dynamically loaded as a plug-in -# ----------------------------------------------------------------------------------------------------------------- -export serverDataDirectory="/var/opt/mssql/data" - -function onBackupDatabase(){ - ( - local OPTIND - local unset flags - while getopts : FLAG; do - case $FLAG in - ? ) flags+="-${OPTARG} ";; - esac - done - shift $((OPTIND-1)) - - _databaseSpec=${1} - _backupFile=${2} - - _hostname=$(getHostname ${_databaseSpec}) - _database=$(getDatabaseName ${_databaseSpec}) - _port=$(getPort ${_databaseSpec}) - _portArg=${_port:+",${_port}"} - _username=$(getLocalOrDBUsername ${_databaseSpec} ${_hostname}) - _password=$(getPassword ${_databaseSpec}) - echoGreen "Backing up '${_hostname}${_port:+:${_port}}${_database:+/${_database}}' to '${_backupFile}' ..." - - #TODO: add support for backing up transaction log as well. - sqlcmd -S ${_hostname}${_portArg} -U ${_username} -P ${_password} -Q "BACKUP DATABASE ${_database} TO DISK = N'${_fileName}' WITH NOFORMAT, NOINIT, SKIP, NOREWIND, NOUNLOAD, STATS = 10" - return ${?} - ) -} - -function onRestoreDatabase(){ - ( - local OPTIND - local unset flags - while getopts : FLAG; do - case $FLAG in - ? ) flags+="-${OPTARG} ";; - esac - done - shift $((OPTIND-1)) - - _databaseSpec=${1} - _fileName=${2} - _adminPassword=${3} - - _hostname=$(getHostname ${flags} ${_databaseSpec}) - _database=$(getDatabaseName ${_databaseSpec}) - _port=$(getPort ${flags} ${_databaseSpec}) - _portArg=${_port:+",${_port}"} - _username=$(getLocalOrDBUsername ${_databaseSpec} ${_hostname}) - _password=$(getPassword ${_databaseSpec}) - echo -e "Restoring '${_fileName}' to '${_hostname}${_port:+:${_port}}${_database:+/${_database}}' ...\n" >&2 - - #force single user mode on database to ensure restore works properly - sqlcmd -S ${_hostname}${_portArg} -U ${_username} -P ${_password} -Q "ALTER DATABASE ${_database} SET SINGLE_USER WITH ROLLBACK AFTER 30;RESTORE DATABASE ${_database} FROM DISK = N'${_fileName}' WITH FILE = 1, NOUNLOAD, REPLACE, STATS=5;ALTER DATABASE ${_database} SET MULTI_USER" - return ${?} - ) -} - -function onStartServer(){ - ( - local OPTIND - local unset flags - while getopts : FLAG; do - case $FLAG in - ? ) flags+="-${OPTARG} ";; - esac - done - shift $((OPTIND-1)) - - _databaseSpec=${1} - - /opt/mssql/bin/sqlservr --accept-eula & - ) -} - -function onStopServer(){ - ( - local OPTIND - local unset flags - while getopts : FLAG; do - case $FLAG in - ? ) flags+="-${OPTARG} ";; - esac - done - shift $((OPTIND-1)) - - _hostname=$(getHostname ${flags} ${_databaseSpec}) - _port=$(getPort ${flags} ${_databaseSpec}) - _portArg=${_port:+",${_port}"} - _username=$(getLocalOrDBUsername ${_databaseSpec} ${_hostname}) - _password=$(getPassword ${_databaseSpec}) - - sqlcmd -S ${_hostname}${_portArg} -U ${_username} -P ${_password} -Q "SHUTDOWN" - ) -} - -function onCleanup(){ - ( - if ! dirIsEmpty ${serverDataDirectory}; then - # Delete the database files and configuration - echo -e "Cleaning up ...\n" >&2 - rm -rf ${serverDataDirectory}/* - else - echo -e "Already clean ...\n" >&2 - fi - ) -} - -function onPingDbServer(){ - ( - local OPTIND - local unset flags - while getopts : FLAG; do - case $FLAG in - ? ) flags+="-${OPTARG} ";; - esac - done - shift $((OPTIND-1)) - - _databaseSpec=${1} - - _hostname=$(getHostname ${flags} ${_databaseSpec}) - _database=$(getDatabaseName ${_databaseSpec}) - _port=$(getPort ${flags} ${_databaseSpec}) - _portArg=${_port:+",${_port}"} - _username=$(getLocalOrDBUsername ${_databaseSpec} ${_hostname}) - _password=$(getPassword ${_databaseSpec}) - - if sqlcmd -S ${_hostname}${_portArg} -U ${_username} -P ${_password} -Q "SELECT 1" >/dev/null 2>&1; then - return 0 - else - return 1 - fi - ) -} - -function onVerifyBackup(){ - ( - local OPTIND - local unset flags - while getopts : FLAG; do - case $FLAG in - ? ) flags+="-${OPTARG} ";; - esac - done - shift $((OPTIND-1)) - - _databaseSpec=${1} - _hostname=$(getHostname -l ${_databaseSpec}) - _database=$(getDatabaseName ${_databaseSpec}) - _port=$(getPort -l ${_databaseSpec}) - _portArg=${_port:+",${_port}"} - _username=$(getLocalOrDBUsername ${_databaseSpec} ${_hostname}) - _password=$(getPassword ${_databaseSpec}) - - tables=$(sqlcmd -S ${_hostname}${_portArg} -U ${_username} -P ${_password} -d ${_database} -Q "SELECT table_name FROM information_schema.tables WHERE TABLE_CATALOG = '${_database}' AND table_type='BASE TABLE';") - rtnCd=${?} - - # Get the size of the restored database - if (( ${rtnCd} == 0 )); then - size=$(getDbSize -l "${_databaseSpec}") - rtnCd=${?} - fi - - if (( ${rtnCd} == 0 )); then - numResults=$(echo "${tables}"| wc -l) - if [[ ! -z "${tables}" ]] && (( numResults >= 1 )); then - # All good - verificationLog="\nThe restored database contained ${numResults} tables, and is ${size} in size." - else - # Not so good - verificationLog="\nNo tables were found in the restored database ${_database}." - rtnCd="3" - fi - fi - - echo ${verificationLog} - return ${rtnCd} - ) -} - -function onGetDbSize(){ - ( - local OPTIND - local unset flags - while getopts : FLAG; do - case $FLAG in - ? ) flags+="-${OPTARG} ";; - esac - done - shift $((OPTIND-1)) - - _databaseSpec=${1} - - _hostname=$(getHostname ${flags} ${_databaseSpec}) - _database=$(getDatabaseName ${_databaseSpec}) - _port=$(getPort ${flags} ${_databaseSpec}) - _portArg=${_port:+",${_port}"} - _username=$(getLocalOrDBUsername ${_databaseSpec} ${_hostname}) - _password=$(getPassword ${_databaseSpec}) - - size=$(sqlcmd -S ${_hostname}${_portArg} -U ${_username} -P ${_password} -d ${_database} -Q "SELECT CONVERT(VARCHAR,SUM(size)*8/1024)+' MB' AS 'size' FROM sys.master_files m INNER JOIN sys.databases d ON d.database_id = m.database_id WHERE d.name = '${_database}' AND m.type_desc = 'ROWS' GROUP BY d.name") - rtnCd=${?} - - echo ${size} - return ${rtnCd} - ) -} - -function getLocalOrDBUsername(){ - ( - _databaseSpec=${1} - _localhost="127.0.0.1" - _hostname=${2} - if [ "$_hostname" == "$_localhost" ]; then - _username=sa - else - _username=$(getUsername ${_databaseSpec}) - fi - echo ${_username} - ) -} -# ================================================================================================================= \ No newline at end of file diff --git a/containers/backup/docker/backup.null.plugin b/containers/backup/docker/backup.null.plugin deleted file mode 100644 index 653bb06a..00000000 --- a/containers/backup/docker/backup.null.plugin +++ /dev/null @@ -1,208 +0,0 @@ -#!/bin/bash -# ================================================================================================================= -# Null Backup and Restore Functions: -# - Dynamically loaded as a plug-in -# - Refer to existing plug-ins for implementation examples. -# ----------------------------------------------------------------------------------------------------------------- -export serverDataDirectory="/var/lib/pgsql/data" - -function onBackupDatabase(){ - ( - local OPTIND - local unset flags - while getopts : FLAG; do - case $FLAG in - ? ) flags+="-${OPTARG} ";; - esac - done - shift $((OPTIND-1)) - - _databaseSpec=${1} - _backupFile=${2} - - _hostname=$(getHostname ${_databaseSpec}) - _database=$(getDatabaseName ${_databaseSpec}) - _port=$(getPort ${_databaseSpec}) - _portArg=${_port:+"--port ${_port}"} - _username=$(getUsername ${_databaseSpec}) - _password=$(getPassword ${_databaseSpec}) - echoGreen "Backing up '${_hostname}${_port:+:${_port}}${_database:+/${_database}}' to '${_backupFile}' ..." - - echoRed "[backup.null.plugin] onBackupDatabase - Not Implemented" - # echoGreen "Starting database backup ..." - # Add your database specific backup operation(s) here. - return ${?} - ) -} - -function onRestoreDatabase(){ - ( - local OPTIND - local unset flags - while getopts : FLAG; do - case $FLAG in - ? ) flags+="-${OPTARG} ";; - esac - done - shift $((OPTIND-1)) - - _databaseSpec=${1} - _fileName=${2} - _adminPassword=${3} - - _hostname=$(getHostname ${flags} ${_databaseSpec}) - _database=$(getDatabaseName ${_databaseSpec}) - _port=$(getPort ${flags} ${_databaseSpec}) - _portArg=${_port:+"--port ${_port}"} - _username=$(getUsername ${_databaseSpec}) - _password=$(getPassword ${_databaseSpec}) - echo -e "Restoring '${_fileName}' to '${_hostname}${_port:+:${_port}}${_database:+/${_database}}' ...\n" >&2 - - echoRed "[backup.null.plugin] onRestoreDatabase - Not Implemented" - # Add your database specific restore operation(s) here. - return ${?} - ) -} - -function onStartServer(){ - ( - local OPTIND - local unset flags - while getopts : FLAG; do - case $FLAG in - ? ) flags+="-${OPTARG} ";; - esac - done - shift $((OPTIND-1)) - - _databaseSpec=${1} - - echoRed "[backup.null.plugin] onStartServer - Not Implemented" - # Add your NON-BLOCKING database specific startup operation(s) here. - # - Start the database server as a background job. - ) -} - -function onStopServer(){ - ( - local OPTIND - local unset flags - while getopts : FLAG; do - case $FLAG in - ? ) flags+="-${OPTARG} ";; - esac - done - shift $((OPTIND-1)) - - _databaseSpec=${1} - _username=$(getUsername ${_databaseSpec}) - _password=$(getPassword ${_databaseSpec}) - - echoRed "[backup.null.plugin] onStopServer - Not Implemented" - - # echo "Shutting down..." - # Add your database specific shutdown operation(s) here. - ) -} - -function onCleanup(){ - ( - # Add your database specific cleanup operation(s) here. - echoRed "[backup.null.plugin] onCleanup - Not Implemented" - - # if ! dirIsEmpty ${serverDataDirectory}; then - # # Delete the database files and configuration - # echo -e "Cleaning up ...\n" >&2 - # rm -rf ${serverDataDirectory}/* - # else - # echo -e "Already clean ...\n" >&2 - # fi - ) -} - -function onPingDbServer(){ - ( - local OPTIND - local unset flags - while getopts : FLAG; do - case $FLAG in - ? ) flags+="-${OPTARG} ";; - esac - done - shift $((OPTIND-1)) - - _databaseSpec=${1} - - _hostname=$(getHostname ${flags} ${_databaseSpec}) - _database=$(getDatabaseName ${_databaseSpec}) - _port=$(getPort ${flags} ${_databaseSpec}) - _portArg=${_port:+"--port ${_port}"} - _username=$(getUsername ${_databaseSpec}) - _password=$(getPassword ${_databaseSpec}) - - echoRed "[backup.null.plugin] onPingDbServer - Not Implemented" - # Add your database specific ping operation(s) here. - # if ; then - # return 0 - # else - # return 1 - # fi - ) -} - -function onVerifyBackup(){ - ( - local OPTIND - local unset flags - while getopts : FLAG; do - case $FLAG in - ? ) flags+="-${OPTARG} ";; - esac - done - shift $((OPTIND-1)) - - _databaseSpec=${1} - - _hostname=$(getHostname -l ${_databaseSpec}) - _database=$(getDatabaseName ${_databaseSpec}) - _port=$(getPort -l ${_databaseSpec}) - _portArg=${_port:+"--port ${_port}"} - _username=$(getUsername ${_databaseSpec}) - _password=$(getPassword ${_databaseSpec}) - - echoRed "[backup.null.plugin] onVerifyBackup - Not Implemented" - # Add your database specific verification operation(s) here. - - # echo ${verificationLog} - # return ${rtnCd} - ) -} - -function onGetDbSize(){ - ( - local OPTIND - local unset flags - while getopts : FLAG; do - case $FLAG in - ? ) flags+="-${OPTARG} ";; - esac - done - shift $((OPTIND-1)) - - _databaseSpec=${1} - - _hostname=$(getHostname ${flags} ${_databaseSpec}) - _database=$(getDatabaseName ${_databaseSpec}) - _port=$(getPort ${flags} ${_databaseSpec}) - _portArg=${_port:+"--port ${_port}"} - _username=$(getUsername ${_databaseSpec}) - _password=$(getPassword ${_databaseSpec}) - - echoRed "[backup.null.plugin] onGetDbSize - Not Implemented" - # Add your database specific get size operation(s) here. - - # echo ${size} - # return ${rtnCd} - ) -} -# ================================================================================================================= diff --git a/containers/backup/docker/backup.postgres.plugin b/containers/backup/docker/backup.postgres.plugin deleted file mode 100644 index d4710f67..00000000 --- a/containers/backup/docker/backup.postgres.plugin +++ /dev/null @@ -1,276 +0,0 @@ -#!/bin/bash -# ================================================================================================================= -# Postgres Backup and Restore Functions: -# - Dynamically loaded as a plug-in -# ----------------------------------------------------------------------------------------------------------------- -export serverDataDirectory="/var/lib/pgsql/data" - -function onBackupDatabase(){ - ( - local OPTIND - local unset flags - while getopts : FLAG; do - case $FLAG in - ? ) flags+="-${OPTARG} ";; - esac - done - shift $((OPTIND-1)) - - _databaseSpec=${1} - _backupFile=${2} - - _hostname=$(getHostname ${_databaseSpec}) - _database=$(getDatabaseName ${_databaseSpec}) - _port=$(getPort ${_databaseSpec}) - _portArg=${_port:+"-p ${_port}"} - _username=$(getUsername ${_databaseSpec}) - _password=$(getPassword ${_databaseSpec}) - echoGreen "Backing up '${_hostname}${_port:+:${_port}}${_database:+/${_database}}' to '${_backupFile}' ..." - - export PGPASSWORD=${_password} - pg_dump -Fp -h "${_hostname}" ${_portArg} -U "${_username}" "${_database}" > "${BACKUP_DIR}backup.sql" - pg_dumpall -h "${_hostname}" ${_portArg} -U "${_username}" --roles-only --no-role-passwords > "${BACKUP_DIR}roles.sql" - cat "${BACKUP_DIR}roles.sql" "${BACKUP_DIR}backup.sql" | gzip > ${_backupFile} - rm "${BACKUP_DIR}roles.sql" && rm "${BACKUP_DIR}backup.sql" - return ${PIPESTATUS[0]} - ) -} - -function onRestoreDatabase(){ - ( - local OPTIND - local unset quiet - local unset flags - while getopts :q FLAG; do - case $FLAG in - q ) - quiet=1 - flags+="-${FLAG} " - ;; - ? ) flags+="-${OPTARG} ";; - esac - done - shift $((OPTIND-1)) - - _databaseSpec=${1} - _fileName=${2} - _adminPassword=${3} - - _hostname=$(getHostname ${flags} ${_databaseSpec}) - _database=$(getDatabaseName ${_databaseSpec}) - _port=$(getPort ${flags} ${_databaseSpec}) - _portArg=${_port:+"-p ${_port}"} - _username=$(getUsername ${_databaseSpec}) - _password=$(getPassword ${_databaseSpec}) - echo -e "Restoring '${_fileName}' to '${_hostname}${_port:+:${_port}}${_database:+/${_database}}' ...\n" >&2 - - export PGPASSWORD=${_adminPassword} - _stopOnErrors="-v ON_ERROR_STOP=1" - if ignoreErrors; then - _stopOnErrors="-v ON_ERROR_STOP=0" - fi - _rtnCd=0 - - # Drop - if (( ${_rtnCd} == 0 )); then - psql -h "${_hostname}" ${_portArg} -ac "DROP DATABASE \"${_database}\";" - _rtnCd=${?} - echo - fi - - # Create - if (( ${_rtnCd} == 0 )); then - psql -h "${_hostname}" ${_portArg} -ac "CREATE DATABASE \"${_database}\";" - _rtnCd=${?} - echo - fi - - # Drop Patroni-specific schemas - if (( ${_rtnCd} == 0 )); then - psql -h "${_hostname}" ${_portArg} -a -d ${_database} </dev/null 2>&1 & - ) -} - -function onStopServer(){ - ( - local OPTIND - local unset flags - while getopts : FLAG; do - case $FLAG in - ? ) flags+="-${OPTARG} ";; - esac - done - shift $((OPTIND-1)) - - _databaseSpec=${1} - - # Stop the local PostgreSql instance - pg_ctl stop -D ${serverDataDirectory}/userdata - ) -} - -function onCleanup(){ - ( - if ! dirIsEmpty ${serverDataDirectory}; then - # Delete the database files and configuration - echo -e "Cleaning up ...\n" >&2 - rm -rf ${serverDataDirectory}/* - else - echo -e "Already clean ...\n" >&2 - fi - ) -} - -function onPingDbServer(){ - ( - local OPTIND - local unset flags - while getopts : FLAG; do - case $FLAG in - ? ) flags+="-${OPTARG} ";; - esac - done - shift $((OPTIND-1)) - - _databaseSpec=${1} - - _hostname=$(getHostname ${flags} ${_databaseSpec}) - _database=$(getDatabaseName ${_databaseSpec}) - _port=$(getPort ${flags} ${_databaseSpec}) - _portArg=${_port:+"-p ${_port}"} - _username=$(getUsername ${_databaseSpec}) - _password=$(getPassword ${_databaseSpec}) - - if PGPASSWORD=${_password} psql -h ${_hostname} ${_portArg} -U ${_username} -q -d ${_database} -c 'SELECT 1' >/dev/null 2>&1; then - return 0 - else - return 1 - fi - ) -} - -function onVerifyBackup(){ - ( - local OPTIND - local unset flags - while getopts : FLAG; do - case $FLAG in - ? ) flags+="-${OPTARG} ";; - esac - done - shift $((OPTIND-1)) - - _databaseSpec=${1} - - _hostname=$(getHostname -l ${_databaseSpec}) - _database=$(getDatabaseName ${_databaseSpec}) - _port=$(getPort -l ${_databaseSpec}) - _portArg=${_port:+"-p ${_port}"} - _username=$(getUsername ${_databaseSpec}) - _password=$(getPassword ${_databaseSpec}) - - debugMsg "backup.postgres.plugin - onVerifyBackup" - tables=$(psql -h "${_hostname}" ${_portArg} -d "${_database}" -t -c "SELECT table_name FROM information_schema.tables WHERE table_schema='${TABLE_SCHEMA}' AND table_type='BASE TABLE';") - rtnCd=${?} - - # Get the size of the restored database - if (( ${rtnCd} == 0 )); then - size=$(getDbSize -l "${_databaseSpec}") - rtnCd=${?} - fi - - if (( ${rtnCd} == 0 )); then - numResults=$(echo "${tables}"| wc -l) - if [[ ! -z "${tables}" ]] && (( numResults >= 1 )); then - # All good - verificationLog="\nThe restored database contained ${numResults} tables, and is ${size} in size." - else - # Not so good - verificationLog="\nNo tables were found in the restored database." - rtnCd="3" - fi - fi - - echo ${verificationLog} - return ${rtnCd} - ) -} - -function onGetDbSize(){ - ( - local OPTIND - local unset flags - while getopts : FLAG; do - case $FLAG in - ? ) flags+="-${OPTARG} ";; - esac - done - shift $((OPTIND-1)) - - _databaseSpec=${1} - - _hostname=$(getHostname ${flags} ${_databaseSpec}) - _database=$(getDatabaseName ${_databaseSpec}) - _port=$(getPort ${flags} ${_databaseSpec}) - _portArg=${_port:+"-p ${_port}"} - _username=$(getUsername ${_databaseSpec}) - _password=$(getPassword ${_databaseSpec}) - - size=$(PGPASSWORD=${_password} psql -h "${_hostname}" ${_portArg} -U "${_username}" -d "${_database}" -t -c "SELECT pg_size_pretty(pg_database_size(current_database())) as size;") - rtnCd=${?} - - echo ${size} - return ${rtnCd} - ) -} -# ================================================================================================================= diff --git a/containers/backup/docker/backup.s3 b/containers/backup/docker/backup.s3 deleted file mode 100644 index ec73cc4f..00000000 --- a/containers/backup/docker/backup.s3 +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -function s3Backup() { - local db_backup="$1" - local minio_alias=minio_s3 - local minio_container=backup-container-minio-1 - - if [[ $S3_ENDPOINT ]]; then - if ! mc alias ls | grep -o "^$minio_alias" > /dev/null; then - echo "Creating $minio_alias.." - mc alias set $minio_alias $S3_ENDPOINT $S3_USER $S3_PASSWORD - fi - - if ! mc ls $minio_alias/$S3_BUCKET &> /dev/null; then - echo "Creating $S3_BUCKET bucket.." - mc mb $minio_alias/$S3_BUCKET - fi - - mc cp "$db_backup" $minio_alias/$S3_BUCKET - fi -} diff --git a/containers/backup/docker/backup.server.utils b/containers/backup/docker/backup.server.utils deleted file mode 100644 index 9e938a15..00000000 --- a/containers/backup/docker/backup.server.utils +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -# ================================================================================================================= -# Backup Server Utility Functions: -# ----------------------------------------------------------------------------------------------------------------- -function startCron(){ - logInfo "Starting backup server in cron mode ..." - listSettings - echoBlue "Starting go-crond as a background task ...\n" - CRON_CMD="go-crond -v --default-user=${UID} --allow-unprivileged ${BACKUP_CONF}" - exec ${CRON_CMD} & - wait -} - -function startLegacy(){ - ( - while true; do - runBackups - - echoYellow "Sleeping for ${BACKUP_PERIOD} ...\n" - sleep ${BACKUP_PERIOD} - done - ) -} - -function shutDown(){ - jobIds=$(jobs | awk -F '[][]' '{print $2}' ) - for jobId in ${jobIds} ; do - echo "Shutting down background job '${jobId}' ..." - kill %${jobId} - done - - if [ ! -z "${jobIds}" ]; then - echo "Waiting for any background jobs to complete ..." - fi - wait - - exit 0 -} -# ====================================================================================== \ No newline at end of file diff --git a/containers/backup/docker/backup.settings b/containers/backup/docker/backup.settings deleted file mode 100644 index 47ad2035..00000000 --- a/containers/backup/docker/backup.settings +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash -# ====================================================================================== -# Default Settings -# -------------------------------------------------------------------------------------- -export BACKUP_FILE_EXTENSION=".sql.gz" -export IN_PROGRESS_BACKUP_FILE_EXTENSION=".sql.gz.in_progress" -export DEFAULT_PORT=${POSTGRESQL_PORT_NUM:-5432} -export DATABASE_SERVICE_NAME=${DATABASE_SERVICE_NAME:-postgresql} -export POSTGRESQL_DATABASE=${POSTGRESQL_DATABASE:-my_postgres_db} -export TABLE_SCHEMA=${TABLE_SCHEMA:-public} - -# Supports: -# - daily -# - rolling -export BACKUP_STRATEGY=$(echo "${BACKUP_STRATEGY:-rolling}" | tr '[:upper:]' '[:lower:]') -export BACKUP_PERIOD=${BACKUP_PERIOD:-1d} -export ROOT_BACKUP_DIR=${ROOT_BACKUP_DIR:-${BACKUP_DIR:-/backups/}} -export BACKUP_CONF=${BACKUP_CONF:-backup.conf} - -# Used to prune the total number of backup when using the daily backup strategy. -# Default provides for one full month of backups -export NUM_BACKUPS=${NUM_BACKUPS:-31} - -# Used to prune the total number of backup when using the rolling backup strategy. -# Defaults provide for: -# - A week's worth of daily backups -# - A month's worth of weekly backups -# - The previous month's backup -export DAILY_BACKUPS=${DAILY_BACKUPS:-6} -export WEEKLY_BACKUPS=${WEEKLY_BACKUPS:-4} -export MONTHLY_BACKUPS=${MONTHLY_BACKUPS:-1} - -# Modes: -export ONCE="once" -export SCHEDULED="scheduled" -export RESTORE="restore" -export VERIFY="verify" -export CRON="cron" -export LEGACY="legacy" -export ERROR="error" -export SCHEDULED_VERIFY="scheduled-verify" -export PRUNE="prune" - -# Supported Database Containers -export UNKNOWN_DB="null" -export MONGO_DB="mongo" -export POSTGRE_DB="postgres" -export MSSQL_DB="mssql" -export MARIA_DB="mariadb" -export CONTAINER_TYPE="$(getContainerType)" - -# Other: -export DATABASE_SERVER_TIMEOUT=${DATABASE_SERVER_TIMEOUT:-120} -export DATABASE_SERVER_TIMEOUT_NS=$((DATABASE_SERVER_TIMEOUT * 1000000000)) -# ====================================================================================== diff --git a/containers/backup/docker/backup.sh b/containers/backup/docker/backup.sh deleted file mode 100755 index 1d41f210..00000000 --- a/containers/backup/docker/backup.sh +++ /dev/null @@ -1,144 +0,0 @@ -#!/bin/bash - -# ====================================================================================== -# Imports -# -------------------------------------------------------------------------------------- -. ./backup.usage # Usage information -. ./backup.logging # Logging functions -. ./backup.config.utils # Configuration functions -. ./backup.container.utils # Container Utility Functions -. ./backup.s3 # S3 Support functions -. ./backup.ftp # FTP Support functions -. ./backup.misc.utils # General Utility Functions -. ./backup.file.utils # File Utility Functions -. ./backup.utils # Primary Database Backup and Restore Functions -. ./backup.server.utils # Backup Server Utility Functions -. ./backup.settings # Default Settings -# ====================================================================================== - -# ====================================================================================== -# Initialization: -# -------------------------------------------------------------------------------------- -trap shutDown EXIT TERM - -# Load database plug-in based on the container type ... -. ./backup.${CONTAINER_TYPE}.plugin > /dev/null 2>&1 -if [[ ${?} != 0 ]]; then - echoRed "backup.${CONTAINER_TYPE}.plugin not found." - - # Default to null plugin. - export CONTAINER_TYPE=${UNKNOWN_DB} - . ./backup.${CONTAINER_TYPE}.plugin > /dev/null 2>&1 -fi - -while getopts nclr:v:f:1spha:I FLAG; do - case $FLAG in - n) - # Allow null database plugin ... - # Without this flag loading the null plugin is considered a configuration error. - # The null plugin can be used for testing. - export _allowNullPlugin=1 - ;; - c) - echoBlue "\nListing configuration settings ..." - listSettings - exit 0 - ;; - l) - listExistingBackups ${ROOT_BACKUP_DIR} - exit 0 - ;; - r) - # Trigger restore mode ... - export _restoreDatabase=${OPTARG} - ;; - v) - # Trigger verify mode ... - export _verifyBackup=${OPTARG} - ;; - f) - # Optionally specify the backup file to verify or restore from ... - export _fromBackup=${OPTARG} - ;; - 1) - export RUN_ONCE=1 - ;; - s) - export SCHEDULED_RUN=1 - ;; - p) - export RUN_PRUNE=1 - ;; - a) - export _adminPassword=${OPTARG} - ;; - I) - export IGNORE_ERRORS=1 - ;; - h) - usage - ;; - \?) - echo -e \\n"Invalid option: -${OPTARG}"\\n - usage - ;; - esac -done -shift $((OPTIND-1)) -# ====================================================================================== - -# ====================================================================================== -# Main Script -# -------------------------------------------------------------------------------------- -case $(getMode) in - ${ONCE}) - runBackups - echoGreen "Single backup run complete.\n" - ;; - - ${SCHEDULED}) - runBackups - echoGreen "Scheduled backup run complete.\n" - ;; - - ${RESTORE}) - unset restoreFlags - if isScripted; then - restoreFlags="-q" - fi - - if validateOperation "${_restoreDatabase}" "${RESTORE}"; then - restoreDatabase ${restoreFlags} "${_restoreDatabase}" "${_fromBackup}" - fi - ;; - - ${VERIFY}) - verifyBackups "${_verifyBackup}" "${_fromBackup}" - ;; - - ${SCHEDULED_VERIFY}) - verifyBackups -q "${_verifyBackup}" "${_fromBackup}" - ;; - - ${CRON}) - startCron - ;; - - ${LEGACY}) - startLegacy - ;; - - ${PRUNE}) - prune - ;; - - ${ERROR}) - echoRed "A configuration error has occurred, review the details above." - usage - ;; - *) - echoYellow "Unrecognized operational mode; ${_mode}" - usage - ;; -esac -# ====================================================================================== diff --git a/containers/backup/docker/backup.usage b/containers/backup/docker/backup.usage deleted file mode 100644 index 70ada874..00000000 --- a/containers/backup/docker/backup.usage +++ /dev/null @@ -1,138 +0,0 @@ -#!/bin/bash -# ================================================================================================================= -# Usage: -# ----------------------------------------------------------------------------------------------------------------- -function usage () { - cat <<-EOF - - Automated backup script for PostgreSQL, MongoDB and MSSQL databases. - - There are two modes of scheduling backups: - - Cron Mode: - - Allows one or more schedules to be defined as cron tabs in ${BACKUP_CONF}. - - If cron (go-crond) is installed (which is handled by the Docker file) and at least one cron tab is defined, the script will startup in Cron Mode, - otherwise it will default to Legacy Mode. - - Refer to ${BACKUP_CONF} for additional details and exples of using cron scheduling. - - - Legacy Mode: - - Uses a simple sleep command to set the schedule based on the setting of BACKUP_PERIOD; defaults to ${BACKUP_PERIOD} - - Refer to the project documentation for additional details on how to use this script. - - https://github.com/BCDevOps/backup-container - - Usage: - $0 [options] - - Standard Options: - ================= - -h prints this usage documentation. - - -1 run once. - Performs a single set of backups and exits. - - -s run in scheduled/silent (no questions asked) mode. - A flag to be used by cron scheduled backups to indicate they are being run on a schedule. - Requires cron (go-crond) to be installed and at least one cron tab to be defined in ${BACKUP_CONF} - Refer to ${BACKUP_CONF} for additional details and examples of using cron scheduling. - - -l lists existing backups. - Great for listing the available backups for a restore. - - -c lists the current configuration settings and exits. - Great for confirming the current settings, and listing the databases included in the backup schedule. - - -p prune backups - Used to manually prune backups. - This can be used with the '-f' option, see below, to prune specific backups or sets of backups. - Use caution when using the '-f' option. - - -I ignore errors - This flag can be used with the Restore Options, when restoring a postgres database, to continue the - database restoration process when errors are encountered. By default the postgres restoration script - stops on the first error. - - Verify Options: - ================ - The verify process performs the following basic operations: - - Start a local database server instance. - - Restore the selected backup locally, watching for errors. - - Run a table query on the restored database as a simple test to ensure tables were restored - and queries against the database succeed without error. - - Stop the local database server instance. - - Delete the local database and configuration. - - -v ; in the form =/, or =:/ - where defaults to container database type if omitted - must be one of "postgres" or "mongo" - must be specified in a mixed database container project - - Triggers verify mode and starts verify mode on the specified database. - - Example: - $0 -v postgresql=postgresql:5432/TheOrgBook_Database - - Would start the verification process on the database using the most recent backup for the database. - - $0 -v all - - Verify the most recent backup of all databases. - - -f ; an OPTIONAL filter to use to find/identify the backup file to restore. - Refer to the same option under 'Restore Options' for details. - - Restore Options: - ================ - The restore process performs the following basic operations: - - Drop and recreate the selected database. - - Grant the database user access to the recreated database - - Restore the database from the selected backup file - - Have the 'Admin' (postgres or mongo) password handy, the script will ask you for it during the restore. - - When in restore mode, the script will list the settings it will use and wait for your confirmation to continue. - This provides you with an opportunity to ensure you have selected the correct database and backup file - for the job. - - Restore mode will allow you to restore a database to a different location (host, and/or database name) provided - it can contact the host and you can provide the appropriate credentials. If you choose to do this, you will need - to provide a file filter using the '-f' option, since the script will likely not be able to determine which backup - file you would want to use. This functionality provides a convenient way to test your backups or migrate your - database/data without affecting the original database. - - -r ; in the form =/, or =:/ - where defaults to container database type if omitted - must be one of "postgres" or "mongo" - must be specified in a mixed database container project - - Triggers restore mode and starts restore mode on the specified database. - - Example: - $0 -r postgresql:5432/TheOrgBook_Database/postgres - - Would start the restore process on the database using the most recent backup for the database. - - -f ; an OPTIONAL filter to use to find/identify the backup file to restore. - This can be a full or partial file specification. When only part of a filename is specified the restore process - attempts to find the most recent backup matching the filter. - If not specified, the restore process attempts to locate the most recent backup file for the specified database. - - Examples: - $0 -r postgresql=wallet-db/test_db/postgres -f wallet-db-tob_holder - - Would try to find the latest backup matching on the partial file name provided. - - $0 -r wallet-db/test_db/postgres -f /backups/daily/2018-11-07/wallet-db-tob_holder_2018-11-07_23-59-35.sql.gz - - Would use the specific backup file. - - $0 -r wallet-db/test_db/postgres -f wallet-db-tob_holder_2018-11-07_23-59-35.sql.gz - - Would use the specific backup file regardless of its location in the root backup folder. - - -s OPTIONAL flag. Use with caution. Could cause unintentional data loss. - Run the restore in scripted/scheduled mode. In this mode the restore will not ask you to confirm the settings, - nor will ask you for the 'Admin' password. It will simply attempt to restore a database from a backup. - It's up to you to ensure it's targeting the correct database and using the correct backup file. - - -a ; an OPTIONAL flag used to specify the 'Admin' password. - Use with the '-s' flag to specify the 'Admin' password. Under normal usage conditions it's better to supply the - password when prompted so it is not visible on the console. - -EOF -exit 1 -} -# ================================================================================================================= \ No newline at end of file diff --git a/containers/backup/docker/backup.utils b/containers/backup/docker/backup.utils deleted file mode 100644 index 4c562c1d..00000000 --- a/containers/backup/docker/backup.utils +++ /dev/null @@ -1,289 +0,0 @@ -#!/bin/bash -# ================================================================================================================= -# Primary Database Backup and Restore Functions: -# ----------------------------------------------------------------------------------------------------------------- -function backupDatabase(){ - ( - _databaseSpec=${1} - _fileName=${2} - - _backupFile="${_fileName}${IN_PROGRESS_BACKUP_FILE_EXTENSION}" - - touchBackupFile "${_backupFile}" - onBackupDatabase "${_databaseSpec}" "${_backupFile}" - _rtnCd=${?} - - if (( ${_rtnCd} != 0 )); then - rm -rfvd ${_backupFile} - fi - - return ${_rtnCd} - ) -} - -function restoreDatabase(){ - ( - local OPTIND - local quiet - local localhost - unset quiet - unset localhost - unset flags - while getopts ql FLAG; do - case $FLAG in - q ) - quiet=1 - flags+="-${FLAG} " - ;; - * ) flags+="-${FLAG} ";; - esac - done - shift $((OPTIND-1)) - - _databaseSpec=${1} - _fileName=${2} - _fileName=$(findBackup "${_databaseSpec}" "${_fileName}") - - if [ -z "${quiet}" ]; then - echoBlue "\nRestoring database ..." - echo -e "\nSettings:" - echo "- Database: ${_databaseSpec}" - - if [ ! -z "${_fileName}" ]; then - echo -e "- Backup file: ${_fileName}\n" - else - echoRed "- Backup file: No backup file found or specified. Cannot continue with the restore.\n" - exit 1 - fi - waitForAnyKey - fi - - if [ -z "${quiet}" ] && [ -z "${_adminPassword}" ]; then - # Ask for the Admin Password for the database, if it has not already been provided. - _msg="Admin password (${_databaseSpec}):" - _yellow='\033[1;33m' - _nc='\033[0m' # No Color - _message=$(echo -e "${_yellow}${_msg}${_nc}") - read -r -s -p $"${_message}" _adminPassword - echo -e "\n" - fi - - local startTime=$(date +%s%N) - onRestoreDatabase ${flags} "${_databaseSpec}" "${_fileName}" "${_adminPassword}" - _rtnCd=${?} - - local endTime=$(date +%s%N) - if (( ${_rtnCd} == 0 )); then - echoGreen "\nRestore complete - Elapsed time: $(getElapsedTime ${startTime} ${endTime})\n" - else - echoRed "\nRestore failed.\n" >&2 - fi - - return ${_rtnCd} - ) -} - -function runBackups(){ - ( - echoBlue "\nStarting backup process ..." - databases=$(readConf) - backupDir=$(createBackupFolder) - listSettings "${backupDir}" "${databases}" - - for database in ${databases}; do - if isForContainerType ${database}; then - local startTime=$(date +%s%N) - filename=$(generateFilename "${backupDir}" "${database}") - backupDatabase "${database}" "${filename}" - rtnCd=${?} - local endTime=$(date +%s%N) - local elapsedTime="\n\nElapsed time: $(getElapsedTime ${startTime} ${endTime}) - Status Code: ${rtnCd}" - - if (( ${rtnCd} == 0 )); then - backupPath=$(finalizeBackup "${filename}") - dbSize=$(getDbSize "${database}") - backupSize=$(getFileSize "${backupPath}") - logInfo "Successfully backed up ${database}.\nBackup written to ${backupPath}.\nDatabase Size: ${dbSize}\nBackup Size: ${backupSize}${elapsedTime}" - - s3Backup "${backupPath}" - ftpBackup "${filename}" - pruneBackups "${backupDir}" "${database}" - else - logError "Failed to backup ${database}.${elapsedTime}" - fi - fi - done - - listExistingBackups ${ROOT_BACKUP_DIR} - ) -} - -function startServer(){ - ( - # Start a local server instance ... - onStartServer ${@} - - # Wait for server to start ... - local startTime=$(date +%s%N) - rtnCd=0 - printf "waiting for server to start" - while ! pingDbServer ${@}; do - printf "." - local duration_ns=$(($(date +%s%N) - $startTime)) - if (( ${duration_ns} >= ${DATABASE_SERVER_TIMEOUT_NS} )); then - echoRed "\nThe server failed to start within $(getElapsedTimeFromDuration ${duration}).\n" - rtnCd=1 - break - fi - sleep 1 - done - - echoBlue "\nThe server started in $(getElapsedTimeFromDuration ${duration}).\n" - echo - return ${rtnCd} - ) -} - -function stopServer(){ - ( - onStopServer ${@} - ) -} - -function cleanUp(){ - ( - onCleanup - ) -} - -function pingDbServer(){ - ( - onPingDbServer ${@} - return ${?} - ) -} - -function verifyBackups(){ - ( - local OPTIND - local flags - unset flags - while getopts q FLAG; do - case $FLAG in - * ) flags+="-${FLAG} " ;; - esac - done - shift $((OPTIND-1)) - - _databaseSpec=${1} - _fileName=${2} - if [[ "${_databaseSpec}" == "all" ]]; then - databases=$(readConf -q) - else - databases=${_databaseSpec} - fi - - for database in ${databases}; do - if isForContainerType ${database}; then - verifyBackup ${flags} "${database}" "${_fileName}" - fi - done - ) -} - -function verifyBackup(){ - ( - local OPTIND - local quiet - unset quiet - while getopts q FLAG; do - case $FLAG in - q ) quiet=1 ;; - esac - done - shift $((OPTIND-1)) - - _databaseSpec=${1} - _fileName=${2} - _fileName=$(findBackup "${_databaseSpec}" "${_fileName}") - - echoBlue "\nVerifying backup ..." - echo -e "\nSettings:" - echo "- Database: ${_databaseSpec}" - - if [ ! -z "${_fileName}" ]; then - echo -e "- Backup file: ${_fileName}\n" - else - echoRed "- Backup file: No backup file found or specified. Cannot continue with the backup verification.\n" - exit 0 - fi - - if [ -z "${quiet}" ]; then - waitForAnyKey - fi - - # Make sure the server is not running ... - if pingDbServer -l "${_databaseSpec}"; then - logError "Backup verification failed: ${_fileName}\n\nThe verification server is still running." - exit 1 - fi - - # Make sure things have been cleaned up before we start ... - cleanUp - - local startTime=$(date +%s%N) - startServer -l "${_databaseSpec}" - rtnCd=${?} - - # Restore the database - if (( ${rtnCd} == 0 )); then - if [ -z "${quiet}" ]; then - restoreDatabase -ql "${_databaseSpec}" "${_fileName}" - rtnCd=${?} - else - # Filter out stdout, keep stderr - echo "Restoring from backup ..." - restoreLog=$(restoreDatabase -ql "${_databaseSpec}" "${_fileName}" 2>&1 >/dev/null) - rtnCd=${?} - - if [ ! -z "${restoreLog}" ] && (( ${rtnCd} == 0 )); then - echo ${restoreLog} - unset restoreLog - elif [ ! -z "${restoreLog}" ] && (( ${rtnCd} != 0 )); then - restoreLog="\n\nThe following issues were encountered during backup verification;\n${restoreLog}" - fi - fi - fi - - # Ensure there are tables in the databse and general queries work - if (( ${rtnCd} == 0 )); then - verificationLog=$(onVerifyBackup "${_databaseSpec}") - rtnCd=${?} - fi - - # Stop the database server and clean up ... - stopServer -l "${_databaseSpec}" - cleanUp - - local endTime=$(date +%s%N) - local elapsedTime="\n\nElapsed time: $(getElapsedTime ${startTime} ${endTime}) - Status Code: ${rtnCd}" - - if (( ${rtnCd} == 0 )); then - logInfo "Successfully verified backup: ${_fileName}${verificationLog}${restoreLog}${elapsedTime}" - else - logError "Backup verification failed: ${_fileName}${verificationLog}${restoreLog}${elapsedTime}" - fi - return ${rtnCd} - ) -} - -function getDbSize(){ - ( - size=$(onGetDbSize ${@}) - rtnCd=${?} - - echo ${size} - return ${rtnCd} - ) -} -# ================================================================================================================= diff --git a/containers/backup/docker/uid_entrypoint b/containers/backup/docker/uid_entrypoint deleted file mode 100644 index ba474c91..00000000 --- a/containers/backup/docker/uid_entrypoint +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh -if ! whoami &> /dev/null; then - if [ -w /etc/passwd ]; then - echo "${USER_NAME:-sqlservr}:x:$(id -u):0:${USER_NAME:-sqlservr} user:${HOME}:/sbin/nologin" >> /etc/passwd - fi -fi -exec "$@" \ No newline at end of file diff --git a/containers/backup/templates/backup-cronjob/backup-cronjob.yaml b/containers/backup/templates/backup-cronjob/backup-cronjob.yaml index b19239f1..abde3e96 100644 --- a/containers/backup/templates/backup-cronjob/backup-cronjob.yaml +++ b/containers/backup/templates/backup-cronjob/backup-cronjob.yaml @@ -241,6 +241,38 @@ objects: secretKeyRef: name: "${DATABASE_DEPLOYMENT_NAME}" key: "${DATABASE_PASSWORD_KEY_NAME}" + - name: PG_USER + valueFrom: + secretKeyRef: + key: database-user + name: '${DATABASE_SERVICE_NAME}' + - name: PG_PASSWORD + valueFrom: + secretKeyRef: + key: database-user-password + name: '${DATABASE_SERVICE_NAME}' + - name: PG_ROOT_PASSWORD + valueFrom: + secretKeyRef: + key: database-admin-password + name: '${DATABASE_SERVICE_NAME}' + - name: PG_DATABASE + valueFrom: + secretKeyRef: + key: database-name + name: '${DATABASE_SERVICE_NAME}' + - name: PG_PRIMARY_PORT + value: '5432' + - name: PG_PRIMARY_USER + valueFrom: + secretKeyRef: + key: database-user + name: '${DATABASE_SERVICE_NAME}' + - name: PG_PRIMARY_PASSWORD + valueFrom: + secretKeyRef: + key: database-user-password + name: '${DATABASE_SERVICE_NAME}' volumes: - name: backup persistentVolumeClaim: diff --git a/containers/backup/templates/backup/backup-deploy.yaml b/containers/backup/templates/backup/backup-deploy.yaml index 94efdb3d..f86be53f 100644 --- a/containers/backup/templates/backup/backup-deploy.yaml +++ b/containers/backup/templates/backup/backup-deploy.yaml @@ -170,16 +170,38 @@ objects: value: ${MONGODB_AUTHENTICATION_DATABASE} - name: TABLE_SCHEMA value: ${TABLE_SCHEMA} - - name: DATABASE_USER + - name: PG_USER valueFrom: secretKeyRef: - name: ${DATABASE_DEPLOYMENT_NAME} - key: ${DATABASE_USER_KEY_NAME} - - name: DATABASE_PASSWORD + key: database-user + name: '${DATABASE_SERVICE_NAME}' + - name: PG_PASSWORD valueFrom: secretKeyRef: - name: ${DATABASE_DEPLOYMENT_NAME} - key: ${DATABASE_PASSWORD_KEY_NAME} + key: database-user-password + name: '${DATABASE_SERVICE_NAME}' + - name: PG_ROOT_PASSWORD + valueFrom: + secretKeyRef: + key: database-admin-password + name: '${DATABASE_SERVICE_NAME}' + - name: PG_DATABASE + valueFrom: + secretKeyRef: + key: database-name + name: '${DATABASE_SERVICE_NAME}' + - name: PG_PRIMARY_PORT + value: '5432' + - name: PG_PRIMARY_USER + valueFrom: + secretKeyRef: + key: database-user + name: '${DATABASE_SERVICE_NAME}' + - name: PG_PRIMARY_PASSWORD + valueFrom: + secretKeyRef: + key: database-user-password + name: '${DATABASE_SERVICE_NAME}' - name: FTP_URL valueFrom: secretKeyRef: @@ -445,3 +467,8 @@ parameters: description: The resources Memory limit (in Mi, Gi, etc) for this build. required: true value: 0Mi + - name: DATABASE_SERVICE_NAME + displayName: Database Service Name + required: true + value: 'postgresql' + description: Username for PostgreSQL user that will be used for accessing the database. From b185095277a84976d15bec842e16c8603d2a54e0 Mon Sep 17 00:00:00 2001 From: Kjartan Einarsson Date: Wed, 7 Aug 2024 17:03:28 -0700 Subject: [PATCH 03/12] update db name --- containers/backup/DEVELOPER.md | 2 +- containers/backup/LICENSE | 201 --------------------------- containers/backup/config/backup.conf | 6 +- 3 files changed, 4 insertions(+), 205 deletions(-) delete mode 100644 containers/backup/LICENSE diff --git a/containers/backup/DEVELOPER.md b/containers/backup/DEVELOPER.md index d433d3bd..e0025fe1 100644 --- a/containers/backup/DEVELOPER.md +++ b/containers/backup/DEVELOPER.md @@ -11,7 +11,7 @@ The following outlines the deployment of a simple backup of three PostgreSQL dat Create the image. ```bash -oc -n d83219-tools process -f ./openshift/templates/backup/backup-build.yaml \ +oc -n d83219-tools process -f ./templates/backup/backup-build.yaml \ -p NAME=nert-bkup OUTPUT_IMAGE_TAG=v1 | oc -n d83219-tools create -f - ``` diff --git a/containers/backup/LICENSE b/containers/backup/LICENSE deleted file mode 100644 index 261eeb9e..00000000 --- a/containers/backup/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/containers/backup/config/backup.conf b/containers/backup/config/backup.conf index 62eb5e17..1a37ea38 100644 --- a/containers/backup/config/backup.conf +++ b/containers/backup/config/backup.conf @@ -18,7 +18,7 @@ # # Examples: # - postgres=postgresql/my_database -- postgres=restoration-tracker-db-postgresql-dev-deploy:5432/restoration-tracker +postgres=restoration-tracker-db-postgresql-dev-deploy:5432/restoration-tracker # - mongo=mongodb/my_database # - mongo=mongodb:27017/my_database # - mssql=mssql_server:1433/my_database @@ -31,10 +31,10 @@ # - ./backup.sh -s # # Examples (assuming system's TZ is set to PST): -- 0 1 * * * default ./backup.sh -s +0 1 * * * default ./backup.sh -s # - Run a backup at 1am Pacific every day. # -- 0 4 * * * default ./backup.sh -s -v all +0 4 * * * default ./backup.sh -s -v all # - Verify the most recent backups for all datbases # at 4am Pacific every day. # ----------------------------------------------------------- From 422ce57f89f3e8ad83564044afdca91792b3fe8b Mon Sep 17 00:00:00 2001 From: Kjartan Einarsson Date: Thu, 8 Aug 2024 13:50:18 -0700 Subject: [PATCH 04/12] Db-backup container running --- containers/backup/DEVELOPER.md | 52 +++++++++++++------ .../templates/backup/backup-deploy.yaml | 44 ++++++++-------- 2 files changed, 58 insertions(+), 38 deletions(-) diff --git a/containers/backup/DEVELOPER.md b/containers/backup/DEVELOPER.md index e0025fe1..43bb2457 100644 --- a/containers/backup/DEVELOPER.md +++ b/containers/backup/DEVELOPER.md @@ -11,8 +11,7 @@ The following outlines the deployment of a simple backup of three PostgreSQL dat Create the image. ```bash -oc -n d83219-tools process -f ./templates/backup/backup-build.yaml \ - -p NAME=nert-bkup OUTPUT_IMAGE_TAG=v1 | oc -n d83219-tools create -f - +oc -n d83219-tools process -f ./templates/backup/backup-build.yaml | oc -n d83219-tools create -f - ``` 3. Configure (./config/backup.conf) (listing your database(s), and setting your cron schedule). @@ -44,32 +43,53 @@ Note that underscores should be used in the environment variable names. 5. Create your customized `./openshift/backup-deploy.overrides.param` parameter file, if required. -6. Deploy the app; here the example namespace is `d83219-dev` and the app name is `nert-bkup`: +6. Deploy the app; here the example namespace is `d83219-dev` and the app name is `backup-postgres`: ```bash oc -n d83219-dev create configmap backup-conf --from-file=./config/backup.conf -oc -n d83219-dev label configmap backup-conf app=nert-bkup - -oc -n d83219-dev process -f ./templates/backup/backup-deploy.yaml \ - -p NAME=nert-bkup \ - -p IMAGE_NAMESPACE=d83219-tools \ - -p SOURCE_IMAGE_NAME=nert-bkup \ - -p TAG_NAME=v1 \ - -p BACKUP_VOLUME_NAME=nert-bkup-pvc -p BACKUP_VOLUME_SIZE=2Gi \ - -p VERIFICATION_VOLUME_SIZE=1Gi \ - -p ENVIRONMENT_NAME=dev \ - -p ENVIRONMENT_FRIENDLY_NAME='NERT DB Backups' | oc -n d83219-dev create -f - +oc -n d83219-dev label configmap backup-conf app=backup-postgres + +oc -n d83219-dev process -f ./templates/backup/backup-deploy.yaml | oc -n d83219-dev create -f - ``` To clean up the deployment ```bash -oc -n d83219-dev delete pvc/nert-bkup-pvc pvc/backup-verification secret/nert-bkup secret/ftp-secret dc/nert-bkup networkpolicy/nert-bkup configmap/backup-conf +oc -n d83219-dev delete pvc/backup-postgres-pvc pvc/backup-verification secret/backup-postgres secret/ftp-secret dc/backup-postgres networkpolicy/backup-postgres configmap/backup-conf ``` To clean up the image stream and build configuration ```bash -oc -n d83219-dev delete buildconfig/nert-bkup imagestream/nert-bkup +oc -n d83219-tools delete buildconfig/backup-postgres imagestream/backup-postgres ``` +### NOTE: User Management Role Binding Required in tools env +``` +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: 'system:image-pullers' + namespace: d83219-tools + annotations: + openshift.io/description: >- + Allows all pods in this namespace to pull images from this namespace. It + is auto-managed by a controller; remove subjects to disable. +subjects: + - kind: Group + apiGroup: rbac.authorization.k8s.io + name: 'system:serviceaccounts:d83219-tools' + - kind: Group + apiGroup: rbac.authorization.k8s.io + name: 'system:serviceaccounts:d83219-dev' + - kind: Group + apiGroup: rbac.authorization.k8s.io + name: 'system:serviceaccounts:d83219-test' + - kind: Group + apiGroup: rbac.authorization.k8s.io + name: 'system:serviceaccounts:d83219-prod' +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: 'system:image-puller' +``` \ No newline at end of file diff --git a/containers/backup/templates/backup/backup-deploy.yaml b/containers/backup/templates/backup/backup-deploy.yaml index f86be53f..9fa33e05 100644 --- a/containers/backup/templates/backup/backup-deploy.yaml +++ b/containers/backup/templates/backup/backup-deploy.yaml @@ -174,34 +174,34 @@ objects: valueFrom: secretKeyRef: key: database-user - name: '${DATABASE_SERVICE_NAME}' + name: '${DATABASE_SECRET_REF}' - name: PG_PASSWORD valueFrom: secretKeyRef: key: database-user-password - name: '${DATABASE_SERVICE_NAME}' + name: '${DATABASE_SECRET_REF}' - name: PG_ROOT_PASSWORD valueFrom: secretKeyRef: key: database-admin-password - name: '${DATABASE_SERVICE_NAME}' + name: '${DATABASE_SECRET_REF}' - name: PG_DATABASE valueFrom: secretKeyRef: key: database-name - name: '${DATABASE_SERVICE_NAME}' + name: '${DATABASE_SECRET_REF}' - name: PG_PRIMARY_PORT value: '5432' - name: PG_PRIMARY_USER valueFrom: secretKeyRef: key: database-user - name: '${DATABASE_SERVICE_NAME}' + name: '${DATABASE_SECRET_REF}' - name: PG_PRIMARY_PASSWORD valueFrom: secretKeyRef: key: database-user-password - name: '${DATABASE_SERVICE_NAME}' + name: '${DATABASE_SECRET_REF}' - name: FTP_URL valueFrom: secretKeyRef: @@ -267,27 +267,27 @@ parameters: displayName: Namespace Name description: The name of the namespace being deployed to.. required: true - value: devex-von-image + value: d83219-dev - name: IMAGE_NAMESPACE displayName: Image Namespace description: The namespace of the OpenShift project containing the imagestream for the application. required: true - value: + value: d83219-tools - name: TAG_NAME displayName: Environment TAG name description: The TAG name for this environment, e.g., dev, test, prod required: true - value: dev + value: latest - name: DATABASE_SERVICE_NAME displayName: Database Service Name description: Used for backward compatibility only. Not needed when using the recommended 'backup.conf' configuration. The name of the database service. required: false - value: "" + value: restoration-tracker-db-postgresql-dev-deploy - name: DATABASE_NAME displayName: Database Name description: Used for backward compatibility only. Not needed when using the recommended 'backup.conf' configuration. The name of the database. required: false - value: "" + value: restoration-tracker - name: MONGODB_AUTHENTICATION_DATABASE displayName: MongoDB Authentication Database description: This is only required if you are backing up mongo database with a separate authentication database. @@ -297,7 +297,7 @@ parameters: displayName: Database Deployment Name description: The name associated to the database deployment resources. In particular, this is used to wire up the credentials associated to the database. required: true - value: postgresql + value: restoration-tracker-db-postgresql-dev-deploy - name: DATABASE_USER_KEY_NAME displayName: Database User Key Name description: The database user key name stored in database deployment resources specified by DATABASE_DEPLOYMENT_NAME. @@ -316,7 +316,7 @@ parameters: displayName: Table Schema description: The table schema for your database. Used for Postgres backups. required: true - value: public + value: restoration - name: BACKUP_STRATEGY displayName: Backup Strategy description: The strategy to use for backups; for example daily, or rolling. @@ -361,12 +361,12 @@ parameters: displayName: Friendly Environment Name description: The human readable name of the environment. This variable is used by the webhook integration to identify the environment in which the backup notifications originate. required: false - value: "" + value: NERT DB Backups - name: ENVIRONMENT_NAME displayName: Environment Name (Environment Id) description: The name or Id of the environment. This variable is used by the webhook integration and by the NetworkSecurityPolicies to identify the environment in which the backup notifications originate. required: true - value: "" + value: dev - name: BACKUP_DIR displayName: The root backup directory description: The name of the root backup directory. The backup volume will be mounted to this directory. @@ -416,12 +416,12 @@ parameters: displayName: Backup Volume Name description: The name of the persistent volume used to store the backups. required: true - value: backup + value: backup-postgres-pvc - name: BACKUP_VOLUME_SIZE displayName: Backup Volume Size description: The size of the persistent volume used to store the backups, e.g. 512Mi, 1Gi, 2Gi. Ensure this is sized correctly. Refer to the container documentation for details. required: true - value: 5Gi + value: 4Gi - name: BACKUP_VOLUME_CLASS displayName: Backup Volume Class description: The class of the persistent volume used to store the backups; netapp-file-standard is the recommended default. @@ -436,7 +436,7 @@ parameters: displayName: Backup Volume Size description: The size of the persistent volume used for restoring and verifying backups, e.g. 512Mi, 1Gi, 2Gi. Ensure this is sized correctly. It should be large enough to contain your largest database. required: true - value: 1Gi + value: 2Gi - name: VERIFICATION_VOLUME_CLASS displayName: Backup Volume Class description: The class of the persistent volume used for restoring and verifying backups; netapp-file-standard, netapp-block-standard. @@ -467,8 +467,8 @@ parameters: description: The resources Memory limit (in Mi, Gi, etc) for this build. required: true value: 0Mi - - name: DATABASE_SERVICE_NAME - displayName: Database Service Name + - name: DATABASE_SECRET_REF + displayName: Database Secret Reference + description: The name of the secret containing the database credentials. required: true - value: 'postgresql' - description: Username for PostgreSQL user that will be used for accessing the database. + value: restoration-tracker-creds \ No newline at end of file From dfa65bb684c88e095d80257e679b93b168516b66 Mon Sep 17 00:00:00 2001 From: Kjartan Einarsson Date: Thu, 8 Aug 2024 13:51:39 -0700 Subject: [PATCH 05/12] remove unused files --- .../backup-cronjob/backup-cronjob.yaml | 285 ------------------ containers/backup/templates/nsp/build.yaml | 50 --- containers/backup/templates/nsp/deploy.yaml | 50 --- 3 files changed, 385 deletions(-) delete mode 100644 containers/backup/templates/backup-cronjob/backup-cronjob.yaml delete mode 100644 containers/backup/templates/nsp/build.yaml delete mode 100644 containers/backup/templates/nsp/deploy.yaml diff --git a/containers/backup/templates/backup-cronjob/backup-cronjob.yaml b/containers/backup/templates/backup-cronjob/backup-cronjob.yaml deleted file mode 100644 index abde3e96..00000000 --- a/containers/backup/templates/backup-cronjob/backup-cronjob.yaml +++ /dev/null @@ -1,285 +0,0 @@ ---- -kind: "Template" -apiVersion: "template.openshift.io/v1" -metadata: - name: "{$JOB_NAME}-cronjob-template" - annotations: - description: "Scheduled Task to perform a Database Backup" - tags: "cronjob,backup" -parameters: - - name: "JOB_NAME" - displayName: "Job Name" - description: "Name of the Scheduled Job to Create." - value: "backup-postgres" - required: true - - name: "JOB_PERSISTENT_STORAGE_NAME" - displayName: "Backup Persistent Storage Name" - description: "Pre-Created PVC to use for backup target" - value: "bk-devex-von-tools-a9vlgd1jpsg1" - required: true - - name: "SCHEDULE" - displayName: "Cron Schedule" - description: "Cron Schedule to Execute the Job (using local cluster system TZ)" - # Currently targeting 1:00 AM Daily - value: "0 1 * * *" - required: true - - name: "SOURCE_IMAGE_NAME" - displayName: "Source Image Name" - description: "The name of the image to use for this resource." - required: true - value: "backup-container" - - name: "IMAGE_REGISTRY" - description: "The base OpenShift docker registry" - displayName: "Docker Image Registry" - required: true - # Set value to "docker-registry.default.svc:5000" if using OCP3 - value: "docker.io" - - name: "IMAGE_NAMESPACE" - displayName: "Image Namespace" - description: "The namespace of the OpenShift project containing the imagestream for the application." - required: true - value: "bcgovimages" - - name: "TAG_NAME" - displayName: "Environment TAG name" - description: "The TAG name for this environment, e.g., dev, test, prod" - required: true - value: "dev" - - name: "DATABASE_SERVICE_NAME" - displayName: "Database Service Name" - description: "The name of the database service." - required: true - value: "postgresql" - - name: "DATABASE_DEFAULT_PORT" - displayName: "Database Service Port" - description: "The configured port for the database service" - required: true - value: "5432" - - name: "DATABASE_NAME" - displayName: "Database Name" - description: "The name of the database." - required: true - value: "MyDatabase" - - name: "DATABASE_DEPLOYMENT_NAME" - displayName: "Database Deployment Name" - description: "The name associated to the database deployment resources. In particular, this is used to wire up the credentials associated to the database." - required: true - value: "postgresql" - - name: DATABASE_USER_KEY_NAME - displayName: Database User Key Name - description: - The database user key name stored in database deployment resources specified - by DATABASE_DEPLOYMENT_NAME. - required: true - value: database-user - - name: DATABASE_PASSWORD_KEY_NAME - displayName: Database Password Key Name - description: - The database password key name stored in database deployment resources - specified by DATABASE_DEPLOYMENT_NAME. - required: true - value: database-password - - name: "BACKUP_STRATEGY" - displayName: "Backup Strategy" - description: "The strategy to use for backups; for example daily, or rolling." - required: true - value: "rolling" - - name: "BACKUP_DIR" - displayName: "The root backup directory" - description: "The name of the root backup directory" - required: true - value: "/backups/" - - name: "NUM_BACKUPS" - displayName: "The number of backup files to be retained" - description: "The number of backup files to be retained. Used for the `daily` backup strategy. Ignored when using the `rolling` backup strategy." - required: false - value: "5" - - name: "DAILY_BACKUPS" - displayName: "Number of Daily Backups to Retain" - description: "The number of daily backup files to be retained. Used for the `rolling` backup strategy." - required: false - value: "7" - - name: "WEEKLY_BACKUPS" - displayName: "Number of Weekly Backups to Retain" - description: "The number of weekly backup files to be retained. Used for the `rolling` backup strategy." - required: false - value: "4" - - name: "MONTHLY_BACKUPS" - displayName: "Number of Monthly Backups to Retain" - description: "The number of monthly backup files to be retained. Used for the `rolling` backup strategy." - required: false - value: "1" - - name: "JOB_SERVICE_ACCOUNT" - displayName: "Service Account Name" - description: "Name of the Service Account To Exeucte the Job As." - value: "default" - required: true - - name: "SUCCESS_JOBS_HISTORY_LIMIT" - displayName: "Successful Job History Limit" - description: "The number of successful jobs that will be retained" - value: "5" - required: true - - name: "FAILED_JOBS_HISTORY_LIMIT" - displayName: "Failed Job History Limit" - description: "The number of failed jobs that will be retained" - value: "2" - required: true - - name: "JOB_BACKOFF_LIMIT" - displayName: "Job Backoff Limit" - description: "The number of attempts to try for a successful job outcome" - value: "0" - required: false -objects: -- kind: ConfigMap - apiVersion: v1 - metadata: - name: "${JOB_NAME}-config" - labels: - template: "${JOB_NAME}-config-template" - cronjob: "${JOB_NAME}" - data: - DATABASE_SERVICE_NAME: "${DATABASE_SERVICE_NAME}" - DEFAULT_PORT: "${DATABASE_DEFAULT_PORT}" - POSTGRESQL_DATABASE: "${DATABASE_NAME}" - # BACKUP_STRATEGY: "daily" - BACKUP_STRATEGY: "rolling" - RETENTION.NUM_BACKUPS: "${NUM_BACKUPS}" - RETENTION.DAILY_BACKUPS: "${DAILY_BACKUPS}" - RETENTION.WEEKLY_BACKUPS: "${WEEKLY_BACKUPS}" - RETENTION.MONTHLY_BACKUPS: "${MONTHLY_BACKUPS}" - -- kind: "CronJob" - apiVersion: "batch/v1" - metadata: - name: "${JOB_NAME}" - labels: - template: "${JOB_NAME}-cronjob" - cronjob: "${JOB_NAME}" - spec: - schedule: "${SCHEDULE}" - concurrencyPolicy: "Forbid" - successfulJobsHistoryLimit: "${{SUCCESS_JOBS_HISTORY_LIMIT}}" - failedJobsHistoryLimit: "${{FAILED_JOBS_HISTORY_LIMIT}}" - jobTemplate: - metadata: - labels: - template: "${JOB_NAME}-job" - cronjob: "${JOB_NAME}" - spec: - backoffLimit: ${{JOB_BACKOFF_LIMIT}} - template: - metadata: - labels: - template: "${JOB_NAME}-job" - cronjob: "${JOB_NAME}" - spec: - containers: - - name: "${JOB_NAME}-cronjob" - image: "${IMAGE_REGISTRY}/${IMAGE_NAMESPACE}/${SOURCE_IMAGE_NAME}:${TAG_NAME}" - # image: backup - command: - - "/bin/bash" - - "-c" - - "/backup.sh -1" - volumeMounts: - - mountPath: "${BACKUP_DIR}" - name: "backup" - env: - - name: BACKUP_DIR - value: "${BACKUP_DIR}" - - name: BACKUP_STRATEGY - valueFrom: - configMapKeyRef: - name: "${JOB_NAME}-config" - key: BACKUP_STRATEGY - - name: NUM_BACKUPS - valueFrom: - configMapKeyRef: - name: "${JOB_NAME}-config" - key: RETENTION.NUM_BACKUPS - optional: true - - name: DAILY_BACKUPS - valueFrom: - configMapKeyRef: - name: "${JOB_NAME}-config" - key: RETENTION.DAILY_BACKUPS - optional: true - - name: WEEKLY_BACKUPS - valueFrom: - configMapKeyRef: - name: "${JOB_NAME}-config" - key: RETENTION.WEEKLY_BACKUPS - optional: true - - name: MONTHLY_BACKUPS - valueFrom: - configMapKeyRef: - name: "${JOB_NAME}-config" - key: RETENTION.MONTHLY_BACKUPS - optional: true - - name: DATABASE_SERVICE_NAME - valueFrom: - configMapKeyRef: - name: "${JOB_NAME}-config" - key: DATABASE_SERVICE_NAME - - name: DEFAULT_PORT - valueFrom: - configMapKeyRef: - name: "${JOB_NAME}-config" - key: DEFAULT_PORT - optional: true - - name: POSTGRESQL_DATABASE - valueFrom: - configMapKeyRef: - name: "${JOB_NAME}-config" - key: POSTGRESQL_DATABASE - - name: DATABASE_USER - valueFrom: - secretKeyRef: - name: "${DATABASE_DEPLOYMENT_NAME}" - key: "${DATABASE_USER_KEY_NAME}" - - name: DATABASE_PASSWORD - valueFrom: - secretKeyRef: - name: "${DATABASE_DEPLOYMENT_NAME}" - key: "${DATABASE_PASSWORD_KEY_NAME}" - - name: PG_USER - valueFrom: - secretKeyRef: - key: database-user - name: '${DATABASE_SERVICE_NAME}' - - name: PG_PASSWORD - valueFrom: - secretKeyRef: - key: database-user-password - name: '${DATABASE_SERVICE_NAME}' - - name: PG_ROOT_PASSWORD - valueFrom: - secretKeyRef: - key: database-admin-password - name: '${DATABASE_SERVICE_NAME}' - - name: PG_DATABASE - valueFrom: - secretKeyRef: - key: database-name - name: '${DATABASE_SERVICE_NAME}' - - name: PG_PRIMARY_PORT - value: '5432' - - name: PG_PRIMARY_USER - valueFrom: - secretKeyRef: - key: database-user - name: '${DATABASE_SERVICE_NAME}' - - name: PG_PRIMARY_PASSWORD - valueFrom: - secretKeyRef: - key: database-user-password - name: '${DATABASE_SERVICE_NAME}' - volumes: - - name: backup - persistentVolumeClaim: - claimName: "${JOB_PERSISTENT_STORAGE_NAME}" - restartPolicy: "Never" - terminationGracePeriodSeconds: 30 - activeDeadlineSeconds: 1600 - dnsPolicy: "ClusterFirst" - serviceAccountName: "${JOB_SERVICE_ACCOUNT}" - serviceAccount: "${JOB_SERVICE_ACCOUNT}" diff --git a/containers/backup/templates/nsp/build.yaml b/containers/backup/templates/nsp/build.yaml deleted file mode 100644 index 031e6bc9..00000000 --- a/containers/backup/templates/nsp/build.yaml +++ /dev/null @@ -1,50 +0,0 @@ ---- -kind: Template -apiVersion: "template.openshift.io/v1" -metadata: - name: global-nsp-build-template -objects: - - kind: NetworkPolicy - apiVersion: networking.k8s.io/v1 - metadata: - name: deny-by-default - spec: - description: | - Deny all traffic by default. - podSelector: {} - ingress: [] - - - kind: NetworkSecurityPolicy - apiVersion: security.devops.gov.bc.ca/v1alpha1 - metadata: - name: any-to-any - spec: - description: | - Disable Aporeto policies - Allow all pods within the namespace to communicate. - source: - - - $namespace=${NAMESPACE_NAME}-${ENV_NAME} - destination: - - - $namespace=${NAMESPACE_NAME}-${ENV_NAME} - - - kind: NetworkSecurityPolicy - apiVersion: security.devops.gov.bc.ca/v1alpha1 - metadata: - name: any-to-external - spec: - description: | - Disable Aporeto policies - Allow all pods within the namespace full access to external systems. - source: - - - $namespace=${NAMESPACE_NAME}-${ENV_NAME} - destination: - - - ext:network=any - -parameters: - - name: NAMESPACE_NAME - displayName: The target namespace for the resources. - required: true - value: - - name: ENV_NAME - displayName: Environment Name - description: Environment name. For the build environment this will typically be 'tools' - required: true - value: tools diff --git a/containers/backup/templates/nsp/deploy.yaml b/containers/backup/templates/nsp/deploy.yaml deleted file mode 100644 index 6dab41e6..00000000 --- a/containers/backup/templates/nsp/deploy.yaml +++ /dev/null @@ -1,50 +0,0 @@ ---- -kind: Template -apiVersion: "template.openshift.io/v1" -metadata: - name: global-nsp-build-template -objects: - - kind: NetworkPolicy - apiVersion: networking.k8s.io/v1 - metadata: - name: deny-by-default - spec: - description: | - Deny all traffic by default. - podSelector: {} - ingress: [] - - - kind: NetworkSecurityPolicy - apiVersion: security.devops.gov.bc.ca/v1alpha1 - metadata: - name: any-to-any - spec: - description: | - Disable Aporeto policies - Allow all pods within the namespace to communicate. - source: - - - $namespace=${NAMESPACE_NAME}-${TAG_NAME} - destination: - - - $namespace=${NAMESPACE_NAME}-${TAG_NAME} - - - kind: NetworkSecurityPolicy - apiVersion: security.devops.gov.bc.ca/v1alpha1 - metadata: - name: any-to-external - spec: - description: | - Disable Aporeto policies - Allow all pods within the namespace full access to external systems. - source: - - - $namespace=${NAMESPACE_NAME}-${TAG_NAME} - destination: - - - ext:network=any - -parameters: - - name: NAMESPACE_NAME - displayName: The target namespace for the resources. - required: true - value: - - name: TAG_NAME - displayName: Environment Name - description: Environment name. For the build environment this will typically be 'tools' - required: true - value: dev From 1d31633de7cedd11fc73fbcf41235314684956d0 Mon Sep 17 00:00:00 2001 From: Kjartan Einarsson Date: Thu, 8 Aug 2024 14:02:46 -0700 Subject: [PATCH 06/12] add info to developer md --- containers/backup/DEVELOPER.md | 41 +++++++++++++++++++++++++++++----- 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/containers/backup/DEVELOPER.md b/containers/backup/DEVELOPER.md index 43bb2457..a54a7f05 100644 --- a/containers/backup/DEVELOPER.md +++ b/containers/backup/DEVELOPER.md @@ -27,16 +27,45 @@ postgres=restoration-tracker-db-postgresql:5432/restoration-tracker 4. Configure references to your DB credentials in [backup-deploy.yaml](./openshift/templates/backup/backup-deploy.yaml), replacing the boilerplate `DATABASE_USER` and `DATABASE_PASSWORD` environment variables. ```yaml -- name: EAOFIDER_POSTGRESQL_USER +- name: PG_USER valueFrom: secretKeyRef: - name: eaofider-postgresql - key: "${DATABASE_USER_KEY_NAME}" -- name: EAOFIDER_POSTGRESQL_PASSWORD + key: database-user + name: '${DATABASE_SECRET_REF}' +- name: PG_PASSWORD valueFrom: secretKeyRef: - name: eaofider-postgresql - key: "${DATABASE_PASSWORD_KEY_NAME}" + key: database-user-password + name: '${DATABASE_SECRET_REF}' +- name: PG_ROOT_PASSWORD + valueFrom: + secretKeyRef: + key: database-admin-password + name: '${DATABASE_SECRET_REF}' +- name: PG_DATABASE + valueFrom: + secretKeyRef: + key: database-name + name: '${DATABASE_SECRET_REF}' +- name: PG_PRIMARY_PORT + value: '5432' +- name: PG_PRIMARY_USER + valueFrom: + secretKeyRef: + key: database-user + name: '${DATABASE_SECRET_REF}' +- name: PG_PRIMARY_PASSWORD + valueFrom: + secretKeyRef: + key: database-user-password + name: '${DATABASE_SECRET_REF}' + +... +- name: DATABASE_SECRET_REF + displayName: Database Secret Reference + description: The name of the secret containing the database credentials. + required: true + value: restoration-tracker-creds ``` Note that underscores should be used in the environment variable names. From d76b40a6b9fccbf34307a00eaacc8a1732cbf72d Mon Sep 17 00:00:00 2001 From: Kjartan Einarsson Date: Mon, 12 Aug 2024 10:45:01 -0700 Subject: [PATCH 07/12] update DB username/password --- containers/backup/DEVELOPER.md | 42 +++++-------------- .../templates/backup/backup-deploy.yaml | 4 +- 2 files changed, 12 insertions(+), 34 deletions(-) diff --git a/containers/backup/DEVELOPER.md b/containers/backup/DEVELOPER.md index a54a7f05..a74ed92f 100644 --- a/containers/backup/DEVELOPER.md +++ b/containers/backup/DEVELOPER.md @@ -27,38 +27,16 @@ postgres=restoration-tracker-db-postgresql:5432/restoration-tracker 4. Configure references to your DB credentials in [backup-deploy.yaml](./openshift/templates/backup/backup-deploy.yaml), replacing the boilerplate `DATABASE_USER` and `DATABASE_PASSWORD` environment variables. ```yaml -- name: PG_USER - valueFrom: - secretKeyRef: - key: database-user - name: '${DATABASE_SECRET_REF}' -- name: PG_PASSWORD - valueFrom: - secretKeyRef: - key: database-user-password - name: '${DATABASE_SECRET_REF}' -- name: PG_ROOT_PASSWORD - valueFrom: - secretKeyRef: - key: database-admin-password - name: '${DATABASE_SECRET_REF}' -- name: PG_DATABASE - valueFrom: - secretKeyRef: - key: database-name - name: '${DATABASE_SECRET_REF}' -- name: PG_PRIMARY_PORT - value: '5432' -- name: PG_PRIMARY_USER - valueFrom: - secretKeyRef: - key: database-user - name: '${DATABASE_SECRET_REF}' -- name: PG_PRIMARY_PASSWORD - valueFrom: - secretKeyRef: - key: database-user-password - name: '${DATABASE_SECRET_REF}' + - name: DATABASE_USER + valueFrom: + secretKeyRef: + key: database-user + name: '${DATABASE_SECRET_REF}' + - name: DATABASE_PASSWORD + valueFrom: + secretKeyRef: + key: database-user-password + name: '${DATABASE_SECRET_REF}' ... - name: DATABASE_SECRET_REF diff --git a/containers/backup/templates/backup/backup-deploy.yaml b/containers/backup/templates/backup/backup-deploy.yaml index 9fa33e05..074d6f58 100644 --- a/containers/backup/templates/backup/backup-deploy.yaml +++ b/containers/backup/templates/backup/backup-deploy.yaml @@ -170,12 +170,12 @@ objects: value: ${MONGODB_AUTHENTICATION_DATABASE} - name: TABLE_SCHEMA value: ${TABLE_SCHEMA} - - name: PG_USER + - name: DATABASE_USER valueFrom: secretKeyRef: key: database-user name: '${DATABASE_SECRET_REF}' - - name: PG_PASSWORD + - name: DATABASE_PASSWORD valueFrom: secretKeyRef: key: database-user-password From dab9320f5a0dcd215c80c120d6a835d6a7dd6494 Mon Sep 17 00:00:00 2001 From: Kjartan Einarsson Date: Fri, 16 Aug 2024 12:34:27 -0700 Subject: [PATCH 08/12] update s3 envs for backup-container --- .../templates/backup/backup-deploy.yaml | 40 +++++++++++++++---- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/containers/backup/templates/backup/backup-deploy.yaml b/containers/backup/templates/backup/backup-deploy.yaml index 074d6f58..4f0b5279 100644 --- a/containers/backup/templates/backup/backup-deploy.yaml +++ b/containers/backup/templates/backup/backup-deploy.yaml @@ -174,34 +174,54 @@ objects: valueFrom: secretKeyRef: key: database-user - name: '${DATABASE_SECRET_REF}' + name: ${DATABASE_SECRET_REF} - name: DATABASE_PASSWORD valueFrom: secretKeyRef: key: database-user-password - name: '${DATABASE_SECRET_REF}' + name: ${DATABASE_SECRET_REF} - name: PG_ROOT_PASSWORD valueFrom: secretKeyRef: key: database-admin-password - name: '${DATABASE_SECRET_REF}' + name: ${DATABASE_SECRET_REF} - name: PG_DATABASE valueFrom: secretKeyRef: key: database-name - name: '${DATABASE_SECRET_REF}' + name: ${DATABASE_SECRET_REF} - name: PG_PRIMARY_PORT value: '5432' - name: PG_PRIMARY_USER valueFrom: secretKeyRef: key: database-user - name: '${DATABASE_SECRET_REF}' + name: ${DATABASE_SECRET_REF} - name: PG_PRIMARY_PASSWORD valueFrom: secretKeyRef: key: database-user-password - name: '${DATABASE_SECRET_REF}' + name: ${DATABASE_SECRET_REF} + - name: S3_ENDPOINT + valueFrom: + secretKeyRef: + key: object_store_url + name: ${OBJECT_STORE_SECRETS} + - name: S3_USER + valueFrom: + secretKeyRef: + key: object_store_access_key_id + name: ${OBJECT_STORE_SECRETS} + - name: S3_PASSWORD + valueFrom: + secretKeyRef: + key: object_store_secret_key_id + name: ${OBJECT_STORE_SECRETS} + - name: S3_BUCKET + valueFrom: + secretKeyRef: + key: object_store_bucket_name + name: ${OBJECT_STORE_SECRETS} - name: FTP_URL valueFrom: secretKeyRef: @@ -307,7 +327,7 @@ parameters: displayName: Database Password Key Name description: The database password key name stored in database deployment resources specified by DATABASE_DEPLOYMENT_NAME. required: true - value: database-password + value: database-user-password - name: MSSQL_SA_PASSWORD displayName: MSSQL SA Password description: The database password to use for the local backup database. @@ -336,7 +356,7 @@ parameters: displayName: Ftp URL Hostname description: Ftp URL Hostname. The backup-deploy.overrides.sh will parse this from the supplied FTP_URL, and fetch it from a secret for updates. required: false - value: + value: "" - name: FTP_USER displayName: FTP user name description: FTP user name @@ -347,6 +367,10 @@ parameters: description: FTP password required: false value: "" + - name: OBJECT_STORE_SECRETS + description: Secrets used to read and write to the S3 storage + required: false + value: restoration-tracker-object-store - name: WEBHOOK_URL displayName: Webhook URL description: The URL of the webhook to use for notifications. If not specified, the webhook integration feature is disabled. From f66f79c843e45fd775e5c9dced2866c62fcadb74 Mon Sep 17 00:00:00 2001 From: Kjartan Einarsson Date: Mon, 19 Aug 2024 09:55:45 -0700 Subject: [PATCH 09/12] update s3 params --- .../templates/backup/backup-deploy.yaml | 40 +++++++++++-------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/containers/backup/templates/backup/backup-deploy.yaml b/containers/backup/templates/backup/backup-deploy.yaml index 4f0b5279..bf37bcbd 100644 --- a/containers/backup/templates/backup/backup-deploy.yaml +++ b/containers/backup/templates/backup/backup-deploy.yaml @@ -203,25 +203,13 @@ objects: key: database-user-password name: ${DATABASE_SECRET_REF} - name: S3_ENDPOINT - valueFrom: - secretKeyRef: - key: object_store_url - name: ${OBJECT_STORE_SECRETS} + value: ${S3_ENDPOINT} - name: S3_USER - valueFrom: - secretKeyRef: - key: object_store_access_key_id - name: ${OBJECT_STORE_SECRETS} + value: ${S3_USER} - name: S3_PASSWORD - valueFrom: - secretKeyRef: - key: object_store_secret_key_id - name: ${OBJECT_STORE_SECRETS} + value: ${S3_PASSWORD} - name: S3_BUCKET - valueFrom: - secretKeyRef: - key: object_store_bucket_name - name: ${OBJECT_STORE_SECRETS} + value: ${S3_BUCKET} - name: FTP_URL valueFrom: secretKeyRef: @@ -263,6 +251,26 @@ objects: subPath: ${CONFIG_FILE_NAME} parameters: + - name: S3_ENDPOINT + valueFrom: + secretKeyRef: + key: object_store_url + name: ${OBJECT_STORE_SECRETS} + - name: S3_USER + valueFrom: + secretKeyRef: + key: object_store_access_key_id + name: ${OBJECT_STORE_SECRETS} + - name: S3_PASSWORD + valueFrom: + secretKeyRef: + key: object_store_secret_key_id + name: ${OBJECT_STORE_SECRETS} + - name: S3_BUCKET + valueFrom: + secretKeyRef: + key: object_store_bucket_name + name: ${OBJECT_STORE_SECRETS} - name: NAME displayName: Name description: The name assigned to all of the resources. Use 'backup-{database name}' depending on your database provider From a27305abb5d7b5c68bd5ae44f523b0ce4952a5a2 Mon Sep 17 00:00:00 2001 From: Kjartan Einarsson Date: Mon, 26 Aug 2024 12:35:45 -0700 Subject: [PATCH 10/12] backup container, backing up correctly --- containers/backup/config/backup.conf | 2 +- containers/backup/templates/backup/backup-deploy.yaml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/containers/backup/config/backup.conf b/containers/backup/config/backup.conf index 1a37ea38..4d8f90a4 100644 --- a/containers/backup/config/backup.conf +++ b/containers/backup/config/backup.conf @@ -18,7 +18,7 @@ # # Examples: # - postgres=postgresql/my_database -postgres=restoration-tracker-db-postgresql-dev-deploy:5432/restoration-tracker +postgres=restoration-tracker-db-postgresql-dev-deploy:5432/restoration_tracker # - mongo=mongodb/my_database # - mongo=mongodb:27017/my_database # - mssql=mssql_server:1433/my_database diff --git a/containers/backup/templates/backup/backup-deploy.yaml b/containers/backup/templates/backup/backup-deploy.yaml index bf37bcbd..655ab035 100644 --- a/containers/backup/templates/backup/backup-deploy.yaml +++ b/containers/backup/templates/backup/backup-deploy.yaml @@ -170,15 +170,15 @@ objects: value: ${MONGODB_AUTHENTICATION_DATABASE} - name: TABLE_SCHEMA value: ${TABLE_SCHEMA} - - name: DATABASE_USER + - name: RESTORATION_TRACKER_DB_POSTGRESQL_DEV_DEPLOY_USER valueFrom: secretKeyRef: - key: database-user + key: database-admin name: ${DATABASE_SECRET_REF} - - name: DATABASE_PASSWORD + - name: RESTORATION_TRACKER_DB_POSTGRESQL_DEV_DEPLOY_PASSWORD valueFrom: secretKeyRef: - key: database-user-password + key: database-admin-password name: ${DATABASE_SECRET_REF} - name: PG_ROOT_PASSWORD valueFrom: From 1e2a7028ad031e8e13f5b53bf9ec4efbf0a6ef8c Mon Sep 17 00:00:00 2001 From: Kjartan Einarsson Date: Mon, 26 Aug 2024 13:27:07 -0700 Subject: [PATCH 11/12] s3 bucket envs added --- .../templates/backup/backup-deploy.yaml | 53 +++++++++---------- 1 file changed, 24 insertions(+), 29 deletions(-) diff --git a/containers/backup/templates/backup/backup-deploy.yaml b/containers/backup/templates/backup/backup-deploy.yaml index 655ab035..70058a01 100644 --- a/containers/backup/templates/backup/backup-deploy.yaml +++ b/containers/backup/templates/backup/backup-deploy.yaml @@ -203,13 +203,27 @@ objects: key: database-user-password name: ${DATABASE_SECRET_REF} - name: S3_ENDPOINT - value: ${S3_ENDPOINT} + value: ${S3_ENDPOINT_URL} + - name: S3_ENDPOINT_URL + valueFrom: + secretKeyRef: + key: object_store_url + name: ${OBJECT_STORE_SECRETS} - name: S3_USER - value: ${S3_USER} + valueFrom: + secretKeyRef: + key: object_store_access_key_id + name: ${OBJECT_STORE_SECRETS} - name: S3_PASSWORD - value: ${S3_PASSWORD} + valueFrom: + secretKeyRef: + key: object_store_secret_key_id + name: ${OBJECT_STORE_SECRETS} - name: S3_BUCKET - value: ${S3_BUCKET} + valueFrom: + secretKeyRef: + key: object_store_bucket_name + name: ${OBJECT_STORE_SECRETS} - name: FTP_URL valueFrom: secretKeyRef: @@ -251,26 +265,6 @@ objects: subPath: ${CONFIG_FILE_NAME} parameters: - - name: S3_ENDPOINT - valueFrom: - secretKeyRef: - key: object_store_url - name: ${OBJECT_STORE_SECRETS} - - name: S3_USER - valueFrom: - secretKeyRef: - key: object_store_access_key_id - name: ${OBJECT_STORE_SECRETS} - - name: S3_PASSWORD - valueFrom: - secretKeyRef: - key: object_store_secret_key_id - name: ${OBJECT_STORE_SECRETS} - - name: S3_BUCKET - valueFrom: - secretKeyRef: - key: object_store_bucket_name - name: ${OBJECT_STORE_SECRETS} - name: NAME displayName: Name description: The name assigned to all of the resources. Use 'backup-{database name}' depending on your database provider @@ -375,10 +369,6 @@ parameters: description: FTP password required: false value: "" - - name: OBJECT_STORE_SECRETS - description: Secrets used to read and write to the S3 storage - required: false - value: restoration-tracker-object-store - name: WEBHOOK_URL displayName: Webhook URL description: The URL of the webhook to use for notifications. If not specified, the webhook integration feature is disabled. @@ -503,4 +493,9 @@ parameters: displayName: Database Secret Reference description: The name of the secret containing the database credentials. required: true - value: restoration-tracker-creds \ No newline at end of file + value: restoration-tracker-creds + - name: OBJECT_STORE_SECRETS + displayName: Object Store Secrets + description: Secrets used to read and write to the S3 storage + required: true + value: restoration-tracker-object-store \ No newline at end of file From 1a09139b5f463cf520756eddfca28dbbc81d043d Mon Sep 17 00:00:00 2001 From: Kjartan Einarsson Date: Mon, 26 Aug 2024 14:49:01 -0700 Subject: [PATCH 12/12] clean up s3 envs, point bucket to backup folder with env --- containers/backup/DEVELOPER.md | 8 ++++---- containers/backup/templates/backup/backup-deploy.yaml | 7 ++++++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/containers/backup/DEVELOPER.md b/containers/backup/DEVELOPER.md index a74ed92f..ed15f28e 100644 --- a/containers/backup/DEVELOPER.md +++ b/containers/backup/DEVELOPER.md @@ -27,15 +27,15 @@ postgres=restoration-tracker-db-postgresql:5432/restoration-tracker 4. Configure references to your DB credentials in [backup-deploy.yaml](./openshift/templates/backup/backup-deploy.yaml), replacing the boilerplate `DATABASE_USER` and `DATABASE_PASSWORD` environment variables. ```yaml - - name: DATABASE_USER + - name: RESTORATION_TRACKER_DB_POSTGRESQL_DEV_DEPLOY_USER valueFrom: secretKeyRef: - key: database-user + key: database-admin name: '${DATABASE_SECRET_REF}' - - name: DATABASE_PASSWORD + - name: RESTORATION_TRACKER_DB_POSTGRESQL_DEV_DEPLOY_PASSWORD valueFrom: secretKeyRef: - key: database-user-password + key: database-admin-password name: '${DATABASE_SECRET_REF}' ... diff --git a/containers/backup/templates/backup/backup-deploy.yaml b/containers/backup/templates/backup/backup-deploy.yaml index 70058a01..858dc6f1 100644 --- a/containers/backup/templates/backup/backup-deploy.yaml +++ b/containers/backup/templates/backup/backup-deploy.yaml @@ -203,7 +203,7 @@ objects: key: database-user-password name: ${DATABASE_SECRET_REF} - name: S3_ENDPOINT - value: ${S3_ENDPOINT_URL} + value: https://nrs.objectstore.gov.bc.ca/ - name: S3_ENDPOINT_URL valueFrom: secretKeyRef: @@ -220,6 +220,8 @@ objects: key: object_store_secret_key_id name: ${OBJECT_STORE_SECRETS} - name: S3_BUCKET + value: nerdel/backups/${ENVIRONMENT_NAME}/ + - name: S3_BUCKET_NAME valueFrom: secretKeyRef: key: object_store_bucket_name @@ -248,6 +250,9 @@ objects: value: ${ENVIRONMENT_FRIENDLY_NAME} - name: ENVIRONMENT_NAME value: ${ENVIRONMENT_NAME} + - name: PGDUTY_SVC_KEY + required: false + value: "" resources: requests: cpu: ${CPU_REQUEST}