From b6beaf5ca0d4c9b37f0c9c368a083996c6465077 Mon Sep 17 00:00:00 2001 From: Fred van Dijk Date: Mon, 17 Jun 2024 15:26:29 +0200 Subject: [PATCH] zodbconvert example config and scripts --- devops/zodbconvert/README.txt | 15 +++++++-------- devops/zodbconvert/docker-compose.yml | 15 +++++++++++++++ devops/zodbconvert/export_filestorage.sh | 1 + devops/zodbconvert/export_filestorage_tarball.sh | 11 +++++++++++ .../{relstorage.cfg => from-relstorage.cfg} | 4 ++-- devops/zodbconvert/requirements.txt | 2 ++ devops/zodbconvert/restore_dump_to_postgresql.sh | 2 ++ devops/zodbconvert/to-relstorage.cfg | 11 +++++++++++ 8 files changed, 51 insertions(+), 10 deletions(-) create mode 100644 devops/zodbconvert/docker-compose.yml create mode 100644 devops/zodbconvert/export_filestorage.sh create mode 100755 devops/zodbconvert/export_filestorage_tarball.sh rename devops/zodbconvert/{relstorage.cfg => from-relstorage.cfg} (59%) create mode 100644 devops/zodbconvert/requirements.txt create mode 100755 devops/zodbconvert/restore_dump_to_postgresql.sh create mode 100644 devops/zodbconvert/to-relstorage.cfg diff --git a/devops/zodbconvert/README.txt b/devops/zodbconvert/README.txt index 1f9db88..f8d9daf 100644 --- a/devops/zodbconvert/README.txt +++ b/devops/zodbconvert/README.txt @@ -1,23 +1,22 @@ Convert postgresql to filestorage ================================= -In test and production the plone.org website stores the content data in a +In test and production the website stores the content data in a Postgresql database using the relstorage driver. If you want to create a local filestorage copy of the data to use for local development, you can use the zodbconvert utility. -First make sure you are running a local postgresql server with the Plone -content database. The zodbconvert config file assume you use the (datbase) -settings as configured in the docker-compose.yml in the project root. This -compose file starts a local postgresql container on 127.0.0.1:5432 with -ploneorg/ploneorg/ploneorg as the database/role/password . +First make sure you are running a local postgresql server with the site +content database. The zodbconvert config file assume you use the (database) +settings as configured in the postgres-compose.yml in the project root. This +compose file starts a local postgresql container on 127.0.0.1:5432 zodbconvert writes the data to the ./data directory in the project root, inside ./data/filestorage and ./data/blobs . Now run zodbconvert from the project root: -> ./backend/bin/zodbconvert ./devops/zodbconvert/relstorage.cfg +> ./backend/bin/zodbconvert ./devops/zodbconvert/from-relstorage.cfg -This can take 5-15 minutes, depending on your machine specs. +This can take some minutes, depending on your machine specs. diff --git a/devops/zodbconvert/docker-compose.yml b/devops/zodbconvert/docker-compose.yml new file mode 100644 index 0000000..13dc137 --- /dev/null +++ b/devops/zodbconvert/docker-compose.yml @@ -0,0 +1,15 @@ +version: "3.8" + +services: + + db: + image: postgres:16 + environment: + POSTGRES_USER: zodbconvert + POSTGRES_PASSWORD: zodbconvert + POSTGRES_DB: zodbconvert + command: postgres -c shared_buffers=2GB -c effective_cache_size=4GB -c maintenance_work_mem=1GB -c wal_buffers=16MB -c random_page_cost=1.1 -c effective_io_concurrency=200 -c work_mem=20971kB -c min_wal_size=2GB -c max_wal_size=8GB -c wal_keep_size=1GB -c max_locks_per_transaction=512 + ports: + - "127.0.0.1:5432:5432" + volumes: + - ./data/postgresql:/var/lib/postgresql/data diff --git a/devops/zodbconvert/export_filestorage.sh b/devops/zodbconvert/export_filestorage.sh new file mode 100644 index 0000000..0378ae2 --- /dev/null +++ b/devops/zodbconvert/export_filestorage.sh @@ -0,0 +1 @@ +/bin/zodbconvert -f ./from-relstorage.cfg diff --git a/devops/zodbconvert/export_filestorage_tarball.sh b/devops/zodbconvert/export_filestorage_tarball.sh new file mode 100755 index 0000000..86419f3 --- /dev/null +++ b/devops/zodbconvert/export_filestorage_tarball.sh @@ -0,0 +1,11 @@ +#!/bin/bash +# import pgdump file: +# docker exec -i zodbconvert-db-1 pg_restore -U zodbconvert -d zodbconvert --clean --no-owner -v < $1 + +rm -rf ./data/filestorage ./data-blobstorage +mkdir -p ./data/filestorage ./data/blobstorage +./bin/zodbconvert ./from-relstorage.cfg +cd data +tar cfz ploneorg_devdata.tar.gz filestorage blobstorage +mv ploneorg_devdatatar.gz .. +cd .. diff --git a/devops/zodbconvert/relstorage.cfg b/devops/zodbconvert/from-relstorage.cfg similarity index 59% rename from devops/zodbconvert/relstorage.cfg rename to devops/zodbconvert/from-relstorage.cfg index cc2ce2c..e6ba2e8 100644 --- a/devops/zodbconvert/relstorage.cfg +++ b/devops/zodbconvert/from-relstorage.cfg @@ -2,10 +2,10 @@ blob-dir /tmp/blobcache shared-blob-dir false - dsn dbname='ploneorg' user='ploneorg' host='127.0.0.1' port='5432' password='ploneorg' + dsn dbname='zodbconvert' user='zodbconvert' host='127.0.0.1' port='5432' password='zodbconvert' path ./data/filestorage/Data.fs - blob-dir ./data/blobs + blob-dir ./data/blobstorage \ No newline at end of file diff --git a/devops/zodbconvert/requirements.txt b/devops/zodbconvert/requirements.txt new file mode 100644 index 0000000..7775193 --- /dev/null +++ b/devops/zodbconvert/requirements.txt @@ -0,0 +1,2 @@ +relstorage==4.0.0 +psycopg2==2.9.9 \ No newline at end of file diff --git a/devops/zodbconvert/restore_dump_to_postgresql.sh b/devops/zodbconvert/restore_dump_to_postgresql.sh new file mode 100755 index 0000000..a565052 --- /dev/null +++ b/devops/zodbconvert/restore_dump_to_postgresql.sh @@ -0,0 +1,2 @@ +#!/bin/bash +docker exec -i zodbconvert-db-1 pg_restore -U zodbconvert -d zodbconvert --clean --no-owner -v < $1 diff --git a/devops/zodbconvert/to-relstorage.cfg b/devops/zodbconvert/to-relstorage.cfg new file mode 100644 index 0000000..4232a4a --- /dev/null +++ b/devops/zodbconvert/to-relstorage.cfg @@ -0,0 +1,11 @@ + + path ../../data/filestorage/Data.fs + blob-dir ../../data/blobs + + + blob-dir /tmp/blobcache + shared-blob-dir false + + dsn dbname='zodbconvert' user='zodbconvert' host='127.0.0.1' port='5432' password='zodbconvert' + +