remove pgbouncer and increase pg clients (#2269)

* removed pgbouncer

* increase max number of connections

* set default values to maxsize 50

* set shared memory size to 256MB as recommended everywhere

.env-devel

@@ -12,12 +12,11 @@ DIRECTOR_REGISTRY_CACHING_TTL=900
 DIRECTOR_REGISTRY_CACHING=True
 
 POSTGRES_DB=simcoredb
-POSTGRES_ENDPOINT=pgbouncer:5432
-POSTGRES_HOST=pgbouncer
+POSTGRES_ENDPOINT=postgres:5432
+POSTGRES_HOST=postgres
 POSTGRES_PASSWORD=adminadmin
 POSTGRES_PORT=5432
 POSTGRES_USER=scu
-POSTGRES_LONG_RUNNING_SESSION_HOST=postgres
 
 RABBIT_CHANNELS={"log": "comp.backend.channels.log", "instrumentation": "comp.backend.channels.instrumentation"}
 RABBIT_HOST=rabbit

Makefile

@@ -213,7 +213,7 @@ endif
 define _show_endpoints
 # The following endpoints are available
 echo "http://$(if $(IS_WSL2),$(get_my_ip),127.0.0.1):9081                                                                 - oSparc platform"
-echo "http://$(if $(IS_WSL2),$(get_my_ip),127.0.0.1):18080/?pgsql=pgbouncer&username=scu&db=simcoredb&ns=public  - Postgres DB"
+echo "http://$(if $(IS_WSL2),$(get_my_ip),127.0.0.1):18080/?pgsql=postgres&username=scu&db=simcoredb&ns=public  - Postgres DB"
 echo "http://$(if $(IS_WSL2),$(get_my_ip),127.0.0.1):9000                                                                 - Portainer"
 endef
 

packages/models-library/src/models_library/settings/postgres.py

@@ -17,10 +17,6 @@ class PostgresSettings(BaseSettings):
     # entrypoint
     host: str
     port: PortInt = 5432
-    long_running_session_host: Optional[str] = Field(
-        None,
-        description="host for long running sessions (e.g. listen/notify, prepared statements, etc...)",
-    )
 
     # auth
     user: str
@@ -30,8 +26,8 @@ class PostgresSettings(BaseSettings):
     db: str
 
     # pool connection limits
-    minsize: conint(ge=1) = 10
-    maxsize: conint(ge=1) = 100
+    minsize: conint(ge=1) = 1
+    maxsize: conint(ge=1) = 50
 
     dsn: Optional[PostgresDsn] = Field(None, description="Database Source Name")
 

packages/postgres-database/tests/docker-compose.yml

@@ -1,27 +1,5 @@
 version: "3.7"
 services:
-  pgbouncer:
-    image: "edoburu/pgbouncer:1.15.0@sha256:2f47bf272fa9fdf25c100d11f1972b23af61a351a136d3721bfa6bdb52630426"
-    init: true
-    environment:
-      - DB_HOST=postgres
-      - DB_PORT=5432
-      - DB_USER=test
-      - DB_PASSWORD=test
-      - DB_NAME=test
-      # pgbouncer.ini variables
-      - LISTEN_ADDR=*
-      - LISTEN_PORT=5432
-      - POOL_MODE=transaction
-      - MAX_CLIENT_CONN=90 # we keep some for other OPS higher level services
-      - APPLICATION_NAME_ADD_HOST=1
-      - ADMIN_USERS=${POSTGRES_USER}
-
-    ports:
-      - "6432:5432"
-    depends_on:
-      - postgres
-
   postgres:
     image: "postgres:10.11@sha256:2aef165ab4f30fbb109e88959271d8b57489790ea13a77d27c02d8adb8feb20f"
     init: true
@@ -73,4 +51,4 @@ services:
     ports:
       - 18080:8080
     depends_on:
-      - pgbouncer
+      - postgres

packages/pytest-simcore/src/pytest_simcore/docker_compose.py

@@ -158,10 +158,6 @@ def core_services_selection(request) -> List[str]:
     """ Selection of services from the simcore stack """
     core_services = getattr(request.module, FIXTURE_CONFIG_CORE_SERVICES_SELECTION, [])
 
-    if "postgres" in core_services:
-        assert (
-            "pgbouncer" in core_services
-        ), f"WARNING: the test is missing pgbouncer service in '{FIXTURE_CONFIG_CORE_SERVICES_SELECTION}' within '{request.module.__name__}'. postgres alone is not accessible!!"
     assert (
         core_services
     ), f"Expected at least one service in '{FIXTURE_CONFIG_CORE_SERVICES_SELECTION}' within '{request.module.__name__}'"

packages/pytest-simcore/src/pytest_simcore/simcore_services.py

@@ -15,7 +15,7 @@
 
 log = logging.getLogger(__name__)
 
-SERVICES_TO_SKIP = ["sidecar", "postgres", "pgbouncer", "redis", "rabbit"]
+SERVICES_TO_SKIP = ["sidecar", "postgres", "redis", "rabbit"]
 SERVICE_HEALTHCHECK_ENTRYPOINT = {"director-v2": "/"}
 
 

packages/simcore-sdk/src/simcore_sdk/config/db.py

@@ -15,9 +15,6 @@
         "host": T.Or(T.String, T.Null),
         "port": T.Or(T.ToInt, T.Null),
         "endpoint": T.Or(T.String, T.Null),
-        T.Key("long_running_session_host", default=None, optional=True): T.Or(
-            T.String, T.Null
-        ),
     }
 )
 

packages/simcore-sdk/tests/integration/test_dbmanager.py

@@ -10,7 +10,7 @@
 from simcore_sdk.node_ports import config
 from simcore_sdk.node_ports.dbmanager import DBManager
 
-pytest_simcore_core_services_selection = ["postgres", "pgbouncer"]
+pytest_simcore_core_services_selection = ["postgres"]
 
 pytest_simcore_ops_services_selection = ["minio"]
 

packages/simcore-sdk/tests/integration/test_filemanager.py

@@ -12,7 +12,7 @@
 import pytest
 from simcore_sdk.node_ports import exceptions, filemanager
 
-pytest_simcore_core_services_selection = ["postgres", "pgbouncer", "storage"]
+pytest_simcore_core_services_selection = ["postgres", "storage"]
 
 pytest_simcore_ops_services_selection = ["minio"]
 

packages/simcore-sdk/tests/integration/test_nodeports.py

@@ -17,7 +17,7 @@
 from simcore_sdk.node_ports._item import ItemConcreteValue
 from simcore_sdk.node_ports.nodeports import Nodeports
 
-pytest_simcore_core_services_selection = ["postgres", "pgbouncer", "storage"]
+pytest_simcore_core_services_selection = ["postgres", "storage"]
 
 pytest_simcore_ops_services_selection = ["minio"]
 

packages/simcore-sdk/tests/integration/test_nodeports2.py

@@ -17,7 +17,7 @@
 from simcore_sdk.node_ports_v2.links import ItemConcreteValue
 from simcore_sdk.node_ports_v2.nodeports_v2 import Nodeports
 
-pytest_simcore_core_services_selection = ["postgres", "pgbouncer", "storage"]
+pytest_simcore_core_services_selection = ["postgres", "storage"]
 
 pytest_simcore_ops_services_selection = ["minio"]
 

services/catalog/tests/unit/with_dbs/test_entrypoint_dags.py

@@ -9,7 +9,7 @@
 from simcore_service_catalog.models.schemas.meta import Meta
 from starlette.testclient import TestClient
 
-pytest_simcore_core_services_selection = ["postgres", "pgbouncer"]
+pytest_simcore_core_services_selection = ["postgres"]
 pytest_simcore_ops_services_selection = ["adminer"]
 
 

services/catalog/tests/unit/with_dbs/test_entrypoint_services.py

@@ -27,7 +27,6 @@
 
 pytest_simcore_core_services_selection = [
     "postgres",
-    "pgbouncer",
 ]
 pytest_simcore_ops_services_selection = ["adminer"]
 

services/director-v2/tests/integration/test_computation_api.py

@@ -42,7 +42,6 @@
     "sidecar",
     "storage",
     "postgres",
-    "pgbouncer",
 ]
 pytest_simcore_ops_services_selection = ["minio", "adminer"]
 

services/docker-compose-ops.yml

@@ -30,9 +30,7 @@ services:
     ports:
       - "18080:8080"
     networks:
-      # NOTE: simcore_default allows access to postgres through pgbouncer, postgres_network allows access directly to postgres
       - simcore_default
-      - postgres_network
 
   portainer:
     image: portainer/portainer-ce
@@ -104,6 +102,3 @@ networks:
   computational_services_subnet:
     name: ${SWARM_STACK_NAME:-simcore}_computational_services_subnet
     external: true
-  postgres_network:
-    name: ${SWARM_STACK_NAME:-simcore}_postgres_network
-    external: true

services/docker-compose.local.yml

@@ -55,10 +55,6 @@ services:
       - "8080"
       - "3001:3000"
 
-  pgbouncer:
-    ports:
-      - "6432:5432"
-
   postgres:
     ports:
       - "5432:5432"
@@ -84,6 +80,10 @@ services:
         "log_line_prefix=%m [%p] %q%u@%d/%a ",
         "-c",
         'listen_addresses="*"',
+        "-c",
+        "max_connections=413",
+        "-c",
+        "shared_buffers=256MB"
       ]
 
   rabbit:

services/docker-compose.yml

@@ -264,30 +264,6 @@ services:
     networks:
       - default
 
-  pgbouncer:
-    image: "edoburu/pgbouncer:1.15.0@sha256:2f47bf272fa9fdf25c100d11f1972b23af61a351a136d3721bfa6bdb52630426"
-    init: true
-    environment:
-      - DB_HOST=postgres
-      - DB_PORT=${POSTGRES_PORT}
-      - DB_USER=${POSTGRES_USER}
-      - DB_PASSWORD=${POSTGRES_PASSWORD}
-      - DB_NAME=${POSTGRES_DB}
-      # pgbouncer.ini variables
-      - LISTEN_ADDR=*
-      - LISTEN_PORT=${POSTGRES_PORT}
-      - POOL_MODE=transaction
-      - MAX_CLIENT_CONN=10000 # we keep some for other OPS higher level services
-      - DEFAULT_POOL_SIZE=20
-      - MAX_DB_CONNECTIONS=90
-      - APPLICATION_NAME_ADD_HOST=1
-      - ADMIN_USERS=${POSTGRES_USER}
-    networks:
-      - default
-      - interactive_services_subnet
-      - computational_services_subnet
-      - postgres_network
-
   postgres:
     image: "postgres:10.11@sha256:2aef165ab4f30fbb109e88959271d8b57489790ea13a77d27c02d8adb8feb20f"
     init: true
@@ -298,7 +274,10 @@ services:
     volumes:
       - postgres_data:/var/lib/postgresql/data
     networks:
-      - postgres_network
+      - default
+      - interactive_services_subnet
+      - computational_services_subnet
+    shm_size: ‘256mb’
     healthcheck:
       test:
         [
@@ -330,6 +309,10 @@ services:
         "tcp_keepalives_interval=600",
         "-c",
         "tcp_keepalives_count=5",
+        "-c",
+        "max_connections=413",
+        "-c",
+        "shared_buffers=256MB"
       ]
 
   redis:

services/sidecar/tests/integration/test_sidecar.py

@@ -27,7 +27,7 @@
 #
 # SEE packages/pytest-simcore/src/pytest_simcore/docker_compose.py
 #
-pytest_simcore_core_services_selection = ["postgres", "pgbouncer", "rabbit", "storage"]
+pytest_simcore_core_services_selection = ["postgres", "rabbit", "storage"]
 
 pytest_simcore_ops_services_selection = ["minio", "adminer"]
 

services/storage/src/simcore_service_storage/data/docker-prod-config.yaml

@@ -4,16 +4,16 @@ port: 8080
 testing: False
 monitoring_enabled: ${STORAGE_MONITORING_ENABLED}
 test_datcore:
-    token_key: ${BF_API_KEY}
-    token_secret: ${BF_API_SECRET}
+  token_key: ${BF_API_KEY}
+  token_secret: ${BF_API_SECRET}
 postgres:
   db: ${POSTGRES_DB}
   user: ${POSTGRES_USER}
   password: ${POSTGRES_PASSWORD}
   host: ${POSTGRES_HOST}
   port: ${POSTGRES_PORT}
-  minsize: 10
-  maxsize: 100
+  minsize: 1
+  maxsize: 50
 s3:
   endpoint: ${S3_ENDPOINT}
   access_key: ${S3_ACCESS_KEY}

services/storage/tests/conftest.py

@@ -125,7 +125,7 @@ def postgres_service(docker_services, docker_ip):
         password=PASS,
         database=DATABASE,
         host=docker_ip,
-        port=docker_services.port_for("pgbouncer", 5432),
+        port=docker_services.port_for("postgres", 5432),
     )
 
     # Wait until service is responsive.

services/storage/tests/docker-compose.yml

@@ -1,27 +1,5 @@
 version: "3.4"
 services:
-  pgbouncer:
-    image: "edoburu/pgbouncer:1.15.0@sha256:2f47bf272fa9fdf25c100d11f1972b23af61a351a136d3721bfa6bdb52630426"
-    init: true
-    restart: always
-    environment:
-      - DB_HOST=postgres
-      - DB_PORT=5432
-      - DB_USER=${POSTGRES_USER:-admin}
-      - DB_PASSWORD=${POSTGRES_PASSWORD:-admin}
-      - DB_NAME=${POSTGRES_DB:-aio_login_tests}
-      # pgbouncer.ini variables
-      - LISTEN_ADDR=*
-      - LISTEN_PORT=${POSTGRES_PORT}
-      - POOL_MODE=transaction
-      - MAX_CLIENT_CONN=10000 # we keep some for other OPS higher level services
-      - DEFAULT_POOL_SIZE=80
-      - MAX_DB_CONNECTIONS=90
-      - APPLICATION_NAME_ADD_HOST=1
-      - ADMIN_USERS=${POSTGRES_USER:-admin}
-      - VERBOSE=1
-    ports:
-      - "6432:5432"
   postgres:
     image: postgres:10.11@sha256:2aef165ab4f30fbb109e88959271d8b57489790ea13a77d27c02d8adb8feb20f
     restart: always

services/web/server/src/simcore_service_webserver/computation_comp_tasks_listening_task.py

@@ -15,13 +15,13 @@
 from models_library.projects_nodes import NodeID
 from models_library.projects_state import RunningState
 from pydantic.types import PositiveInt
+from servicelib.application_keys import APP_DB_ENGINE_KEY
 from servicelib.logging_utils import log_decorator
 from servicelib.utils import logged_gather
 from simcore_postgres_database.webserver_models import DB_CHANNEL_NAME, projects
 from sqlalchemy.sql import select
 
 from .computation_api import convert_state_from_db
-from .db import APP_LONG_RUNNING_DB_ENGINE_KEY
 from .projects import projects_api, projects_exceptions
 from .projects.projects_utils import project_get_depending_nodes
 
@@ -170,7 +170,7 @@ async def comp_tasks_listening_task(app: web.Application) -> None:
     while True:
         try:
             # create a special connection here
-            db_engine = app[APP_LONG_RUNNING_DB_ENGINE_KEY]
+            db_engine = app[APP_DB_ENGINE_KEY]
             log.info("listening to comp_task events...")
             await listen(app, db_engine)
         except asyncio.CancelledError:

services/web/server/src/simcore_service_webserver/config/server-defaults.yaml

@@ -22,7 +22,6 @@ db:
     database: simcoredb
     endpoint: postgres:5432
     host: postgres
-    long_running_session_host: postgres
     maxsize: 5
     minsize: 1
     password: simcore

services/web/server/src/simcore_service_webserver/config/server-docker-dev.yaml

@@ -28,10 +28,9 @@ db:
     user: ${POSTGRES_USER}
     password: ${POSTGRES_PASSWORD}
     host: ${POSTGRES_HOST}
-    long_running_session_host: ${POSTGRES_LONG_RUNNING_SESSION_HOST}
     port: ${POSTGRES_PORT}
-    minsize: 10
-    maxsize: 100
+    minsize: 1
+    maxsize: 50
 resource_manager:
   enabled: True
   resource_deletion_timeout_seconds: ${WEBSERVER_RESOURCES_DELETION_TIMEOUT_SECONDS}

services/web/server/src/simcore_service_webserver/config/server-docker-prod.yaml

@@ -28,10 +28,9 @@ db:
     user: ${POSTGRES_USER}
     password: ${POSTGRES_PASSWORD}
     host: ${POSTGRES_HOST}
-    long_running_session_host: ${POSTGRES_LONG_RUNNING_SESSION_HOST}
     port: ${POSTGRES_PORT}
-    minsize: 10
-    maxsize: 100
+    minsize: 1
+    maxsize: 50
 resource_manager:
   enabled: True
   resource_deletion_timeout_seconds: ${WEBSERVER_RESOURCES_DELETION_TIMEOUT_SECONDS}