Make Processing single category option configurable

docs/src/installation/configurations.md

@@ -78,7 +78,7 @@ The following are the different configuration options that are accepted.
 | `DUCK_DB_THREAD_LIMIT` | `DUCK_DB_THREAD_LIMIT` | `[HDX]` | None | Duck DB max threads limit ,n of your cores eg : 2 | CONDITIONAL |
 | `HDX_SOFT_TASK_LIMIT` | `HDX_SOFT_TASK_LIMIT` | `[HDX]` | `18000` | Soft task time limit signal for celery workers in seconds.It will gently remind celery to finish up the task and terminate, Defaults to 5 Hour| OPTIONAL |
 | `HDX_HARD_TASK_LIMIT` | `HDX_HARD_TASK_LIMIT` | `[HDX]` | `21600` | Hard task time limit signal for celery workers in seconds. It will immediately kill the celery task.Defaults to 6 Hour| OPTIONAL |
-
+| `PROCESS_SINGLE_CATEGORY_IN_POSTGRES` | `PROCESS_SINGLE_CATEGORY_IN_POSTGRES` | `[HDX]` | False | Recommended for workers with low memery or CPU usage , This will process single category request like buildings only , Roads only in postgres itself and avoid extraction from duckdb| OPTIONAL |
 
 ## Which Service uses which settings?
 
@@ -126,6 +126,7 @@ The following are the different configuration options that are accepted.
 | `DUCK_DB_THREAD_LIMIT` | `[HDX]` | Yes | Yes |
 | `HDX_SOFT_TASK_LIMIT` | `[HDX]` | No | Yes |
 | `HDX_HARD_TASK_LIMIT` | `[HDX]` | No | Yes |
+| `PROCESS_SINGLE_CATEGORY_IN_POSTGRES` | `[HDX]` | No | Yes |
 
 
 

src/app.py

@@ -55,7 +55,7 @@
 )
 from src.config import EXPORT_PATH as export_path
 from src.config import INDEX_THRESHOLD as index_threshold
-from src.config import POLYGON_STATISTICS_API_URL
+from src.config import POLYGON_STATISTICS_API_URL, PROCESS_SINGLE_CATEGORY_IN_POSTGRES
 from src.config import USE_CONNECTION_POOLING as use_connection_pooling
 from src.config import USE_S3_TO_UPLOAD, get_db_connection_params, level
 from src.config import logger as logging
@@ -1587,7 +1587,8 @@ def process_hdx_tags(self):
             for cat_type in list(category.values())[0].types
         ]
         where_0_category = None
-        if len(self.params.categories) == 1:
+
+        if len(self.params.categories) == 1 and PROCESS_SINGLE_CATEGORY_IN_POSTGRES:
             where_0_category = list(self.params.categories[0].values())[0].where
 
         table_names = self.types_to_tables(list(set(table_type)))

src/config.py

@@ -202,6 +202,9 @@ def not_raises(func, *args, **kwargs):
     "HDX", "ENABLE_HDX_EXPORTS", fallback=False
 )
 
+PROCESS_SINGLE_CATEGORY_IN_POSTGRES = os.environ.get(
+    "PROCESS_SINGLE_CATEGORY_IN_POSTGRES"
+) or config.getboolean("HDX", "PROCESS_SINGLE_CATEGORY_IN_POSTGRES", fallback=False)
 
 if ENABLE_HDX_EXPORTS:
     HDX_SITE = os.environ.get("HDX_SITE") or config.getboolean(