Add duckdb thread limit feature

hotosm · Jan 11, 2024 · 011bcd4 · 011bcd4
1 parent 8d7f395
commit 011bcd4
Show file tree

Hide file tree

Showing 4 changed files with 22 additions and 13 deletions.
diff --git a/API/hdx.py b/API/hdx.py
@@ -248,7 +248,7 @@ async def process_custom_requests(
                                     "source",
                                 ],
                                 "where": "tags['building'] IS NOT NULL",
-                                "formats": ["shp"],
+                                "formats": ["geojson"],
                             }
                         },
                         {
@@ -271,7 +271,7 @@ async def process_custom_requests(
                                     "source",
                                 ],
                                 "where": "tags['highway'] IS NOT NULL",
-                                "formats": ["shp"],
+                                "formats": ["geojson"],
                             }
                         },
                         {
@@ -295,7 +295,7 @@ async def process_custom_requests(
                                     "source",
                                 ],
                                 "where": "tags['waterway'] IS NOT NULL OR tags['water'] IS NOT NULL OR tags['natural'] IN ('water','wetland','bay')",
-                                "formats": ["shp"],
+                                "formats": ["geojson"],
                             }
                         },
                         {
@@ -325,7 +325,7 @@ async def process_custom_requests(
                                     "source",
                                 ],
                                 "where": "tags['amenity'] IS NOT NULL OR tags['man_made'] IS NOT NULL OR tags['shop'] IS NOT NULL OR tags['tourism'] IS NOT NULL",
-                                "formats": ["shp"],
+                                "formats": ["geojson"],
                             }
                         },
                         {
@@ -352,7 +352,7 @@ async def process_custom_requests(
                                     "source",
                                 ],
                                 "where": "tags['aeroway'] IS NOT NULL OR tags['building'] = 'aerodrome' OR tags['emergency:helipad'] IS NOT NULL OR tags['emergency'] = 'landing_site'",
-                                "formats": ["shp"],
+                                "formats": ["geojson"],
                             }
                         },
                         {
@@ -377,7 +377,7 @@ async def process_custom_requests(
                                     "source",
                                 ],
                                 "where": "tags['amenity'] = 'ferry_terminal' OR tags['building'] = 'ferry_terminal' OR tags['port'] IS NOT NULL",
-                                "formats": ["shp"],
+                                "formats": ["geojson"],
                             }
                         },
                         {
@@ -398,7 +398,7 @@ async def process_custom_requests(
                                     "source",
                                 ],
                                 "where": "tags['amenity'] IN ('kindergarten', 'school', 'college', 'university') OR building IN ('kindergarten', 'school', 'college', 'university')",
-                                "formats": ["shp"],
+                                "formats": ["geojson"],
                             }
                         },
                         {
@@ -421,7 +421,7 @@ async def process_custom_requests(
                                     "source",
                                 ],
                                 "where": "tags['healthcare'] IS NOT NULL OR tags['amenity'] IN ('doctors', 'dentist', 'clinic', 'hospital', 'pharmacy')",
-                                "formats": ["shp"],
+                                "formats": ["geojson"],
                             }
                         },
                         {
@@ -443,7 +443,7 @@ async def process_custom_requests(
                                     "source",
                                 ],
                                 "where": "tags['place'] IN ('isolated_dwelling', 'town', 'village', 'hamlet', 'city')",
-                                "formats": ["shp"],
+                                "formats": ["geojson"],
                             }
                         },
                         {
@@ -468,7 +468,7 @@ async def process_custom_requests(
                                     "source",
                                 ],
                                 "where": "tags['amenity'] IN ('mobile_money_agent','bureau_de_change','bank','microfinance','atm','sacco','money_transfer','post_office')",
-                                "formats": ["shp"],
+                                "formats": ["geojson"],
                             }
                         },
                         {
@@ -494,7 +494,7 @@ async def process_custom_requests(
                                     "source",
                                 ],
                                 "where": "tags['railway'] IN ('rail','station')",
-                                "formats": ["shp"],
+                                "formats": ["geojson"],
                             }
                         },
                     ],

diff --git a/docs/src/installation/configurations.md b/docs/src/installation/configurations.md
@@ -73,7 +73,7 @@ The following are the different configuration options that are accepted.
 | `HDX_OWNER_ORG` | `HDX_OWNER_ORG` | `[HDX]` | None | Your HDX organization ID| CONDITIONAL |
 | `HDX_MAINTAINER` | `HDX_MAINTAINER` | `[HDX]` | None | Your HDX Maintainer ID | CONDITIONAL |
 | `DUCK_DB_MEMORY_LIMIT` | `DUCK_DB_MEMORY_LIMIT` | `[HDX]` | None | Duck DB max memory limit , 80 % of your RAM eg : '5GB'| CONDITIONAL |
-
+| `DUCK_DB_THREAD_LIMIT` | `DUCK_DB_THREAD_LIMIT` | `[HDX]` | None | Duck DB max threads limit ,n of your cores eg : 2 | CONDITIONAL |
 
 
 ## Which Service uses which settings?
@@ -117,6 +117,8 @@ The following are the different configuration options that are accepted.
 | `HDX_OWNER_ORG` | `[HDX]` | Yes | Yes |
 | `HDX_MAINTAINER` | `[HDX]` | Yes | Yes |
 | `DUCK_DB_MEMORY_LIMIT` | `[HDX]` | Yes | Yes |
+| `DUCK_DB_THREAD_LIMIT` | `[HDX]` | Yes | Yes |
+
 
 
 

diff --git a/src/app.py b/src/app.py
@@ -90,6 +90,7 @@
 
     from src.config import (
         DUCK_DB_MEMORY_LIMIT,
+        DUCK_DB_THREAD_LIMIT,
         HDX_MAINTAINER,
         HDX_OWNER_ORG,
         HDX_URL_PREFIX,
@@ -1141,6 +1142,9 @@ def __init__(self, db_path, temp_dir=None):
 
         if DUCK_DB_MEMORY_LIMIT:
             con.sql(f"""SET memory_limit = '{DUCK_DB_MEMORY_LIMIT}'""")
+        if DUCK_DB_THREAD_LIMIT:
+            con.sql(f"""SET threads to {DUCK_DB_MEMORY_LIMIT}""")
+
         con.sql("""SET enable_progress_bar = true""")
 
     def run_query(self, query, attach_pgsql=False, load_spatial=False):
@@ -1244,7 +1248,7 @@ def types_to_tables(self, type_list: list):
 
         return list(table_set)
 
-    def format_where_clause(self,where_clause):
+    def format_where_clause(self, where_clause):
         """
         Formats the where_clause by replacing the first occurrence of the pattern.
 

diff --git a/src/config.py b/src/config.py
@@ -247,6 +247,9 @@ def not_raises(func, *args, **kwargs):
     DUCK_DB_MEMORY_LIMIT = os.environ.get("DUCK_DB_MEMORY_LIMIT") or config.get(
         "HDX", "DUCK_DB_MEMORY_LIMIT", fallback=None
     )
+    DUCK_DB_THREAD_LIMIT = os.environ.get("DUCK_DB_THREAD_LIMIT") or config.get(
+        "HDX", "DUCK_DB_THREAD_LIMIT", fallback=None
+    )
 
 
 def get_db_connection_params() -> dict: