From 2492ffb77fcb08bf72d7e6e693b6110a2a23b0d9 Mon Sep 17 00:00:00 2001
From: kshitijrajsharma <skshitizraj@gmail.com>
Date: Sun, 31 Dec 2023 09:27:06 +0545
Subject: [PATCH] Run api even if hdx api is not reachable , Display failed
 info in the API

---
 API/tasks.py             | 28 ++++++++++++++++++++++++----
 src/app.py               |  8 +++++++-
 src/config.py            | 22 +++++++++++++++++++---
 src/validation/models.py | 18 ++++++++++--------
 4 files changed, 60 insertions(+), 16 deletions(-)

diff --git a/API/tasks.py b/API/tasks.py
index c110cadc..27a538c3 100644
--- a/API/tasks.py
+++ b/API/tasks.py
@@ -1,4 +1,3 @@
-import html
 import json
 from datetime import datetime
 
@@ -39,7 +38,28 @@ def get_task_status(
     Returns:
 
         id: Id of the task
-        status : SUCCESS / PENDING
+        status : Possible values includes:
+
+                PENDING
+
+                    The task is waiting for execution.
+
+                STARTED
+
+                    The task has been started.
+
+                RETRY
+
+                    The task is to be retried, possibly because of failure.
+
+                FAILURE
+
+                    The task raised an exception, or has exceeded the retry limit. The result attribute then contains the exception raised by the task.
+
+                SUCCESS
+
+                    The task executed successfully. The result attribute then contains the tasks return value.
+
         result : Result of task
 
     Successful task will have additional nested json inside
@@ -49,8 +69,8 @@ def get_task_status(
     task_response_result = None
     if task_result.status == "SUCCESS":
         task_response_result = task_result.result
-    if task_result.status == "FAILED":
-        task_response_result = html.escape(task_result.traceback)
+    if task_result.state != "SUCCESS":
+        task_response_result = str(task_result.info)
 
     result = {
         "id": task_id,
diff --git a/src/app.py b/src/app.py
index 1b511c90..2ebcf609 100644
--- a/src/app.py
+++ b/src/app.py
@@ -1786,7 +1786,13 @@ def upload_dataset(self, dump_config_to_s3=False):
                 )
 
             self.dataset.set_reference_period(datetime.now())
-            self.dataset.create_in_hdx(allow_no_resources=True)
+            try:
+                self.dataset.create_in_hdx(allow_no_resources=True)
+                dataset_info["hdx_upload"] = "SUCCESS"
+            except Exception as ex:
+                logging.error(ex)
+                dataset_info["hdx_upload"] = "FAILED"
+
             dataset_info["name"] = self.dataset["name"]
             dataset_info["hdx_url"] = f"{HDX_URL_PREFIX}/dataset/{self.dataset['name']}"
             dataset_info["resources"] = self.resources
diff --git a/src/config.py b/src/config.py
index 26535b49..710add0e 100644
--- a/src/config.py
+++ b/src/config.py
@@ -77,6 +77,15 @@
 )
 
 
+def not_raises(func, *args, **kwargs):
+    try:
+        func(*args, **kwargs)
+        return True
+    except Exception as ex:
+        logging.error(ex)
+        return False
+
+
 ####################
 
 ### EXPORT_UPLOAD CONFIG BLOCK
@@ -204,7 +213,7 @@
         logging.debug(HDX_URL_PREFIX)
     except Exception as e:
         logging.error(
-            f"Error creating HDX configuration: {e}, Disabling the hdx exports feature"
+            "Error creating HDX configuration: %s, Disabling the hdx exports feature", e
         )
         ENABLE_HDX_EXPORTS = False
 
@@ -221,13 +230,20 @@
     ALLOWED_HDX_TAGS = parse_list(
         os.environ.get("ALLOWED_HDX_TAGS")
         or config.get("HDX", "ALLOWED_HDX_TAGS", fallback=None)
-        or Vocabulary.approved_tags()
+        or (
+            Vocabulary.approved_tags() if not_raises(Vocabulary.approved_tags) else None
+        )
     )
     ALLOWED_HDX_UPDATE_FREQUENCIES = parse_list(
         os.environ.get("ALLOWED_HDX_UPDATE_FREQUENCIES")
         or config.get("HDX", "ALLOWED_HDX_UPDATE_FREQUENCIES", fallback=None)
-        or Dataset.list_valid_update_frequencies()
+        or (
+            Dataset.list_valid_update_frequencies()
+            if not_raises(Dataset.list_valid_update_frequencies)
+            else None
+        )
     )
+
     DUCK_DB_MEMORY_LIMIT = os.environ.get("DUCK_DB_MEMORY_LIMIT") or config.get(
         "HDX", "DUCK_DB_MEMORY_LIMIT", fallback=None
     )
diff --git a/src/validation/models.py b/src/validation/models.py
index ac4bfe42..ff6c38f2 100644
--- a/src/validation/models.py
+++ b/src/validation/models.py
@@ -364,10 +364,11 @@ def validate_tags(cls, value):
         """
         if value:
             for item in value:
-                if item.strip() not in ALLOWED_HDX_TAGS:
-                    raise ValueError(
-                        f"Invalid tag {item.strip()} , Should be within {ALLOWED_HDX_TAGS}"
-                    )
+                if ALLOWED_HDX_TAGS:
+                    if item.strip() not in ALLOWED_HDX_TAGS:
+                        raise ValueError(
+                            f"Invalid tag {item.strip()} , Should be within {ALLOWED_HDX_TAGS}"
+                        )
         return value
 
 
@@ -538,10 +539,11 @@ def validate_frequency(cls, value):
         Returns:
             _type_: _description_
         """
-        if value.strip() not in ALLOWED_HDX_UPDATE_FREQUENCIES:
-            raise ValueError(
-                f"Invalid update frequency , Should be within {ALLOWED_HDX_UPDATE_FREQUENCIES}"
-            )
+        if ALLOWED_HDX_UPDATE_FREQUENCIES:
+            if value.strip() not in ALLOWED_HDX_UPDATE_FREQUENCIES:
+                raise ValueError(
+                    f"Invalid update frequency , Should be within {ALLOWED_HDX_UPDATE_FREQUENCIES}"
+                )
         return value.strip()