Handle crawler timeout gracefully

.dockerignore

@@ -1,3 +1,5 @@
-__pycache__/
-*.py[cod]
-*$py.class
+*
+!metaphor
+!README.md
+!pyproject.toml
+!poetry.lock

metaphor/__main__.py

@@ -1,7 +1,7 @@
-import argparse
 import subprocess
 from importlib import import_module
 
+from metaphor.common.cli import parse_args
 from metaphor.common.logger import get_logger
 
 logger = get_logger()
@@ -19,19 +19,13 @@
 def main():
     print_packages()
 
-    parser = argparse.ArgumentParser(description="Metaphor Connectors")
-
-    parser.add_argument(
-        "name", help="Name of the connector, e.g. snowflake or bigquery"
-    )
-    parser.add_argument("config", help="Path to the config file")
-    args = parser.parse_args()
+    args = parse_args()
 
     package_main = getattr(import_module(f"metaphor.{args.name}"), "main", None)
     if package_main is None:
         raise ValueError(f"Unable to load {args.package}:main")
-
     logger.info(f"Executing {args.name} connector with config file {args.config}")
+
     package_main(args.config)
 
 

metaphor/common/cli.py

@@ -1,3 +1,4 @@
+import argparse
 from typing import Type
 
 from metaphor.common.base_config import BaseConfig
@@ -6,6 +7,17 @@
 from metaphor.common.runner import run_connector
 
 
+def parse_args():
+    parser = argparse.ArgumentParser(description="Metaphor Connectors")
+
+    parser.add_argument(
+        "name", help="Name of the connector, e.g. snowflake or bigquery"
+    )
+    parser.add_argument("config", help="Path to the config file")
+    args = parser.parse_args()
+    return args
+
+
 def cli_main(extractor_cls: Type[BaseExtractor], config_file: str):
     base_config = BaseConfig.from_yaml_file(config_file)
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "metaphor-connectors"
-version = "0.14.101"
+version = "0.14.102"
 license = "Apache-2.0"
 description = "A collection of Python-based 'connectors' that extract metadata from various sources to ingest into the Metaphor app."
 authors = ["Metaphor <[email protected]>"]