Drop job_id from dbt cloud config (#619)

MetaphorData · Oct 10, 2023 · cefc69d · cefc69d
1 parent 88ed5c4
commit cefc69d
Show file tree

Hide file tree

Showing 4 changed files with 3 additions and 40 deletions.
diff --git a/metaphor/dbt/cloud/config.py b/metaphor/dbt/cloud/config.py
@@ -1,7 +1,6 @@
 from dataclasses import field as dataclass_field
-from typing import List, Optional
+from typing import List
 
-from pydantic import root_validator
 from pydantic.dataclasses import dataclass
 
 from metaphor.common.base_config import BaseConfig
@@ -17,11 +16,8 @@ class DbtCloudConfig(BaseConfig):
     # Service token for dbt cloud
     service_token: str
 
-    # (Deprecated. Use job_ids instead) dbt cloud job ID
-    job_id: Optional[int] = None
-
     # dbt cloud job IDs
-    job_ids: List[int] = dataclass_field(default_factory=lambda: [])
+    job_ids: List[int]
 
     # map meta field to ownerships
     meta_ownerships: List[MetaOwnership] = dataclass_field(default_factory=lambda: [])
@@ -31,10 +27,3 @@ class DbtCloudConfig(BaseConfig):
 
     # Base URL for dbt instance
     base_url: str = "https://cloud.getdbt.com"
-
-    @root_validator
-    def set_job_ids_or_job_id(cls, values):
-        if values.get("job_id") is None and not values.get("job_ids"):
-            raise ValueError("Must set job_ids")
-
-        return values
diff --git a/metaphor/dbt/cloud/extractor.py b/metaphor/dbt/cloud/extractor.py
@@ -24,11 +24,7 @@ def from_config_file(config_file: str) -> "DbtCloudExtractor":
     def __init__(self, config: DbtCloudConfig):
         super().__init__(config, "dbt cloud metadata crawler", Platform.DBT_MODEL)
         self._account_id = config.account_id
-        self._job_ids = (
-            [config.job_id]
-            if config.job_id and len(config.job_ids) == 0
-            else config.job_ids
-        )
+        self._job_ids = config.job_ids
         self._service_token = config.service_token
         self._meta_ownerships = config.meta_ownerships
         self._meta_tags = config.meta_tags

diff --git a/tests/dbt/cloud/config_legacy.yml b/tests/dbt/cloud/config_legacy.yml
diff --git a/tests/dbt/cloud/test_config.py b/tests/dbt/cloud/test_config.py
@@ -7,24 +7,8 @@ def test_yaml_config(test_root_dir):
 
     assert config == DbtCloudConfig(
         account_id=1234,
-        job_id=None,
         job_ids=[5678, 9012],
         service_token="token",
         base_url="https://cloud.metaphor.getdbt.com",
         output=OutputConfig(),
     )
-
-
-def test_yaml_config_legacy(test_root_dir):
-    config = DbtCloudConfig.from_yaml_file(
-        f"{test_root_dir}/dbt/cloud/config_legacy.yml"
-    )
-
-    assert config == DbtCloudConfig(
-        account_id=1234,
-        job_id=5678,
-        job_ids=[],
-        service_token="token",
-        base_url="https://cloud.metaphor.getdbt.com",
-        output=OutputConfig(),
-    )