diff --git a/.github/workflows/publish-datahub-jars.yml b/.github/workflows/publish-datahub-jars.yml
index 5aec66bc33bb67..393f9d993e2a2f 100644
--- a/.github/workflows/publish-datahub-jars.yml
+++ b/.github/workflows/publish-datahub-jars.yml
@@ -201,7 +201,7 @@ jobs:
permissions:
id-token: write
contents: read
- needs: ["check-secret", "setup"]
+ needs: ["check-secret", "setup", "publish"]
if: ${{ needs.check-secret.outputs.publish-enabled == 'true' }}
steps:
- uses: acryldata/sane-checkout-action@v3
diff --git a/datahub-web-react/src/app/entity/dataset/profile/schema/components/StructuredPropValues.tsx b/datahub-web-react/src/app/entity/dataset/profile/schema/components/StructuredPropValues.tsx
index 4cba36b9375dbc..91379ce97fe22f 100644
--- a/datahub-web-react/src/app/entity/dataset/profile/schema/components/StructuredPropValues.tsx
+++ b/datahub-web-react/src/app/entity/dataset/profile/schema/components/StructuredPropValues.tsx
@@ -24,11 +24,11 @@ const StructuredPropValues = ({ schemaFieldEntity, propColumn }: Props) => {
const entityRegistry = useEntityRegistry();
const property = schemaFieldEntity.structuredProperties?.properties?.find(
- (prop) => prop.structuredProperty.urn === propColumn?.entity.urn,
+ (prop) => prop.structuredProperty.urn === propColumn?.entity?.urn,
);
const propRow = property ? mapStructuredPropertyToPropertyRow(property) : undefined;
const values = propRow?.values;
- const isRichText = propRow?.dataType?.info.type === StdDataType.RichText;
+ const isRichText = propRow?.dataType?.info?.type === StdDataType.RichText;
const hasMoreValues = values && values.length > 2;
const displayedValues = hasMoreValues ? values.slice(0, 1) : values;
diff --git a/datahub-web-react/src/app/entity/shared/containers/profile/header/StructuredPropertyBadge.tsx b/datahub-web-react/src/app/entity/shared/containers/profile/header/StructuredPropertyBadge.tsx
index 64953928990ee5..fc7892f6ba6cca 100644
--- a/datahub-web-react/src/app/entity/shared/containers/profile/header/StructuredPropertyBadge.tsx
+++ b/datahub-web-react/src/app/entity/shared/containers/profile/header/StructuredPropertyBadge.tsx
@@ -41,8 +41,8 @@ const StructuredPropertyBadge = ({ structuredProperties }: Props) => {
if (!badgeStructuredProperty) return null;
- const propertyValue = propRow?.values[0].value;
- const relatedDescription = propRow?.structuredProperty.definition.allowedValues?.find(
+ const propertyValue = propRow?.values[0]?.value;
+ const relatedDescription = propRow?.structuredProperty?.definition?.allowedValues?.find(
(v) => getStructuredPropertyValue(v.value) === propertyValue,
)?.description;
@@ -56,7 +56,7 @@ const StructuredPropertyBadge = ({ structuredProperties }: Props) => {
Value
- {propRow?.values[0].value}
+ {propRow?.values[0]?.value}
{relatedDescription && (
@@ -79,7 +79,7 @@ const StructuredPropertyBadge = ({ structuredProperties }: Props) => {
>
{
property,
currentProperties,
);
- const isRichText = propertyRow?.dataType?.info.type === StdDataType.RichText;
+ const isRichText = propertyRow?.dataType?.info?.type === StdDataType.RichText;
const values = propertyRow?.values;
const hasMultipleValues = values && values.length > 1;
const propertyName = getDisplayName(property.entity as StructuredPropertyEntity);
diff --git a/datahub-web-react/src/app/entity/shared/entityForm/Form.tsx b/datahub-web-react/src/app/entity/shared/entityForm/Form.tsx
index 05fd91d4680ac7..b5f3cd770078a0 100644
--- a/datahub-web-react/src/app/entity/shared/entityForm/Form.tsx
+++ b/datahub-web-react/src/app/entity/shared/entityForm/Form.tsx
@@ -57,7 +57,7 @@ function Form({ formUrn }: Props) {
const title = formAssociation?.form?.info?.name;
const associatedUrn = formAssociation?.associatedUrn;
const description = formAssociation?.form?.info?.description;
- const owners = formAssociation?.form.ownership?.owners;
+ const owners = formAssociation?.form?.ownership?.owners;
return (
diff --git a/datahub-web-react/src/app/entity/shared/tabs/Properties/Edit/EditStructuredPropertyModal.tsx b/datahub-web-react/src/app/entity/shared/tabs/Properties/Edit/EditStructuredPropertyModal.tsx
index 1714f0d1872e37..92691b1ad2239a 100644
--- a/datahub-web-react/src/app/entity/shared/tabs/Properties/Edit/EditStructuredPropertyModal.tsx
+++ b/datahub-web-react/src/app/entity/shared/tabs/Properties/Edit/EditStructuredPropertyModal.tsx
@@ -99,7 +99,7 @@ export default function EditStructuredPropertyModal({
return (
{
const searchAcrossEntities = data?.searchAcrossEntities;
const noOfProperties = searchAcrossEntities?.searchResults?.length;
- const badgeProperty = searchAcrossEntities?.searchResults.find(
+ const badgeProperty = searchAcrossEntities?.searchResults?.find(
(prop) => (prop.entity as StructuredPropertyEntity).settings?.showAsAssetBadge,
)?.entity;
diff --git a/datahub-web-react/src/app/govern/structuredProperties/ViewAdvancedOptions.tsx b/datahub-web-react/src/app/govern/structuredProperties/ViewAdvancedOptions.tsx
index 1f08995e237ec5..25f1d672390426 100644
--- a/datahub-web-react/src/app/govern/structuredProperties/ViewAdvancedOptions.tsx
+++ b/datahub-web-react/src/app/govern/structuredProperties/ViewAdvancedOptions.tsx
@@ -32,7 +32,7 @@ const ViewAdvancedOptions = ({ propEntity }: Props) => {
{propEntity && (
Qualified Name
- {propEntity?.definition.qualifiedName}
+ {propEntity?.definition?.qualifiedName}
)}
diff --git a/datahub-web-react/src/app/govern/structuredProperties/ViewStructuredPropsDrawer.tsx b/datahub-web-react/src/app/govern/structuredProperties/ViewStructuredPropsDrawer.tsx
index 2fd36a29c8e760..bc91a90989d2c8 100644
--- a/datahub-web-react/src/app/govern/structuredProperties/ViewStructuredPropsDrawer.tsx
+++ b/datahub-web-react/src/app/govern/structuredProperties/ViewStructuredPropsDrawer.tsx
@@ -40,9 +40,9 @@ const ViewStructuredPropsDrawer = ({
const selectedPropEntity = selectedProperty && (selectedProperty?.entity as StructuredPropertyEntity);
- const allowedValues = selectedPropEntity?.definition.allowedValues;
+ const allowedValues = selectedPropEntity?.definition?.allowedValues;
- const allowedTypes = selectedPropEntity?.definition.typeQualifier?.allowedTypes;
+ const allowedTypes = selectedPropEntity?.definition?.typeQualifier?.allowedTypes;
const propType = getValueTypeLabel(
selectedPropEntity.definition.valueType.urn,
diff --git a/datahub-web-react/src/app/ingest/source/executions/ExecutionRequestDetailsModal.tsx b/datahub-web-react/src/app/ingest/source/executions/ExecutionRequestDetailsModal.tsx
index f624d7e6ca796f..a7e6f516bb7943 100644
--- a/datahub-web-react/src/app/ingest/source/executions/ExecutionRequestDetailsModal.tsx
+++ b/datahub-web-react/src/app/ingest/source/executions/ExecutionRequestDetailsModal.tsx
@@ -156,7 +156,7 @@ export const ExecutionDetailsModal = ({ urn, open, onClose }: Props) => {
(status && {getExecutionRequestSummaryText(status)}) ||
undefined;
- const recipeJson = data?.executionRequest?.input.arguments?.find((arg) => arg.key === 'recipe')?.value;
+ const recipeJson = data?.executionRequest?.input?.arguments?.find((arg) => arg.key === 'recipe')?.value;
let recipeYaml: string;
try {
recipeYaml = recipeJson && YAML.stringify(JSON.parse(recipeJson), 8, 2).trim();
diff --git a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_api.py b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_api.py
index 7f4e0f520b7a5e..d913b7e42065d2 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_api.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_api.py
@@ -1,6 +1,7 @@
import concurrent.futures
import json
import logging
+import re
import warnings
from collections import defaultdict
from enum import Enum
@@ -609,32 +610,6 @@ def extract_all_queries(self) -> List[Dict[str, Any]]:
return self.execute_query(query=jobs_query)
- def get_source_by_id(self, source_id: str) -> Optional[Dict]:
- """
- Fetch source details by ID.
- """
- response = self.get(
- url=f"/source/{source_id}",
- )
- return response if response else None
-
- def get_source_for_dataset(self, schema: str, dataset: str) -> Optional[Dict]:
- """
- Get source information for a dataset given its schema and name.
- """
- dataset_id = self.get_dataset_id(schema, dataset)
- if not dataset_id:
- return None
-
- catalog_entry = self.get(
- url=f"/catalog/{dataset_id}",
- )
- if not catalog_entry or "path" not in catalog_entry:
- return None
-
- source_id = catalog_entry["path"][0]
- return self.get_source_by_id(source_id)
-
def get_tags_for_resource(self, resource_id: str) -> Optional[List[str]]:
"""
Get Dremio tags for a given resource_id.
@@ -673,55 +648,119 @@ def get_description_for_resource(self, resource_id: str) -> Optional[str]:
)
return None
- def get_containers_for_location(
- self, resource_id: str, path: List[str]
- ) -> List[Dict[str, str]]:
- containers = []
+ def _check_pattern_match(
+ self,
+ pattern: str,
+ paths: List[str],
+ allow_prefix: bool = True,
+ ) -> bool:
+ """
+ Helper method to check if a pattern matches any of the paths.
+ Handles hierarchical matching where each level is matched independently.
+ Also handles prefix matching for partial paths.
+ """
+ if pattern == ".*":
+ return True
- def traverse_path(location_id: str, entity_path: List[str]) -> List:
- nonlocal containers
- try:
- response = self.get(url=f"/catalog/{location_id}")
- if (
- response.get("entityType")
- == DremioEntityContainerType.FOLDER.value.lower()
- ):
- containers.append(
- {
- "id": location_id,
- "name": entity_path[-1],
- "path": entity_path[:-1],
- "container_type": DremioEntityContainerType.FOLDER,
- }
- )
+ # Convert the pattern to regex with proper anchoring
+ regex_pattern = pattern
+ if pattern.startswith("^"):
+ # Already has start anchor
+ regex_pattern = pattern.replace(".", r"\.") # Escape dots
+ regex_pattern = regex_pattern.replace(
+ r"\.*", ".*"
+ ) # Convert .* to wildcard
+ else:
+ # Add start anchor and handle dots
+ regex_pattern = "^" + pattern.replace(".", r"\.").replace(r"\.*", ".*")
+
+ # Handle end matching
+ if not pattern.endswith(".*"):
+ if pattern.endswith("$"):
+ # Keep explicit end anchor
+ pass
+ elif not allow_prefix:
+ # Add end anchor for exact matching
+ regex_pattern = regex_pattern + "$"
+
+ for path in paths:
+ if re.match(regex_pattern, path, re.IGNORECASE):
+ return True
- for container in response.get("children", []):
- if (
- container.get("type")
- == DremioEntityContainerType.CONTAINER.value
- ):
- traverse_path(container.get("id"), container.get("path"))
+ return False
- except Exception as exc:
- logging.info(
- "Location {} contains no tables or views. Skipping...".format(id)
- )
- self.report.warning(
- message="Failed to get tables or views",
- context=f"{id}",
- exc=exc,
- )
+ def should_include_container(self, path: List[str], name: str) -> bool:
+ """
+ Helper method to check if a container should be included based on schema patterns.
+ Used by both get_all_containers and get_containers_for_location.
+ """
+ path_components = path + [name] if path else [name]
+ full_path = ".".join(path_components)
- return containers
+ # Default allow everything case
+ if self.allow_schema_pattern == [".*"] and not self.deny_schema_pattern:
+ self.report.report_container_scanned(full_path)
+ return True
- return traverse_path(location_id=resource_id, entity_path=path)
+ # Check deny patterns first
+ if self.deny_schema_pattern:
+ for pattern in self.deny_schema_pattern:
+ if self._check_pattern_match(
+ pattern=pattern,
+ paths=[full_path],
+ allow_prefix=False,
+ ):
+ self.report.report_container_filtered(full_path)
+ return False
+
+ # Check allow patterns
+ for pattern in self.allow_schema_pattern:
+ # For patterns with wildcards, check if this path is a parent of the pattern
+ if "*" in pattern:
+ pattern_parts = pattern.split(".")
+ path_parts = path_components
+
+ # If pattern has exact same number of parts, check each component
+ if len(pattern_parts) == len(path_parts):
+ matches = True
+ for p_part, c_part in zip(pattern_parts, path_parts):
+ if p_part != "*" and p_part.lower() != c_part.lower():
+ matches = False
+ break
+ if matches:
+ self.report.report_container_scanned(full_path)
+ return True
+ # Otherwise check if current path is prefix match
+ else:
+ # Remove the trailing wildcard if present
+ if pattern_parts[-1] == "*":
+ pattern_parts = pattern_parts[:-1]
+
+ for i in range(len(path_parts)):
+ current_path = ".".join(path_parts[: i + 1])
+ pattern_prefix = ".".join(pattern_parts[: i + 1])
+
+ if pattern_prefix.startswith(current_path):
+ self.report.report_container_scanned(full_path)
+ return True
+
+ # Direct pattern matching
+ if self._check_pattern_match(
+ pattern=pattern,
+ paths=[full_path],
+ allow_prefix=True,
+ ):
+ self.report.report_container_scanned(full_path)
+ return True
+
+ self.report.report_container_filtered(full_path)
+ return False
def get_all_containers(self):
"""
- Query the Dremio sources API and return source information.
+ Query the Dremio sources API and return filtered source information.
"""
containers = []
-
response = self.get(url="/catalog")
def process_source(source):
@@ -731,34 +770,41 @@ def process_source(source):
)
source_config = source_resp.get("config", {})
- if source_config.get("database"):
- db = source_config.get("database")
- else:
- db = source_config.get("databaseName", "")
-
- return {
- "id": source.get("id"),
- "name": source.get("path")[0],
- "path": [],
- "container_type": DremioEntityContainerType.SOURCE,
- "source_type": source_resp.get("type"),
- "root_path": source_config.get("rootPath"),
- "database_name": db,
- }
+ db = source_config.get(
+ "database", source_config.get("databaseName", "")
+ )
+
+ if self.should_include_container([], source.get("path")[0]):
+ return {
+ "id": source.get("id"),
+ "name": source.get("path")[0],
+ "path": [],
+ "container_type": DremioEntityContainerType.SOURCE,
+ "source_type": source_resp.get("type"),
+ "root_path": source_config.get("rootPath"),
+ "database_name": db,
+ }
else:
- return {
- "id": source.get("id"),
- "name": source.get("path")[0],
- "path": [],
- "container_type": DremioEntityContainerType.SPACE,
- }
+ if self.should_include_container([], source.get("path")[0]):
+ return {
+ "id": source.get("id"),
+ "name": source.get("path")[0],
+ "path": [],
+ "container_type": DremioEntityContainerType.SPACE,
+ }
+ return None
def process_source_and_containers(source):
container = process_source(source)
+ if not container:
+ return []
+
+ # Get sub-containers
sub_containers = self.get_containers_for_location(
resource_id=container.get("id"),
path=[container.get("name")],
)
+
return [container] + sub_containers
# Use ThreadPoolExecutor to parallelize the processing of sources
@@ -771,7 +817,16 @@ def process_source_and_containers(source):
}
for future in concurrent.futures.as_completed(future_to_source):
- containers.extend(future.result())
+ source = future_to_source[future]
+ try:
+ containers.extend(future.result())
+ except Exception as exc:
+ logger.error(f"Error processing source: {exc}")
+ self.report.warning(
+ message="Failed to process source",
+ context=f"{source}",
+ exc=exc,
+ )
return containers
@@ -785,3 +840,55 @@ def get_context_for_vds(self, resource_id: str) -> str:
)
else:
return ""
+
+ def get_containers_for_location(
+ self, resource_id: str, path: List[str]
+ ) -> List[Dict[str, str]]:
+ containers = []
+
+ def traverse_path(location_id: str, entity_path: List[str]) -> List:
+ nonlocal containers
+ try:
+ response = self.get(url=f"/catalog/{location_id}")
+
+ # Check if current folder should be included
+ if (
+ response.get("entityType")
+ == DremioEntityContainerType.FOLDER.value.lower()
+ ):
+ folder_name = entity_path[-1]
+ folder_path = entity_path[:-1]
+
+ if self.should_include_container(folder_path, folder_name):
+ containers.append(
+ {
+ "id": location_id,
+ "name": folder_name,
+ "path": folder_path,
+ "container_type": DremioEntityContainerType.FOLDER,
+ }
+ )
+
+ # Recursively process child containers
+ for container in response.get("children", []):
+ if (
+ container.get("type")
+ == DremioEntityContainerType.CONTAINER.value
+ ):
+ traverse_path(container.get("id"), container.get("path"))
+
+ except Exception as exc:
+ logging.info(
+ "Location {} contains no tables or views. Skipping...".format(
+ location_id
+ )
+ )
+ self.report.warning(
+ message="Failed to get tables or views",
+ context=f"{location_id}",
+ exc=exc,
+ )
+
+ return containers
+
+ return traverse_path(location_id=resource_id, entity_path=path)
diff --git a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_datahub_source_mapping.py b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_datahub_source_mapping.py
index 928c145e5eb505..e5d6b8e40fb3d8 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_datahub_source_mapping.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_datahub_source_mapping.py
@@ -31,6 +31,7 @@ class DremioToDataHubSourceTypeMapping:
"SNOWFLAKE": "snowflake",
"SYNAPSE": "mssql",
"TERADATA": "teradata",
+ "VERTICA": "vertica",
}
DATABASE_SOURCE_TYPES = {
@@ -52,6 +53,7 @@ class DremioToDataHubSourceTypeMapping:
"SNOWFLAKE",
"SYNAPSE",
"TERADATA",
+ "VERTICA",
}
FILE_OBJECT_STORAGE_TYPES = {
diff --git a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_reporting.py b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_reporting.py
index 926dbd42eb2673..c8eb035461ca16 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_reporting.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/dremio/dremio_reporting.py
@@ -14,12 +14,27 @@ class DremioSourceReport(
):
num_containers_failed: int = 0
num_datasets_failed: int = 0
+ containers_scanned: int = 0
+ containers_filtered: int = 0
def report_upstream_latency(self, start_time: datetime, end_time: datetime) -> None:
# recording total combined latency is not very useful, keeping this method as a placeholder
# for future implementation of min / max / percentiles etc.
pass
+ def report_container_scanned(self, name: str) -> None:
+ """
+ Record that a container was successfully scanned
+ """
+ self.containers_scanned += 1
+
+ def report_container_filtered(self, container_name: str) -> None:
+ """
+ Record that a container was filtered out
+ """
+ self.containers_filtered += 1
+ self.report_dropped(container_name)
+
def report_entity_scanned(self, name: str, ent_type: str = "View") -> None:
"""
Entity could be a view or a table
diff --git a/metadata-ingestion/src/datahub/ingestion/source/kafka/kafka_connect.py b/metadata-ingestion/src/datahub/ingestion/source/kafka/kafka_connect.py
index 0b201278142e3a..23a99ccb310e13 100644
--- a/metadata-ingestion/src/datahub/ingestion/source/kafka/kafka_connect.py
+++ b/metadata-ingestion/src/datahub/ingestion/source/kafka/kafka_connect.py
@@ -282,10 +282,6 @@ class JdbcParser:
query: str
transforms: list
- def report_warning(self, key: str, reason: str) -> None:
- logger.warning(f"{key}: {reason}")
- self.report.report_warning(key, reason)
-
def get_parser(
self,
connector_manifest: ConnectorManifest,
@@ -355,9 +351,9 @@ def default_get_lineages(
source_table = f"{table_name_tuple[-2]}.{source_table}"
else:
include_source_dataset = False
- self.report_warning(
- self.connector_manifest.name,
- f"could not find schema for table {source_table}",
+ self.report.warning(
+ "Could not find schema for table"
+ f"{self.connector_manifest.name} : {source_table}",
)
dataset_name: str = get_dataset_name(database_name, source_table)
lineage = KafkaConnectLineage(
@@ -457,9 +453,9 @@ def _extract_lineages(self):
target_platform=KAFKA,
)
lineages.append(lineage)
- self.report_warning(
+ self.report.warning(
+ "Could not find input dataset, the connector has query configuration set",
self.connector_manifest.name,
- "could not find input dataset, the connector has query configuration set",
)
self.connector_manifest.lineages = lineages
return
@@ -535,24 +531,24 @@ def _extract_lineages(self):
include_source_dataset=False,
)
)
- self.report_warning(
- self.connector_manifest.name,
- f"could not find input dataset, for connector topics {topic_names}",
+ self.report.warning(
+ "Could not find input dataset for connector topics",
+ f"{self.connector_manifest.name} : {topic_names}",
)
self.connector_manifest.lineages = lineages
return
else:
include_source_dataset = True
if SINGLE_TRANSFORM and UNKNOWN_TRANSFORM:
- self.report_warning(
- self.connector_manifest.name,
- f"could not find input dataset, connector has unknown transform - {transforms[0]['type']}",
+ self.report.warning(
+ "Could not find input dataset, connector has unknown transform",
+ f"{self.connector_manifest.name} : {transforms[0]['type']}",
)
include_source_dataset = False
if not SINGLE_TRANSFORM and UNKNOWN_TRANSFORM:
- self.report_warning(
+ self.report.warning(
+ "Could not find input dataset, connector has one or more unknown transforms",
self.connector_manifest.name,
- "could not find input dataset, connector has one or more unknown transforms",
)
include_source_dataset = False
lineages = self.default_get_lineages(
@@ -753,8 +749,10 @@ def _extract_lineages(self):
lineages.append(lineage)
self.connector_manifest.lineages = lineages
except Exception as e:
- self.report.report_warning(
- self.connector_manifest.name, f"Error resolving lineage: {e}"
+ self.report.warning(
+ "Error resolving lineage for connector",
+ self.connector_manifest.name,
+ exc=e,
)
return
@@ -783,10 +781,6 @@ class BQParser:
defaultDataset: Optional[str] = None
version: str = "v1"
- def report_warning(self, key: str, reason: str) -> None:
- logger.warning(f"{key}: {reason}")
- self.report.report_warning(key, reason)
-
def get_parser(
self,
connector_manifest: ConnectorManifest,
@@ -917,9 +911,9 @@ def _extract_lineages(self):
transformed_topic = self.apply_transformations(topic, transforms)
dataset_table = self.get_dataset_table_for_topic(transformed_topic, parser)
if dataset_table is None:
- self.report_warning(
- self.connector_manifest.name,
- f"could not find target dataset for topic {transformed_topic}, please check your connector configuration",
+ self.report.warning(
+ "Could not find target dataset for topic, please check your connector configuration"
+ f"{self.connector_manifest.name} : {transformed_topic} ",
)
continue
target_dataset = f"{project}.{dataset_table}"
@@ -954,10 +948,6 @@ class SnowflakeParser:
schema_name: str
topics_to_tables: Dict[str, str]
- def report_warning(self, key: str, reason: str) -> None:
- logger.warning(f"{key}: {reason}")
- self.report.report_warning(key, reason)
-
def get_table_name_from_topic_name(self, topic_name: str) -> str:
"""
This function converts the topic name to a valid Snowflake table name using some rules.
@@ -1105,8 +1095,10 @@ def _extract_lineages(self):
)
self.connector_manifest.lineages = lineages
except Exception as e:
- self.report.report_warning(
- self.connector_manifest.name, f"Error resolving lineage: {e}"
+ self.report.warning(
+ "Error resolving lineage for connector",
+ self.connector_manifest.name,
+ exc=e,
)
return
@@ -1155,7 +1147,7 @@ def __init__(self, config: KafkaConnectSourceConfig, ctx: PipelineContext):
)
self.session.auth = (self.config.username, self.config.password)
- test_response = self.session.get(f"{self.config.connect_uri}")
+ test_response = self.session.get(f"{self.config.connect_uri}/connectors")
test_response.raise_for_status()
logger.info(f"Connection to {self.config.connect_uri} is ok")
if not jpype.isJVMStarted():
@@ -1178,13 +1170,16 @@ def get_connectors_manifest(self) -> List[ConnectorManifest]:
payload = connector_response.json()
- for c in payload:
- connector_url = f"{self.config.connect_uri}/connectors/{c}"
- connector_response = self.session.get(connector_url)
- manifest = connector_response.json()
- connector_manifest = ConnectorManifest(**manifest)
- if not self.config.connector_patterns.allowed(connector_manifest.name):
- self.report.report_dropped(connector_manifest.name)
+ for connector_name in payload:
+ connector_url = f"{self.config.connect_uri}/connectors/{connector_name}"
+ connector_manifest = self._get_connector_manifest(
+ connector_name, connector_url
+ )
+ if (
+ connector_manifest is None
+ or not self.config.connector_patterns.allowed(connector_manifest.name)
+ ):
+ self.report.report_dropped(connector_name)
continue
if self.config.provided_configs:
@@ -1195,19 +1190,11 @@ def get_connectors_manifest(self) -> List[ConnectorManifest]:
connector_manifest.lineages = list()
connector_manifest.url = connector_url
- topics = self.session.get(
- f"{self.config.connect_uri}/connectors/{c}/topics",
- ).json()
-
- connector_manifest.topic_names = topics[c]["topics"]
+ connector_manifest.topic_names = self._get_connector_topics(connector_name)
# Populate Source Connector metadata
if connector_manifest.type == SOURCE:
- tasks = self.session.get(
- f"{self.config.connect_uri}/connectors/{c}/tasks",
- ).json()
-
- connector_manifest.tasks = tasks
+ connector_manifest.tasks = self._get_connector_tasks(connector_name)
# JDBC source connector lineages
if connector_manifest.config.get(CONNECTOR_CLASS).__eq__(
@@ -1246,7 +1233,7 @@ def get_connectors_manifest(self) -> List[ConnectorManifest]:
)
continue
- for topic in topics:
+ for topic in connector_manifest.topic_names:
lineage = KafkaConnectLineage(
source_dataset=target_connector.source_dataset,
source_platform=target_connector.source_platform,
@@ -1286,6 +1273,49 @@ def get_connectors_manifest(self) -> List[ConnectorManifest]:
return connectors_manifest
+ def _get_connector_manifest(
+ self, connector_name: str, connector_url: str
+ ) -> Optional[ConnectorManifest]:
+ try:
+ connector_response = self.session.get(connector_url)
+ connector_response.raise_for_status()
+ except Exception as e:
+ self.report.warning(
+ "Failed to get connector details", connector_name, exc=e
+ )
+ return None
+ manifest = connector_response.json()
+ connector_manifest = ConnectorManifest(**manifest)
+ return connector_manifest
+
+ def _get_connector_tasks(self, connector_name: str) -> dict:
+ try:
+ response = self.session.get(
+ f"{self.config.connect_uri}/connectors/{connector_name}/tasks",
+ )
+ response.raise_for_status()
+ except Exception as e:
+ self.report.warning(
+ "Error getting connector tasks", context=connector_name, exc=e
+ )
+ return {}
+
+ return response.json()
+
+ def _get_connector_topics(self, connector_name: str) -> List[str]:
+ try:
+ response = self.session.get(
+ f"{self.config.connect_uri}/connectors/{connector_name}/topics",
+ )
+ response.raise_for_status()
+ except Exception as e:
+ self.report.warning(
+ "Error getting connector topics", context=connector_name, exc=e
+ )
+ return []
+
+ return response.json()[connector_name]["topics"]
+
def construct_flow_workunit(self, connector: ConnectorManifest) -> MetadataWorkUnit:
connector_name = connector.name
connector_type = connector.type
diff --git a/metadata-ingestion/tests/integration/dremio/dremio_mces_golden.json b/metadata-ingestion/tests/integration/dremio/dremio_mces_golden.json
index 3a8fce62f4bb36..41d47a574a211b 100644
--- a/metadata-ingestion/tests/integration/dremio/dremio_mces_golden.json
+++ b/metadata-ingestion/tests/integration/dremio/dremio_mces_golden.json
@@ -15,7 +15,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -31,7 +31,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -49,7 +49,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -65,7 +65,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -85,7 +85,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -105,7 +105,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -121,7 +121,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -139,7 +139,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -155,7 +155,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -175,7 +175,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -195,7 +195,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -211,7 +211,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -229,7 +229,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -245,7 +245,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -265,7 +265,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -285,7 +285,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -301,7 +301,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -319,7 +319,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -335,7 +335,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -355,7 +355,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -375,7 +375,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -391,7 +391,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -409,7 +409,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -425,7 +425,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -445,7 +445,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -465,7 +465,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -481,7 +481,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -497,7 +497,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -515,7 +515,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -531,7 +531,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -555,117 +555,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
- "lastRunId": "no-run-id-provided"
- }
-},
-{
- "entityType": "container",
- "entityUrn": "urn:li:container:6d76c46c4893c111f0006794f4a16482",
- "changeType": "UPSERT",
- "aspectName": "containerProperties",
- "aspect": {
- "json": {
- "customProperties": {},
- "name": "warehouse",
- "qualifiedName": "s3.warehouse",
- "description": "",
- "env": "PROD"
- }
- },
- "systemMetadata": {
- "lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
- "lastRunId": "no-run-id-provided"
- }
-},
-{
- "entityType": "container",
- "entityUrn": "urn:li:container:6d76c46c4893c111f0006794f4a16482",
- "changeType": "UPSERT",
- "aspectName": "container",
- "aspect": {
- "json": {
- "container": "urn:li:container:63a316133b08a091e919dc8c7a828a4d"
- }
- },
- "systemMetadata": {
- "lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
- "lastRunId": "no-run-id-provided"
- }
-},
-{
- "entityType": "container",
- "entityUrn": "urn:li:container:6d76c46c4893c111f0006794f4a16482",
- "changeType": "UPSERT",
- "aspectName": "dataPlatformInstance",
- "aspect": {
- "json": {
- "platform": "urn:li:dataPlatform:dremio"
- }
- },
- "systemMetadata": {
- "lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
- "lastRunId": "no-run-id-provided"
- }
-},
-{
- "entityType": "container",
- "entityUrn": "urn:li:container:6d76c46c4893c111f0006794f4a16482",
- "changeType": "UPSERT",
- "aspectName": "subTypes",
- "aspect": {
- "json": {
- "typeNames": [
- "Dremio Folder"
- ]
- }
- },
- "systemMetadata": {
- "lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
- "lastRunId": "no-run-id-provided"
- }
-},
-{
- "entityType": "container",
- "entityUrn": "urn:li:container:6d76c46c4893c111f0006794f4a16482",
- "changeType": "UPSERT",
- "aspectName": "status",
- "aspect": {
- "json": {
- "removed": false
- }
- },
- "systemMetadata": {
- "lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
- "lastRunId": "no-run-id-provided"
- }
-},
-{
- "entityType": "container",
- "entityUrn": "urn:li:container:6d76c46c4893c111f0006794f4a16482",
- "changeType": "UPSERT",
- "aspectName": "browsePathsV2",
- "aspect": {
- "json": {
- "path": [
- {
- "id": "Sources"
- },
- {
- "id": "urn:li:container:63a316133b08a091e919dc8c7a828a4d",
- "urn": "urn:li:container:63a316133b08a091e919dc8c7a828a4d"
- }
- ]
- }
- },
- "systemMetadata": {
- "lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -685,7 +575,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -701,7 +591,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -717,7 +607,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -735,7 +625,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -751,7 +641,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -775,7 +665,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -795,7 +685,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -811,7 +701,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -827,7 +717,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -845,7 +735,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -861,7 +751,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -885,7 +775,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -905,7 +795,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -921,7 +811,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -937,7 +827,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -955,7 +845,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -971,7 +861,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -995,7 +885,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1015,7 +905,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1031,7 +921,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1047,7 +937,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1065,7 +955,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1081,7 +971,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1105,7 +995,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1125,7 +1015,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1141,7 +1031,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1157,7 +1047,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1175,7 +1065,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1191,7 +1081,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1215,7 +1105,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1235,7 +1125,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1251,7 +1141,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1267,7 +1157,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1285,7 +1175,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1301,7 +1191,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1325,7 +1215,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1345,7 +1235,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1361,7 +1251,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1377,7 +1267,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1395,7 +1285,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1411,7 +1301,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1435,7 +1325,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1455,7 +1345,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1471,7 +1361,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1487,7 +1377,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1505,7 +1395,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1521,7 +1411,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1549,7 +1439,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1569,7 +1459,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1585,7 +1475,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1601,7 +1491,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1619,7 +1509,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1635,7 +1525,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1663,7 +1553,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1683,7 +1573,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1699,7 +1589,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1715,7 +1605,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1733,7 +1623,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1749,7 +1639,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1781,7 +1671,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1801,7 +1691,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1817,7 +1707,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1833,7 +1723,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1851,7 +1741,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1867,7 +1757,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1903,7 +1793,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1927,7 +1817,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1945,7 +1835,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1961,7 +1851,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1977,7 +1867,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -1995,7 +1885,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -2025,7 +1915,7 @@
},
"fields": [
{
- "fieldPath": "F",
+ "fieldPath": "A",
"nullable": true,
"type": {
"type": {
@@ -2037,7 +1927,7 @@
"isPartOfKey": false
},
{
- "fieldPath": "G",
+ "fieldPath": "B",
"nullable": true,
"type": {
"type": {
@@ -2049,7 +1939,7 @@
"isPartOfKey": false
},
{
- "fieldPath": "H",
+ "fieldPath": "C",
"nullable": true,
"type": {
"type": {
@@ -2061,7 +1951,7 @@
"isPartOfKey": false
},
{
- "fieldPath": "I",
+ "fieldPath": "D",
"nullable": true,
"type": {
"type": {
@@ -2073,7 +1963,7 @@
"isPartOfKey": false
},
{
- "fieldPath": "A",
+ "fieldPath": "E",
"nullable": true,
"type": {
"type": {
@@ -2085,7 +1975,7 @@
"isPartOfKey": false
},
{
- "fieldPath": "B",
+ "fieldPath": "F",
"nullable": true,
"type": {
"type": {
@@ -2097,7 +1987,7 @@
"isPartOfKey": false
},
{
- "fieldPath": "C",
+ "fieldPath": "G",
"nullable": true,
"type": {
"type": {
@@ -2109,7 +1999,7 @@
"isPartOfKey": false
},
{
- "fieldPath": "D",
+ "fieldPath": "H",
"nullable": true,
"type": {
"type": {
@@ -2121,7 +2011,7 @@
"isPartOfKey": false
},
{
- "fieldPath": "E",
+ "fieldPath": "I",
"nullable": true,
"type": {
"type": {
@@ -2137,7 +2027,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -2153,7 +2043,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -2177,7 +2067,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -2201,7 +2091,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -2219,7 +2109,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -2235,7 +2125,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -2251,7 +2141,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -2263,13 +2153,13 @@
"aspect": {
"json": {
"materialized": false,
- "viewLogic": "SELECT * FROM \"mysql\".northwind.customers",
+ "viewLogic": "SELECT * FROM mysql.northwind.customers",
"viewLanguage": "SQL"
}
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -2299,31 +2189,31 @@
},
"fields": [
{
- "fieldPath": "email_address",
+ "fieldPath": "priority",
"nullable": true,
"type": {
"type": {
- "com.linkedin.schema.StringType": {}
+ "com.linkedin.schema.NumberType": {}
}
},
- "nativeDataType": "character varying(65536)",
+ "nativeDataType": "float(24)",
"recursive": false,
"isPartOfKey": false
},
{
- "fieldPath": "priority",
+ "fieldPath": "email_address",
"nullable": true,
"type": {
"type": {
- "com.linkedin.schema.NumberType": {}
+ "com.linkedin.schema.StringType": {}
}
},
- "nativeDataType": "float(24)",
+ "nativeDataType": "character varying(65536)",
"recursive": false,
"isPartOfKey": false
},
{
- "fieldPath": "company",
+ "fieldPath": "first_name",
"nullable": true,
"type": {
"type": {
@@ -2335,19 +2225,19 @@
"isPartOfKey": false
},
{
- "fieldPath": "id",
+ "fieldPath": "last_name",
"nullable": true,
"type": {
"type": {
- "com.linkedin.schema.NumberType": {}
+ "com.linkedin.schema.StringType": {}
}
},
- "nativeDataType": "integer(32)",
+ "nativeDataType": "character varying(65536)",
"recursive": false,
"isPartOfKey": false
},
{
- "fieldPath": "first_name",
+ "fieldPath": "company",
"nullable": true,
"type": {
"type": {
@@ -2359,14 +2249,14 @@
"isPartOfKey": false
},
{
- "fieldPath": "last_name",
+ "fieldPath": "id",
"nullable": true,
"type": {
"type": {
- "com.linkedin.schema.StringType": {}
+ "com.linkedin.schema.NumberType": {}
}
},
- "nativeDataType": "character varying(65536)",
+ "nativeDataType": "integer(32)",
"recursive": false,
"isPartOfKey": false
}
@@ -2375,7 +2265,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -2391,7 +2281,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -2419,7 +2309,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -2443,7 +2333,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -2461,7 +2351,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -2477,7 +2367,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -2493,7 +2383,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -2505,13 +2395,13 @@
"aspect": {
"json": {
"materialized": false,
- "viewLogic": "SELECT * FROM \"mysql\".metagalaxy.\"metadata_aspect\"",
+ "viewLogic": "SELECT * FROM mysql.metagalaxy.metadata_aspect",
"viewLanguage": "SQL"
}
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -2541,7 +2431,7 @@
},
"fields": [
{
- "fieldPath": "createdby",
+ "fieldPath": "urn",
"nullable": true,
"type": {
"type": {
@@ -2553,7 +2443,7 @@
"isPartOfKey": false
},
{
- "fieldPath": "createdfor",
+ "fieldPath": "aspect",
"nullable": true,
"type": {
"type": {
@@ -2565,19 +2455,19 @@
"isPartOfKey": false
},
{
- "fieldPath": "urn",
+ "fieldPath": "version",
"nullable": true,
"type": {
"type": {
- "com.linkedin.schema.StringType": {}
+ "com.linkedin.schema.NumberType": {}
}
},
- "nativeDataType": "character varying(65536)",
+ "nativeDataType": "bigint(64)",
"recursive": false,
"isPartOfKey": false
},
{
- "fieldPath": "aspect",
+ "fieldPath": "metadata",
"nullable": true,
"type": {
"type": {
@@ -2589,19 +2479,19 @@
"isPartOfKey": false
},
{
- "fieldPath": "version",
+ "fieldPath": "createdon",
"nullable": true,
"type": {
"type": {
- "com.linkedin.schema.NumberType": {}
+ "com.linkedin.schema.DateType": {}
}
},
- "nativeDataType": "bigint(64)",
+ "nativeDataType": "timestamp(23)",
"recursive": false,
"isPartOfKey": false
},
{
- "fieldPath": "metadata",
+ "fieldPath": "createdby",
"nullable": true,
"type": {
"type": {
@@ -2613,14 +2503,14 @@
"isPartOfKey": false
},
{
- "fieldPath": "createdon",
+ "fieldPath": "createdfor",
"nullable": true,
"type": {
"type": {
- "com.linkedin.schema.DateType": {}
+ "com.linkedin.schema.StringType": {}
}
},
- "nativeDataType": "timestamp(23)",
+ "nativeDataType": "character varying(65536)",
"recursive": false,
"isPartOfKey": false
}
@@ -2629,7 +2519,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -2645,7 +2535,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -2673,7 +2563,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -2697,7 +2587,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -2715,7 +2605,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -2731,7 +2621,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -2747,7 +2637,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -2759,13 +2649,13 @@
"aspect": {
"json": {
"materialized": false,
- "viewLogic": "SELECT * FROM \"mysql\".metagalaxy.\"metadata_index\"",
+ "viewLogic": "SELECT * FROM mysql.metagalaxy.metadata_index",
"viewLanguage": "SQL"
}
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -2795,19 +2685,19 @@
},
"fields": [
{
- "fieldPath": "id",
+ "fieldPath": "doubleVal",
"nullable": true,
"type": {
"type": {
"com.linkedin.schema.NumberType": {}
}
},
- "nativeDataType": "bigint(64)",
+ "nativeDataType": "double(53)",
"recursive": false,
"isPartOfKey": false
},
{
- "fieldPath": "urn",
+ "fieldPath": "stringVal",
"nullable": true,
"type": {
"type": {
@@ -2831,7 +2721,7 @@
"isPartOfKey": false
},
{
- "fieldPath": "stringVal",
+ "fieldPath": "path",
"nullable": true,
"type": {
"type": {
@@ -2843,19 +2733,19 @@
"isPartOfKey": false
},
{
- "fieldPath": "doubleVal",
+ "fieldPath": "aspect",
"nullable": true,
"type": {
"type": {
- "com.linkedin.schema.NumberType": {}
+ "com.linkedin.schema.StringType": {}
}
},
- "nativeDataType": "double(53)",
+ "nativeDataType": "character varying(65536)",
"recursive": false,
"isPartOfKey": false
},
{
- "fieldPath": "path",
+ "fieldPath": "urn",
"nullable": true,
"type": {
"type": {
@@ -2867,14 +2757,14 @@
"isPartOfKey": false
},
{
- "fieldPath": "aspect",
+ "fieldPath": "id",
"nullable": true,
"type": {
"type": {
- "com.linkedin.schema.StringType": {}
+ "com.linkedin.schema.NumberType": {}
}
},
- "nativeDataType": "character varying(65536)",
+ "nativeDataType": "bigint(64)",
"recursive": false,
"isPartOfKey": false
}
@@ -2883,7 +2773,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -2899,7 +2789,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -2927,7 +2817,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -2951,7 +2841,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -2969,7 +2859,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -2985,7 +2875,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -3001,7 +2891,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -3013,13 +2903,13 @@
"aspect": {
"json": {
"materialized": false,
- "viewLogic": "SELECT * FROM \"mysql\".metagalaxy.\"metadata_index_view\"",
+ "viewLogic": "SELECT * FROM mysql.metagalaxy.metadata_index_view",
"viewLanguage": "SQL"
}
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -3061,14 +2951,14 @@
"isPartOfKey": false
},
{
- "fieldPath": "urn",
+ "fieldPath": "doubleVal",
"nullable": true,
"type": {
"type": {
- "com.linkedin.schema.StringType": {}
+ "com.linkedin.schema.NumberType": {}
}
},
- "nativeDataType": "character varying(65536)",
+ "nativeDataType": "double(53)",
"recursive": false,
"isPartOfKey": false
},
@@ -3085,14 +2975,14 @@
"isPartOfKey": false
},
{
- "fieldPath": "doubleVal",
+ "fieldPath": "urn",
"nullable": true,
"type": {
"type": {
- "com.linkedin.schema.NumberType": {}
+ "com.linkedin.schema.StringType": {}
}
},
- "nativeDataType": "double(53)",
+ "nativeDataType": "character varying(65536)",
"recursive": false,
"isPartOfKey": false
}
@@ -3101,7 +2991,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -3117,7 +3007,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -3145,7 +3035,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -3169,7 +3059,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -3187,7 +3077,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -3203,7 +3093,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -3219,7 +3109,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -3231,13 +3121,13 @@
"aspect": {
"json": {
"materialized": false,
- "viewLogic": "SELECT * FROM \"mysql\".northwind.orders",
+ "viewLogic": "SELECT * FROM mysql.northwind.orders",
"viewLanguage": "SQL"
}
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -3279,26 +3169,26 @@
"isPartOfKey": false
},
{
- "fieldPath": "description",
+ "fieldPath": "id",
"nullable": true,
"type": {
"type": {
- "com.linkedin.schema.StringType": {}
+ "com.linkedin.schema.NumberType": {}
}
},
- "nativeDataType": "character varying(65536)",
+ "nativeDataType": "integer(32)",
"recursive": false,
"isPartOfKey": false
},
{
- "fieldPath": "id",
+ "fieldPath": "description",
"nullable": true,
"type": {
"type": {
- "com.linkedin.schema.NumberType": {}
+ "com.linkedin.schema.StringType": {}
}
},
- "nativeDataType": "integer(32)",
+ "nativeDataType": "character varying(65536)",
"recursive": false,
"isPartOfKey": false
}
@@ -3307,7 +3197,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -3323,7 +3213,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -3351,7 +3241,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -3375,7 +3265,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -3393,7 +3283,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -3409,7 +3299,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -3425,7 +3315,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -3437,13 +3327,13 @@
"aspect": {
"json": {
"materialized": false,
- "viewLogic": "SELECT * FROM s3.warehouse.\"sample.parquet\"",
+ "viewLogic": "SELECT * FROM s3.warehouse",
"viewLanguage": "SQL"
}
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -3473,7 +3363,7 @@
},
"fields": [
{
- "fieldPath": "salary",
+ "fieldPath": "age",
"nullable": true,
"type": {
"type": {
@@ -3485,31 +3375,31 @@
"isPartOfKey": false
},
{
- "fieldPath": "age",
+ "fieldPath": "name",
"nullable": true,
"type": {
"type": {
- "com.linkedin.schema.NumberType": {}
+ "com.linkedin.schema.StringType": {}
}
},
- "nativeDataType": "bigint(64)",
+ "nativeDataType": "character varying(65536)",
"recursive": false,
"isPartOfKey": false
},
{
- "fieldPath": "name",
+ "fieldPath": "id",
"nullable": true,
"type": {
"type": {
- "com.linkedin.schema.StringType": {}
+ "com.linkedin.schema.NumberType": {}
}
},
- "nativeDataType": "character varying(65536)",
+ "nativeDataType": "bigint(64)",
"recursive": false,
"isPartOfKey": false
},
{
- "fieldPath": "id",
+ "fieldPath": "salary",
"nullable": true,
"type": {
"type": {
@@ -3525,7 +3415,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -3541,7 +3431,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -3569,21 +3459,21 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
- "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse.sample.parquet,PROD)",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD)",
"changeType": "UPSERT",
"aspectName": "datasetProperties",
"aspect": {
"json": {
"customProperties": {},
- "externalUrl": "http://localhost:9047/source/\"s3\"/\"warehouse\".\"sample.parquet\"",
- "name": "sample.parquet",
- "qualifiedName": "s3.warehouse.sample.parquet",
+ "externalUrl": "http://localhost:9047/source/\"s3\"/\"warehouse\"",
+ "name": "warehouse",
+ "qualifiedName": "s3.warehouse",
"description": "",
"created": {
"time": 0
@@ -3593,13 +3483,13 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
- "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse.sample.parquet,PROD)",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD)",
"changeType": "UPSERT",
"aspectName": "subTypes",
"aspect": {
@@ -3611,13 +3501,13 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
- "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse.sample.parquet,PROD)",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD)",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
@@ -3627,34 +3517,34 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
- "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse.sample.parquet,PROD)",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD)",
"changeType": "UPSERT",
"aspectName": "container",
"aspect": {
"json": {
- "container": "urn:li:container:6d76c46c4893c111f0006794f4a16482"
+ "container": "urn:li:container:63a316133b08a091e919dc8c7a828a4d"
}
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
- "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse.sample.parquet,PROD)",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD)",
"changeType": "UPSERT",
"aspectName": "schemaMetadata",
"aspect": {
"json": {
- "schemaName": "s3.warehouse.sample.parquet",
+ "schemaName": "s3.warehouse",
"platform": "urn:li:dataPlatform:dremio",
"version": 0,
"created": {
@@ -3725,13 +3615,13 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
- "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse.sample.parquet,PROD)",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD)",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
@@ -3741,13 +3631,13 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
- "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse.sample.parquet,PROD)",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD)",
"changeType": "UPSERT",
"aspectName": "upstreamLineage",
"aspect": {
@@ -3758,7 +3648,7 @@
"time": 0,
"actor": "urn:li:corpuser:unknown"
},
- "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./warehouse/sample.parquet,PROD)",
+ "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples.//warehouse,PROD)",
"type": "COPY"
}
]
@@ -3766,13 +3656,13 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
- "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse.sample.parquet,PROD)",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD)",
"changeType": "UPSERT",
"aspectName": "browsePathsV2",
"aspect": {
@@ -3784,17 +3674,13 @@
{
"id": "urn:li:container:63a316133b08a091e919dc8c7a828a4d",
"urn": "urn:li:container:63a316133b08a091e919dc8c7a828a4d"
- },
- {
- "id": "urn:li:container:6d76c46c4893c111f0006794f4a16482",
- "urn": "urn:li:container:6d76c46c4893c111f0006794f4a16482"
}
]
}
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -3818,7 +3704,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -3836,7 +3722,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -3852,7 +3738,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -3868,7 +3754,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -3910,19 +3796,19 @@
"isPartOfKey": false
},
{
- "fieldPath": "createdfor",
+ "fieldPath": "version",
"nullable": true,
"type": {
"type": {
- "com.linkedin.schema.StringType": {}
+ "com.linkedin.schema.NumberType": {}
}
},
- "nativeDataType": "character varying(65536)",
+ "nativeDataType": "bigint(64)",
"recursive": false,
"isPartOfKey": false
},
{
- "fieldPath": "createdby",
+ "fieldPath": "metadata",
"nullable": true,
"type": {
"type": {
@@ -3946,7 +3832,7 @@
"isPartOfKey": false
},
{
- "fieldPath": "metadata",
+ "fieldPath": "createdby",
"nullable": true,
"type": {
"type": {
@@ -3958,14 +3844,14 @@
"isPartOfKey": false
},
{
- "fieldPath": "version",
+ "fieldPath": "createdfor",
"nullable": true,
"type": {
"type": {
- "com.linkedin.schema.NumberType": {}
+ "com.linkedin.schema.StringType": {}
}
},
- "nativeDataType": "bigint(64)",
+ "nativeDataType": "character varying(65536)",
"recursive": false,
"isPartOfKey": false
},
@@ -3986,7 +3872,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -4002,7 +3888,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -4027,7 +3913,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -4055,7 +3941,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -4079,7 +3965,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -4097,7 +3983,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -4113,7 +3999,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -4129,7 +4015,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -4171,38 +4057,38 @@
"isPartOfKey": false
},
{
- "fieldPath": "id",
+ "fieldPath": "stringVal",
"nullable": true,
"type": {
"type": {
- "com.linkedin.schema.NumberType": {}
+ "com.linkedin.schema.StringType": {}
}
},
- "nativeDataType": "bigint(64)",
+ "nativeDataType": "character varying(65536)",
"recursive": false,
"isPartOfKey": false
},
{
- "fieldPath": "urn",
+ "fieldPath": "longVal",
"nullable": true,
"type": {
"type": {
- "com.linkedin.schema.StringType": {}
+ "com.linkedin.schema.NumberType": {}
}
},
- "nativeDataType": "character varying(65536)",
+ "nativeDataType": "bigint(64)",
"recursive": false,
"isPartOfKey": false
},
{
- "fieldPath": "aspect",
+ "fieldPath": "id",
"nullable": true,
"type": {
"type": {
- "com.linkedin.schema.StringType": {}
+ "com.linkedin.schema.NumberType": {}
}
},
- "nativeDataType": "character varying(65536)",
+ "nativeDataType": "bigint(64)",
"recursive": false,
"isPartOfKey": false
},
@@ -4219,19 +4105,19 @@
"isPartOfKey": false
},
{
- "fieldPath": "longVal",
+ "fieldPath": "aspect",
"nullable": true,
"type": {
"type": {
- "com.linkedin.schema.NumberType": {}
+ "com.linkedin.schema.StringType": {}
}
},
- "nativeDataType": "bigint(64)",
+ "nativeDataType": "character varying(65536)",
"recursive": false,
"isPartOfKey": false
},
{
- "fieldPath": "stringVal",
+ "fieldPath": "urn",
"nullable": true,
"type": {
"type": {
@@ -4247,7 +4133,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -4263,7 +4149,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -4288,7 +4174,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -4316,7 +4202,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -4340,7 +4226,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -4358,7 +4244,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -4374,7 +4260,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -4390,7 +4276,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -4420,31 +4306,31 @@
},
"fields": [
{
- "fieldPath": "path",
+ "fieldPath": "id",
"nullable": true,
"type": {
"type": {
- "com.linkedin.schema.StringType": {}
+ "com.linkedin.schema.NumberType": {}
}
},
- "nativeDataType": "character varying(65536)",
+ "nativeDataType": "bigint(64)",
"recursive": false,
"isPartOfKey": false
},
{
- "fieldPath": "id",
+ "fieldPath": "urn",
"nullable": true,
"type": {
"type": {
- "com.linkedin.schema.NumberType": {}
+ "com.linkedin.schema.StringType": {}
}
},
- "nativeDataType": "bigint(64)",
+ "nativeDataType": "character varying(65536)",
"recursive": false,
"isPartOfKey": false
},
{
- "fieldPath": "urn",
+ "fieldPath": "path",
"nullable": true,
"type": {
"type": {
@@ -4472,7 +4358,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -4488,7 +4374,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -4513,7 +4399,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -4541,7 +4427,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -4565,7 +4451,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -4583,7 +4469,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -4599,7 +4485,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -4615,7 +4501,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -4645,19 +4531,19 @@
},
"fields": [
{
- "fieldPath": "id",
+ "fieldPath": "priority",
"nullable": true,
"type": {
"type": {
"com.linkedin.schema.NumberType": {}
}
},
- "nativeDataType": "integer(32)",
+ "nativeDataType": "float(24)",
"recursive": false,
"isPartOfKey": false
},
{
- "fieldPath": "company",
+ "fieldPath": "email_address",
"nullable": true,
"type": {
"type": {
@@ -4669,7 +4555,7 @@
"isPartOfKey": false
},
{
- "fieldPath": "last_name",
+ "fieldPath": "first_name",
"nullable": true,
"type": {
"type": {
@@ -4681,7 +4567,7 @@
"isPartOfKey": false
},
{
- "fieldPath": "first_name",
+ "fieldPath": "last_name",
"nullable": true,
"type": {
"type": {
@@ -4693,7 +4579,7 @@
"isPartOfKey": false
},
{
- "fieldPath": "email_address",
+ "fieldPath": "company",
"nullable": true,
"type": {
"type": {
@@ -4705,14 +4591,14 @@
"isPartOfKey": false
},
{
- "fieldPath": "priority",
+ "fieldPath": "id",
"nullable": true,
"type": {
"type": {
"com.linkedin.schema.NumberType": {}
}
},
- "nativeDataType": "float(24)",
+ "nativeDataType": "integer(32)",
"recursive": false,
"isPartOfKey": false
}
@@ -4721,7 +4607,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -4737,7 +4623,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -4762,7 +4648,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -4790,7 +4676,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -4814,7 +4700,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -4832,7 +4718,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -4848,7 +4734,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -4864,7 +4750,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -4894,7 +4780,7 @@
},
"fields": [
{
- "fieldPath": "customer_id",
+ "fieldPath": "id",
"nullable": true,
"type": {
"type": {
@@ -4906,26 +4792,26 @@
"isPartOfKey": false
},
{
- "fieldPath": "id",
+ "fieldPath": "description",
"nullable": true,
"type": {
"type": {
- "com.linkedin.schema.NumberType": {}
+ "com.linkedin.schema.StringType": {}
}
},
- "nativeDataType": "integer(32)",
+ "nativeDataType": "character varying(65536)",
"recursive": false,
"isPartOfKey": false
},
{
- "fieldPath": "description",
+ "fieldPath": "customer_id",
"nullable": true,
"type": {
"type": {
- "com.linkedin.schema.StringType": {}
+ "com.linkedin.schema.NumberType": {}
}
},
- "nativeDataType": "character varying(65536)",
+ "nativeDataType": "integer(32)",
"recursive": false,
"isPartOfKey": false
}
@@ -4934,7 +4820,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -4950,7 +4836,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -4975,7 +4861,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -5003,7 +4889,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -5027,7 +4913,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -5045,7 +4931,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -5061,7 +4947,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -5077,7 +4963,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -5107,7 +4993,7 @@
},
"fields": [
{
- "fieldPath": "F",
+ "fieldPath": "D",
"nullable": true,
"type": {
"type": {
@@ -5119,7 +5005,7 @@
"isPartOfKey": false
},
{
- "fieldPath": "G",
+ "fieldPath": "C",
"nullable": true,
"type": {
"type": {
@@ -5131,7 +5017,7 @@
"isPartOfKey": false
},
{
- "fieldPath": "H",
+ "fieldPath": "B",
"nullable": true,
"type": {
"type": {
@@ -5143,7 +5029,7 @@
"isPartOfKey": false
},
{
- "fieldPath": "I",
+ "fieldPath": "A",
"nullable": true,
"type": {
"type": {
@@ -5155,7 +5041,7 @@
"isPartOfKey": false
},
{
- "fieldPath": "A",
+ "fieldPath": "H",
"nullable": true,
"type": {
"type": {
@@ -5167,7 +5053,7 @@
"isPartOfKey": false
},
{
- "fieldPath": "B",
+ "fieldPath": "G",
"nullable": true,
"type": {
"type": {
@@ -5179,7 +5065,7 @@
"isPartOfKey": false
},
{
- "fieldPath": "C",
+ "fieldPath": "F",
"nullable": true,
"type": {
"type": {
@@ -5191,7 +5077,7 @@
"isPartOfKey": false
},
{
- "fieldPath": "D",
+ "fieldPath": "E",
"nullable": true,
"type": {
"type": {
@@ -5203,7 +5089,7 @@
"isPartOfKey": false
},
{
- "fieldPath": "E",
+ "fieldPath": "I",
"nullable": true,
"type": {
"type": {
@@ -5219,7 +5105,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -5235,7 +5121,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -5260,7 +5146,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -5288,7 +5174,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -5312,7 +5198,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -5330,7 +5216,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -5346,7 +5232,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -5362,7 +5248,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -5552,7 +5438,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -5568,7 +5454,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -5593,7 +5479,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -5625,7 +5511,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -5649,7 +5535,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -5667,7 +5553,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -5683,7 +5569,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -5699,7 +5585,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -5781,7 +5667,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -5797,7 +5683,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -5822,7 +5708,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -5854,7 +5740,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -5878,7 +5764,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -5896,7 +5782,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -5912,7 +5798,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -5928,7 +5814,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -5958,7 +5844,7 @@
},
"fields": [
{
- "fieldPath": "cp_start_date_sk",
+ "fieldPath": "cp_catalog_page_number",
"nullable": true,
"type": {
"type": {
@@ -5970,19 +5856,19 @@
"isPartOfKey": false
},
{
- "fieldPath": "cp_catalog_page_sk",
+ "fieldPath": "cp_type",
"nullable": true,
"type": {
"type": {
- "com.linkedin.schema.NumberType": {}
+ "com.linkedin.schema.StringType": {}
}
},
- "nativeDataType": "bigint(64)",
+ "nativeDataType": "character varying(65536)",
"recursive": false,
"isPartOfKey": false
},
{
- "fieldPath": "cp_catalog_page_id",
+ "fieldPath": "cp_description",
"nullable": true,
"type": {
"type": {
@@ -5994,7 +5880,7 @@
"isPartOfKey": false
},
{
- "fieldPath": "cp_end_date_sk",
+ "fieldPath": "cp_catalog_number",
"nullable": true,
"type": {
"type": {
@@ -6006,19 +5892,19 @@
"isPartOfKey": false
},
{
- "fieldPath": "cp_department",
+ "fieldPath": "cp_end_date_sk",
"nullable": true,
"type": {
"type": {
- "com.linkedin.schema.StringType": {}
+ "com.linkedin.schema.NumberType": {}
}
},
- "nativeDataType": "character varying(65536)",
+ "nativeDataType": "bigint(64)",
"recursive": false,
"isPartOfKey": false
},
{
- "fieldPath": "cp_catalog_number",
+ "fieldPath": "cp_start_date_sk",
"nullable": true,
"type": {
"type": {
@@ -6030,31 +5916,31 @@
"isPartOfKey": false
},
{
- "fieldPath": "cp_catalog_page_number",
+ "fieldPath": "cp_catalog_page_id",
"nullable": true,
"type": {
"type": {
- "com.linkedin.schema.NumberType": {}
+ "com.linkedin.schema.StringType": {}
}
},
- "nativeDataType": "bigint(64)",
+ "nativeDataType": "character varying(65536)",
"recursive": false,
"isPartOfKey": false
},
{
- "fieldPath": "cp_description",
+ "fieldPath": "cp_catalog_page_sk",
"nullable": true,
"type": {
"type": {
- "com.linkedin.schema.StringType": {}
+ "com.linkedin.schema.NumberType": {}
}
},
- "nativeDataType": "character varying(65536)",
+ "nativeDataType": "bigint(64)",
"recursive": false,
"isPartOfKey": false
},
{
- "fieldPath": "cp_type",
+ "fieldPath": "cp_department",
"nullable": true,
"type": {
"type": {
@@ -6070,7 +5956,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -6086,7 +5972,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -6111,7 +5997,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -6151,7 +6037,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -6191,22 +6077,22 @@
{
"upstreamType": "FIELD_SET",
"upstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD),createdfor)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD),version)"
],
"downstreamType": "FIELD",
"downstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD),createdfor)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD),version)"
],
"confidenceScore": 1.0
},
{
"upstreamType": "FIELD_SET",
"upstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD),createdby)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD),metadata)"
],
"downstreamType": "FIELD",
"downstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD),createdby)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD),metadata)"
],
"confidenceScore": 1.0
},
@@ -6224,22 +6110,22 @@
{
"upstreamType": "FIELD_SET",
"upstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD),metadata)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD),createdby)"
],
"downstreamType": "FIELD",
"downstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD),metadata)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD),createdby)"
],
"confidenceScore": 1.0
},
{
"upstreamType": "FIELD_SET",
"upstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD),version)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_aspect,PROD),createdfor)"
],
"downstreamType": "FIELD",
"downstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD),version)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD),createdfor)"
],
"confidenceScore": 1.0
},
@@ -6259,7 +6145,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -6299,33 +6185,33 @@
{
"upstreamType": "FIELD_SET",
"upstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),id)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),stringVal)"
],
"downstreamType": "FIELD",
"downstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD),id)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD),stringVal)"
],
"confidenceScore": 1.0
},
{
"upstreamType": "FIELD_SET",
"upstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),urn)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),longVal)"
],
"downstreamType": "FIELD",
"downstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD),urn)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD),longVal)"
],
"confidenceScore": 1.0
},
{
"upstreamType": "FIELD_SET",
"upstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),aspect)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),id)"
],
"downstreamType": "FIELD",
"downstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD),aspect)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD),id)"
],
"confidenceScore": 1.0
},
@@ -6343,22 +6229,22 @@
{
"upstreamType": "FIELD_SET",
"upstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),longVal)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),aspect)"
],
"downstreamType": "FIELD",
"downstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD),longVal)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD),aspect)"
],
"confidenceScore": 1.0
},
{
"upstreamType": "FIELD_SET",
"upstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),stringVal)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index,PROD),urn)"
],
"downstreamType": "FIELD",
"downstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD),stringVal)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD),urn)"
],
"confidenceScore": 1.0
}
@@ -6367,7 +6253,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -6396,33 +6282,33 @@
{
"upstreamType": "FIELD_SET",
"upstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD),path)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD),id)"
],
"downstreamType": "FIELD",
"downstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index_view,PROD),path)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index_view,PROD),id)"
],
"confidenceScore": 1.0
},
{
"upstreamType": "FIELD_SET",
"upstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD),id)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD),urn)"
],
"downstreamType": "FIELD",
"downstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index_view,PROD),id)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index_view,PROD),urn)"
],
"confidenceScore": 1.0
},
{
"upstreamType": "FIELD_SET",
"upstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD),urn)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,metagalaxy.metadata_index_view,PROD),path)"
],
"downstreamType": "FIELD",
"downstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index_view,PROD),urn)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index_view,PROD),path)"
],
"confidenceScore": 1.0
},
@@ -6442,7 +6328,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -6471,66 +6357,66 @@
{
"upstreamType": "FIELD_SET",
"upstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD),id)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD),priority)"
],
"downstreamType": "FIELD",
"downstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD),id)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD),priority)"
],
"confidenceScore": 1.0
},
{
"upstreamType": "FIELD_SET",
"upstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD),company)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD),email_address)"
],
"downstreamType": "FIELD",
"downstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD),company)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD),email_address)"
],
"confidenceScore": 1.0
},
{
"upstreamType": "FIELD_SET",
"upstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD),last_name)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD),first_name)"
],
"downstreamType": "FIELD",
"downstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD),last_name)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD),first_name)"
],
"confidenceScore": 1.0
},
{
"upstreamType": "FIELD_SET",
"upstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD),first_name)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD),last_name)"
],
"downstreamType": "FIELD",
"downstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD),first_name)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD),last_name)"
],
"confidenceScore": 1.0
},
{
"upstreamType": "FIELD_SET",
"upstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD),email_address)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD),company)"
],
"downstreamType": "FIELD",
"downstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD),email_address)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD),company)"
],
"confidenceScore": 1.0
},
{
"upstreamType": "FIELD_SET",
"upstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD),priority)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.customers,PROD),id)"
],
"downstreamType": "FIELD",
"downstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD),priority)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.customers,PROD),id)"
],
"confidenceScore": 1.0
}
@@ -6539,7 +6425,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -6568,33 +6454,33 @@
{
"upstreamType": "FIELD_SET",
"upstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD),customer_id)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD),id)"
],
"downstreamType": "FIELD",
"downstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.orders,PROD),customer_id)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.orders,PROD),id)"
],
"confidenceScore": 1.0
},
{
"upstreamType": "FIELD_SET",
"upstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD),id)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD),description)"
],
"downstreamType": "FIELD",
"downstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.orders,PROD),id)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.orders,PROD),description)"
],
"confidenceScore": 1.0
},
{
"upstreamType": "FIELD_SET",
"upstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD),description)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,northwind.orders,PROD),customer_id)"
],
"downstreamType": "FIELD",
"downstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.orders,PROD),description)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.orders,PROD),customer_id)"
],
"confidenceScore": 1.0
}
@@ -6603,13 +6489,13 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
- "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse.sample.parquet,PROD)",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD)",
"changeType": "UPSERT",
"aspectName": "upstreamLineage",
"aspect": {
@@ -6624,7 +6510,7 @@
"time": 0,
"actor": "urn:li:corpuser:_ingestion"
},
- "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./warehouse/sample.parquet,PROD)",
+ "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples.//warehouse,PROD)",
"type": "COPY"
}
],
@@ -6632,44 +6518,44 @@
{
"upstreamType": "FIELD_SET",
"upstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./warehouse/sample.parquet,PROD),id)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples.//warehouse,PROD),id)"
],
"downstreamType": "FIELD",
"downstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse.sample.parquet,PROD),id)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD),id)"
],
"confidenceScore": 1.0
},
{
"upstreamType": "FIELD_SET",
"upstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./warehouse/sample.parquet,PROD),name)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples.//warehouse,PROD),name)"
],
"downstreamType": "FIELD",
"downstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse.sample.parquet,PROD),name)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD),name)"
],
"confidenceScore": 1.0
},
{
"upstreamType": "FIELD_SET",
"upstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./warehouse/sample.parquet,PROD),age)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples.//warehouse,PROD),age)"
],
"downstreamType": "FIELD",
"downstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse.sample.parquet,PROD),age)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD),age)"
],
"confidenceScore": 1.0
},
{
"upstreamType": "FIELD_SET",
"upstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./warehouse/sample.parquet,PROD),salary)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples.//warehouse,PROD),salary)"
],
"downstreamType": "FIELD",
"downstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse.sample.parquet,PROD),salary)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD),salary)"
],
"confidenceScore": 1.0
}
@@ -6678,7 +6564,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -6852,7 +6738,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -6927,7 +6813,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -6956,99 +6842,99 @@
{
"upstreamType": "FIELD_SET",
"upstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),F)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),D)"
],
"downstreamType": "FIELD",
"downstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),F)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),D)"
],
"confidenceScore": 1.0
},
{
"upstreamType": "FIELD_SET",
"upstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),G)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),C)"
],
"downstreamType": "FIELD",
"downstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),G)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),C)"
],
"confidenceScore": 1.0
},
{
"upstreamType": "FIELD_SET",
"upstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),H)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),B)"
],
"downstreamType": "FIELD",
"downstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),H)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),B)"
],
"confidenceScore": 1.0
},
{
"upstreamType": "FIELD_SET",
"upstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),I)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),A)"
],
"downstreamType": "FIELD",
"downstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),I)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),A)"
],
"confidenceScore": 1.0
},
{
"upstreamType": "FIELD_SET",
"upstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),A)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),H)"
],
"downstreamType": "FIELD",
"downstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),A)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),H)"
],
"confidenceScore": 1.0
},
{
"upstreamType": "FIELD_SET",
"upstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),B)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),G)"
],
"downstreamType": "FIELD",
"downstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),B)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),G)"
],
"confidenceScore": 1.0
},
{
"upstreamType": "FIELD_SET",
"upstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),C)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),F)"
],
"downstreamType": "FIELD",
"downstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),C)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),F)"
],
"confidenceScore": 1.0
},
{
"upstreamType": "FIELD_SET",
"upstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),D)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),E)"
],
"downstreamType": "FIELD",
"downstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),D)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),E)"
],
"confidenceScore": 1.0
},
{
"upstreamType": "FIELD_SET",
"upstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),E)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),I)"
],
"downstreamType": "FIELD",
"downstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),E)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),I)"
],
"confidenceScore": 1.0
}
@@ -7057,7 +6943,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -7086,183 +6972,770 @@
{
"upstreamType": "FIELD_SET",
"upstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_start_date_sk)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_page_number)"
],
"downstreamType": "FIELD",
"downstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_start_date_sk)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_page_number)"
],
"confidenceScore": 1.0
},
{
"upstreamType": "FIELD_SET",
"upstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_page_sk)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_type)"
],
"downstreamType": "FIELD",
"downstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_page_sk)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_type)"
],
"confidenceScore": 1.0
},
{
"upstreamType": "FIELD_SET",
"upstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_page_id)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_description)"
],
"downstreamType": "FIELD",
"downstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_page_id)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_description)"
],
"confidenceScore": 1.0
},
{
"upstreamType": "FIELD_SET",
"upstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_end_date_sk)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_number)"
],
"downstreamType": "FIELD",
"downstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_end_date_sk)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_number)"
],
"confidenceScore": 1.0
},
{
"upstreamType": "FIELD_SET",
"upstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_department)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_end_date_sk)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_end_date_sk)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_start_date_sk)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_start_date_sk)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_page_id)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_page_id)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_page_sk)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_page_sk)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_department)"
],
"downstreamType": "FIELD",
"downstreams": [
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_department)"
],
"confidenceScore": 1.0
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.customers,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "upstreamLineage",
+ "aspect": {
+ "json": {
+ "upstreams": [
+ {
+ "auditStamp": {
+ "time": 1697353200000,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "dataset": "urn:li:dataset:(urn:li:dataPlatform:dremio,mysql.northwind.customers,PROD)",
+ "type": "VIEW",
+ "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.customers%2CPROD%29"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "query",
+ "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.customers%2CPROD%29",
+ "changeType": "UPSERT",
+ "aspectName": "queryProperties",
+ "aspect": {
+ "json": {
+ "statement": {
+ "value": "SELECT\n *\nFROM mysql.northwind.customers",
+ "language": "SQL"
+ },
+ "source": "SYSTEM",
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "lastModified": {
+ "time": 1697353200000,
+ "actor": "urn:li:corpuser:_ingestion"
+ }
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "query",
+ "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.customers%2CPROD%29",
+ "changeType": "UPSERT",
+ "aspectName": "querySubjects",
+ "aspect": {
+ "json": {
+ "subjects": [
+ {
+ "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,mysql.northwind.customers,PROD)"
+ },
+ {
+ "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.customers,PROD)"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "query",
+ "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.customers%2CPROD%29",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_aspect,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "upstreamLineage",
+ "aspect": {
+ "json": {
+ "upstreams": [
+ {
+ "auditStamp": {
+ "time": 1697353200000,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "dataset": "urn:li:dataset:(urn:li:dataPlatform:dremio,mysql.metagalaxy.metadata_aspect,PROD)",
+ "type": "VIEW",
+ "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.metadata_aspect%2CPROD%29"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "query",
+ "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.metadata_aspect%2CPROD%29",
+ "changeType": "UPSERT",
+ "aspectName": "queryProperties",
+ "aspect": {
+ "json": {
+ "statement": {
+ "value": "SELECT\n *\nFROM mysql.metagalaxy.metadata_aspect",
+ "language": "SQL"
+ },
+ "source": "SYSTEM",
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "lastModified": {
+ "time": 1697353200000,
+ "actor": "urn:li:corpuser:_ingestion"
+ }
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "query",
+ "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.metadata_aspect%2CPROD%29",
+ "changeType": "UPSERT",
+ "aspectName": "querySubjects",
+ "aspect": {
+ "json": {
+ "subjects": [
+ {
+ "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,mysql.metagalaxy.metadata_aspect,PROD)"
+ },
+ {
+ "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_aspect,PROD)"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "query",
+ "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.metadata_aspect%2CPROD%29",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_index,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "upstreamLineage",
+ "aspect": {
+ "json": {
+ "upstreams": [
+ {
+ "auditStamp": {
+ "time": 1697353200000,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "dataset": "urn:li:dataset:(urn:li:dataPlatform:dremio,mysql.metagalaxy.metadata_index,PROD)",
+ "type": "VIEW",
+ "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.metadata_index%2CPROD%29"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "query",
+ "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.metadata_index%2CPROD%29",
+ "changeType": "UPSERT",
+ "aspectName": "queryProperties",
+ "aspect": {
+ "json": {
+ "statement": {
+ "value": "SELECT\n *\nFROM mysql.metagalaxy.metadata_index",
+ "language": "SQL"
+ },
+ "source": "SYSTEM",
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "lastModified": {
+ "time": 1697353200000,
+ "actor": "urn:li:corpuser:_ingestion"
+ }
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "query",
+ "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.metadata_index%2CPROD%29",
+ "changeType": "UPSERT",
+ "aspectName": "querySubjects",
+ "aspect": {
+ "json": {
+ "subjects": [
+ {
+ "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,mysql.metagalaxy.metadata_index,PROD)"
+ },
+ {
+ "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_index,PROD)"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "query",
+ "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.metadata_index%2CPROD%29",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_index_view,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "upstreamLineage",
+ "aspect": {
+ "json": {
+ "upstreams": [
+ {
+ "auditStamp": {
+ "time": 1697353200000,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "dataset": "urn:li:dataset:(urn:li:dataPlatform:dremio,mysql.metagalaxy.metadata_index_view,PROD)",
+ "type": "VIEW",
+ "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.metadata_index_view%2CPROD%29"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "query",
+ "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.metadata_index_view%2CPROD%29",
+ "changeType": "UPSERT",
+ "aspectName": "queryProperties",
+ "aspect": {
+ "json": {
+ "statement": {
+ "value": "SELECT\n *\nFROM mysql.metagalaxy.metadata_index_view",
+ "language": "SQL"
+ },
+ "source": "SYSTEM",
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "lastModified": {
+ "time": 1697353200000,
+ "actor": "urn:li:corpuser:_ingestion"
+ }
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "query",
+ "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.metadata_index_view%2CPROD%29",
+ "changeType": "UPSERT",
+ "aspectName": "querySubjects",
+ "aspect": {
+ "json": {
+ "subjects": [
+ {
+ "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,mysql.metagalaxy.metadata_index_view,PROD)"
+ },
+ {
+ "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_index_view,PROD)"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "query",
+ "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.metadata_index_view%2CPROD%29",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.orders,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "upstreamLineage",
+ "aspect": {
+ "json": {
+ "upstreams": [
+ {
+ "auditStamp": {
+ "time": 1697353200000,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "dataset": "urn:li:dataset:(urn:li:dataPlatform:dremio,mysql.northwind.orders,PROD)",
+ "type": "VIEW",
+ "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.orders%2CPROD%29"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "query",
+ "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.orders%2CPROD%29",
+ "changeType": "UPSERT",
+ "aspectName": "queryProperties",
+ "aspect": {
+ "json": {
+ "statement": {
+ "value": "SELECT\n *\nFROM mysql.northwind.orders",
+ "language": "SQL"
+ },
+ "source": "SYSTEM",
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "lastModified": {
+ "time": 1697353200000,
+ "actor": "urn:li:corpuser:_ingestion"
+ }
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "query",
+ "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.orders%2CPROD%29",
+ "changeType": "UPSERT",
+ "aspectName": "querySubjects",
+ "aspect": {
+ "json": {
+ "subjects": [
+ {
+ "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,mysql.northwind.orders,PROD)"
},
+ {
+ "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.orders,PROD)"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "query",
+ "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.orders%2CPROD%29",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.raw,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "upstreamLineage",
+ "aspect": {
+ "json": {
+ "upstreams": [
+ {
+ "auditStamp": {
+ "time": 1697353200000,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "dataset": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD)",
+ "type": "VIEW",
+ "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.raw%2CPROD%29"
+ }
+ ],
+ "fineGrainedLineages": [
{
"upstreamType": "FIELD_SET",
"upstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_number)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD),id)"
],
"downstreamType": "FIELD",
"downstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_number)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.raw,PROD),id)"
],
- "confidenceScore": 1.0
+ "confidenceScore": 0.9,
+ "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.raw%2CPROD%29"
},
{
"upstreamType": "FIELD_SET",
"upstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_page_number)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD),name)"
],
"downstreamType": "FIELD",
"downstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_page_number)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.raw,PROD),name)"
],
- "confidenceScore": 1.0
+ "confidenceScore": 0.9,
+ "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.raw%2CPROD%29"
},
{
"upstreamType": "FIELD_SET",
"upstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_description)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD),age)"
],
"downstreamType": "FIELD",
"downstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_description)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.raw,PROD),age)"
],
- "confidenceScore": 1.0
+ "confidenceScore": 0.9,
+ "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.raw%2CPROD%29"
},
{
"upstreamType": "FIELD_SET",
"upstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_type)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD),salary)"
],
"downstreamType": "FIELD",
"downstreams": [
- "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_type)"
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.raw,PROD),salary)"
],
- "confidenceScore": 1.0
+ "confidenceScore": 0.9,
+ "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.raw%2CPROD%29"
}
]
}
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
{
- "entityType": "dataset",
- "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)",
+ "entityType": "query",
+ "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.raw%2CPROD%29",
"changeType": "UPSERT",
- "aspectName": "datasetProfile",
+ "aspectName": "queryProperties",
"aspect": {
"json": {
- "timestampMillis": 1697353200000,
- "partitionSpec": {
- "partition": "FULL_TABLE_SNAPSHOT",
- "type": "FULL_TABLE"
+ "statement": {
+ "value": "SELECT\n *\nFROM s3.warehouse",
+ "language": "SQL"
},
- "rowCount": 3834,
- "columnCount": 9,
- "fieldProfiles": [
+ "source": "SYSTEM",
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "lastModified": {
+ "time": 1697353200000,
+ "actor": "urn:li:corpuser:_ingestion"
+ }
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "query",
+ "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.raw%2CPROD%29",
+ "changeType": "UPSERT",
+ "aspectName": "querySubjects",
+ "aspect": {
+ "json": {
+ "subjects": [
{
- "fieldPath": "F",
- "uniqueCount": 61,
- "nullCount": 0
+ "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD)"
},
{
- "fieldPath": "G",
- "uniqueCount": 40,
- "nullCount": 0
+ "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD),age)"
},
{
- "fieldPath": "H",
- "uniqueCount": 91,
- "nullCount": 0
+ "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD),id)"
},
{
- "fieldPath": "I",
- "uniqueCount": 85,
- "nullCount": 0
+ "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD),name)"
},
{
- "fieldPath": "A",
- "uniqueCount": 2,
- "nullCount": 0
+ "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD),salary)"
},
{
- "fieldPath": "B",
- "uniqueCount": 2,
- "nullCount": 0
+ "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.raw,PROD)"
},
{
- "fieldPath": "C",
- "uniqueCount": 3834,
- "nullCount": 0
+ "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.raw,PROD),id)"
},
{
- "fieldPath": "D",
- "uniqueCount": 76,
- "nullCount": 0
+ "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.raw,PROD),name)"
},
{
- "fieldPath": "E",
- "uniqueCount": 192,
- "nullCount": 0
+ "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.raw,PROD),age)"
+ },
+ {
+ "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.raw,PROD),salary)"
}
]
}
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "query",
+ "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.raw%2CPROD%29",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
- "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index_view,PROD)",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_index_view,PROD)",
"changeType": "UPSERT",
"aspectName": "datasetProfile",
"aspect": {
@@ -7275,23 +7748,62 @@
"rowCount": 0,
"columnCount": 4,
"fieldProfiles": [
+ {
+ "fieldPath": "id",
+ "uniqueCount": 0,
+ "nullCount": 0
+ },
+ {
+ "fieldPath": "doubleVal",
+ "uniqueCount": 0,
+ "nullCount": 0
+ },
{
"fieldPath": "path",
"uniqueCount": 0,
"nullCount": 0
},
+ {
+ "fieldPath": "urn",
+ "uniqueCount": 0,
+ "nullCount": 0
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.orders,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "datasetProfile",
+ "aspect": {
+ "json": {
+ "timestampMillis": 1697353200000,
+ "partitionSpec": {
+ "partition": "FULL_TABLE_SNAPSHOT",
+ "type": "FULL_TABLE"
+ },
+ "rowCount": 0,
+ "columnCount": 3,
+ "fieldProfiles": [
{
"fieldPath": "id",
"uniqueCount": 0,
"nullCount": 0
},
{
- "fieldPath": "urn",
+ "fieldPath": "description",
"uniqueCount": 0,
"nullCount": 0
},
{
- "fieldPath": "doubleVal",
+ "fieldPath": "customer_id",
"uniqueCount": 0,
"nullCount": 0
}
@@ -7300,13 +7812,63 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.raw,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "datasetProfile",
+ "aspect": {
+ "json": {
+ "timestampMillis": 1697353200000,
+ "partitionSpec": {
+ "partition": "FULL_TABLE_SNAPSHOT",
+ "type": "FULL_TABLE"
+ },
+ "rowCount": 4,
+ "columnCount": 4,
+ "fieldProfiles": [
+ {
+ "fieldPath": "age",
+ "uniqueCount": 4,
+ "nullCount": 0,
+ "mean": "32.5",
+ "stdev": "6.454972243679028"
+ },
+ {
+ "fieldPath": "name",
+ "uniqueCount": 4,
+ "nullCount": 0
+ },
+ {
+ "fieldPath": "id",
+ "uniqueCount": 4,
+ "nullCount": 0,
+ "mean": "2.5",
+ "stdev": "1.2909944487358056"
+ },
+ {
+ "fieldPath": "salary",
+ "uniqueCount": 4,
+ "nullCount": 0,
+ "mean": "65000.0",
+ "stdev": "12909.944487358056"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
- "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD)",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_aspect,PROD)",
"changeType": "UPSERT",
"aspectName": "datasetProfile",
"aspect": {
@@ -7316,56 +7878,56 @@
"partition": "FULL_TABLE_SNAPSHOT",
"type": "FULL_TABLE"
},
- "rowCount": 0,
+ "rowCount": 2,
"columnCount": 7,
"fieldProfiles": [
{
- "fieldPath": "doubleVal",
- "uniqueCount": 0,
+ "fieldPath": "urn",
+ "uniqueCount": 1,
"nullCount": 0
},
{
- "fieldPath": "id",
- "uniqueCount": 0,
+ "fieldPath": "aspect",
+ "uniqueCount": 2,
"nullCount": 0
},
{
- "fieldPath": "urn",
- "uniqueCount": 0,
+ "fieldPath": "version",
+ "uniqueCount": 1,
"nullCount": 0
},
{
- "fieldPath": "aspect",
- "uniqueCount": 0,
+ "fieldPath": "metadata",
+ "uniqueCount": 2,
"nullCount": 0
},
{
- "fieldPath": "path",
- "uniqueCount": 0,
+ "fieldPath": "createdon",
+ "uniqueCount": 1,
"nullCount": 0
},
{
- "fieldPath": "longVal",
- "uniqueCount": 0,
+ "fieldPath": "createdby",
+ "uniqueCount": 1,
"nullCount": 0
},
{
- "fieldPath": "stringVal",
+ "fieldPath": "createdfor",
"uniqueCount": 0,
- "nullCount": 0
+ "nullCount": 2
}
]
}
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
- "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.northwind.orders,PROD)",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.customers,PROD)",
"changeType": "UPSERT",
"aspectName": "datasetProfile",
"aspect": {
@@ -7375,86 +7937,55 @@
"partition": "FULL_TABLE_SNAPSHOT",
"type": "FULL_TABLE"
},
- "rowCount": 0,
- "columnCount": 3,
+ "rowCount": 5,
+ "columnCount": 6,
"fieldProfiles": [
{
- "fieldPath": "customer_id",
- "uniqueCount": 0,
- "nullCount": 0
+ "fieldPath": "priority",
+ "uniqueCount": 3,
+ "nullCount": 1,
+ "mean": "4.175000011920929",
+ "stdev": "0.4924429489953036"
},
{
- "fieldPath": "id",
- "uniqueCount": 0,
+ "fieldPath": "email_address",
+ "uniqueCount": 5,
"nullCount": 0
},
{
- "fieldPath": "description",
- "uniqueCount": 0,
+ "fieldPath": "first_name",
+ "uniqueCount": 5,
"nullCount": 0
- }
- ]
- }
- },
- "systemMetadata": {
- "lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
- "lastRunId": "no-run-id-provided"
- }
-},
-{
- "entityType": "dataset",
- "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.raw,PROD)",
- "changeType": "UPSERT",
- "aspectName": "datasetProfile",
- "aspect": {
- "json": {
- "timestampMillis": 1697353200000,
- "partitionSpec": {
- "partition": "FULL_TABLE_SNAPSHOT",
- "type": "FULL_TABLE"
- },
- "rowCount": 4,
- "columnCount": 4,
- "fieldProfiles": [
- {
- "fieldPath": "salary",
- "uniqueCount": 4,
- "nullCount": 0,
- "mean": "65000.0",
- "stdev": "12909.944487358056"
},
{
- "fieldPath": "age",
- "uniqueCount": 4,
- "nullCount": 0,
- "mean": "32.5",
- "stdev": "6.454972243679028"
+ "fieldPath": "last_name",
+ "uniqueCount": 5,
+ "nullCount": 0
},
{
- "fieldPath": "name",
- "uniqueCount": 4,
+ "fieldPath": "company",
+ "uniqueCount": 5,
"nullCount": 0
},
{
"fieldPath": "id",
- "uniqueCount": 4,
+ "uniqueCount": 5,
"nullCount": 0,
- "mean": "2.5",
- "stdev": "1.2909944487358056"
+ "mean": "3.0",
+ "stdev": "1.5811388300841898"
}
]
}
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
- "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_index_view,PROD)",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index_view,PROD)",
"changeType": "UPSERT",
"aspectName": "datasetProfile",
"aspect": {
@@ -7492,13 +8023,13 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
- "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.customers,PROD)",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.orders,PROD)",
"changeType": "UPSERT",
"aspectName": "datasetProfile",
"aspect": {
@@ -7508,41 +8039,22 @@
"partition": "FULL_TABLE_SNAPSHOT",
"type": "FULL_TABLE"
},
- "rowCount": 5,
- "columnCount": 6,
+ "rowCount": 0,
+ "columnCount": 3,
"fieldProfiles": [
{
- "fieldPath": "email_address",
- "uniqueCount": 5,
- "nullCount": 0
- },
- {
- "fieldPath": "priority",
- "uniqueCount": 3,
- "nullCount": 1,
- "mean": "4.175000011920929",
- "stdev": "0.4924429489953036"
- },
- {
- "fieldPath": "company",
- "uniqueCount": 5,
+ "fieldPath": "customer_id",
+ "uniqueCount": 0,
"nullCount": 0
},
{
"fieldPath": "id",
- "uniqueCount": 5,
- "nullCount": 0,
- "mean": "3.0",
- "stdev": "1.5811388300841898"
- },
- {
- "fieldPath": "first_name",
- "uniqueCount": 5,
+ "uniqueCount": 0,
"nullCount": 0
},
{
- "fieldPath": "last_name",
- "uniqueCount": 5,
+ "fieldPath": "description",
+ "uniqueCount": 0,
"nullCount": 0
}
]
@@ -7550,13 +8062,13 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
- "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.warehouse,PROD)",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_index,PROD)",
"changeType": "UPSERT",
"aspectName": "datasetProfile",
"aspect": {
@@ -7566,52 +8078,42 @@
"partition": "FULL_TABLE_SNAPSHOT",
"type": "FULL_TABLE"
},
- "rowCount": 3834,
- "columnCount": 9,
+ "rowCount": 0,
+ "columnCount": 7,
"fieldProfiles": [
{
- "fieldPath": "F",
- "uniqueCount": 61,
- "nullCount": 0
- },
- {
- "fieldPath": "G",
- "uniqueCount": 40,
- "nullCount": 0
- },
- {
- "fieldPath": "H",
- "uniqueCount": 91,
+ "fieldPath": "doubleVal",
+ "uniqueCount": 0,
"nullCount": 0
},
{
- "fieldPath": "I",
- "uniqueCount": 85,
+ "fieldPath": "stringVal",
+ "uniqueCount": 0,
"nullCount": 0
},
{
- "fieldPath": "A",
- "uniqueCount": 2,
+ "fieldPath": "longVal",
+ "uniqueCount": 0,
"nullCount": 0
},
{
- "fieldPath": "B",
- "uniqueCount": 2,
+ "fieldPath": "id",
+ "uniqueCount": 0,
"nullCount": 0
},
{
- "fieldPath": "C",
- "uniqueCount": 3834,
+ "fieldPath": "path",
+ "uniqueCount": 0,
"nullCount": 0
},
{
- "fieldPath": "D",
- "uniqueCount": 76,
+ "fieldPath": "aspect",
+ "uniqueCount": 0,
"nullCount": 0
},
{
- "fieldPath": "E",
- "uniqueCount": 192,
+ "fieldPath": "urn",
+ "uniqueCount": 0,
"nullCount": 0
}
]
@@ -7619,13 +8121,13 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
- "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_aspect,PROD)",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_index,PROD)",
"changeType": "UPSERT",
"aspectName": "datasetProfile",
"aspect": {
@@ -7635,42 +8137,42 @@
"partition": "FULL_TABLE_SNAPSHOT",
"type": "FULL_TABLE"
},
- "rowCount": 2,
+ "rowCount": 0,
"columnCount": 7,
"fieldProfiles": [
{
- "fieldPath": "createdby",
- "uniqueCount": 1,
+ "fieldPath": "doubleVal",
+ "uniqueCount": 0,
"nullCount": 0
},
{
- "fieldPath": "createdfor",
+ "fieldPath": "stringVal",
"uniqueCount": 0,
- "nullCount": 2
+ "nullCount": 0
},
{
- "fieldPath": "urn",
- "uniqueCount": 1,
+ "fieldPath": "longVal",
+ "uniqueCount": 0,
"nullCount": 0
},
{
- "fieldPath": "aspect",
- "uniqueCount": 2,
+ "fieldPath": "path",
+ "uniqueCount": 0,
"nullCount": 0
},
{
- "fieldPath": "version",
- "uniqueCount": 1,
+ "fieldPath": "aspect",
+ "uniqueCount": 0,
"nullCount": 0
},
{
- "fieldPath": "metadata",
- "uniqueCount": 2,
+ "fieldPath": "urn",
+ "uniqueCount": 0,
"nullCount": 0
},
{
- "fieldPath": "createdon",
- "uniqueCount": 1,
+ "fieldPath": "id",
+ "uniqueCount": 0,
"nullCount": 0
}
]
@@ -7678,13 +8180,13 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
- "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD)",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)",
"changeType": "UPSERT",
"aspectName": "datasetProfile",
"aspect": {
@@ -7694,42 +8196,52 @@
"partition": "FULL_TABLE_SNAPSHOT",
"type": "FULL_TABLE"
},
- "rowCount": 2,
- "columnCount": 7,
+ "rowCount": 3834,
+ "columnCount": 9,
"fieldProfiles": [
{
- "fieldPath": "urn",
- "uniqueCount": 1,
+ "fieldPath": "D",
+ "uniqueCount": 76,
"nullCount": 0
},
{
- "fieldPath": "createdfor",
- "uniqueCount": 0,
- "nullCount": 2
+ "fieldPath": "C",
+ "uniqueCount": 3834,
+ "nullCount": 0
},
{
- "fieldPath": "createdby",
- "uniqueCount": 1,
+ "fieldPath": "B",
+ "uniqueCount": 2,
"nullCount": 0
},
{
- "fieldPath": "createdon",
- "uniqueCount": 1,
+ "fieldPath": "A",
+ "uniqueCount": 2,
"nullCount": 0
},
{
- "fieldPath": "metadata",
- "uniqueCount": 2,
+ "fieldPath": "H",
+ "uniqueCount": 91,
"nullCount": 0
},
{
- "fieldPath": "version",
- "uniqueCount": 1,
+ "fieldPath": "G",
+ "uniqueCount": 40,
+ "nullCount": 0
+ },
+ {
+ "fieldPath": "F",
+ "uniqueCount": 61,
+ "nullCount": 0
+ },
+ {
+ "fieldPath": "E",
+ "uniqueCount": 192,
"nullCount": 0
},
{
- "fieldPath": "aspect",
- "uniqueCount": 2,
+ "fieldPath": "I",
+ "uniqueCount": 85,
"nullCount": 0
}
]
@@ -7737,13 +8249,13 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
- "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse.sample.parquet,PROD)",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.s3.warehouse,PROD)",
"changeType": "UPSERT",
"aspectName": "datasetProfile",
"aspect": {
@@ -7787,13 +8299,13 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
- "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.orders,PROD)",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.mysql.metagalaxy.metadata_aspect,PROD)",
"changeType": "UPSERT",
"aspectName": "datasetProfile",
"aspect": {
@@ -7803,22 +8315,42 @@
"partition": "FULL_TABLE_SNAPSHOT",
"type": "FULL_TABLE"
},
- "rowCount": 0,
- "columnCount": 3,
+ "rowCount": 2,
+ "columnCount": 7,
"fieldProfiles": [
{
- "fieldPath": "customer_id",
- "uniqueCount": 0,
+ "fieldPath": "urn",
+ "uniqueCount": 1,
"nullCount": 0
},
{
- "fieldPath": "description",
- "uniqueCount": 0,
+ "fieldPath": "version",
+ "uniqueCount": 1,
"nullCount": 0
},
{
- "fieldPath": "id",
+ "fieldPath": "metadata",
+ "uniqueCount": 2,
+ "nullCount": 0
+ },
+ {
+ "fieldPath": "createdon",
+ "uniqueCount": 1,
+ "nullCount": 0
+ },
+ {
+ "fieldPath": "createdby",
+ "uniqueCount": 1,
+ "nullCount": 0
+ },
+ {
+ "fieldPath": "createdfor",
"uniqueCount": 0,
+ "nullCount": 2
+ },
+ {
+ "fieldPath": "aspect",
+ "uniqueCount": 2,
"nullCount": 0
}
]
@@ -7826,13 +8358,13 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
- "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.test_folder.metadata_index,PROD)",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.space.warehouse,PROD)",
"changeType": "UPSERT",
"aspectName": "datasetProfile",
"aspect": {
@@ -7842,42 +8374,52 @@
"partition": "FULL_TABLE_SNAPSHOT",
"type": "FULL_TABLE"
},
- "rowCount": 0,
- "columnCount": 7,
+ "rowCount": 3834,
+ "columnCount": 9,
"fieldProfiles": [
{
- "fieldPath": "id",
- "uniqueCount": 0,
+ "fieldPath": "A",
+ "uniqueCount": 2,
"nullCount": 0
},
{
- "fieldPath": "urn",
- "uniqueCount": 0,
+ "fieldPath": "B",
+ "uniqueCount": 2,
"nullCount": 0
},
{
- "fieldPath": "longVal",
- "uniqueCount": 0,
+ "fieldPath": "C",
+ "uniqueCount": 3834,
"nullCount": 0
},
{
- "fieldPath": "stringVal",
- "uniqueCount": 0,
+ "fieldPath": "D",
+ "uniqueCount": 76,
"nullCount": 0
},
{
- "fieldPath": "doubleVal",
- "uniqueCount": 0,
+ "fieldPath": "E",
+ "uniqueCount": 192,
"nullCount": 0
},
{
- "fieldPath": "path",
- "uniqueCount": 0,
+ "fieldPath": "F",
+ "uniqueCount": 61,
"nullCount": 0
},
{
- "fieldPath": "aspect",
- "uniqueCount": 0,
+ "fieldPath": "G",
+ "uniqueCount": 40,
+ "nullCount": 0
+ },
+ {
+ "fieldPath": "H",
+ "uniqueCount": 91,
+ "nullCount": 0
+ },
+ {
+ "fieldPath": "I",
+ "uniqueCount": 85,
"nullCount": 0
}
]
@@ -7885,7 +8427,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -7905,45 +8447,45 @@
"columnCount": 6,
"fieldProfiles": [
{
- "fieldPath": "id",
- "uniqueCount": 5,
- "nullCount": 0,
- "mean": "3.0",
- "stdev": "1.5811388300841898"
+ "fieldPath": "priority",
+ "uniqueCount": 3,
+ "nullCount": 1,
+ "mean": "4.175000011920929",
+ "stdev": "0.4924429489953036"
},
{
- "fieldPath": "company",
+ "fieldPath": "email_address",
"uniqueCount": 5,
"nullCount": 0
},
{
- "fieldPath": "last_name",
+ "fieldPath": "first_name",
"uniqueCount": 5,
"nullCount": 0
},
{
- "fieldPath": "first_name",
+ "fieldPath": "last_name",
"uniqueCount": 5,
"nullCount": 0
},
{
- "fieldPath": "email_address",
+ "fieldPath": "company",
"uniqueCount": 5,
"nullCount": 0
},
{
- "fieldPath": "priority",
- "uniqueCount": 3,
- "nullCount": 1,
- "mean": "4.175000011920929",
- "stdev": "0.4924429489953036"
+ "fieldPath": "id",
+ "uniqueCount": 5,
+ "nullCount": 0,
+ "mean": "3.0",
+ "stdev": "1.5811388300841898"
}
]
}
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -8032,7 +8574,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -8082,7 +8624,7 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
},
@@ -8102,34 +8644,20 @@
"columnCount": 9,
"fieldProfiles": [
{
- "fieldPath": "cp_start_date_sk",
- "uniqueCount": 91,
- "nullCount": 286,
- "mean": "2451880.7982769064",
- "stdev": "634.1320559175408"
- },
- {
- "fieldPath": "cp_catalog_page_sk",
- "uniqueCount": 30000,
- "nullCount": 0,
- "mean": "15000.5",
- "stdev": "8660.398374208891"
+ "fieldPath": "cp_catalog_page_number",
+ "uniqueCount": 277,
+ "nullCount": 294,
+ "mean": "138.8378442065576",
+ "stdev": "80.01625232480262"
},
{
- "fieldPath": "cp_catalog_page_id",
- "uniqueCount": 30000,
+ "fieldPath": "cp_type",
+ "uniqueCount": 4,
"nullCount": 0
},
{
- "fieldPath": "cp_end_date_sk",
- "uniqueCount": 97,
- "nullCount": 302,
- "mean": "2451940.632601522",
- "stdev": "634.7367986145963"
- },
- {
- "fieldPath": "cp_department",
- "uniqueCount": 2,
+ "fieldPath": "cp_description",
+ "uniqueCount": 29718,
"nullCount": 0
},
{
@@ -8140,20 +8668,34 @@
"stdev": "31.27039684588463"
},
{
- "fieldPath": "cp_catalog_page_number",
- "uniqueCount": 277,
- "nullCount": 294,
- "mean": "138.8378442065576",
- "stdev": "80.01625232480262"
+ "fieldPath": "cp_end_date_sk",
+ "uniqueCount": 97,
+ "nullCount": 302,
+ "mean": "2451940.632601522",
+ "stdev": "634.7367986145963"
},
{
- "fieldPath": "cp_description",
- "uniqueCount": 29718,
+ "fieldPath": "cp_start_date_sk",
+ "uniqueCount": 91,
+ "nullCount": 286,
+ "mean": "2451880.7982769064",
+ "stdev": "634.1320559175408"
+ },
+ {
+ "fieldPath": "cp_catalog_page_id",
+ "uniqueCount": 30000,
"nullCount": 0
},
{
- "fieldPath": "cp_type",
- "uniqueCount": 4,
+ "fieldPath": "cp_catalog_page_sk",
+ "uniqueCount": 30000,
+ "nullCount": 0,
+ "mean": "15000.5",
+ "stdev": "8660.398374208891"
+ },
+ {
+ "fieldPath": "cp_department",
+ "uniqueCount": 2,
"nullCount": 0
}
]
@@ -8161,7 +8703,103 @@
},
"systemMetadata": {
"lastObserved": 1697353200000,
- "runId": "dremio-2023_10_15-07_00_00-bo12f3",
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "query",
+ "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.customers%2CPROD%29",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "query",
+ "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.metadata_aspect%2CPROD%29",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "query",
+ "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.metadata_index%2CPROD%29",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "query",
+ "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.metadata_index_view%2CPROD%29",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "query",
+ "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.orders%2CPROD%29",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "query",
+ "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Cdremio.space.test_folder.raw%2CPROD%29",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-5p79wf",
"lastRunId": "no-run-id-provided"
}
}
diff --git a/metadata-ingestion/tests/integration/dremio/dremio_platform_instance_mces_golden.json b/metadata-ingestion/tests/integration/dremio/dremio_platform_instance_mces_golden.json
new file mode 100644
index 00000000000000..81e9c38d1bbee8
--- /dev/null
+++ b/metadata-ingestion/tests/integration/dremio/dremio_platform_instance_mces_golden.json
@@ -0,0 +1,8002 @@
+[
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:5fc7ba11cb45461f55fb834da2141c46",
+ "changeType": "UPSERT",
+ "aspectName": "containerProperties",
+ "aspect": {
+ "json": {
+ "customProperties": {},
+ "name": "s3",
+ "qualifiedName": "s3",
+ "description": "",
+ "env": "PROD"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:5fc7ba11cb45461f55fb834da2141c46",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio",
+ "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:5fc7ba11cb45461f55fb834da2141c46",
+ "changeType": "UPSERT",
+ "aspectName": "subTypes",
+ "aspect": {
+ "json": {
+ "typeNames": [
+ "Dremio Source"
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:5fc7ba11cb45461f55fb834da2141c46",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:5fc7ba11cb45461f55fb834da2141c46",
+ "changeType": "UPSERT",
+ "aspectName": "browsePathsV2",
+ "aspect": {
+ "json": {
+ "path": [
+ {
+ "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+ "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ },
+ {
+ "id": "Sources"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:85534727d17f56b5996dabbf0fda04a2",
+ "changeType": "UPSERT",
+ "aspectName": "containerProperties",
+ "aspect": {
+ "json": {
+ "customProperties": {},
+ "name": "mysql",
+ "qualifiedName": "mysql",
+ "description": "",
+ "env": "PROD"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:85534727d17f56b5996dabbf0fda04a2",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio",
+ "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:85534727d17f56b5996dabbf0fda04a2",
+ "changeType": "UPSERT",
+ "aspectName": "subTypes",
+ "aspect": {
+ "json": {
+ "typeNames": [
+ "Dremio Source"
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:85534727d17f56b5996dabbf0fda04a2",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:85534727d17f56b5996dabbf0fda04a2",
+ "changeType": "UPSERT",
+ "aspectName": "browsePathsV2",
+ "aspect": {
+ "json": {
+ "path": [
+ {
+ "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+ "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ },
+ {
+ "id": "Sources"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:007d12a4a241c87924b54e1e35990234",
+ "changeType": "UPSERT",
+ "aspectName": "containerProperties",
+ "aspect": {
+ "json": {
+ "customProperties": {},
+ "name": "Samples",
+ "qualifiedName": "Samples",
+ "description": "",
+ "env": "PROD"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:007d12a4a241c87924b54e1e35990234",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio",
+ "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:007d12a4a241c87924b54e1e35990234",
+ "changeType": "UPSERT",
+ "aspectName": "subTypes",
+ "aspect": {
+ "json": {
+ "typeNames": [
+ "Dremio Source"
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:007d12a4a241c87924b54e1e35990234",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:007d12a4a241c87924b54e1e35990234",
+ "changeType": "UPSERT",
+ "aspectName": "browsePathsV2",
+ "aspect": {
+ "json": {
+ "path": [
+ {
+ "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+ "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ },
+ {
+ "id": "Sources"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:083ad9a0c7aa64b4ebc5f470646c25ab",
+ "changeType": "UPSERT",
+ "aspectName": "containerProperties",
+ "aspect": {
+ "json": {
+ "customProperties": {},
+ "name": "@admin",
+ "qualifiedName": "@admin",
+ "description": "# Wikis & Labels\n\n\n\nYou are reading the wiki for your home space! You can create and edit this information for any source, space, or folder.\n\nThis sidebar always shows the wiki for the current source, space or folder you are browsing.\n\nWhen browsing or previewing datasets, click on the `Open details panel` button to create a wiki or add labels to that dataset.\n\n**Tip:** You can hide the wiki by clicking on the sidebar icon on upper right hand side.",
+ "env": "PROD"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:083ad9a0c7aa64b4ebc5f470646c25ab",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio",
+ "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:083ad9a0c7aa64b4ebc5f470646c25ab",
+ "changeType": "UPSERT",
+ "aspectName": "subTypes",
+ "aspect": {
+ "json": {
+ "typeNames": [
+ "Dremio Space"
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:083ad9a0c7aa64b4ebc5f470646c25ab",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:083ad9a0c7aa64b4ebc5f470646c25ab",
+ "changeType": "UPSERT",
+ "aspectName": "browsePathsV2",
+ "aspect": {
+ "json": {
+ "path": [
+ {
+ "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+ "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ },
+ {
+ "id": "Spaces"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7",
+ "changeType": "UPSERT",
+ "aspectName": "containerProperties",
+ "aspect": {
+ "json": {
+ "customProperties": {},
+ "name": "space",
+ "qualifiedName": "space",
+ "description": "",
+ "env": "PROD"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio",
+ "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7",
+ "changeType": "UPSERT",
+ "aspectName": "subTypes",
+ "aspect": {
+ "json": {
+ "typeNames": [
+ "Dremio Space"
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7",
+ "changeType": "UPSERT",
+ "aspectName": "browsePathsV2",
+ "aspect": {
+ "json": {
+ "path": [
+ {
+ "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+ "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ },
+ {
+ "id": "Spaces"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5",
+ "changeType": "UPSERT",
+ "aspectName": "containerProperties",
+ "aspect": {
+ "json": {
+ "customProperties": {},
+ "name": "test_folder",
+ "qualifiedName": "space.test_folder",
+ "description": "",
+ "env": "PROD"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5",
+ "changeType": "UPSERT",
+ "aspectName": "container",
+ "aspect": {
+ "json": {
+ "container": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio",
+ "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5",
+ "changeType": "UPSERT",
+ "aspectName": "subTypes",
+ "aspect": {
+ "json": {
+ "typeNames": [
+ "Dremio Folder"
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5",
+ "changeType": "UPSERT",
+ "aspectName": "browsePathsV2",
+ "aspect": {
+ "json": {
+ "path": [
+ {
+ "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+ "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ },
+ {
+ "id": "Spaces"
+ },
+ {
+ "id": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7",
+ "urn": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:d3fda876e164a89fd73b14594fd88992",
+ "changeType": "UPSERT",
+ "aspectName": "containerProperties",
+ "aspect": {
+ "json": {
+ "customProperties": {},
+ "name": "dataCharmer",
+ "qualifiedName": "mysql.dataCharmer",
+ "description": "",
+ "env": "PROD"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:d3fda876e164a89fd73b14594fd88992",
+ "changeType": "UPSERT",
+ "aspectName": "container",
+ "aspect": {
+ "json": {
+ "container": "urn:li:container:85534727d17f56b5996dabbf0fda04a2"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:d3fda876e164a89fd73b14594fd88992",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio",
+ "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:d3fda876e164a89fd73b14594fd88992",
+ "changeType": "UPSERT",
+ "aspectName": "subTypes",
+ "aspect": {
+ "json": {
+ "typeNames": [
+ "Dremio Folder"
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:d3fda876e164a89fd73b14594fd88992",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:d3fda876e164a89fd73b14594fd88992",
+ "changeType": "UPSERT",
+ "aspectName": "browsePathsV2",
+ "aspect": {
+ "json": {
+ "path": [
+ {
+ "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+ "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ },
+ {
+ "id": "Sources"
+ },
+ {
+ "id": "urn:li:container:85534727d17f56b5996dabbf0fda04a2",
+ "urn": "urn:li:container:85534727d17f56b5996dabbf0fda04a2"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:67653e01aa4da4fcfd6675a591700e89",
+ "changeType": "UPSERT",
+ "aspectName": "containerProperties",
+ "aspect": {
+ "json": {
+ "customProperties": {},
+ "name": "metagalaxy",
+ "qualifiedName": "mysql.metagalaxy",
+ "description": "",
+ "env": "PROD"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:67653e01aa4da4fcfd6675a591700e89",
+ "changeType": "UPSERT",
+ "aspectName": "container",
+ "aspect": {
+ "json": {
+ "container": "urn:li:container:85534727d17f56b5996dabbf0fda04a2"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:67653e01aa4da4fcfd6675a591700e89",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio",
+ "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:67653e01aa4da4fcfd6675a591700e89",
+ "changeType": "UPSERT",
+ "aspectName": "subTypes",
+ "aspect": {
+ "json": {
+ "typeNames": [
+ "Dremio Folder"
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:67653e01aa4da4fcfd6675a591700e89",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:67653e01aa4da4fcfd6675a591700e89",
+ "changeType": "UPSERT",
+ "aspectName": "browsePathsV2",
+ "aspect": {
+ "json": {
+ "path": [
+ {
+ "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+ "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ },
+ {
+ "id": "Sources"
+ },
+ {
+ "id": "urn:li:container:85534727d17f56b5996dabbf0fda04a2",
+ "urn": "urn:li:container:85534727d17f56b5996dabbf0fda04a2"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:2a7bcb9914ba7d78cb4ec2a15259fdaf",
+ "changeType": "UPSERT",
+ "aspectName": "containerProperties",
+ "aspect": {
+ "json": {
+ "customProperties": {},
+ "name": "mysql",
+ "qualifiedName": "mysql.mysql",
+ "description": "",
+ "env": "PROD"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:2a7bcb9914ba7d78cb4ec2a15259fdaf",
+ "changeType": "UPSERT",
+ "aspectName": "container",
+ "aspect": {
+ "json": {
+ "container": "urn:li:container:85534727d17f56b5996dabbf0fda04a2"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:2a7bcb9914ba7d78cb4ec2a15259fdaf",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio",
+ "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:2a7bcb9914ba7d78cb4ec2a15259fdaf",
+ "changeType": "UPSERT",
+ "aspectName": "subTypes",
+ "aspect": {
+ "json": {
+ "typeNames": [
+ "Dremio Folder"
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:2a7bcb9914ba7d78cb4ec2a15259fdaf",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:2a7bcb9914ba7d78cb4ec2a15259fdaf",
+ "changeType": "UPSERT",
+ "aspectName": "browsePathsV2",
+ "aspect": {
+ "json": {
+ "path": [
+ {
+ "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+ "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ },
+ {
+ "id": "Sources"
+ },
+ {
+ "id": "urn:li:container:85534727d17f56b5996dabbf0fda04a2",
+ "urn": "urn:li:container:85534727d17f56b5996dabbf0fda04a2"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:7a2433f85d4b3102265c99dfd7146d2f",
+ "changeType": "UPSERT",
+ "aspectName": "containerProperties",
+ "aspect": {
+ "json": {
+ "customProperties": {},
+ "name": "northwind",
+ "qualifiedName": "mysql.northwind",
+ "description": "",
+ "env": "PROD"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:7a2433f85d4b3102265c99dfd7146d2f",
+ "changeType": "UPSERT",
+ "aspectName": "container",
+ "aspect": {
+ "json": {
+ "container": "urn:li:container:85534727d17f56b5996dabbf0fda04a2"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:7a2433f85d4b3102265c99dfd7146d2f",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio",
+ "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:7a2433f85d4b3102265c99dfd7146d2f",
+ "changeType": "UPSERT",
+ "aspectName": "subTypes",
+ "aspect": {
+ "json": {
+ "typeNames": [
+ "Dremio Folder"
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:7a2433f85d4b3102265c99dfd7146d2f",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:7a2433f85d4b3102265c99dfd7146d2f",
+ "changeType": "UPSERT",
+ "aspectName": "browsePathsV2",
+ "aspect": {
+ "json": {
+ "path": [
+ {
+ "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+ "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ },
+ {
+ "id": "Sources"
+ },
+ {
+ "id": "urn:li:container:85534727d17f56b5996dabbf0fda04a2",
+ "urn": "urn:li:container:85534727d17f56b5996dabbf0fda04a2"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:ed04c22c5f26a90cdd328acf1d4c5791",
+ "changeType": "UPSERT",
+ "aspectName": "containerProperties",
+ "aspect": {
+ "json": {
+ "customProperties": {},
+ "name": "performance_schema",
+ "qualifiedName": "mysql.performance_schema",
+ "description": "",
+ "env": "PROD"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:ed04c22c5f26a90cdd328acf1d4c5791",
+ "changeType": "UPSERT",
+ "aspectName": "container",
+ "aspect": {
+ "json": {
+ "container": "urn:li:container:85534727d17f56b5996dabbf0fda04a2"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:ed04c22c5f26a90cdd328acf1d4c5791",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio",
+ "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:ed04c22c5f26a90cdd328acf1d4c5791",
+ "changeType": "UPSERT",
+ "aspectName": "subTypes",
+ "aspect": {
+ "json": {
+ "typeNames": [
+ "Dremio Folder"
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:ed04c22c5f26a90cdd328acf1d4c5791",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:ed04c22c5f26a90cdd328acf1d4c5791",
+ "changeType": "UPSERT",
+ "aspectName": "browsePathsV2",
+ "aspect": {
+ "json": {
+ "path": [
+ {
+ "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+ "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ },
+ {
+ "id": "Sources"
+ },
+ {
+ "id": "urn:li:container:85534727d17f56b5996dabbf0fda04a2",
+ "urn": "urn:li:container:85534727d17f56b5996dabbf0fda04a2"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:2b72d75cbfc34d112da228f20f924e9c",
+ "changeType": "UPSERT",
+ "aspectName": "containerProperties",
+ "aspect": {
+ "json": {
+ "customProperties": {},
+ "name": "test_cases",
+ "qualifiedName": "mysql.test_cases",
+ "description": "",
+ "env": "PROD"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:2b72d75cbfc34d112da228f20f924e9c",
+ "changeType": "UPSERT",
+ "aspectName": "container",
+ "aspect": {
+ "json": {
+ "container": "urn:li:container:85534727d17f56b5996dabbf0fda04a2"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:2b72d75cbfc34d112da228f20f924e9c",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio",
+ "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:2b72d75cbfc34d112da228f20f924e9c",
+ "changeType": "UPSERT",
+ "aspectName": "subTypes",
+ "aspect": {
+ "json": {
+ "typeNames": [
+ "Dremio Folder"
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:2b72d75cbfc34d112da228f20f924e9c",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:2b72d75cbfc34d112da228f20f924e9c",
+ "changeType": "UPSERT",
+ "aspectName": "browsePathsV2",
+ "aspect": {
+ "json": {
+ "path": [
+ {
+ "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+ "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ },
+ {
+ "id": "Sources"
+ },
+ {
+ "id": "urn:li:container:85534727d17f56b5996dabbf0fda04a2",
+ "urn": "urn:li:container:85534727d17f56b5996dabbf0fda04a2"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715",
+ "changeType": "UPSERT",
+ "aspectName": "containerProperties",
+ "aspect": {
+ "json": {
+ "customProperties": {},
+ "name": "samples.dremio.com",
+ "qualifiedName": "Samples.samples.dremio.com",
+ "description": "",
+ "env": "PROD"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715",
+ "changeType": "UPSERT",
+ "aspectName": "container",
+ "aspect": {
+ "json": {
+ "container": "urn:li:container:007d12a4a241c87924b54e1e35990234"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio",
+ "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715",
+ "changeType": "UPSERT",
+ "aspectName": "subTypes",
+ "aspect": {
+ "json": {
+ "typeNames": [
+ "Dremio Folder"
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715",
+ "changeType": "UPSERT",
+ "aspectName": "browsePathsV2",
+ "aspect": {
+ "json": {
+ "path": [
+ {
+ "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+ "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ },
+ {
+ "id": "Sources"
+ },
+ {
+ "id": "urn:li:container:007d12a4a241c87924b54e1e35990234",
+ "urn": "urn:li:container:007d12a4a241c87924b54e1e35990234"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:55c3b773b40bb744b4e5946db4e13455",
+ "changeType": "UPSERT",
+ "aspectName": "containerProperties",
+ "aspect": {
+ "json": {
+ "customProperties": {},
+ "name": "Dremio University",
+ "qualifiedName": "Samples.samples.dremio.com.Dremio University",
+ "description": "",
+ "env": "PROD"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:55c3b773b40bb744b4e5946db4e13455",
+ "changeType": "UPSERT",
+ "aspectName": "container",
+ "aspect": {
+ "json": {
+ "container": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:55c3b773b40bb744b4e5946db4e13455",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio",
+ "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:55c3b773b40bb744b4e5946db4e13455",
+ "changeType": "UPSERT",
+ "aspectName": "subTypes",
+ "aspect": {
+ "json": {
+ "typeNames": [
+ "Dremio Folder"
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:55c3b773b40bb744b4e5946db4e13455",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:55c3b773b40bb744b4e5946db4e13455",
+ "changeType": "UPSERT",
+ "aspectName": "browsePathsV2",
+ "aspect": {
+ "json": {
+ "path": [
+ {
+ "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+ "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ },
+ {
+ "id": "Sources"
+ },
+ {
+ "id": "urn:li:container:007d12a4a241c87924b54e1e35990234",
+ "urn": "urn:li:container:007d12a4a241c87924b54e1e35990234"
+ },
+ {
+ "id": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715",
+ "urn": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:df1130913ed9cfec6c7afb9fc58b9554",
+ "changeType": "UPSERT",
+ "aspectName": "containerProperties",
+ "aspect": {
+ "json": {
+ "customProperties": {},
+ "name": "tpcds_sf1000",
+ "qualifiedName": "Samples.samples.dremio.com.tpcds_sf1000",
+ "description": "",
+ "env": "PROD"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:df1130913ed9cfec6c7afb9fc58b9554",
+ "changeType": "UPSERT",
+ "aspectName": "container",
+ "aspect": {
+ "json": {
+ "container": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:df1130913ed9cfec6c7afb9fc58b9554",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio",
+ "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:df1130913ed9cfec6c7afb9fc58b9554",
+ "changeType": "UPSERT",
+ "aspectName": "subTypes",
+ "aspect": {
+ "json": {
+ "typeNames": [
+ "Dremio Folder"
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:df1130913ed9cfec6c7afb9fc58b9554",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:df1130913ed9cfec6c7afb9fc58b9554",
+ "changeType": "UPSERT",
+ "aspectName": "browsePathsV2",
+ "aspect": {
+ "json": {
+ "path": [
+ {
+ "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+ "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ },
+ {
+ "id": "Sources"
+ },
+ {
+ "id": "urn:li:container:007d12a4a241c87924b54e1e35990234",
+ "urn": "urn:li:container:007d12a4a241c87924b54e1e35990234"
+ },
+ {
+ "id": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715",
+ "urn": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:2ebbe0028345d0b8d147aed919b1024c",
+ "changeType": "UPSERT",
+ "aspectName": "containerProperties",
+ "aspect": {
+ "json": {
+ "customProperties": {},
+ "name": "catalog_page",
+ "qualifiedName": "Samples.samples.dremio.com.tpcds_sf1000.catalog_page",
+ "description": "",
+ "env": "PROD"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:2ebbe0028345d0b8d147aed919b1024c",
+ "changeType": "UPSERT",
+ "aspectName": "container",
+ "aspect": {
+ "json": {
+ "container": "urn:li:container:df1130913ed9cfec6c7afb9fc58b9554"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:2ebbe0028345d0b8d147aed919b1024c",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio",
+ "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:2ebbe0028345d0b8d147aed919b1024c",
+ "changeType": "UPSERT",
+ "aspectName": "subTypes",
+ "aspect": {
+ "json": {
+ "typeNames": [
+ "Dremio Folder"
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:2ebbe0028345d0b8d147aed919b1024c",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:2ebbe0028345d0b8d147aed919b1024c",
+ "changeType": "UPSERT",
+ "aspectName": "browsePathsV2",
+ "aspect": {
+ "json": {
+ "path": [
+ {
+ "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+ "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ },
+ {
+ "id": "Sources"
+ },
+ {
+ "id": "urn:li:container:007d12a4a241c87924b54e1e35990234",
+ "urn": "urn:li:container:007d12a4a241c87924b54e1e35990234"
+ },
+ {
+ "id": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715",
+ "urn": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715"
+ },
+ {
+ "id": "urn:li:container:df1130913ed9cfec6c7afb9fc58b9554",
+ "urn": "urn:li:container:df1130913ed9cfec6c7afb9fc58b9554"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:25745bd0b919d9f4e402df43a1ee0ca8",
+ "changeType": "UPSERT",
+ "aspectName": "containerProperties",
+ "aspect": {
+ "json": {
+ "customProperties": {},
+ "name": "1ab266d5-18eb-4780-711d-0fa337fa6c00",
+ "qualifiedName": "Samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00",
+ "description": "",
+ "env": "PROD"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:25745bd0b919d9f4e402df43a1ee0ca8",
+ "changeType": "UPSERT",
+ "aspectName": "container",
+ "aspect": {
+ "json": {
+ "container": "urn:li:container:2ebbe0028345d0b8d147aed919b1024c"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:25745bd0b919d9f4e402df43a1ee0ca8",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio",
+ "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:25745bd0b919d9f4e402df43a1ee0ca8",
+ "changeType": "UPSERT",
+ "aspectName": "subTypes",
+ "aspect": {
+ "json": {
+ "typeNames": [
+ "Dremio Folder"
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:25745bd0b919d9f4e402df43a1ee0ca8",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:25745bd0b919d9f4e402df43a1ee0ca8",
+ "changeType": "UPSERT",
+ "aspectName": "browsePathsV2",
+ "aspect": {
+ "json": {
+ "path": [
+ {
+ "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+ "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ },
+ {
+ "id": "Sources"
+ },
+ {
+ "id": "urn:li:container:007d12a4a241c87924b54e1e35990234",
+ "urn": "urn:li:container:007d12a4a241c87924b54e1e35990234"
+ },
+ {
+ "id": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715",
+ "urn": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715"
+ },
+ {
+ "id": "urn:li:container:df1130913ed9cfec6c7afb9fc58b9554",
+ "urn": "urn:li:container:df1130913ed9cfec6c7afb9fc58b9554"
+ },
+ {
+ "id": "urn:li:container:2ebbe0028345d0b8d147aed919b1024c",
+ "urn": "urn:li:container:2ebbe0028345d0b8d147aed919b1024c"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.warehouse,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "datasetProperties",
+ "aspect": {
+ "json": {
+ "customProperties": {},
+ "externalUrl": "http://localhost:9047/space/\"space\"/\"warehouse\"",
+ "name": "warehouse",
+ "qualifiedName": "space.warehouse",
+ "description": "",
+ "created": {
+ "time": 0
+ },
+ "tags": []
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.warehouse,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "subTypes",
+ "aspect": {
+ "json": {
+ "typeNames": [
+ "View"
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.warehouse,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio",
+ "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.warehouse,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "container",
+ "aspect": {
+ "json": {
+ "container": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.warehouse,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "viewProperties",
+ "aspect": {
+ "json": {
+ "materialized": false,
+ "viewLogic": "SELECT * from Samples.\"samples.dremio.com\".\"NYC-weather.csv\"",
+ "viewLanguage": "SQL"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.warehouse,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "schemaMetadata",
+ "aspect": {
+ "json": {
+ "schemaName": "space.warehouse",
+ "platform": "urn:li:dataPlatform:dremio",
+ "version": 0,
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "lastModified": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "hash": "",
+ "platformSchema": {
+ "com.linkedin.schema.MySqlDDL": {
+ "tableSchema": ""
+ }
+ },
+ "fields": [
+ {
+ "fieldPath": "I",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "A",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "B",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "C",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "D",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "E",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "F",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "G",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "H",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.warehouse,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.warehouse,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "browsePathsV2",
+ "aspect": {
+ "json": {
+ "path": [
+ {
+ "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+ "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ },
+ {
+ "id": "Spaces"
+ },
+ {
+ "id": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7",
+ "urn": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.customers,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "datasetProperties",
+ "aspect": {
+ "json": {
+ "customProperties": {},
+ "externalUrl": "http://localhost:9047/space/\"space\"/\"test_folder\".\"customers\"",
+ "name": "customers",
+ "qualifiedName": "space.test_folder.customers",
+ "description": "",
+ "created": {
+ "time": 0
+ },
+ "tags": []
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.customers,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "subTypes",
+ "aspect": {
+ "json": {
+ "typeNames": [
+ "View"
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.customers,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio",
+ "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.customers,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "container",
+ "aspect": {
+ "json": {
+ "container": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.customers,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "viewProperties",
+ "aspect": {
+ "json": {
+ "materialized": false,
+ "viewLogic": "SELECT * FROM mysql.northwind.customers",
+ "viewLanguage": "SQL"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.customers,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "schemaMetadata",
+ "aspect": {
+ "json": {
+ "schemaName": "space.test_folder.customers",
+ "platform": "urn:li:dataPlatform:dremio",
+ "version": 0,
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "lastModified": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "hash": "",
+ "platformSchema": {
+ "com.linkedin.schema.MySqlDDL": {
+ "tableSchema": ""
+ }
+ },
+ "fields": [
+ {
+ "fieldPath": "first_name",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "last_name",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "id",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.NumberType": {}
+ }
+ },
+ "nativeDataType": "integer(32)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "company",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "priority",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.NumberType": {}
+ }
+ },
+ "nativeDataType": "float(24)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "email_address",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.customers,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.customers,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "browsePathsV2",
+ "aspect": {
+ "json": {
+ "path": [
+ {
+ "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+ "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ },
+ {
+ "id": "Spaces"
+ },
+ {
+ "id": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7",
+ "urn": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7"
+ },
+ {
+ "id": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5",
+ "urn": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_aspect,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "datasetProperties",
+ "aspect": {
+ "json": {
+ "customProperties": {},
+ "externalUrl": "http://localhost:9047/space/\"space\"/\"test_folder\".\"metadata_aspect\"",
+ "name": "metadata_aspect",
+ "qualifiedName": "space.test_folder.metadata_aspect",
+ "description": "",
+ "created": {
+ "time": 0
+ },
+ "tags": []
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_aspect,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "subTypes",
+ "aspect": {
+ "json": {
+ "typeNames": [
+ "View"
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_aspect,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio",
+ "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_aspect,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "container",
+ "aspect": {
+ "json": {
+ "container": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_aspect,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "viewProperties",
+ "aspect": {
+ "json": {
+ "materialized": false,
+ "viewLogic": "SELECT * FROM mysql.metagalaxy.metadata_aspect",
+ "viewLanguage": "SQL"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_aspect,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "schemaMetadata",
+ "aspect": {
+ "json": {
+ "schemaName": "space.test_folder.metadata_aspect",
+ "platform": "urn:li:dataPlatform:dremio",
+ "version": 0,
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "lastModified": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "hash": "",
+ "platformSchema": {
+ "com.linkedin.schema.MySqlDDL": {
+ "tableSchema": ""
+ }
+ },
+ "fields": [
+ {
+ "fieldPath": "urn",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "createdby",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "createdon",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.DateType": {}
+ }
+ },
+ "nativeDataType": "timestamp(23)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "metadata",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "version",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.NumberType": {}
+ }
+ },
+ "nativeDataType": "bigint(64)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "createdfor",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "aspect",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_aspect,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_aspect,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "browsePathsV2",
+ "aspect": {
+ "json": {
+ "path": [
+ {
+ "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+ "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ },
+ {
+ "id": "Spaces"
+ },
+ {
+ "id": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7",
+ "urn": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7"
+ },
+ {
+ "id": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5",
+ "urn": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "datasetProperties",
+ "aspect": {
+ "json": {
+ "customProperties": {},
+ "externalUrl": "http://localhost:9047/space/\"space\"/\"test_folder\".\"metadata_index\"",
+ "name": "metadata_index",
+ "qualifiedName": "space.test_folder.metadata_index",
+ "description": "",
+ "created": {
+ "time": 0
+ },
+ "tags": []
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "subTypes",
+ "aspect": {
+ "json": {
+ "typeNames": [
+ "View"
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio",
+ "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "container",
+ "aspect": {
+ "json": {
+ "container": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "viewProperties",
+ "aspect": {
+ "json": {
+ "materialized": false,
+ "viewLogic": "SELECT * FROM mysql.metagalaxy.metadata_index",
+ "viewLanguage": "SQL"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "schemaMetadata",
+ "aspect": {
+ "json": {
+ "schemaName": "space.test_folder.metadata_index",
+ "platform": "urn:li:dataPlatform:dremio",
+ "version": 0,
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "lastModified": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "hash": "",
+ "platformSchema": {
+ "com.linkedin.schema.MySqlDDL": {
+ "tableSchema": ""
+ }
+ },
+ "fields": [
+ {
+ "fieldPath": "id",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.NumberType": {}
+ }
+ },
+ "nativeDataType": "bigint(64)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "urn",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "aspect",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "path",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "longVal",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.NumberType": {}
+ }
+ },
+ "nativeDataType": "bigint(64)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "stringVal",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "doubleVal",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.NumberType": {}
+ }
+ },
+ "nativeDataType": "double(53)",
+ "recursive": false,
+ "isPartOfKey": false
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "browsePathsV2",
+ "aspect": {
+ "json": {
+ "path": [
+ {
+ "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+ "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ },
+ {
+ "id": "Spaces"
+ },
+ {
+ "id": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7",
+ "urn": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7"
+ },
+ {
+ "id": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5",
+ "urn": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index_view,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "datasetProperties",
+ "aspect": {
+ "json": {
+ "customProperties": {},
+ "externalUrl": "http://localhost:9047/space/\"space\"/\"test_folder\".\"metadata_index_view\"",
+ "name": "metadata_index_view",
+ "qualifiedName": "space.test_folder.metadata_index_view",
+ "description": "",
+ "created": {
+ "time": 0
+ },
+ "tags": []
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index_view,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "subTypes",
+ "aspect": {
+ "json": {
+ "typeNames": [
+ "View"
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index_view,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio",
+ "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index_view,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "container",
+ "aspect": {
+ "json": {
+ "container": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index_view,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "viewProperties",
+ "aspect": {
+ "json": {
+ "materialized": false,
+ "viewLogic": "SELECT * FROM mysql.metagalaxy.metadata_index_view",
+ "viewLanguage": "SQL"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index_view,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "schemaMetadata",
+ "aspect": {
+ "json": {
+ "schemaName": "space.test_folder.metadata_index_view",
+ "platform": "urn:li:dataPlatform:dremio",
+ "version": 0,
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "lastModified": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "hash": "",
+ "platformSchema": {
+ "com.linkedin.schema.MySqlDDL": {
+ "tableSchema": ""
+ }
+ },
+ "fields": [
+ {
+ "fieldPath": "doubleVal",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.NumberType": {}
+ }
+ },
+ "nativeDataType": "double(53)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "path",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "urn",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "id",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.NumberType": {}
+ }
+ },
+ "nativeDataType": "bigint(64)",
+ "recursive": false,
+ "isPartOfKey": false
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index_view,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index_view,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "browsePathsV2",
+ "aspect": {
+ "json": {
+ "path": [
+ {
+ "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+ "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ },
+ {
+ "id": "Spaces"
+ },
+ {
+ "id": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7",
+ "urn": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7"
+ },
+ {
+ "id": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5",
+ "urn": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.orders,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "datasetProperties",
+ "aspect": {
+ "json": {
+ "customProperties": {},
+ "externalUrl": "http://localhost:9047/space/\"space\"/\"test_folder\".\"orders\"",
+ "name": "orders",
+ "qualifiedName": "space.test_folder.orders",
+ "description": "",
+ "created": {
+ "time": 0
+ },
+ "tags": []
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.orders,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "subTypes",
+ "aspect": {
+ "json": {
+ "typeNames": [
+ "View"
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.orders,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio",
+ "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.orders,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "container",
+ "aspect": {
+ "json": {
+ "container": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.orders,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "viewProperties",
+ "aspect": {
+ "json": {
+ "materialized": false,
+ "viewLogic": "SELECT * FROM mysql.northwind.orders",
+ "viewLanguage": "SQL"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.orders,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "schemaMetadata",
+ "aspect": {
+ "json": {
+ "schemaName": "space.test_folder.orders",
+ "platform": "urn:li:dataPlatform:dremio",
+ "version": 0,
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "lastModified": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "hash": "",
+ "platformSchema": {
+ "com.linkedin.schema.MySqlDDL": {
+ "tableSchema": ""
+ }
+ },
+ "fields": [
+ {
+ "fieldPath": "customer_id",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.NumberType": {}
+ }
+ },
+ "nativeDataType": "integer(32)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "description",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "id",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.NumberType": {}
+ }
+ },
+ "nativeDataType": "integer(32)",
+ "recursive": false,
+ "isPartOfKey": false
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.orders,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.orders,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "browsePathsV2",
+ "aspect": {
+ "json": {
+ "path": [
+ {
+ "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+ "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ },
+ {
+ "id": "Spaces"
+ },
+ {
+ "id": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7",
+ "urn": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7"
+ },
+ {
+ "id": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5",
+ "urn": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "datasetProperties",
+ "aspect": {
+ "json": {
+ "customProperties": {},
+ "externalUrl": "http://localhost:9047/space/\"space\"/\"test_folder\".\"raw\"",
+ "name": "raw",
+ "qualifiedName": "space.test_folder.raw",
+ "description": "",
+ "created": {
+ "time": 0
+ },
+ "tags": []
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "subTypes",
+ "aspect": {
+ "json": {
+ "typeNames": [
+ "View"
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio",
+ "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "container",
+ "aspect": {
+ "json": {
+ "container": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "viewProperties",
+ "aspect": {
+ "json": {
+ "materialized": false,
+ "viewLogic": "SELECT * FROM s3.warehouse",
+ "viewLanguage": "SQL"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "schemaMetadata",
+ "aspect": {
+ "json": {
+ "schemaName": "space.test_folder.raw",
+ "platform": "urn:li:dataPlatform:dremio",
+ "version": 0,
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "lastModified": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "hash": "",
+ "platformSchema": {
+ "com.linkedin.schema.MySqlDDL": {
+ "tableSchema": ""
+ }
+ },
+ "fields": [
+ {
+ "fieldPath": "salary",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.NumberType": {}
+ }
+ },
+ "nativeDataType": "bigint(64)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "age",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.NumberType": {}
+ }
+ },
+ "nativeDataType": "bigint(64)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "name",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "id",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.NumberType": {}
+ }
+ },
+ "nativeDataType": "bigint(64)",
+ "recursive": false,
+ "isPartOfKey": false
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "browsePathsV2",
+ "aspect": {
+ "json": {
+ "path": [
+ {
+ "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+ "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ },
+ {
+ "id": "Spaces"
+ },
+ {
+ "id": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7",
+ "urn": "urn:li:container:4d7b71bc17cedc7e6e894cbb2bfe10f7"
+ },
+ {
+ "id": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5",
+ "urn": "urn:li:container:17f24d8e67c4130e5309c70421e90fd5"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "datasetProperties",
+ "aspect": {
+ "json": {
+ "customProperties": {},
+ "externalUrl": "http://localhost:9047/source/\"s3\"/\"warehouse\"",
+ "name": "warehouse",
+ "qualifiedName": "s3.warehouse",
+ "description": "",
+ "created": {
+ "time": 0
+ },
+ "tags": []
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "subTypes",
+ "aspect": {
+ "json": {
+ "typeNames": [
+ "Table"
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio",
+ "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "container",
+ "aspect": {
+ "json": {
+ "container": "urn:li:container:5fc7ba11cb45461f55fb834da2141c46"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "schemaMetadata",
+ "aspect": {
+ "json": {
+ "schemaName": "s3.warehouse",
+ "platform": "urn:li:dataPlatform:dremio",
+ "version": 0,
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "lastModified": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "hash": "",
+ "platformSchema": {
+ "com.linkedin.schema.MySqlDDL": {
+ "tableSchema": ""
+ }
+ },
+ "fields": [
+ {
+ "fieldPath": "id",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.NumberType": {}
+ }
+ },
+ "nativeDataType": "bigint(64)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "name",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "age",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.NumberType": {}
+ }
+ },
+ "nativeDataType": "bigint(64)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "salary",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.NumberType": {}
+ }
+ },
+ "nativeDataType": "bigint(64)",
+ "recursive": false,
+ "isPartOfKey": false
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "upstreamLineage",
+ "aspect": {
+ "json": {
+ "upstreams": [
+ {
+ "auditStamp": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples.//warehouse,PROD)",
+ "type": "COPY"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "browsePathsV2",
+ "aspect": {
+ "json": {
+ "path": [
+ {
+ "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+ "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ },
+ {
+ "id": "Sources"
+ },
+ {
+ "id": "urn:li:container:5fc7ba11cb45461f55fb834da2141c46",
+ "urn": "urn:li:container:5fc7ba11cb45461f55fb834da2141c46"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_aspect,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "datasetProperties",
+ "aspect": {
+ "json": {
+ "customProperties": {},
+ "externalUrl": "http://localhost:9047/source/\"mysql\"/\"metagalaxy\".\"metadata_aspect\"",
+ "name": "metadata_aspect",
+ "qualifiedName": "mysql.metagalaxy.metadata_aspect",
+ "description": "",
+ "created": {
+ "time": 0
+ },
+ "tags": []
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_aspect,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "subTypes",
+ "aspect": {
+ "json": {
+ "typeNames": [
+ "Table"
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_aspect,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio",
+ "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_aspect,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "container",
+ "aspect": {
+ "json": {
+ "container": "urn:li:container:67653e01aa4da4fcfd6675a591700e89"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_aspect,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "schemaMetadata",
+ "aspect": {
+ "json": {
+ "schemaName": "mysql.metagalaxy.metadata_aspect",
+ "platform": "urn:li:dataPlatform:dremio",
+ "version": 0,
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "lastModified": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "hash": "",
+ "platformSchema": {
+ "com.linkedin.schema.MySqlDDL": {
+ "tableSchema": ""
+ }
+ },
+ "fields": [
+ {
+ "fieldPath": "metadata",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "createdon",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.DateType": {}
+ }
+ },
+ "nativeDataType": "timestamp(23)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "createdby",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "createdfor",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "urn",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "aspect",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "version",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.NumberType": {}
+ }
+ },
+ "nativeDataType": "bigint(64)",
+ "recursive": false,
+ "isPartOfKey": false
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_aspect,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_aspect,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "upstreamLineage",
+ "aspect": {
+ "json": {
+ "upstreams": [
+ {
+ "auditStamp": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_aspect,PROD)",
+ "type": "COPY"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_aspect,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "browsePathsV2",
+ "aspect": {
+ "json": {
+ "path": [
+ {
+ "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+ "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ },
+ {
+ "id": "Sources"
+ },
+ {
+ "id": "urn:li:container:85534727d17f56b5996dabbf0fda04a2",
+ "urn": "urn:li:container:85534727d17f56b5996dabbf0fda04a2"
+ },
+ {
+ "id": "urn:li:container:67653e01aa4da4fcfd6675a591700e89",
+ "urn": "urn:li:container:67653e01aa4da4fcfd6675a591700e89"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "datasetProperties",
+ "aspect": {
+ "json": {
+ "customProperties": {},
+ "externalUrl": "http://localhost:9047/source/\"mysql\"/\"metagalaxy\".\"metadata_index\"",
+ "name": "metadata_index",
+ "qualifiedName": "mysql.metagalaxy.metadata_index",
+ "description": "",
+ "created": {
+ "time": 0
+ },
+ "tags": []
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "subTypes",
+ "aspect": {
+ "json": {
+ "typeNames": [
+ "Table"
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio",
+ "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "container",
+ "aspect": {
+ "json": {
+ "container": "urn:li:container:67653e01aa4da4fcfd6675a591700e89"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "schemaMetadata",
+ "aspect": {
+ "json": {
+ "schemaName": "mysql.metagalaxy.metadata_index",
+ "platform": "urn:li:dataPlatform:dremio",
+ "version": 0,
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "lastModified": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "hash": "",
+ "platformSchema": {
+ "com.linkedin.schema.MySqlDDL": {
+ "tableSchema": ""
+ }
+ },
+ "fields": [
+ {
+ "fieldPath": "path",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "aspect",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "urn",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "id",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.NumberType": {}
+ }
+ },
+ "nativeDataType": "bigint(64)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "stringVal",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "longVal",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.NumberType": {}
+ }
+ },
+ "nativeDataType": "bigint(64)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "doubleVal",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.NumberType": {}
+ }
+ },
+ "nativeDataType": "double(53)",
+ "recursive": false,
+ "isPartOfKey": false
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "upstreamLineage",
+ "aspect": {
+ "json": {
+ "upstreams": [
+ {
+ "auditStamp": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_index,PROD)",
+ "type": "COPY"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "browsePathsV2",
+ "aspect": {
+ "json": {
+ "path": [
+ {
+ "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+ "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ },
+ {
+ "id": "Sources"
+ },
+ {
+ "id": "urn:li:container:85534727d17f56b5996dabbf0fda04a2",
+ "urn": "urn:li:container:85534727d17f56b5996dabbf0fda04a2"
+ },
+ {
+ "id": "urn:li:container:67653e01aa4da4fcfd6675a591700e89",
+ "urn": "urn:li:container:67653e01aa4da4fcfd6675a591700e89"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index_view,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "datasetProperties",
+ "aspect": {
+ "json": {
+ "customProperties": {},
+ "externalUrl": "http://localhost:9047/source/\"mysql\"/\"metagalaxy\".\"metadata_index_view\"",
+ "name": "metadata_index_view",
+ "qualifiedName": "mysql.metagalaxy.metadata_index_view",
+ "description": "",
+ "created": {
+ "time": 0
+ },
+ "tags": []
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index_view,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "subTypes",
+ "aspect": {
+ "json": {
+ "typeNames": [
+ "Table"
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index_view,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio",
+ "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index_view,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "container",
+ "aspect": {
+ "json": {
+ "container": "urn:li:container:67653e01aa4da4fcfd6675a591700e89"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index_view,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "schemaMetadata",
+ "aspect": {
+ "json": {
+ "schemaName": "mysql.metagalaxy.metadata_index_view",
+ "platform": "urn:li:dataPlatform:dremio",
+ "version": 0,
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "lastModified": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "hash": "",
+ "platformSchema": {
+ "com.linkedin.schema.MySqlDDL": {
+ "tableSchema": ""
+ }
+ },
+ "fields": [
+ {
+ "fieldPath": "id",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.NumberType": {}
+ }
+ },
+ "nativeDataType": "bigint(64)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "urn",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "path",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "doubleVal",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.NumberType": {}
+ }
+ },
+ "nativeDataType": "double(53)",
+ "recursive": false,
+ "isPartOfKey": false
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index_view,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index_view,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "upstreamLineage",
+ "aspect": {
+ "json": {
+ "upstreams": [
+ {
+ "auditStamp": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_index_view,PROD)",
+ "type": "COPY"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index_view,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "browsePathsV2",
+ "aspect": {
+ "json": {
+ "path": [
+ {
+ "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+ "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ },
+ {
+ "id": "Sources"
+ },
+ {
+ "id": "urn:li:container:85534727d17f56b5996dabbf0fda04a2",
+ "urn": "urn:li:container:85534727d17f56b5996dabbf0fda04a2"
+ },
+ {
+ "id": "urn:li:container:67653e01aa4da4fcfd6675a591700e89",
+ "urn": "urn:li:container:67653e01aa4da4fcfd6675a591700e89"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.customers,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "datasetProperties",
+ "aspect": {
+ "json": {
+ "customProperties": {},
+ "externalUrl": "http://localhost:9047/source/\"mysql\"/\"northwind\".\"customers\"",
+ "name": "customers",
+ "qualifiedName": "mysql.northwind.customers",
+ "description": "",
+ "created": {
+ "time": 0
+ },
+ "tags": []
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.customers,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "subTypes",
+ "aspect": {
+ "json": {
+ "typeNames": [
+ "Table"
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.customers,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio",
+ "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.customers,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "container",
+ "aspect": {
+ "json": {
+ "container": "urn:li:container:7a2433f85d4b3102265c99dfd7146d2f"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.customers,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "schemaMetadata",
+ "aspect": {
+ "json": {
+ "schemaName": "mysql.northwind.customers",
+ "platform": "urn:li:dataPlatform:dremio",
+ "version": 0,
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "lastModified": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "hash": "",
+ "platformSchema": {
+ "com.linkedin.schema.MySqlDDL": {
+ "tableSchema": ""
+ }
+ },
+ "fields": [
+ {
+ "fieldPath": "priority",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.NumberType": {}
+ }
+ },
+ "nativeDataType": "float(24)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "email_address",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "first_name",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "company",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "id",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.NumberType": {}
+ }
+ },
+ "nativeDataType": "integer(32)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "last_name",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.customers,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.customers,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "upstreamLineage",
+ "aspect": {
+ "json": {
+ "upstreams": [
+ {
+ "auditStamp": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.northwind.customers,PROD)",
+ "type": "COPY"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.customers,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "browsePathsV2",
+ "aspect": {
+ "json": {
+ "path": [
+ {
+ "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+ "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ },
+ {
+ "id": "Sources"
+ },
+ {
+ "id": "urn:li:container:85534727d17f56b5996dabbf0fda04a2",
+ "urn": "urn:li:container:85534727d17f56b5996dabbf0fda04a2"
+ },
+ {
+ "id": "urn:li:container:7a2433f85d4b3102265c99dfd7146d2f",
+ "urn": "urn:li:container:7a2433f85d4b3102265c99dfd7146d2f"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.orders,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "datasetProperties",
+ "aspect": {
+ "json": {
+ "customProperties": {},
+ "externalUrl": "http://localhost:9047/source/\"mysql\"/\"northwind\".\"orders\"",
+ "name": "orders",
+ "qualifiedName": "mysql.northwind.orders",
+ "description": "",
+ "created": {
+ "time": 0
+ },
+ "tags": []
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.orders,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "subTypes",
+ "aspect": {
+ "json": {
+ "typeNames": [
+ "Table"
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.orders,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio",
+ "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.orders,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "container",
+ "aspect": {
+ "json": {
+ "container": "urn:li:container:7a2433f85d4b3102265c99dfd7146d2f"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.orders,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "schemaMetadata",
+ "aspect": {
+ "json": {
+ "schemaName": "mysql.northwind.orders",
+ "platform": "urn:li:dataPlatform:dremio",
+ "version": 0,
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "lastModified": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "hash": "",
+ "platformSchema": {
+ "com.linkedin.schema.MySqlDDL": {
+ "tableSchema": ""
+ }
+ },
+ "fields": [
+ {
+ "fieldPath": "description",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "customer_id",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.NumberType": {}
+ }
+ },
+ "nativeDataType": "integer(32)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "id",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.NumberType": {}
+ }
+ },
+ "nativeDataType": "integer(32)",
+ "recursive": false,
+ "isPartOfKey": false
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.orders,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.orders,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "upstreamLineage",
+ "aspect": {
+ "json": {
+ "upstreams": [
+ {
+ "auditStamp": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.northwind.orders,PROD)",
+ "type": "COPY"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.orders,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "browsePathsV2",
+ "aspect": {
+ "json": {
+ "path": [
+ {
+ "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+ "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ },
+ {
+ "id": "Sources"
+ },
+ {
+ "id": "urn:li:container:85534727d17f56b5996dabbf0fda04a2",
+ "urn": "urn:li:container:85534727d17f56b5996dabbf0fda04a2"
+ },
+ {
+ "id": "urn:li:container:7a2433f85d4b3102265c99dfd7146d2f",
+ "urn": "urn:li:container:7a2433f85d4b3102265c99dfd7146d2f"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "datasetProperties",
+ "aspect": {
+ "json": {
+ "customProperties": {},
+ "externalUrl": "http://localhost:9047/source/\"Samples\"/\"samples.dremio.com\".\"NYC-weather.csv\"",
+ "name": "NYC-weather.csv",
+ "qualifiedName": "Samples.samples.dremio.com.NYC-weather.csv",
+ "description": "",
+ "created": {
+ "time": 0
+ },
+ "tags": []
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "subTypes",
+ "aspect": {
+ "json": {
+ "typeNames": [
+ "Table"
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio",
+ "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "container",
+ "aspect": {
+ "json": {
+ "container": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "schemaMetadata",
+ "aspect": {
+ "json": {
+ "schemaName": "Samples.samples.dremio.com.NYC-weather.csv",
+ "platform": "urn:li:dataPlatform:dremio",
+ "version": 0,
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "lastModified": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "hash": "",
+ "platformSchema": {
+ "com.linkedin.schema.MySqlDDL": {
+ "tableSchema": ""
+ }
+ },
+ "fields": [
+ {
+ "fieldPath": "E",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "G",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "H",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "I",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "F",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "A",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "B",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "C",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "D",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "upstreamLineage",
+ "aspect": {
+ "json": {
+ "upstreams": [
+ {
+ "auditStamp": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD)",
+ "type": "COPY"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "browsePathsV2",
+ "aspect": {
+ "json": {
+ "path": [
+ {
+ "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+ "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ },
+ {
+ "id": "Sources"
+ },
+ {
+ "id": "urn:li:container:007d12a4a241c87924b54e1e35990234",
+ "urn": "urn:li:container:007d12a4a241c87924b54e1e35990234"
+ },
+ {
+ "id": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715",
+ "urn": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "datasetProperties",
+ "aspect": {
+ "json": {
+ "customProperties": {},
+ "externalUrl": "http://localhost:9047/source/\"Samples\"/\"samples.dremio.com\".\"Dremio University\".\"googleplaystore.csv\"",
+ "name": "googleplaystore.csv",
+ "qualifiedName": "Samples.samples.dremio.com.Dremio University.googleplaystore.csv",
+ "description": "",
+ "created": {
+ "time": 0
+ },
+ "tags": []
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "subTypes",
+ "aspect": {
+ "json": {
+ "typeNames": [
+ "Table"
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio",
+ "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "container",
+ "aspect": {
+ "json": {
+ "container": "urn:li:container:55c3b773b40bb744b4e5946db4e13455"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "schemaMetadata",
+ "aspect": {
+ "json": {
+ "schemaName": "Samples.samples.dremio.com.Dremio University.googleplaystore.csv",
+ "platform": "urn:li:dataPlatform:dremio",
+ "version": 0,
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "lastModified": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "hash": "",
+ "platformSchema": {
+ "com.linkedin.schema.MySqlDDL": {
+ "tableSchema": ""
+ }
+ },
+ "fields": [
+ {
+ "fieldPath": "F",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "G",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "H",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "I",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "J",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "K",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "L",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "M",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "A",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "B",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "C",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "D",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "E",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "upstreamLineage",
+ "aspect": {
+ "json": {
+ "upstreams": [
+ {
+ "auditStamp": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD)",
+ "type": "COPY"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "browsePathsV2",
+ "aspect": {
+ "json": {
+ "path": [
+ {
+ "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+ "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ },
+ {
+ "id": "Sources"
+ },
+ {
+ "id": "urn:li:container:007d12a4a241c87924b54e1e35990234",
+ "urn": "urn:li:container:007d12a4a241c87924b54e1e35990234"
+ },
+ {
+ "id": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715",
+ "urn": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715"
+ },
+ {
+ "id": "urn:li:container:55c3b773b40bb744b4e5946db4e13455",
+ "urn": "urn:li:container:55c3b773b40bb744b4e5946db4e13455"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "datasetProperties",
+ "aspect": {
+ "json": {
+ "customProperties": {},
+ "externalUrl": "http://localhost:9047/source/\"Samples\"/\"samples.dremio.com\".\"Dremio University\".\"oracle-departments.xlsx\"",
+ "name": "oracle-departments.xlsx",
+ "qualifiedName": "Samples.samples.dremio.com.Dremio University.oracle-departments.xlsx",
+ "description": "",
+ "created": {
+ "time": 0
+ },
+ "tags": []
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "subTypes",
+ "aspect": {
+ "json": {
+ "typeNames": [
+ "Table"
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio",
+ "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "container",
+ "aspect": {
+ "json": {
+ "container": "urn:li:container:55c3b773b40bb744b4e5946db4e13455"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "schemaMetadata",
+ "aspect": {
+ "json": {
+ "schemaName": "Samples.samples.dremio.com.Dremio University.oracle-departments.xlsx",
+ "platform": "urn:li:dataPlatform:dremio",
+ "version": 0,
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "lastModified": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "hash": "",
+ "platformSchema": {
+ "com.linkedin.schema.MySqlDDL": {
+ "tableSchema": ""
+ }
+ },
+ "fields": [
+ {
+ "fieldPath": "DEPARTMENT_ID",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.NumberType": {}
+ }
+ },
+ "nativeDataType": "double(53)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "DEPARTMENT_NAME",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "MANAGER_ID",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.NumberType": {}
+ }
+ },
+ "nativeDataType": "double(53)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "LOCATION_ID",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.NumberType": {}
+ }
+ },
+ "nativeDataType": "double(53)",
+ "recursive": false,
+ "isPartOfKey": false
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "upstreamLineage",
+ "aspect": {
+ "json": {
+ "upstreams": [
+ {
+ "auditStamp": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD)",
+ "type": "COPY"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "browsePathsV2",
+ "aspect": {
+ "json": {
+ "path": [
+ {
+ "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+ "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ },
+ {
+ "id": "Sources"
+ },
+ {
+ "id": "urn:li:container:007d12a4a241c87924b54e1e35990234",
+ "urn": "urn:li:container:007d12a4a241c87924b54e1e35990234"
+ },
+ {
+ "id": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715",
+ "urn": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715"
+ },
+ {
+ "id": "urn:li:container:55c3b773b40bb744b4e5946db4e13455",
+ "urn": "urn:li:container:55c3b773b40bb744b4e5946db4e13455"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "datasetProperties",
+ "aspect": {
+ "json": {
+ "customProperties": {},
+ "externalUrl": "http://localhost:9047/source/\"Samples\"/\"samples.dremio.com\".\"tpcds_sf1000\".\"catalog_page\".\"1ab266d5-18eb-4780-711d-0fa337fa6c00\".\"0_0_0.parquet\"",
+ "name": "0_0_0.parquet",
+ "qualifiedName": "Samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet",
+ "description": "",
+ "created": {
+ "time": 0
+ },
+ "tags": []
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "subTypes",
+ "aspect": {
+ "json": {
+ "typeNames": [
+ "Table"
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio",
+ "instance": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "container",
+ "aspect": {
+ "json": {
+ "container": "urn:li:container:25745bd0b919d9f4e402df43a1ee0ca8"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "schemaMetadata",
+ "aspect": {
+ "json": {
+ "schemaName": "Samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet",
+ "platform": "urn:li:dataPlatform:dremio",
+ "version": 0,
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "lastModified": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "hash": "",
+ "platformSchema": {
+ "com.linkedin.schema.MySqlDDL": {
+ "tableSchema": ""
+ }
+ },
+ "fields": [
+ {
+ "fieldPath": "cp_catalog_page_sk",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.NumberType": {}
+ }
+ },
+ "nativeDataType": "bigint(64)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "cp_catalog_page_id",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "cp_start_date_sk",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.NumberType": {}
+ }
+ },
+ "nativeDataType": "bigint(64)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "cp_end_date_sk",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.NumberType": {}
+ }
+ },
+ "nativeDataType": "bigint(64)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "cp_department",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "cp_catalog_number",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.NumberType": {}
+ }
+ },
+ "nativeDataType": "bigint(64)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "cp_catalog_page_number",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.NumberType": {}
+ }
+ },
+ "nativeDataType": "bigint(64)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "cp_description",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "cp_type",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "upstreamLineage",
+ "aspect": {
+ "json": {
+ "upstreams": [
+ {
+ "auditStamp": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD)",
+ "type": "COPY"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "browsePathsV2",
+ "aspect": {
+ "json": {
+ "path": [
+ {
+ "id": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)",
+ "urn": "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ },
+ {
+ "id": "Sources"
+ },
+ {
+ "id": "urn:li:container:007d12a4a241c87924b54e1e35990234",
+ "urn": "urn:li:container:007d12a4a241c87924b54e1e35990234"
+ },
+ {
+ "id": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715",
+ "urn": "urn:li:container:bbca630ddf6a79e03fa681adc3fa1715"
+ },
+ {
+ "id": "urn:li:container:df1130913ed9cfec6c7afb9fc58b9554",
+ "urn": "urn:li:container:df1130913ed9cfec6c7afb9fc58b9554"
+ },
+ {
+ "id": "urn:li:container:2ebbe0028345d0b8d147aed919b1024c",
+ "urn": "urn:li:container:2ebbe0028345d0b8d147aed919b1024c"
+ },
+ {
+ "id": "urn:li:container:25745bd0b919d9f4e402df43a1ee0ca8",
+ "urn": "urn:li:container:25745bd0b919d9f4e402df43a1ee0ca8"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_aspect,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "upstreamLineage",
+ "aspect": {
+ "json": {
+ "upstreams": [
+ {
+ "auditStamp": {
+ "time": 1697353200000,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_aspect,PROD)",
+ "type": "COPY"
+ }
+ ],
+ "fineGrainedLineages": [
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_aspect,PROD),metadata)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_aspect,PROD),metadata)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_aspect,PROD),createdon)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_aspect,PROD),createdon)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_aspect,PROD),createdby)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_aspect,PROD),createdby)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_aspect,PROD),createdfor)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_aspect,PROD),createdfor)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_aspect,PROD),urn)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_aspect,PROD),urn)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_aspect,PROD),aspect)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_aspect,PROD),aspect)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_aspect,PROD),version)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_aspect,PROD),version)"
+ ],
+ "confidenceScore": 1.0
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "upstreamLineage",
+ "aspect": {
+ "json": {
+ "upstreams": [
+ {
+ "auditStamp": {
+ "time": 1697353200000,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_index,PROD)",
+ "type": "COPY"
+ }
+ ],
+ "fineGrainedLineages": [
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_index,PROD),path)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index,PROD),path)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_index,PROD),aspect)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index,PROD),aspect)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_index,PROD),urn)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index,PROD),urn)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_index,PROD),id)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index,PROD),id)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_index,PROD),stringVal)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index,PROD),stringVal)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_index,PROD),longVal)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index,PROD),longVal)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_index,PROD),doubleVal)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index,PROD),doubleVal)"
+ ],
+ "confidenceScore": 1.0
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index_view,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "upstreamLineage",
+ "aspect": {
+ "json": {
+ "upstreams": [
+ {
+ "auditStamp": {
+ "time": 1697353200000,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_index_view,PROD)",
+ "type": "COPY"
+ }
+ ],
+ "fineGrainedLineages": [
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_index_view,PROD),id)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index_view,PROD),id)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_index_view,PROD),urn)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index_view,PROD),urn)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_index_view,PROD),path)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index_view,PROD),path)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.metagalaxy.metadata_index_view,PROD),doubleVal)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.metagalaxy.metadata_index_view,PROD),doubleVal)"
+ ],
+ "confidenceScore": 1.0
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.customers,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "upstreamLineage",
+ "aspect": {
+ "json": {
+ "upstreams": [
+ {
+ "auditStamp": {
+ "time": 1697353200000,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.northwind.customers,PROD)",
+ "type": "COPY"
+ }
+ ],
+ "fineGrainedLineages": [
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.northwind.customers,PROD),priority)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.customers,PROD),priority)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.northwind.customers,PROD),email_address)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.customers,PROD),email_address)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.northwind.customers,PROD),first_name)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.customers,PROD),first_name)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.northwind.customers,PROD),company)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.customers,PROD),company)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.northwind.customers,PROD),id)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.customers,PROD),id)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.northwind.customers,PROD),last_name)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.customers,PROD),last_name)"
+ ],
+ "confidenceScore": 1.0
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.orders,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "upstreamLineage",
+ "aspect": {
+ "json": {
+ "upstreams": [
+ {
+ "auditStamp": {
+ "time": 1697353200000,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "dataset": "urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.northwind.orders,PROD)",
+ "type": "COPY"
+ }
+ ],
+ "fineGrainedLineages": [
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.northwind.orders,PROD),description)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.orders,PROD),description)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.northwind.orders,PROD),customer_id)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.orders,PROD),customer_id)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:mysql,test-platform.northwind.orders,PROD),id)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.mysql.northwind.orders,PROD),id)"
+ ],
+ "confidenceScore": 1.0
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "upstreamLineage",
+ "aspect": {
+ "json": {
+ "upstreams": [
+ {
+ "auditStamp": {
+ "time": 1697353200000,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples.//warehouse,PROD)",
+ "type": "COPY"
+ }
+ ],
+ "fineGrainedLineages": [
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples.//warehouse,PROD),id)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD),id)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples.//warehouse,PROD),name)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD),name)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples.//warehouse,PROD),age)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD),age)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples.//warehouse,PROD),salary)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD),salary)"
+ ],
+ "confidenceScore": 1.0
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "upstreamLineage",
+ "aspect": {
+ "json": {
+ "upstreams": [
+ {
+ "auditStamp": {
+ "time": 1697353200000,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD)",
+ "type": "COPY"
+ }
+ ],
+ "fineGrainedLineages": [
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),F)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),F)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),G)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),G)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),H)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),H)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),I)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),I)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),J)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),J)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),K)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),K)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),L)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),L)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),M)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),M)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),A)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),A)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),B)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),B)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),C)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),C)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),D)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),D)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),E)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),E)"
+ ],
+ "confidenceScore": 1.0
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "upstreamLineage",
+ "aspect": {
+ "json": {
+ "upstreams": [
+ {
+ "auditStamp": {
+ "time": 1697353200000,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD)",
+ "type": "COPY"
+ }
+ ],
+ "fineGrainedLineages": [
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD),DEPARTMENT_ID)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD),DEPARTMENT_ID)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD),DEPARTMENT_NAME)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD),DEPARTMENT_NAME)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD),MANAGER_ID)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD),MANAGER_ID)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD),LOCATION_ID)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD),LOCATION_ID)"
+ ],
+ "confidenceScore": 1.0
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "upstreamLineage",
+ "aspect": {
+ "json": {
+ "upstreams": [
+ {
+ "auditStamp": {
+ "time": 1697353200000,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD)",
+ "type": "COPY"
+ }
+ ],
+ "fineGrainedLineages": [
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),E)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),E)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),G)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),G)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),H)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),H)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),I)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),I)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),F)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),F)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),A)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),A)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),B)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),B)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),C)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),C)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),D)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),D)"
+ ],
+ "confidenceScore": 1.0
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "upstreamLineage",
+ "aspect": {
+ "json": {
+ "upstreams": [
+ {
+ "auditStamp": {
+ "time": 1697353200000,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD)",
+ "type": "COPY"
+ }
+ ],
+ "fineGrainedLineages": [
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_page_sk)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_page_sk)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_page_id)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_page_id)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_start_date_sk)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_start_date_sk)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_end_date_sk)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_end_date_sk)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_department)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_department)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_number)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_number)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_page_number)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_page_number)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_description)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_description)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_type)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_type)"
+ ],
+ "confidenceScore": 1.0
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.customers,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "upstreamLineage",
+ "aspect": {
+ "json": {
+ "upstreams": [
+ {
+ "auditStamp": {
+ "time": 1697353200000,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "dataset": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.mysql.northwind.customers,PROD)",
+ "type": "VIEW",
+ "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.customers%2CPROD%29"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "query",
+ "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.customers%2CPROD%29",
+ "changeType": "UPSERT",
+ "aspectName": "queryProperties",
+ "aspect": {
+ "json": {
+ "statement": {
+ "value": "SELECT\n *\nFROM mysql.northwind.customers",
+ "language": "SQL"
+ },
+ "source": "SYSTEM",
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "lastModified": {
+ "time": 1697353200000,
+ "actor": "urn:li:corpuser:_ingestion"
+ }
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "query",
+ "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.customers%2CPROD%29",
+ "changeType": "UPSERT",
+ "aspectName": "querySubjects",
+ "aspect": {
+ "json": {
+ "subjects": [
+ {
+ "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.mysql.northwind.customers,PROD)"
+ },
+ {
+ "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.customers,PROD)"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "query",
+ "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.customers%2CPROD%29",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_aspect,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "upstreamLineage",
+ "aspect": {
+ "json": {
+ "upstreams": [
+ {
+ "auditStamp": {
+ "time": 1697353200000,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "dataset": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.mysql.metagalaxy.metadata_aspect,PROD)",
+ "type": "VIEW",
+ "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.metadata_aspect%2CPROD%29"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "query",
+ "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.metadata_aspect%2CPROD%29",
+ "changeType": "UPSERT",
+ "aspectName": "queryProperties",
+ "aspect": {
+ "json": {
+ "statement": {
+ "value": "SELECT\n *\nFROM mysql.metagalaxy.metadata_aspect",
+ "language": "SQL"
+ },
+ "source": "SYSTEM",
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "lastModified": {
+ "time": 1697353200000,
+ "actor": "urn:li:corpuser:_ingestion"
+ }
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "query",
+ "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.metadata_aspect%2CPROD%29",
+ "changeType": "UPSERT",
+ "aspectName": "querySubjects",
+ "aspect": {
+ "json": {
+ "subjects": [
+ {
+ "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.mysql.metagalaxy.metadata_aspect,PROD)"
+ },
+ {
+ "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_aspect,PROD)"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "query",
+ "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.metadata_aspect%2CPROD%29",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "upstreamLineage",
+ "aspect": {
+ "json": {
+ "upstreams": [
+ {
+ "auditStamp": {
+ "time": 1697353200000,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "dataset": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.mysql.metagalaxy.metadata_index,PROD)",
+ "type": "VIEW",
+ "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.metadata_index%2CPROD%29"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "query",
+ "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.metadata_index%2CPROD%29",
+ "changeType": "UPSERT",
+ "aspectName": "queryProperties",
+ "aspect": {
+ "json": {
+ "statement": {
+ "value": "SELECT\n *\nFROM mysql.metagalaxy.metadata_index",
+ "language": "SQL"
+ },
+ "source": "SYSTEM",
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "lastModified": {
+ "time": 1697353200000,
+ "actor": "urn:li:corpuser:_ingestion"
+ }
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "query",
+ "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.metadata_index%2CPROD%29",
+ "changeType": "UPSERT",
+ "aspectName": "querySubjects",
+ "aspect": {
+ "json": {
+ "subjects": [
+ {
+ "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.mysql.metagalaxy.metadata_index,PROD)"
+ },
+ {
+ "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index,PROD)"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "query",
+ "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.metadata_index%2CPROD%29",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index_view,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "upstreamLineage",
+ "aspect": {
+ "json": {
+ "upstreams": [
+ {
+ "auditStamp": {
+ "time": 1697353200000,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "dataset": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.mysql.metagalaxy.metadata_index_view,PROD)",
+ "type": "VIEW",
+ "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.metadata_index_view%2CPROD%29"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "query",
+ "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.metadata_index_view%2CPROD%29",
+ "changeType": "UPSERT",
+ "aspectName": "queryProperties",
+ "aspect": {
+ "json": {
+ "statement": {
+ "value": "SELECT\n *\nFROM mysql.metagalaxy.metadata_index_view",
+ "language": "SQL"
+ },
+ "source": "SYSTEM",
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "lastModified": {
+ "time": 1697353200000,
+ "actor": "urn:li:corpuser:_ingestion"
+ }
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "query",
+ "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.metadata_index_view%2CPROD%29",
+ "changeType": "UPSERT",
+ "aspectName": "querySubjects",
+ "aspect": {
+ "json": {
+ "subjects": [
+ {
+ "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.mysql.metagalaxy.metadata_index_view,PROD)"
+ },
+ {
+ "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.metadata_index_view,PROD)"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "query",
+ "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.metadata_index_view%2CPROD%29",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.orders,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "upstreamLineage",
+ "aspect": {
+ "json": {
+ "upstreams": [
+ {
+ "auditStamp": {
+ "time": 1697353200000,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "dataset": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.mysql.northwind.orders,PROD)",
+ "type": "VIEW",
+ "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.orders%2CPROD%29"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "query",
+ "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.orders%2CPROD%29",
+ "changeType": "UPSERT",
+ "aspectName": "queryProperties",
+ "aspect": {
+ "json": {
+ "statement": {
+ "value": "SELECT\n *\nFROM mysql.northwind.orders",
+ "language": "SQL"
+ },
+ "source": "SYSTEM",
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "lastModified": {
+ "time": 1697353200000,
+ "actor": "urn:li:corpuser:_ingestion"
+ }
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "query",
+ "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.orders%2CPROD%29",
+ "changeType": "UPSERT",
+ "aspectName": "querySubjects",
+ "aspect": {
+ "json": {
+ "subjects": [
+ {
+ "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.mysql.northwind.orders,PROD)"
+ },
+ {
+ "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.orders,PROD)"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "query",
+ "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.orders%2CPROD%29",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "upstreamLineage",
+ "aspect": {
+ "json": {
+ "upstreams": [
+ {
+ "auditStamp": {
+ "time": 1697353200000,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "dataset": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD)",
+ "type": "VIEW",
+ "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.raw%2CPROD%29"
+ }
+ ],
+ "fineGrainedLineages": [
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD),id)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD),id)"
+ ],
+ "confidenceScore": 0.9,
+ "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.raw%2CPROD%29"
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD),name)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD),name)"
+ ],
+ "confidenceScore": 0.9,
+ "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.raw%2CPROD%29"
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD),age)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD),age)"
+ ],
+ "confidenceScore": 0.9,
+ "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.raw%2CPROD%29"
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD),salary)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD),salary)"
+ ],
+ "confidenceScore": 0.9,
+ "query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.raw%2CPROD%29"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "query",
+ "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.raw%2CPROD%29",
+ "changeType": "UPSERT",
+ "aspectName": "queryProperties",
+ "aspect": {
+ "json": {
+ "statement": {
+ "value": "SELECT\n *\nFROM s3.warehouse",
+ "language": "SQL"
+ },
+ "source": "SYSTEM",
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "lastModified": {
+ "time": 1697353200000,
+ "actor": "urn:li:corpuser:_ingestion"
+ }
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "query",
+ "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.raw%2CPROD%29",
+ "changeType": "UPSERT",
+ "aspectName": "querySubjects",
+ "aspect": {
+ "json": {
+ "subjects": [
+ {
+ "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD)"
+ },
+ {
+ "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD),age)"
+ },
+ {
+ "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD),id)"
+ },
+ {
+ "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD),name)"
+ },
+ {
+ "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.s3.warehouse,PROD),salary)"
+ },
+ {
+ "entity": "urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD)"
+ },
+ {
+ "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD),id)"
+ },
+ {
+ "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD),name)"
+ },
+ {
+ "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD),age)"
+ },
+ {
+ "entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,test-platform.dremio.space.test_folder.raw,PROD),salary)"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "query",
+ "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.raw%2CPROD%29",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "query",
+ "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.customers%2CPROD%29",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "query",
+ "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.metadata_aspect%2CPROD%29",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "query",
+ "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.metadata_index%2CPROD%29",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "query",
+ "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.metadata_index_view%2CPROD%29",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "query",
+ "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.orders%2CPROD%29",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "query",
+ "entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Adremio%2Ctest-platform.dremio.space.test_folder.raw%2CPROD%29",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-t5gf20",
+ "lastRunId": "no-run-id-provided"
+ }
+}
+]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/dremio/dremio_platform_instance_to_file.yml b/metadata-ingestion/tests/integration/dremio/dremio_platform_instance_to_file.yml
new file mode 100644
index 00000000000000..923dae139681a4
--- /dev/null
+++ b/metadata-ingestion/tests/integration/dremio/dremio_platform_instance_to_file.yml
@@ -0,0 +1,26 @@
+source:
+ type: dremio
+ config:
+ # Coordinates
+ hostname: localhost
+ port: 9047
+ tls: false
+
+ # Credentials
+ authentication_method: password
+ username: admin
+ password: "2310Admin1234!@"
+
+ platform_instance: test-platform
+
+ include_query_lineage: false
+
+ source_mappings:
+ - platform: s3
+ source_name: samples
+ platform_instance: s3_test_samples
+
+sink:
+ type: file
+ config:
+ filename: "./dremio_mces.json"
diff --git a/metadata-ingestion/tests/integration/dremio/dremio_schema_filter_mces_golden.json b/metadata-ingestion/tests/integration/dremio/dremio_schema_filter_mces_golden.json
new file mode 100644
index 00000000000000..f4dfed45ccf0c5
--- /dev/null
+++ b/metadata-ingestion/tests/integration/dremio/dremio_schema_filter_mces_golden.json
@@ -0,0 +1,2327 @@
+[
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68",
+ "changeType": "UPSERT",
+ "aspectName": "containerProperties",
+ "aspect": {
+ "json": {
+ "customProperties": {},
+ "name": "Samples",
+ "qualifiedName": "Samples",
+ "description": "",
+ "env": "PROD"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68",
+ "changeType": "UPSERT",
+ "aspectName": "subTypes",
+ "aspect": {
+ "json": {
+ "typeNames": [
+ "Dremio Source"
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68",
+ "changeType": "UPSERT",
+ "aspectName": "browsePathsV2",
+ "aspect": {
+ "json": {
+ "path": [
+ {
+ "id": "Sources"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83",
+ "changeType": "UPSERT",
+ "aspectName": "containerProperties",
+ "aspect": {
+ "json": {
+ "customProperties": {},
+ "name": "samples.dremio.com",
+ "qualifiedName": "Samples.samples.dremio.com",
+ "description": "",
+ "env": "PROD"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83",
+ "changeType": "UPSERT",
+ "aspectName": "container",
+ "aspect": {
+ "json": {
+ "container": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83",
+ "changeType": "UPSERT",
+ "aspectName": "subTypes",
+ "aspect": {
+ "json": {
+ "typeNames": [
+ "Dremio Folder"
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83",
+ "changeType": "UPSERT",
+ "aspectName": "browsePathsV2",
+ "aspect": {
+ "json": {
+ "path": [
+ {
+ "id": "Sources"
+ },
+ {
+ "id": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68",
+ "urn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:687c0496e464bc4c0de935cb1da1becf",
+ "changeType": "UPSERT",
+ "aspectName": "containerProperties",
+ "aspect": {
+ "json": {
+ "customProperties": {},
+ "name": "Dremio University",
+ "qualifiedName": "Samples.samples.dremio.com.Dremio University",
+ "description": "",
+ "env": "PROD"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:687c0496e464bc4c0de935cb1da1becf",
+ "changeType": "UPSERT",
+ "aspectName": "container",
+ "aspect": {
+ "json": {
+ "container": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:687c0496e464bc4c0de935cb1da1becf",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:687c0496e464bc4c0de935cb1da1becf",
+ "changeType": "UPSERT",
+ "aspectName": "subTypes",
+ "aspect": {
+ "json": {
+ "typeNames": [
+ "Dremio Folder"
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:687c0496e464bc4c0de935cb1da1becf",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:687c0496e464bc4c0de935cb1da1becf",
+ "changeType": "UPSERT",
+ "aspectName": "browsePathsV2",
+ "aspect": {
+ "json": {
+ "path": [
+ {
+ "id": "Sources"
+ },
+ {
+ "id": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68",
+ "urn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68"
+ },
+ {
+ "id": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83",
+ "urn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f",
+ "changeType": "UPSERT",
+ "aspectName": "containerProperties",
+ "aspect": {
+ "json": {
+ "customProperties": {},
+ "name": "tpcds_sf1000",
+ "qualifiedName": "Samples.samples.dremio.com.tpcds_sf1000",
+ "description": "",
+ "env": "PROD"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f",
+ "changeType": "UPSERT",
+ "aspectName": "container",
+ "aspect": {
+ "json": {
+ "container": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f",
+ "changeType": "UPSERT",
+ "aspectName": "subTypes",
+ "aspect": {
+ "json": {
+ "typeNames": [
+ "Dremio Folder"
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f",
+ "changeType": "UPSERT",
+ "aspectName": "browsePathsV2",
+ "aspect": {
+ "json": {
+ "path": [
+ {
+ "id": "Sources"
+ },
+ {
+ "id": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68",
+ "urn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68"
+ },
+ {
+ "id": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83",
+ "urn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:41ea3e8314dd9dedc00d6f47c69e3400",
+ "changeType": "UPSERT",
+ "aspectName": "containerProperties",
+ "aspect": {
+ "json": {
+ "customProperties": {},
+ "name": "catalog_page",
+ "qualifiedName": "Samples.samples.dremio.com.tpcds_sf1000.catalog_page",
+ "description": "",
+ "env": "PROD"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:41ea3e8314dd9dedc00d6f47c69e3400",
+ "changeType": "UPSERT",
+ "aspectName": "container",
+ "aspect": {
+ "json": {
+ "container": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:41ea3e8314dd9dedc00d6f47c69e3400",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:41ea3e8314dd9dedc00d6f47c69e3400",
+ "changeType": "UPSERT",
+ "aspectName": "subTypes",
+ "aspect": {
+ "json": {
+ "typeNames": [
+ "Dremio Folder"
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:41ea3e8314dd9dedc00d6f47c69e3400",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:41ea3e8314dd9dedc00d6f47c69e3400",
+ "changeType": "UPSERT",
+ "aspectName": "browsePathsV2",
+ "aspect": {
+ "json": {
+ "path": [
+ {
+ "id": "Sources"
+ },
+ {
+ "id": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68",
+ "urn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68"
+ },
+ {
+ "id": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83",
+ "urn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83"
+ },
+ {
+ "id": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f",
+ "urn": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:fd0949800e3c7cc7ce5de373fd737e0b",
+ "changeType": "UPSERT",
+ "aspectName": "containerProperties",
+ "aspect": {
+ "json": {
+ "customProperties": {},
+ "name": "1ab266d5-18eb-4780-711d-0fa337fa6c00",
+ "qualifiedName": "Samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00",
+ "description": "",
+ "env": "PROD"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:fd0949800e3c7cc7ce5de373fd737e0b",
+ "changeType": "UPSERT",
+ "aspectName": "container",
+ "aspect": {
+ "json": {
+ "container": "urn:li:container:41ea3e8314dd9dedc00d6f47c69e3400"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:fd0949800e3c7cc7ce5de373fd737e0b",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:fd0949800e3c7cc7ce5de373fd737e0b",
+ "changeType": "UPSERT",
+ "aspectName": "subTypes",
+ "aspect": {
+ "json": {
+ "typeNames": [
+ "Dremio Folder"
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:fd0949800e3c7cc7ce5de373fd737e0b",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "container",
+ "entityUrn": "urn:li:container:fd0949800e3c7cc7ce5de373fd737e0b",
+ "changeType": "UPSERT",
+ "aspectName": "browsePathsV2",
+ "aspect": {
+ "json": {
+ "path": [
+ {
+ "id": "Sources"
+ },
+ {
+ "id": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68",
+ "urn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68"
+ },
+ {
+ "id": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83",
+ "urn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83"
+ },
+ {
+ "id": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f",
+ "urn": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f"
+ },
+ {
+ "id": "urn:li:container:41ea3e8314dd9dedc00d6f47c69e3400",
+ "urn": "urn:li:container:41ea3e8314dd9dedc00d6f47c69e3400"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "datasetProperties",
+ "aspect": {
+ "json": {
+ "customProperties": {},
+ "externalUrl": "http://localhost:9047/source/\"Samples\"/\"samples.dremio.com\".\"NYC-weather.csv\"",
+ "name": "NYC-weather.csv",
+ "qualifiedName": "Samples.samples.dremio.com.NYC-weather.csv",
+ "description": "",
+ "created": {
+ "time": 0
+ },
+ "tags": []
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "subTypes",
+ "aspect": {
+ "json": {
+ "typeNames": [
+ "Table"
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "container",
+ "aspect": {
+ "json": {
+ "container": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "schemaMetadata",
+ "aspect": {
+ "json": {
+ "schemaName": "Samples.samples.dremio.com.NYC-weather.csv",
+ "platform": "urn:li:dataPlatform:dremio",
+ "version": 0,
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "lastModified": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "hash": "",
+ "platformSchema": {
+ "com.linkedin.schema.MySqlDDL": {
+ "tableSchema": ""
+ }
+ },
+ "fields": [
+ {
+ "fieldPath": "I",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "H",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "G",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "F",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "E",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "D",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "C",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "B",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "A",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "upstreamLineage",
+ "aspect": {
+ "json": {
+ "upstreams": [
+ {
+ "auditStamp": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD)",
+ "type": "COPY"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "browsePathsV2",
+ "aspect": {
+ "json": {
+ "path": [
+ {
+ "id": "Sources"
+ },
+ {
+ "id": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68",
+ "urn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68"
+ },
+ {
+ "id": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83",
+ "urn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "datasetProperties",
+ "aspect": {
+ "json": {
+ "customProperties": {},
+ "externalUrl": "http://localhost:9047/source/\"Samples\"/\"samples.dremio.com\".\"Dremio University\".\"googleplaystore.csv\"",
+ "name": "googleplaystore.csv",
+ "qualifiedName": "Samples.samples.dremio.com.Dremio University.googleplaystore.csv",
+ "description": "",
+ "created": {
+ "time": 0
+ },
+ "tags": []
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "subTypes",
+ "aspect": {
+ "json": {
+ "typeNames": [
+ "Table"
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "container",
+ "aspect": {
+ "json": {
+ "container": "urn:li:container:687c0496e464bc4c0de935cb1da1becf"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "schemaMetadata",
+ "aspect": {
+ "json": {
+ "schemaName": "Samples.samples.dremio.com.Dremio University.googleplaystore.csv",
+ "platform": "urn:li:dataPlatform:dremio",
+ "version": 0,
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "lastModified": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "hash": "",
+ "platformSchema": {
+ "com.linkedin.schema.MySqlDDL": {
+ "tableSchema": ""
+ }
+ },
+ "fields": [
+ {
+ "fieldPath": "E",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "A",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "B",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "C",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "D",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "F",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "G",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "H",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "I",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "J",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "K",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "L",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "M",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "upstreamLineage",
+ "aspect": {
+ "json": {
+ "upstreams": [
+ {
+ "auditStamp": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD)",
+ "type": "COPY"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "browsePathsV2",
+ "aspect": {
+ "json": {
+ "path": [
+ {
+ "id": "Sources"
+ },
+ {
+ "id": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68",
+ "urn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68"
+ },
+ {
+ "id": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83",
+ "urn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83"
+ },
+ {
+ "id": "urn:li:container:687c0496e464bc4c0de935cb1da1becf",
+ "urn": "urn:li:container:687c0496e464bc4c0de935cb1da1becf"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "datasetProperties",
+ "aspect": {
+ "json": {
+ "customProperties": {},
+ "externalUrl": "http://localhost:9047/source/\"Samples\"/\"samples.dremio.com\".\"Dremio University\".\"oracle-departments.xlsx\"",
+ "name": "oracle-departments.xlsx",
+ "qualifiedName": "Samples.samples.dremio.com.Dremio University.oracle-departments.xlsx",
+ "description": "",
+ "created": {
+ "time": 0
+ },
+ "tags": []
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "subTypes",
+ "aspect": {
+ "json": {
+ "typeNames": [
+ "Table"
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "container",
+ "aspect": {
+ "json": {
+ "container": "urn:li:container:687c0496e464bc4c0de935cb1da1becf"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "schemaMetadata",
+ "aspect": {
+ "json": {
+ "schemaName": "Samples.samples.dremio.com.Dremio University.oracle-departments.xlsx",
+ "platform": "urn:li:dataPlatform:dremio",
+ "version": 0,
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "lastModified": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "hash": "",
+ "platformSchema": {
+ "com.linkedin.schema.MySqlDDL": {
+ "tableSchema": ""
+ }
+ },
+ "fields": [
+ {
+ "fieldPath": "LOCATION_ID",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.NumberType": {}
+ }
+ },
+ "nativeDataType": "double(53)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "MANAGER_ID",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.NumberType": {}
+ }
+ },
+ "nativeDataType": "double(53)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "DEPARTMENT_NAME",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "DEPARTMENT_ID",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.NumberType": {}
+ }
+ },
+ "nativeDataType": "double(53)",
+ "recursive": false,
+ "isPartOfKey": false
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "upstreamLineage",
+ "aspect": {
+ "json": {
+ "upstreams": [
+ {
+ "auditStamp": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD)",
+ "type": "COPY"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "browsePathsV2",
+ "aspect": {
+ "json": {
+ "path": [
+ {
+ "id": "Sources"
+ },
+ {
+ "id": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68",
+ "urn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68"
+ },
+ {
+ "id": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83",
+ "urn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83"
+ },
+ {
+ "id": "urn:li:container:687c0496e464bc4c0de935cb1da1becf",
+ "urn": "urn:li:container:687c0496e464bc4c0de935cb1da1becf"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "datasetProperties",
+ "aspect": {
+ "json": {
+ "customProperties": {},
+ "externalUrl": "http://localhost:9047/source/\"Samples\"/\"samples.dremio.com\".\"tpcds_sf1000\".\"catalog_page\".\"1ab266d5-18eb-4780-711d-0fa337fa6c00\".\"0_0_0.parquet\"",
+ "name": "0_0_0.parquet",
+ "qualifiedName": "Samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet",
+ "description": "",
+ "created": {
+ "time": 0
+ },
+ "tags": []
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "subTypes",
+ "aspect": {
+ "json": {
+ "typeNames": [
+ "Table"
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "dataPlatformInstance",
+ "aspect": {
+ "json": {
+ "platform": "urn:li:dataPlatform:dremio"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "container",
+ "aspect": {
+ "json": {
+ "container": "urn:li:container:fd0949800e3c7cc7ce5de373fd737e0b"
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "schemaMetadata",
+ "aspect": {
+ "json": {
+ "schemaName": "Samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet",
+ "platform": "urn:li:dataPlatform:dremio",
+ "version": 0,
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "lastModified": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "hash": "",
+ "platformSchema": {
+ "com.linkedin.schema.MySqlDDL": {
+ "tableSchema": ""
+ }
+ },
+ "fields": [
+ {
+ "fieldPath": "cp_start_date_sk",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.NumberType": {}
+ }
+ },
+ "nativeDataType": "bigint(64)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "cp_catalog_page_id",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "cp_catalog_page_sk",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.NumberType": {}
+ }
+ },
+ "nativeDataType": "bigint(64)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "cp_department",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "cp_catalog_number",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.NumberType": {}
+ }
+ },
+ "nativeDataType": "bigint(64)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "cp_end_date_sk",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.NumberType": {}
+ }
+ },
+ "nativeDataType": "bigint(64)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "cp_catalog_page_number",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.NumberType": {}
+ }
+ },
+ "nativeDataType": "bigint(64)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "cp_description",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ },
+ {
+ "fieldPath": "cp_type",
+ "nullable": true,
+ "type": {
+ "type": {
+ "com.linkedin.schema.StringType": {}
+ }
+ },
+ "nativeDataType": "character varying(65536)",
+ "recursive": false,
+ "isPartOfKey": false
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "status",
+ "aspect": {
+ "json": {
+ "removed": false
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "upstreamLineage",
+ "aspect": {
+ "json": {
+ "upstreams": [
+ {
+ "auditStamp": {
+ "time": 0,
+ "actor": "urn:li:corpuser:unknown"
+ },
+ "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD)",
+ "type": "COPY"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "browsePathsV2",
+ "aspect": {
+ "json": {
+ "path": [
+ {
+ "id": "Sources"
+ },
+ {
+ "id": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68",
+ "urn": "urn:li:container:e8cccb9f7a06aeafad68f76e30c62f68"
+ },
+ {
+ "id": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83",
+ "urn": "urn:li:container:56c2e18fbc5786016aacecb7f7d64e83"
+ },
+ {
+ "id": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f",
+ "urn": "urn:li:container:bf1ee664b5c9fa9610f731399062a47f"
+ },
+ {
+ "id": "urn:li:container:41ea3e8314dd9dedc00d6f47c69e3400",
+ "urn": "urn:li:container:41ea3e8314dd9dedc00d6f47c69e3400"
+ },
+ {
+ "id": "urn:li:container:fd0949800e3c7cc7ce5de373fd737e0b",
+ "urn": "urn:li:container:fd0949800e3c7cc7ce5de373fd737e0b"
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "upstreamLineage",
+ "aspect": {
+ "json": {
+ "upstreams": [
+ {
+ "auditStamp": {
+ "time": 1697353200000,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD)",
+ "type": "COPY"
+ }
+ ],
+ "fineGrainedLineages": [
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),E)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),E)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),A)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),A)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),B)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),B)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),C)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),C)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),D)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),D)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),F)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),F)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),G)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),G)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),H)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),H)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),I)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),I)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),J)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),J)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),K)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),K)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),L)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),L)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/googleplaystore.csv,PROD),M)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.googleplaystore.csv,PROD),M)"
+ ],
+ "confidenceScore": 1.0
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "upstreamLineage",
+ "aspect": {
+ "json": {
+ "upstreams": [
+ {
+ "auditStamp": {
+ "time": 1697353200000,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD)",
+ "type": "COPY"
+ }
+ ],
+ "fineGrainedLineages": [
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD),LOCATION_ID)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD),LOCATION_ID)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD),MANAGER_ID)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD),MANAGER_ID)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD),DEPARTMENT_NAME)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD),DEPARTMENT_NAME)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/Dremio University/oracle-departments.xlsx,PROD),DEPARTMENT_ID)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.dremio university.oracle-departments.xlsx,PROD),DEPARTMENT_ID)"
+ ],
+ "confidenceScore": 1.0
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "upstreamLineage",
+ "aspect": {
+ "json": {
+ "upstreams": [
+ {
+ "auditStamp": {
+ "time": 1697353200000,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD)",
+ "type": "COPY"
+ }
+ ],
+ "fineGrainedLineages": [
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),I)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),I)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),H)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),H)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),G)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),G)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),F)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),F)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),E)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),E)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),D)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),D)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),C)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),C)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),B)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),B)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/NYC-weather.csv,PROD),A)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.nyc-weather.csv,PROD),A)"
+ ],
+ "confidenceScore": 1.0
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+},
+{
+ "entityType": "dataset",
+ "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD)",
+ "changeType": "UPSERT",
+ "aspectName": "upstreamLineage",
+ "aspect": {
+ "json": {
+ "upstreams": [
+ {
+ "auditStamp": {
+ "time": 1697353200000,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "created": {
+ "time": 0,
+ "actor": "urn:li:corpuser:_ingestion"
+ },
+ "dataset": "urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD)",
+ "type": "COPY"
+ }
+ ],
+ "fineGrainedLineages": [
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_start_date_sk)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_start_date_sk)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_page_id)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_page_id)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_page_sk)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_page_sk)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_department)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_department)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_number)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_number)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_end_date_sk)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_end_date_sk)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_catalog_page_number)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_catalog_page_number)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_description)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_description)"
+ ],
+ "confidenceScore": 1.0
+ },
+ {
+ "upstreamType": "FIELD_SET",
+ "upstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:s3,s3_test_samples./samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet,PROD),cp_type)"
+ ],
+ "downstreamType": "FIELD",
+ "downstreams": [
+ "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:dremio,dremio.samples.samples.dremio.com.tpcds_sf1000.catalog_page.1ab266d5-18eb-4780-711d-0fa337fa6c00.0_0_0.parquet,PROD),cp_type)"
+ ],
+ "confidenceScore": 1.0
+ }
+ ]
+ }
+ },
+ "systemMetadata": {
+ "lastObserved": 1697353200000,
+ "runId": "dremio-2023_10_15-07_00_00-vftab1",
+ "lastRunId": "no-run-id-provided"
+ }
+}
+]
\ No newline at end of file
diff --git a/metadata-ingestion/tests/integration/dremio/dremio_schema_filter_to_file.yml b/metadata-ingestion/tests/integration/dremio/dremio_schema_filter_to_file.yml
new file mode 100644
index 00000000000000..2b3cea465eebe2
--- /dev/null
+++ b/metadata-ingestion/tests/integration/dremio/dremio_schema_filter_to_file.yml
@@ -0,0 +1,28 @@
+source:
+ type: dremio
+ config:
+ # Coordinates
+ hostname: localhost
+ port: 9047
+ tls: false
+
+ # Credentials
+ authentication_method: password
+ username: admin
+ password: "2310Admin1234!@"
+
+ include_query_lineage: false
+
+ source_mappings:
+ - platform: s3
+ source_name: samples
+ platform_instance: s3_test_samples
+
+ schema_pattern:
+ allow:
+ - "Samples"
+
+sink:
+ type: file
+ config:
+ filename: "./dremio_mces.json"
diff --git a/metadata-ingestion/tests/integration/dremio/test_dremio.py b/metadata-ingestion/tests/integration/dremio/test_dremio.py
index cc3a7e19bc93e2..401f487d8a14b8 100644
--- a/metadata-ingestion/tests/integration/dremio/test_dremio.py
+++ b/metadata-ingestion/tests/integration/dremio/test_dremio.py
@@ -1,6 +1,7 @@
import json
import os
import subprocess
+from typing import Dict
import boto3
import pytest
@@ -75,9 +76,10 @@ def create_spaces_and_folders(headers):
def create_sample_source(headers):
- url = f"{DREMIO_HOST}/apiv2/source/Samples"
+ url = f"{DREMIO_HOST}/api/v3/catalog"
payload = {
+ "entityType": "source",
"config": {
"externalBucketList": ["samples.dremio.com"],
"credentialType": "NONE",
@@ -95,14 +97,15 @@ def create_sample_source(headers):
"type": "S3",
}
- response = requests.put(url, headers=headers, data=json.dumps(payload))
+ response = requests.post(url, headers=headers, data=json.dumps(payload))
assert response.status_code == 200, f"Failed to add dataset: {response.text}"
def create_s3_source(headers):
- url = f"{DREMIO_HOST}/apiv2/source/s3"
+ url = f"{DREMIO_HOST}/api/v3/catalog"
payload = {
+ "entityType": "source",
"name": "s3",
"config": {
"credentialType": "ACCESS_KEY",
@@ -139,24 +142,25 @@ def create_s3_source(headers):
"metadataPolicy": {
"deleteUnavailableDatasets": True,
"autoPromoteDatasets": False,
- "namesRefreshMillis": 3600000,
- "datasetDefinitionRefreshAfterMillis": 3600000,
- "datasetDefinitionExpireAfterMillis": 10800000,
- "authTTLMillis": 86400000,
- "updateMode": "PREFETCH_QUERIED",
+ "namesRefreshMs": 3600000,
+ "datasetRefreshAfterMs": 3600000,
+ "datasetExpireAfterMs": 10800000,
+ "authTTLMs": 86400000,
+ "datasetUpdateMode": "PREFETCH_QUERIED",
},
"type": "S3",
"accessControlList": {"userControls": [], "roleControls": []},
}
- response = requests.put(url, headers=headers, data=json.dumps(payload))
+ response = requests.post(url, headers=headers, data=json.dumps(payload))
assert response.status_code == 200, f"Failed to add s3 datasource: {response.text}"
def create_mysql_source(headers):
- url = f"{DREMIO_HOST}/apiv2/source/mysql"
+ url = f"{DREMIO_HOST}/api/v3/catalog"
payload = {
+ "entityType": "source",
"config": {
"username": "root",
"password": "rootpwd123",
@@ -169,7 +173,7 @@ def create_mysql_source(headers):
"maxIdleConns": 8,
"idleTimeSec": 60,
},
- "name": "mysql-source",
+ "name": "mysql",
"accelerationRefreshPeriod": 3600000,
"accelerationGracePeriod": 10800000,
"accelerationActivePolicyType": "PERIOD",
@@ -177,72 +181,121 @@ def create_mysql_source(headers):
"accelerationRefreshOnDataChanges": False,
"metadataPolicy": {
"deleteUnavailableDatasets": True,
- "namesRefreshMillis": 3600000,
- "datasetDefinitionRefreshAfterMillis": 3600000,
- "datasetDefinitionExpireAfterMillis": 10800000,
- "authTTLMillis": 86400000,
- "updateMode": "PREFETCH_QUERIED",
+ "namesRefreshMs": 3600000,
+ "datasetRefreshAfterMs": 3600000,
+ "datasetExpireAfterMs": 10800000,
+ "authTTLMs": 86400000,
+ "datasetUpdateMode": "PREFETCH_QUERIED",
},
"type": "MYSQL",
}
- response = requests.put(url, headers=headers, data=json.dumps(payload))
+ response = requests.post(url, headers=headers, data=json.dumps(payload))
assert (
response.status_code == 200
), f"Failed to add mysql datasource: {response.text}"
def upload_dataset(headers):
- url = f"{DREMIO_HOST}/apiv2/source/s3/file_format/warehouse/sample.parquet"
- payload = {"ignoreOtherFileFormats": False, "type": "Parquet"}
+ url = f"{DREMIO_HOST}/api/v3/catalog/dremio%3A%2Fs3%2Fwarehouse"
+ payload = {
+ "entityType": "dataset",
+ "type": "PHYSICAL_DATASET",
+ "path": [
+ "s3",
+ "warehouse",
+ ],
+ "format": {"type": "Parquet"},
+ }
- response = requests.put(url, headers=headers, data=json.dumps(payload))
+ response = requests.post(url, headers=headers, data=json.dumps(payload))
assert response.status_code == 200, f"Failed to add dataset: {response.text}"
- url = f"{DREMIO_HOST}/apiv2/source/Samples/file_format/samples.dremio.com/NYC-weather.csv"
+ url = f"{DREMIO_HOST}/api/v3/catalog/dremio%3A%2FSamples%2Fsamples.dremio.com%2FNYC-weather.csv"
payload = {
- "fieldDelimiter": ",",
- "quote": '"',
- "comment": "#",
- "lineDelimiter": "\r\n",
- "escape": '"',
- "extractHeader": False,
- "trimHeader": True,
- "skipFirstLine": False,
- "type": "Text",
+ "entityType": "dataset",
+ "type": "PHYSICAL_DATASET",
+ "path": [
+ "Samples",
+ "samples.dremio.com",
+ "NYC-weather.csv",
+ ],
+ "format": {
+ "fieldDelimiter": ",",
+ "quote": '"',
+ "comment": "#",
+ "lineDelimiter": "\r\n",
+ "escape": '"',
+ "extractHeader": False,
+ "trimHeader": True,
+ "skipFirstLine": False,
+ "type": "Text",
+ },
}
- response = requests.put(url, headers=headers, data=json.dumps(payload))
+ response = requests.post(url, headers=headers, data=json.dumps(payload))
assert response.status_code == 200, f"Failed to add dataset: {response.text}"
- url = f"{DREMIO_HOST}/apiv2/source/Samples/file_format/samples.dremio.com/Dremio%20University/oracle-departments.xlsx"
+ url = f"{DREMIO_HOST}/api/v3/catalog/dremio%3A%2FSamples%2Fsamples.dremio.com%2FDremio%20University%2Foracle-departments.xlsx"
- payload = {"extractHeader": True, "hasMergedCells": False, "type": "Excel"}
+ payload = {
+ "entityType": "dataset",
+ "type": "PHYSICAL_DATASET",
+ "path": [
+ "Samples",
+ "samples.dremio.com",
+ "Dremio University",
+ "oracle-departments.xlsx",
+ ],
+ "format": {"extractHeader": True, "hasMergedCells": False, "type": "Excel"},
+ }
- response = requests.put(url, headers=headers, data=json.dumps(payload))
+ response = requests.post(url, headers=headers, data=json.dumps(payload))
assert response.status_code == 200, f"Failed to add dataset: {response.text}"
- url = f"{DREMIO_HOST}/apiv2/source/Samples/file_format/samples.dremio.com/Dremio%20University/googleplaystore.csv"
+ url = f"{DREMIO_HOST}/api/v3/catalog/dremio%3A%2FSamples%2Fsamples.dremio.com%2FDremio%20University%2Fgoogleplaystore.csv"
payload = {
- "fieldDelimiter": ",",
- "quote": '"',
- "comment": "#",
- "lineDelimiter": "\r\n",
- "escape": '"',
- "extractHeader": False,
- "trimHeader": True,
- "skipFirstLine": False,
- "type": "Text",
+ "entityType": "dataset",
+ "type": "PHYSICAL_DATASET",
+ "path": [
+ "Samples",
+ "samples.dremio.com",
+ "Dremio University",
+ "googleplaystore.csv",
+ ],
+ "format": {
+ "fieldDelimiter": ",",
+ "quote": '"',
+ "comment": "#",
+ "lineDelimiter": "\r\n",
+ "escape": '"',
+ "extractHeader": False,
+ "trimHeader": True,
+ "skipFirstLine": False,
+ "type": "Text",
+ },
}
- response = requests.put(url, headers=headers, data=json.dumps(payload))
+ response = requests.post(url, headers=headers, data=json.dumps(payload))
assert response.status_code == 200, f"Failed to add dataset: {response.text}"
- url = f"{DREMIO_HOST}/apiv2/source/Samples/file_format/samples.dremio.com/tpcds_sf1000/catalog_page/1ab266d5-18eb-4780-711d-0fa337fa6c00/0_0_0.parquet"
- payload = {"ignoreOtherFileFormats": False, "type": "Parquet"}
+ url = f"{DREMIO_HOST}/api/v3/catalog/dremio%3A%2FSamples%2Fsamples.dremio.com%2Ftpcds_sf1000%2Fcatalog_page%2F1ab266d5-18eb-4780-711d-0fa337fa6c00%2F0_0_0.parquet"
+ payload = {
+ "entityType": "dataset",
+ "type": "PHYSICAL_DATASET",
+ "path": [
+ "Samples",
+ "samples.dremio.com",
+ "tpcds_sf1000",
+ "catalog_page",
+ "1ab266d5-18eb-4780-711d-0fa337fa6c00",
+ "0_0_0.parquet",
+ ],
+ "format": {"type": "Parquet"},
+ }
- response = requests.put(url, headers=headers, data=json.dumps(payload))
+ response = requests.post(url, headers=headers, data=json.dumps(payload))
assert response.status_code == 200, f"Failed to add dataset: {response.text}"
@@ -253,7 +306,7 @@ def create_view(headers):
"entityType": "dataset",
"type": "VIRTUAL_DATASET",
"path": ["space", "test_folder", "raw"],
- "sql": 'SELECT * FROM s3.warehouse."sample.parquet"',
+ "sql": "SELECT * FROM s3.warehouse",
}
response = requests.post(url, headers=headers, data=json.dumps(payload))
assert response.status_code == 200, f"Failed to create view: {response.text}"
@@ -273,7 +326,7 @@ def create_view(headers):
"entityType": "dataset",
"type": "VIRTUAL_DATASET",
"path": ["space", "test_folder", "customers"],
- "sql": 'SELECT * FROM "mysql".northwind.customers',
+ "sql": "SELECT * FROM mysql.northwind.customers",
"sqlContext": ["mysql", "northwind"],
}
response = requests.post(url, headers=headers, data=json.dumps(payload))
@@ -283,7 +336,7 @@ def create_view(headers):
"entityType": "dataset",
"type": "VIRTUAL_DATASET",
"path": ["space", "test_folder", "orders"],
- "sql": 'SELECT * FROM "mysql".northwind.orders',
+ "sql": "SELECT * FROM mysql.northwind.orders",
"sqlContext": ["mysql", "northwind"],
}
response = requests.post(url, headers=headers, data=json.dumps(payload))
@@ -293,7 +346,7 @@ def create_view(headers):
"entityType": "dataset",
"type": "VIRTUAL_DATASET",
"path": ["space", "test_folder", "metadata_aspect"],
- "sql": 'SELECT * FROM "mysql".metagalaxy."metadata_aspect"',
+ "sql": "SELECT * FROM mysql.metagalaxy.metadata_aspect",
"sqlContext": ["mysql", "metagalaxy"],
}
response = requests.post(url, headers=headers, data=json.dumps(payload))
@@ -303,7 +356,7 @@ def create_view(headers):
"entityType": "dataset",
"type": "VIRTUAL_DATASET",
"path": ["space", "test_folder", "metadata_index"],
- "sql": 'SELECT * FROM "mysql".metagalaxy."metadata_index"',
+ "sql": "SELECT * FROM mysql.metagalaxy.metadata_index",
"sqlContext": ["mysql", "metagalaxy"],
}
response = requests.post(url, headers=headers, data=json.dumps(payload))
@@ -313,7 +366,7 @@ def create_view(headers):
"entityType": "dataset",
"type": "VIRTUAL_DATASET",
"path": ["space", "test_folder", "metadata_index_view"],
- "sql": 'SELECT * FROM "mysql".metagalaxy."metadata_index_view"',
+ "sql": "SELECT * FROM mysql.metagalaxy.metadata_index_view",
"sqlContext": ["mysql", "metagalaxy"],
}
response = requests.post(url, headers=headers, data=json.dumps(payload))
@@ -422,14 +475,119 @@ def test_dremio_ingest(
pytestconfig,
tmp_path,
):
- # Run the metadata ingestion pipeline.
+ # Run the metadata ingestion pipeline with specific output file
config_file = (test_resources_dir / "dremio_to_file.yml").resolve()
+ output_path = tmp_path / "dremio_mces.json"
+
run_datahub_cmd(["ingest", "-c", f"{config_file}"], tmp_path=tmp_path)
- # Verify the output.
+ # Verify the output
mce_helpers.check_golden_file(
pytestconfig,
- output_path=tmp_path / "dremio_mces.json",
+ output_path=output_path,
golden_path=test_resources_dir / "dremio_mces_golden.json",
ignore_paths=[],
)
+
+
+@freeze_time(FROZEN_TIME)
+@pytest.mark.integration
+def test_dremio_platform_instance_urns(
+ test_resources_dir,
+ dremio_setup,
+ pytestconfig,
+ tmp_path,
+):
+ config_file = (
+ test_resources_dir / "dremio_platform_instance_to_file.yml"
+ ).resolve()
+ output_path = tmp_path / "dremio_mces.json"
+
+ run_datahub_cmd(["ingest", "-c", f"{config_file}"], tmp_path=tmp_path)
+
+ with output_path.open() as f:
+ content = f.read()
+ # Skip if file is empty or just contains brackets
+ if not content or content.strip() in ("[]", "[", "]"):
+ pytest.fail(f"Output file is empty or invalid: {content}")
+
+ try:
+ # Try to load as JSON Lines first
+ mces = []
+ for line in content.splitlines():
+ line = line.strip()
+ if line and line not in ("[", "]"): # Skip empty lines and bare brackets
+ mce = json.loads(line)
+ mces.append(mce)
+ except json.JSONDecodeError:
+ # If that fails, try loading as a single JSON array
+ try:
+ mces = json.loads(content)
+ except json.JSONDecodeError as e:
+ print(f"Failed to parse file content: {content}")
+ raise e
+
+ # Verify MCEs
+ assert len(mces) > 0, "No MCEs found in output file"
+
+ # Verify the platform instances
+ for mce in mces:
+ if "entityType" not in mce:
+ continue
+
+ # Check dataset URN structure
+ if mce["entityType"] == "dataset" and "entityUrn" in mce:
+ assert (
+ "test-platform.dremio" in mce["entityUrn"]
+ ), f"Platform instance missing in dataset URN: {mce['entityUrn']}"
+
+ # Check aspects for both datasets and containers
+ if "aspectName" in mce:
+ # Check dataPlatformInstance aspect
+ if mce["aspectName"] == "dataPlatformInstance":
+ aspect = mce["aspect"]
+ if not isinstance(aspect, Dict) or "json" not in aspect:
+ continue
+
+ aspect_json = aspect["json"]
+ if not isinstance(aspect_json, Dict):
+ continue
+
+ if "instance" not in aspect_json:
+ continue
+
+ instance = aspect_json["instance"]
+ expected_instance = "urn:li:dataPlatformInstance:(urn:li:dataPlatform:dremio,test-platform)"
+ assert (
+ instance == expected_instance
+ ), f"Invalid platform instance format: {instance}"
+
+ # Verify against golden file
+ mce_helpers.check_golden_file(
+ pytestconfig,
+ output_path=output_path,
+ golden_path=test_resources_dir / "dremio_platform_instance_mces_golden.json",
+ ignore_paths=[],
+ )
+
+
+@freeze_time(FROZEN_TIME)
+@pytest.mark.integration
+def test_dremio_schema_filter(
+ test_resources_dir,
+ dremio_setup,
+ pytestconfig,
+ tmp_path,
+):
+ config_file = (test_resources_dir / "dremio_schema_filter_to_file.yml").resolve()
+ output_path = tmp_path / "dremio_mces.json"
+
+ run_datahub_cmd(["ingest", "-c", f"{config_file}"], tmp_path=tmp_path)
+
+ # Verify against golden file
+ mce_helpers.check_golden_file(
+ pytestconfig,
+ output_path=output_path,
+ golden_path=test_resources_dir / "dremio_schema_filter_mces_golden.json",
+ ignore_paths=[],
+ )
diff --git a/metadata-ingestion/tests/unit/dremio/test_dremio_schema_filter.py b/metadata-ingestion/tests/unit/dremio/test_dremio_schema_filter.py
new file mode 100644
index 00000000000000..27df05296b7521
--- /dev/null
+++ b/metadata-ingestion/tests/unit/dremio/test_dremio_schema_filter.py
@@ -0,0 +1,123 @@
+from unittest.mock import Mock
+
+import pytest
+
+from datahub.ingestion.source.dremio.dremio_api import DremioAPIOperations
+from datahub.ingestion.source.dremio.dremio_config import DremioSourceConfig
+from datahub.ingestion.source.dremio.dremio_reporting import DremioSourceReport
+
+
+class TestDremioContainerFiltering:
+ @pytest.fixture
+ def dremio_api(self, monkeypatch):
+ # Mock the requests.Session
+ mock_session = Mock()
+ monkeypatch.setattr("requests.Session", Mock(return_value=mock_session))
+
+ # Mock the authentication response
+ mock_session.post.return_value.json.return_value = {"token": "dummy-token"}
+ mock_session.post.return_value.status_code = 200
+
+ config = DremioSourceConfig(
+ hostname="dummy-host",
+ port=9047,
+ tls=False,
+ authentication_method="password",
+ username="dummy-user",
+ password="dummy-password",
+ schema_pattern=dict(allow=[".*"], deny=[]),
+ )
+ report = DremioSourceReport()
+ return DremioAPIOperations(config, report)
+
+ def test_basic_allow_pattern(self, dremio_api):
+ """Test basic allow pattern matching"""
+ dremio_api.allow_schema_pattern = ["test"]
+ dremio_api.deny_schema_pattern = []
+
+ assert dremio_api.should_include_container([], "test")
+ assert dremio_api.should_include_container(["test"], "subfolder")
+ assert not dremio_api.should_include_container([], "prod_space")
+
+ def test_basic_deny_pattern(self, dremio_api):
+ """Test basic deny pattern matching"""
+ dremio_api.allow_schema_pattern = [".*"]
+ dremio_api.deny_schema_pattern = ["test_space.*"]
+
+ assert not dremio_api.should_include_container([], "test_space")
+ assert not dremio_api.should_include_container(["test_space"], "subfolder")
+ assert dremio_api.should_include_container([], "prod_space")
+
+ def test_hierarchical_matching(self, dremio_api):
+ """Test matching with hierarchical paths"""
+ dremio_api.allow_schema_pattern = ["prod.data.*"]
+ dremio_api.deny_schema_pattern = []
+
+ assert dremio_api.should_include_container([], "prod")
+ assert dremio_api.should_include_container(["prod"], "data")
+ assert dremio_api.should_include_container(["prod", "data"], "sales")
+ assert not dremio_api.should_include_container([], "dev")
+ assert not dremio_api.should_include_container(["dev"], "data")
+
+ def test_allow_and_deny_patterns(self, dremio_api):
+ """Test combination of allow and deny patterns"""
+ dremio_api.allow_schema_pattern = ["prod.*"]
+ dremio_api.deny_schema_pattern = ["prod.internal.*"]
+
+ assert dremio_api.should_include_container([], "prod")
+ assert dremio_api.should_include_container(["prod"], "public")
+ assert dremio_api.should_include_container(["prod", "public"], "next")
+ assert not dremio_api.should_include_container(["prod"], "internal")
+ assert not dremio_api.should_include_container(["prod", "internal"], "secrets")
+
+ def test_wildcard_patterns(self, dremio_api):
+ """Test wildcard pattern handling"""
+ dremio_api.allow_schema_pattern = [".*"]
+ dremio_api.deny_schema_pattern = []
+
+ assert dremio_api.should_include_container([], "any_space")
+ assert dremio_api.should_include_container(["any_space"], "any_folder")
+
+ # Test with specific wildcard in middle
+ dremio_api.allow_schema_pattern = ["prod.*.public"]
+ assert dremio_api.should_include_container(["prod", "customer"], "public")
+ assert not dremio_api.should_include_container(["prod", "customer"], "private")
+
+ def test_case_insensitive_matching(self, dremio_api):
+ """Test case-insensitive pattern matching"""
+ dremio_api.allow_schema_pattern = ["PROD.*"]
+ dremio_api.deny_schema_pattern = []
+
+ assert dremio_api.should_include_container([], "prod")
+ assert dremio_api.should_include_container([], "PROD")
+ assert dremio_api.should_include_container(["prod"], "DATA")
+ assert dremio_api.should_include_container(["PROD"], "data")
+
+ def test_empty_patterns(self, dremio_api):
+ """Test behavior with empty patterns"""
+ dremio_api.allow_schema_pattern = [".*"]
+ dremio_api.deny_schema_pattern = []
+
+ # Should allow everything when allow pattern is empty
+ assert dremio_api.should_include_container([], "any_space")
+ assert dremio_api.should_include_container(["any_space"], "any_folder")
+
+ def test_partial_path_matching(self, dremio_api):
+ """Test matching behavior with partial paths"""
+ dremio_api.allow_schema_pattern = ["^pr.*.data.*"]
+ dremio_api.deny_schema_pattern = []
+
+ assert dremio_api.should_include_container(["prod"], "data")
+ # Should match the partial path even though pattern doesn't have wildcards
+ assert dremio_api.should_include_container(["prod", "data"], "sales")
+ assert not dremio_api.should_include_container([], "dev")
+ assert not dremio_api.should_include_container(["dev", "data"], "sales")
+
+ def test_partial_start_end_chars(self, dremio_api):
+ """Test matching behavior with partial paths"""
+ dremio_api.allow_schema_pattern = ["pr.*.data$"]
+ dremio_api.deny_schema_pattern = []
+
+ assert dremio_api.should_include_container(["prod"], "data")
+ # Should match the partial path even though pattern doesn't have wildcards
+ assert not dremio_api.should_include_container(["prod", "data"], "sales")