Skip to content

Commit

Permalink
test: fixed trino source
Browse files Browse the repository at this point in the history
  • Loading branch information
@sagar-salvi-apptware
sagar-salvi-apptware committed Dec 26, 2024
1 parent dd14114 commit deaab5c
Show file tree
Hide file tree
Showing 2 changed files with 268 additions and 64 deletions.
166 changes: 134 additions & 32 deletions metadata-ingestion/tests/integration/trino/trino_hive_instance_mces_golden.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,22 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:46baa6eebd802861e5ee3d043456e171",
"changeType": "UPSERT",
"aspectName": "container",
"aspect": {
"json": {
"container": "urn:li:container:f311add3fdc7c16e8a50a63fe1dcce8b"
}
},
"systemMetadata": {
"lastObserved": 1632398400000,
"runId": "trino-hive-instance-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:46baa6eebd802861e5ee3d043456e171",
Expand Down Expand Up @@ -169,22 +185,6 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:46baa6eebd802861e5ee3d043456e171",
"changeType": "UPSERT",
"aspectName": "container",
"aspect": {
"json": {
"container": "urn:li:container:f311add3fdc7c16e8a50a63fe1dcce8b"
}
},
"systemMetadata": {
"lastObserved": 1632398400000,
"runId": "trino-hive-instance-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "container",
"entityUrn": "urn:li:container:46baa6eebd802861e5ee3d043456e171",
Expand Down Expand Up @@ -246,7 +246,7 @@
"numrows": "1",
"rawdatasize": "32",
"totalsize": "33",
"transient_lastddltime": "1724180599"
"transient_lastddltime": "1735206396"
},
"name": "array_struct_test",
"description": "This table has array of structs",
Expand Down Expand Up @@ -507,7 +507,7 @@
"numrows": "3",
"rawdatasize": "94",
"totalsize": "97",
"transient_lastddltime": "1724180605"
"transient_lastddltime": "1735206403"
},
"name": "classification_test",
"tags": []
Expand Down Expand Up @@ -766,7 +766,7 @@
"numrows": "0",
"rawdatasize": "0",
"totalsize": "0",
"transient_lastddltime": "1724180602"
"transient_lastddltime": "1735206400"
},
"name": "map_test",
"tags": []
Expand Down Expand Up @@ -993,7 +993,7 @@
"numrows": "0",
"rawdatasize": "0",
"totalsize": "0",
"transient_lastddltime": "1724180602"
"transient_lastddltime": "1735206400"
},
"name": "nested_struct_test",
"tags": []
Expand Down Expand Up @@ -1264,7 +1264,7 @@
{
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
"customProperties": {
"transient_lastddltime": "1724180591"
"transient_lastddltime": "1735206384"
},
"name": "pokes",
"tags": []
Expand Down Expand Up @@ -1499,7 +1499,7 @@
"numrows": "0",
"rawdatasize": "0",
"totalsize": "0",
"transient_lastddltime": "1724180595"
"transient_lastddltime": "1735206390"
},
"name": "struct_test",
"tags": []
Expand Down Expand Up @@ -1750,7 +1750,7 @@
"customProperties": {
"numfiles": "0",
"totalsize": "0",
"transient_lastddltime": "1724180601"
"transient_lastddltime": "1735206399"
},
"name": "struct_test_view_materialized",
"tags": []
Expand Down Expand Up @@ -2004,7 +2004,7 @@
"numrows": "0",
"rawdatasize": "0",
"totalsize": "0",
"transient_lastddltime": "1724180595"
"transient_lastddltime": "1735206390"
},
"name": "_test_table_underscore",
"tags": []
Expand Down Expand Up @@ -2227,7 +2227,7 @@
"numrows": "0",
"rawdatasize": "0",
"totalsize": "0",
"transient_lastddltime": "1724180602"
"transient_lastddltime": "1735206400"
},
"name": "union_test",
"tags": []
Expand Down Expand Up @@ -2529,9 +2529,9 @@
{
"com.linkedin.pegasus2avro.dataset.DatasetProperties": {
"customProperties": {
"transient_lastddltime": "1724180602",
"view_definition": "SELECT \"property_id\", \"service\"\nFROM \"db1\".\"array_struct_test\"",
"is_view": "True"
"transient_lastddltime": "1735206400",
"is_view": "True",
"view_definition": "SELECT \"property_id\", \"service\"\nFROM \"db1\".\"array_struct_test\""
},
"name": "array_struct_test_view",
"tags": []
Expand Down Expand Up @@ -2758,11 +2758,16 @@
"upstreams": [
{
"auditStamp": {
"time": 1632398400000,
"actor": "urn:li:corpuser:_ingestion"
},
"created": {
"time": 0,
"actor": "urn:li:corpuser:unknown"
"actor": "urn:li:corpuser:_ingestion"
},
"dataset": "urn:li:dataset:(urn:li:dataPlatform:trino,production_warehouse.hivedb.db1.array_struct_test,PROD)",
"type": "VIEW"
"type": "VIEW",
"query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Atrino%2Cproduction_warehouse.hivedb.db1.array_struct_test_view%2CPROD%29"
}
],
"fineGrainedLineages": [
Expand All @@ -2775,7 +2780,8 @@
"downstreams": [
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:trino,production_warehouse.hivedb.db1.array_struct_test_view,PROD),property_id)"
],
"confidenceScore": 1.0
"confidenceScore": 0.9,
"query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Atrino%2Cproduction_warehouse.hivedb.db1.array_struct_test_view%2CPROD%29"
},
{
"upstreamType": "FIELD_SET",
Expand All @@ -2786,7 +2792,8 @@
"downstreams": [
"urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:trino,production_warehouse.hivedb.db1.array_struct_test_view,PROD),service)"
],
"confidenceScore": 1.0
"confidenceScore": 0.9,
"query": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Atrino%2Cproduction_warehouse.hivedb.db1.array_struct_test_view%2CPROD%29"
}
]
}
Expand All @@ -2797,6 +2804,85 @@
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "query",
"entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Atrino%2Cproduction_warehouse.hivedb.db1.array_struct_test_view%2CPROD%29",
"changeType": "UPSERT",
"aspectName": "queryProperties",
"aspect": {
"json": {
"statement": {
"value": "SELECT\n \"property_id\",\n \"service\"\nFROM \"db1\".\"array_struct_test\"",
"language": "SQL"
},
"source": "SYSTEM",
"created": {
"time": 0,
"actor": "urn:li:corpuser:_ingestion"
},
"lastModified": {
"time": 1632398400000,
"actor": "urn:li:corpuser:_ingestion"
}
}
},
"systemMetadata": {
"lastObserved": 1632398400000,
"runId": "trino-hive-instance-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "query",
"entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Atrino%2Cproduction_warehouse.hivedb.db1.array_struct_test_view%2CPROD%29",
"changeType": "UPSERT",
"aspectName": "querySubjects",
"aspect": {
"json": {
"subjects": [
{
"entity": "urn:li:dataset:(urn:li:dataPlatform:trino,production_warehouse.hivedb.db1.array_struct_test,PROD)"
},
{
"entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:trino,production_warehouse.hivedb.db1.array_struct_test,PROD),property_id)"
},
{
"entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:trino,production_warehouse.hivedb.db1.array_struct_test,PROD),service)"
},
{
"entity": "urn:li:dataset:(urn:li:dataPlatform:trino,production_warehouse.hivedb.db1.array_struct_test_view,PROD)"
},
{
"entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:trino,production_warehouse.hivedb.db1.array_struct_test_view,PROD),property_id)"
},
{
"entity": "urn:li:schemaField:(urn:li:dataset:(urn:li:dataPlatform:trino,production_warehouse.hivedb.db1.array_struct_test_view,PROD),service)"
}
]
}
},
"systemMetadata": {
"lastObserved": 1632398400000,
"runId": "trino-hive-instance-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "query",
"entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Atrino%2Cproduction_warehouse.hivedb.db1.array_struct_test_view%2CPROD%29",
"changeType": "UPSERT",
"aspectName": "dataPlatformInstance",
"aspect": {
"json": {
"platform": "urn:li:dataPlatform:trino"
}
},
"systemMetadata": {
"lastObserved": 1632398400000,
"runId": "trino-hive-instance-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "dataset",
"entityUrn": "urn:li:dataset:(urn:li:dataPlatform:glue,local_server.db1._test_table_underscore,PROD)",
Expand Down Expand Up @@ -2956,5 +3042,21 @@
"runId": "trino-hive-instance-test",
"lastRunId": "no-run-id-provided"
}
},
{
"entityType": "query",
"entityUrn": "urn:li:query:view_urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Atrino%2Cproduction_warehouse.hivedb.db1.array_struct_test_view%2CPROD%29",
"changeType": "UPSERT",
"aspectName": "status",
"aspect": {
"json": {
"removed": false
}
},
"systemMetadata": {
"lastObserved": 1632398400000,
"runId": "trino-hive-instance-test",
"lastRunId": "no-run-id-provided"
}
}
]
Loading

0 comments on commit deaab5c

Please sign in to comment.