diff --git a/poetry.lock b/poetry.lock index 369d4ad00..e6babece1 100644 --- a/poetry.lock +++ b/poetry.lock @@ -59,7 +59,7 @@ wrapt = [ name = "attrs" version = "22.1.0" description = "Classes Without Boilerplate" -category = "dev" +category = "main" optional = false python-versions = ">=3.5" files = [ @@ -148,6 +148,32 @@ urllib3 = {version = ">=1.25.4,<2.2.0 || >2.2.0,<3", markers = "python_version > [package.extras] crt = ["awscrt (==0.22.0)"] +[[package]] +name = "cattrs" +version = "23.1.2" +description = "Composable complex class support for attrs and dataclasses." +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "cattrs-23.1.2-py3-none-any.whl", hash = "sha256:b2bb14311ac17bed0d58785e5a60f022e5431aca3932e3fc5cc8ed8639de50a4"}, + {file = "cattrs-23.1.2.tar.gz", hash = "sha256:db1c821b8c537382b2c7c66678c3790091ca0275ac486c76f3c8f3920e83c657"}, +] + +[package.dependencies] +attrs = ">=20" +exceptiongroup = {version = "*", markers = "python_version < \"3.11\""} +typing_extensions = {version = ">=4.1.0", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.2.0,<5.0.0)"] +cbor2 = ["cbor2 (>=5.4.6,<6.0.0)"] +msgpack = ["msgpack (>=1.0.2,<2.0.0)"] +orjson = ["orjson (>=3.5.2,<4.0.0)"] +pyyaml = ["PyYAML (>=6.0,<7.0)"] +tomlkit = ["tomlkit (>=0.11.4,<0.12.0)"] +ujson = ["ujson (>=5.4.0,<6.0.0)"] + [[package]] name = "certifi" version = "2018.8.24" @@ -424,7 +450,7 @@ files = [ name = "exceptiongroup" version = "1.0.4" description = "Backport of PEP 654 (exception groups)" -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -764,6 +790,24 @@ files = [ {file = "lazy_object_proxy-1.8.0-pp39-pypy39_pp73-any.whl", hash = "sha256:ce58b2b3734c73e68f0e30e4e725264d4d6be95818ec0a0be4bb6bf9a7e79aa8"}, ] +[[package]] +name = "looker-sdk" +version = "25.0.0" +description = "Looker REST API" +category = "main" +optional = false +python-versions = ">=3.6" +files = [ + {file = "looker_sdk-25.0.0-py3-none-any.whl", hash = "sha256:495704818b0af0cc1c18b9cfe16bc2429ca7d4a1a9f0f47fc6ae93c1c6bd6e33"}, + {file = "looker_sdk-25.0.0.tar.gz", hash = "sha256:4930288792a4024a5e661ff7756aa5e1fab1357a53aa6e32f8a7d16f94dd506d"}, +] + +[package.dependencies] +attrs = {version = ">=20.1.0", markers = "python_version >= \"3.7\""} +cattrs = {version = ">=1.3", markers = "python_version >= \"3.7\""} +requests = ">=2.22" +typing-extensions = ">=4.1.1" + [[package]] name = "markupsafe" version = "3.0.0" @@ -2016,4 +2060,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "5bab345108ce44441fd4b60849db7e239221d1d4fd99aa0a95466d80832fd5b5" +content-hash = "a1a2304aba8367254170aacd89c6a2fb5b5bb34a47a0fdec9188c8126354d0d4" diff --git a/pyproject.toml b/pyproject.toml index e90558561..30601e930 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,7 @@ tabulate = "^0.9.0" pydantic = "^2.8.2" pyyaml = "^6.0.2" boto3 = "^1.35.35" +looker-sdk = "^25.0.0" [tool.poetry.group.dev.dependencies] pylint = "^2.15.4" diff --git a/utils/dashboard_export.py b/utils/dashboard_export.py new file mode 100644 index 000000000..e3102b3bf --- /dev/null +++ b/utils/dashboard_export.py @@ -0,0 +1,61 @@ +import json + +import looker_sdk +import tqdm +from looker_sdk import error, models40 +from slugify import slugify + +sdk = looker_sdk.init40() # or init31() for the older v3.1 API + + +def get_dashboard_list(): + dashboards_query = models40.WriteQuery( + model="system__activity", + view="dashboard", + fields=[ + "dashboard.id", + "dashboard.title", + "dashboard_element.count", + "dashboard_element.count_text", + "query.count", + ], + pivots=None, + fill_fields=None, + filters={ + "dashboard.deleted_date": "NULL", + "dashboard.moved_to_trash": "No", + }, + filter_expression=None, + sorts=["query.count desc"], + limit="5000", + ) + + get_dashboards_query = sdk.create_query(body=dashboards_query) + + return json.loads(sdk.run_query(query_id=get_dashboards_query.id, result_format="json", cache=True)) + + +def export_dashboards(): + dashboard_list = get_dashboard_list() + + count = 0 + for dashboard in tqdm.tqdm(dashboard_list): + dashboard_id = str(dashboard["dashboard.id"]) + title = slugify(dashboard["dashboard.title"]) + try: + dashboard_lookml = sdk.dashboard_lookml(dashboard_id=dashboard_id)['lookml'] + with open(f'dashboards/{dashboard_id}-{title}.dashboard.lookml', 'w') as fp: + fp.write(dashboard_lookml) + except error.SDKError: + count += 1 + print(f"Broken dashboard, dashboard LookML was not imported for dashboard {dashboard_id}.") + + print(f"Total: {len(dashboard_list)} dashboards") + print(f"Failed to extract {count} lookml") + + +if __name__ == '__main__': + # Make sure that the environment variables defined in + # https://github.com/looker-open-source/sdk-codegen?tab=readme-ov-file#environment-variable-configuration have been + # set. + export_dashboards()