From 75707e163dc9cb58328350c04cc4603280a31410 Mon Sep 17 00:00:00 2001 From: Dan Wolfson Date: Fri, 30 Aug 2024 14:10:03 -0500 Subject: [PATCH] Added docker compose configuration, minor fixes in common routines and changed structure to accomodate Compose. Signed-off-by: Dan Wolfson --- README.md | 4 + .../administration-labs}/README.md | 0 .../managing-servers.ipynb | 0 .../understanding-cohorts.ipynb | 0 .../understanding-platform-services.ipynb | 0 .../understanding-server-config.ipynb | 0 .../asset-management-labs}/README.md | 2 +- .../automated-curation.ipynb | 0 .../building-a-data-catalog.ipynb | 0 .../improving-data-quality.ipynb | 0 .../asset-management-labs}/open-lineage.ipynb | 0 .../understanding-an-asset.ipynb | 0 .../common}/common-functions.ipynb | 99 +++-- coco-jupyter-labs/common/config_coco_core.py | 298 +++++++++++++ .../common/config_coco_datalake.py | 415 ++++++++++++++++++ .../common/config_coco_development.py | 368 ++++++++++++++++ .../common}/environment-check.ipynb | 0 .../common}/globals.ipynb | 4 +- coco-jupyter-labs/common/globals.py | 153 +++++++ .../analyze-performance-results.ipynb | 0 .../run-conformance-test-suite.ipynb | 0 .../run-performance-test-suite.ipynb | 0 .../run-ui-test-server.ipynb | 0 .../governance-server-operation.ipynb | 0 .../metadata-representations.ipynb | 0 .../simple-cohort-demo.ipynb | 0 .../egeria-server-config.ipynb | 0 .../egeria-server-start.ipynb | 0 .../images}/CTS-Cohort.png | Bin .../asset-zones-for-building-catalog.png | Bin .../images}/automated-curation-scenario-1.png | Bin .../images}/automated-curation-scenario-2.png | Bin .../images}/automated-curation-scenario-3.png | Bin .../automated-curation-scenario-3b.png | Bin .../images}/automated-curation-scenario-4.png | Bin .../images}/automated-curation-scenario-5.png | Bin .../images}/automated-curation-scenario-6.png | Bin .../images}/automated-curation-scenario-7.png | Bin .../images}/automated-curation-scenario.png | Bin .../images}/callie-quartile.png | Bin .../coco-pharmaceuticals-systems-cohorts.png | Bin ...tems-omag-server-platforms-engine-host.png | Bin ...ag-server-platforms-integration-daemon.png | Bin ...-omag-server-platforms-metadata-server.png | Bin ...tems-omag-server-platforms-view-server.png | Bin ...euticals-systems-omag-server-platforms.png | Bin .../coco-pharmaceuticals-systems.drawio | 0 .../creating-configuration-documents.png | Bin .../images}/database-type-hierarchy.png | Bin .../deploying-configuration-documents.png | Bin .../images}/design-model-to-archive-load.png | Bin .../discovery-engine-configuration.png | Bin .../images}/discovery-servers.png | Bin .../distributed-asset-management-lab.drawio | 0 .../distributed-asset-with-comments.png | Bin .../images}/distributed-discovery-engines.png | Bin .../distributed-engine-services-config.png | Bin .../images}/egeria-as-tool-integrator.png | Bin ...i-asset-catalog-archive-search-results.png | Bin .../egeria-ui-asset-catalog-no-rows.png | Bin ...ia-ui-asset-catalog-search-field-input.png | Bin .../egeria-ui-asset-end-to-end-lineage.png | Bin .../images}/egeria-ui-asset-list.png | Bin .../images}/egeria-ui-asset-search.png | Bin .../egeria-ui-end-to-end-lineage-graph.png | Bin .../images}/egeria-ui-erin-logon.png | Bin .../images}/egeria-ui-login.png | Bin .../images}/egeria-ui-nav-bar.png | Bin .../images}/engine-host.png | Bin .../images}/exchangeDL01-config.png | Bin .../images}/file-asset-hierarchy.png | Bin .../images}/glossary-revision-history.png | Bin .../images}/glossary-structure.png | Bin .../governance-action-process-flow.png | Bin ...vernance-action-process-implementation.png | Bin .../governance-action-request-type.png | Bin .../images}/governance-engine-definition.png | Bin .../governance-engine-lab-platform-part.png | Bin .../governance-engine-lab-platform.png | Bin .../governance-engine-lab-servers-part-2.png | Bin .../governance-engine-lab-servers-part.png | Bin .../images}/governance-engine-lab-servers.png | Bin ...nance-servers-for-automated-onboarding.png | Bin ...e-servers-running-automated-onboarding.png | Bin .../images}/integration-daemon.png | Bin .../images}/jupyter-notebook-play-button.png | Bin .../images}/kafka-server-properties.png | Bin .../images}/omag-server-platform-ready.png | Bin .../images}/open-lineage-server.png | Bin .../open-lineage-service-lab-assets.png | Bin .../images}/open-lineage-services.drawio | 0 .../images}/peter-and-erin.png | Bin .../images}/peter-profile.png | Bin .../images}/simple-api-catalog.png | Bin .../simple-catalog-demo-asset-properties.png | Bin ...simple-catalog-demo-connected-catalogs.png | Bin .../images}/simple-catalog-demo-data-map.png | Bin ...mple-catalog-demo-independent-catalogs.png | Bin .../images}/simple-catalog-demo.png | Bin ...simple-catalog-metadata-origin-comment.png | Bin .../simple-catalog-metadata-origin.png | Bin .../images}/simple-catalog.png | Bin .../images}/simple-data-catalog.png | Bin .../images}/simple-event-catalog.png | Bin .../working-with-standard-models.ipynb | 0 .../labs-in-development}/README.md | 0 .../labs-in-development}/jdbc-connector.ipynb | 0 .../overview}/egeria-demo.ipynb | 0 .../read-me-first.ipynb | 6 +- .../ui-labs}/ui-asset-search.ipynb | 0 .../Coco-Pharma Docker Compose.iml | 11 + coco-lab-docker-compose/Dockerfile | 34 ++ coco-lab-docker-compose/README.md | 49 +++ .../before-notebook.d/config-coco.sh | 18 + coco-lab-docker-compose/coco-lab-setup.yaml | 130 ++++++ .../core_application.properties | 227 ++++++++++ .../datalake_application.properties | 226 ++++++++++ .../dev_application.properties | 226 ++++++++++ requirements.txt | 5 +- 119 files changed, 2221 insertions(+), 54 deletions(-) rename {administration-labs => coco-jupyter-labs/administration-labs}/README.md (100%) rename {administration-labs => coco-jupyter-labs/administration-labs}/managing-servers.ipynb (100%) rename {administration-labs => coco-jupyter-labs/administration-labs}/understanding-cohorts.ipynb (100%) rename {administration-labs => coco-jupyter-labs/administration-labs}/understanding-platform-services.ipynb (100%) rename {administration-labs => coco-jupyter-labs/administration-labs}/understanding-server-config.ipynb (100%) rename {asset-management-labs => coco-jupyter-labs/asset-management-labs}/README.md (94%) rename {asset-management-labs => coco-jupyter-labs/asset-management-labs}/automated-curation.ipynb (100%) rename {asset-management-labs => coco-jupyter-labs/asset-management-labs}/building-a-data-catalog.ipynb (100%) rename {asset-management-labs => coco-jupyter-labs/asset-management-labs}/improving-data-quality.ipynb (100%) rename {asset-management-labs => coco-jupyter-labs/asset-management-labs}/open-lineage.ipynb (100%) rename {asset-management-labs => coco-jupyter-labs/asset-management-labs}/understanding-an-asset.ipynb (100%) rename {common => coco-jupyter-labs/common}/common-functions.ipynb (99%) create mode 100644 coco-jupyter-labs/common/config_coco_core.py create mode 100644 coco-jupyter-labs/common/config_coco_datalake.py create mode 100755 coco-jupyter-labs/common/config_coco_development.py rename {common => coco-jupyter-labs/common}/environment-check.ipynb (100%) rename {common => coco-jupyter-labs/common}/globals.ipynb (96%) mode change 100755 => 100644 create mode 100755 coco-jupyter-labs/common/globals.py rename {conformance-testing-labs => coco-jupyter-labs/conformance-testing-labs}/analyze-performance-results.ipynb (100%) rename {conformance-testing-labs => coco-jupyter-labs/conformance-testing-labs}/run-conformance-test-suite.ipynb (100%) rename {conformance-testing-labs => coco-jupyter-labs/conformance-testing-labs}/run-performance-test-suite.ipynb (100%) rename {conformance-testing-labs => coco-jupyter-labs/conformance-testing-labs}/run-ui-test-server.ipynb (100%) rename {development-labs => coco-jupyter-labs/development-labs}/governance-server-operation.ipynb (100%) rename {development-labs => coco-jupyter-labs/development-labs}/metadata-representations.ipynb (100%) rename {development-labs => coco-jupyter-labs/development-labs}/simple-cohort-demo.ipynb (100%) rename egeria-server-config.ipynb => coco-jupyter-labs/egeria-server-config.ipynb (100%) rename egeria-server-start.ipynb => coco-jupyter-labs/egeria-server-start.ipynb (100%) rename {images => coco-jupyter-labs/images}/CTS-Cohort.png (100%) rename {images => coco-jupyter-labs/images}/asset-zones-for-building-catalog.png (100%) rename {images => coco-jupyter-labs/images}/automated-curation-scenario-1.png (100%) rename {images => coco-jupyter-labs/images}/automated-curation-scenario-2.png (100%) rename {images => coco-jupyter-labs/images}/automated-curation-scenario-3.png (100%) rename {images => coco-jupyter-labs/images}/automated-curation-scenario-3b.png (100%) rename {images => coco-jupyter-labs/images}/automated-curation-scenario-4.png (100%) rename {images => coco-jupyter-labs/images}/automated-curation-scenario-5.png (100%) rename {images => coco-jupyter-labs/images}/automated-curation-scenario-6.png (100%) rename {images => coco-jupyter-labs/images}/automated-curation-scenario-7.png (100%) rename {images => coco-jupyter-labs/images}/automated-curation-scenario.png (100%) rename {images => coco-jupyter-labs/images}/callie-quartile.png (100%) rename {images => coco-jupyter-labs/images}/coco-pharmaceuticals-systems-cohorts.png (100%) rename {images => coco-jupyter-labs/images}/coco-pharmaceuticals-systems-omag-server-platforms-engine-host.png (100%) rename {images => coco-jupyter-labs/images}/coco-pharmaceuticals-systems-omag-server-platforms-integration-daemon.png (100%) rename {images => coco-jupyter-labs/images}/coco-pharmaceuticals-systems-omag-server-platforms-metadata-server.png (100%) rename {images => coco-jupyter-labs/images}/coco-pharmaceuticals-systems-omag-server-platforms-view-server.png (100%) rename {images => coco-jupyter-labs/images}/coco-pharmaceuticals-systems-omag-server-platforms.png (100%) rename {images => coco-jupyter-labs/images}/coco-pharmaceuticals-systems.drawio (100%) rename {images => coco-jupyter-labs/images}/creating-configuration-documents.png (100%) rename {images => coco-jupyter-labs/images}/database-type-hierarchy.png (100%) rename {images => coco-jupyter-labs/images}/deploying-configuration-documents.png (100%) rename {images => coco-jupyter-labs/images}/design-model-to-archive-load.png (100%) rename {images => coco-jupyter-labs/images}/discovery-engine-configuration.png (100%) rename {images => coco-jupyter-labs/images}/discovery-servers.png (100%) rename {images => coco-jupyter-labs/images}/distributed-asset-management-lab.drawio (100%) rename {images => coco-jupyter-labs/images}/distributed-asset-with-comments.png (100%) rename {images => coco-jupyter-labs/images}/distributed-discovery-engines.png (100%) rename {images => coco-jupyter-labs/images}/distributed-engine-services-config.png (100%) rename {images => coco-jupyter-labs/images}/egeria-as-tool-integrator.png (100%) rename {images => coco-jupyter-labs/images}/egeria-ui-asset-catalog-archive-search-results.png (100%) rename {images => coco-jupyter-labs/images}/egeria-ui-asset-catalog-no-rows.png (100%) rename {images => coco-jupyter-labs/images}/egeria-ui-asset-catalog-search-field-input.png (100%) rename {images => coco-jupyter-labs/images}/egeria-ui-asset-end-to-end-lineage.png (100%) rename {images => coco-jupyter-labs/images}/egeria-ui-asset-list.png (100%) rename {images => coco-jupyter-labs/images}/egeria-ui-asset-search.png (100%) rename {images => coco-jupyter-labs/images}/egeria-ui-end-to-end-lineage-graph.png (100%) rename {images => coco-jupyter-labs/images}/egeria-ui-erin-logon.png (100%) rename {images => coco-jupyter-labs/images}/egeria-ui-login.png (100%) rename {images => coco-jupyter-labs/images}/egeria-ui-nav-bar.png (100%) rename {images => coco-jupyter-labs/images}/engine-host.png (100%) rename {images => coco-jupyter-labs/images}/exchangeDL01-config.png (100%) rename {images => coco-jupyter-labs/images}/file-asset-hierarchy.png (100%) rename {images => coco-jupyter-labs/images}/glossary-revision-history.png (100%) rename {images => coco-jupyter-labs/images}/glossary-structure.png (100%) rename {images => coco-jupyter-labs/images}/governance-action-process-flow.png (100%) rename {images => coco-jupyter-labs/images}/governance-action-process-implementation.png (100%) rename {images => coco-jupyter-labs/images}/governance-action-request-type.png (100%) rename {images => coco-jupyter-labs/images}/governance-engine-definition.png (100%) rename {images => coco-jupyter-labs/images}/governance-engine-lab-platform-part.png (100%) rename {images => coco-jupyter-labs/images}/governance-engine-lab-platform.png (100%) rename {images => coco-jupyter-labs/images}/governance-engine-lab-servers-part-2.png (100%) rename {images => coco-jupyter-labs/images}/governance-engine-lab-servers-part.png (100%) rename {images => coco-jupyter-labs/images}/governance-engine-lab-servers.png (100%) rename {images => coco-jupyter-labs/images}/governance-servers-for-automated-onboarding.png (100%) rename {images => coco-jupyter-labs/images}/governance-servers-running-automated-onboarding.png (100%) rename {images => coco-jupyter-labs/images}/integration-daemon.png (100%) rename {images => coco-jupyter-labs/images}/jupyter-notebook-play-button.png (100%) rename {images => coco-jupyter-labs/images}/kafka-server-properties.png (100%) rename {images => coco-jupyter-labs/images}/omag-server-platform-ready.png (100%) rename {images => coco-jupyter-labs/images}/open-lineage-server.png (100%) rename {images => coco-jupyter-labs/images}/open-lineage-service-lab-assets.png (100%) rename {images => coco-jupyter-labs/images}/open-lineage-services.drawio (100%) rename {images => coco-jupyter-labs/images}/peter-and-erin.png (100%) rename {images => coco-jupyter-labs/images}/peter-profile.png (100%) rename {images => coco-jupyter-labs/images}/simple-api-catalog.png (100%) rename {images => coco-jupyter-labs/images}/simple-catalog-demo-asset-properties.png (100%) rename {images => coco-jupyter-labs/images}/simple-catalog-demo-connected-catalogs.png (100%) rename {images => coco-jupyter-labs/images}/simple-catalog-demo-data-map.png (100%) rename {images => coco-jupyter-labs/images}/simple-catalog-demo-independent-catalogs.png (100%) rename {images => coco-jupyter-labs/images}/simple-catalog-demo.png (100%) rename {images => coco-jupyter-labs/images}/simple-catalog-metadata-origin-comment.png (100%) rename {images => coco-jupyter-labs/images}/simple-catalog-metadata-origin.png (100%) rename {images => coco-jupyter-labs/images}/simple-catalog.png (100%) rename {images => coco-jupyter-labs/images}/simple-data-catalog.png (100%) rename {images => coco-jupyter-labs/images}/simple-event-catalog.png (100%) rename {information-architecture => coco-jupyter-labs/information-architecture}/working-with-standard-models.ipynb (100%) rename {labs-in-development => coco-jupyter-labs/labs-in-development}/README.md (100%) rename {labs-in-development => coco-jupyter-labs/labs-in-development}/jdbc-connector.ipynb (100%) rename {overview => coco-jupyter-labs/overview}/egeria-demo.ipynb (100%) rename read-me-first.ipynb => coco-jupyter-labs/read-me-first.ipynb (96%) mode change 100755 => 100644 rename {ui-labs => coco-jupyter-labs/ui-labs}/ui-asset-search.ipynb (100%) create mode 100644 coco-lab-docker-compose/Coco-Pharma Docker Compose.iml create mode 100644 coco-lab-docker-compose/Dockerfile create mode 100644 coco-lab-docker-compose/README.md create mode 100755 coco-lab-docker-compose/before-notebook.d/config-coco.sh create mode 100644 coco-lab-docker-compose/coco-lab-setup.yaml create mode 100755 coco-lab-docker-compose/core_application.properties create mode 100755 coco-lab-docker-compose/datalake_application.properties create mode 100755 coco-lab-docker-compose/dev_application.properties diff --git a/README.md b/README.md index 2732e545..5aee7d2d 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,10 @@ The open metadata labs contain an interactive environment that allow you to experiment with different capabilities of Egeria. As such we often refer to them as **Hands-on Labs**. They are implemented using the Jupyter Notebooks stored in this repository. +The labs can be set up and configured using the Docker Compose script in the **coco-lab-docker-compose** directory or can be +deployed and configured manually. It is easiest to get started using the Docker Compose deployment; the README.md file in the +directory provides more details. + * [Overview of the Labs](https://egeria-project.org/education/open-metadata-labs/overview/) ---- diff --git a/administration-labs/README.md b/coco-jupyter-labs/administration-labs/README.md similarity index 100% rename from administration-labs/README.md rename to coco-jupyter-labs/administration-labs/README.md diff --git a/administration-labs/managing-servers.ipynb b/coco-jupyter-labs/administration-labs/managing-servers.ipynb similarity index 100% rename from administration-labs/managing-servers.ipynb rename to coco-jupyter-labs/administration-labs/managing-servers.ipynb diff --git a/administration-labs/understanding-cohorts.ipynb b/coco-jupyter-labs/administration-labs/understanding-cohorts.ipynb similarity index 100% rename from administration-labs/understanding-cohorts.ipynb rename to coco-jupyter-labs/administration-labs/understanding-cohorts.ipynb diff --git a/administration-labs/understanding-platform-services.ipynb b/coco-jupyter-labs/administration-labs/understanding-platform-services.ipynb similarity index 100% rename from administration-labs/understanding-platform-services.ipynb rename to coco-jupyter-labs/administration-labs/understanding-platform-services.ipynb diff --git a/administration-labs/understanding-server-config.ipynb b/coco-jupyter-labs/administration-labs/understanding-server-config.ipynb similarity index 100% rename from administration-labs/understanding-server-config.ipynb rename to coco-jupyter-labs/administration-labs/understanding-server-config.ipynb diff --git a/asset-management-labs/README.md b/coco-jupyter-labs/asset-management-labs/README.md similarity index 94% rename from asset-management-labs/README.md rename to coco-jupyter-labs/asset-management-labs/README.md index 9d115d60..a9ca22f7 100644 --- a/asset-management-labs/README.md +++ b/coco-jupyter-labs/asset-management-labs/README.md @@ -12,7 +12,7 @@ The asset management notebooks cover the function to manage a catalog of assets ---- -* [Return to the Labs Overiew](..) +* [Return to the Labs Overiew](../..) ---- diff --git a/asset-management-labs/automated-curation.ipynb b/coco-jupyter-labs/asset-management-labs/automated-curation.ipynb similarity index 100% rename from asset-management-labs/automated-curation.ipynb rename to coco-jupyter-labs/asset-management-labs/automated-curation.ipynb diff --git a/asset-management-labs/building-a-data-catalog.ipynb b/coco-jupyter-labs/asset-management-labs/building-a-data-catalog.ipynb similarity index 100% rename from asset-management-labs/building-a-data-catalog.ipynb rename to coco-jupyter-labs/asset-management-labs/building-a-data-catalog.ipynb diff --git a/asset-management-labs/improving-data-quality.ipynb b/coco-jupyter-labs/asset-management-labs/improving-data-quality.ipynb similarity index 100% rename from asset-management-labs/improving-data-quality.ipynb rename to coco-jupyter-labs/asset-management-labs/improving-data-quality.ipynb diff --git a/asset-management-labs/open-lineage.ipynb b/coco-jupyter-labs/asset-management-labs/open-lineage.ipynb similarity index 100% rename from asset-management-labs/open-lineage.ipynb rename to coco-jupyter-labs/asset-management-labs/open-lineage.ipynb diff --git a/asset-management-labs/understanding-an-asset.ipynb b/coco-jupyter-labs/asset-management-labs/understanding-an-asset.ipynb similarity index 100% rename from asset-management-labs/understanding-an-asset.ipynb rename to coco-jupyter-labs/asset-management-labs/understanding-an-asset.ipynb diff --git a/common/common-functions.ipynb b/coco-jupyter-labs/common/common-functions.ipynb similarity index 99% rename from common/common-functions.ipynb rename to coco-jupyter-labs/common/common-functions.ipynb index c412f6ef..f9223fd1 100644 --- a/common/common-functions.ipynb +++ b/coco-jupyter-labs/common/common-functions.ipynb @@ -63,13 +63,13 @@ "#\n", "\n", "# These are the three main platforms used to run Egeria's OMAG Servers\n", - "corePlatformURL = os.environ.get('corePlatformURL', 'https://localhost:9443')\n", + "corePlatformURL = os.environ.get('corePlatformURL', 'https://host.docker.internal:7443')\n", "corePlatformName = \"Core Platform\"\n", "\n", - "dataLakePlatformURL = os.environ.get('dataLakePlatformURL', 'https://localhost:9444')\n", + "dataLakePlatformURL = os.environ.get('dataLakePlatformURL', 'https://host.docker.internal:7444')\n", "dataLakePlatformName = \"Data Lake Platform\"\n", "\n", - "devPlatformURL = os.environ.get('devPlatformURL', 'https://localhost:9445')\n", + "devPlatformURL = os.environ.get('devPlatformURL', 'https://host.docker.internal:7445')\n", "devPlatformName = \"Dev Platform\"\n", "\n", "\n", @@ -306,6 +306,12 @@ { "cell_type": "code", "execution_count": null, + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, "outputs": [], "source": [ "def postAndPrintResult(url, json=None, headers=None):\n", @@ -339,10 +345,7 @@ " except requests.exceptions.RequestException as e:\n", " print (\" ...... FAILED - http request threw an exception: \", e)\n", " return None\n" - ], - "metadata": { - "collapsed": false - } + ] }, { "cell_type": "code", @@ -434,6 +437,12 @@ { "cell_type": "code", "execution_count": null, + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, "outputs": [], "source": [ "\n", @@ -485,14 +494,17 @@ " except Exception as error:\n", " print(\"Exception: %s\" % error)\n", " print(\"Platform \" + serverName + \" (\" + adminPlatformURL + \") is returning an error\")\n" - ], - "metadata": { - "collapsed": false - } + ] }, { "cell_type": "code", "execution_count": null, + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, "outputs": [], "source": [ "#\n", @@ -536,14 +548,17 @@ " }\n", " }\n", " return body" - ], - "metadata": { - "collapsed": false - } + ] }, { "cell_type": "code", "execution_count": null, + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, "outputs": [], "source": [ "#\n", @@ -658,14 +673,17 @@ "\n", " else:\n", " configureBuiltinMetadataRepository(adminPlatformURL, adminUserId, serverName, repositoryType)\n" - ], - "metadata": { - "collapsed": false - } + ] }, { "cell_type": "code", "execution_count": null, + "metadata": { + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, "outputs": [], "source": [ "\n", @@ -822,10 +840,7 @@ " }\n", " url = adminCommandURLRoot + engineServerName + '/engine-services/' + engineServiceURLMarker\n", " issuePost(url, requestBody)" - ], - "metadata": { - "collapsed": false - } + ] }, { "cell_type": "code", @@ -851,11 +866,14 @@ { "cell_type": "code", "execution_count": null, - "outputs": [], - "source": [], "metadata": { - "collapsed": false - } + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } + }, + "outputs": [], + "source": [] }, { "cell_type": "code", @@ -1847,24 +1865,6 @@ " propertyIndent = \" \"\n", " if printSchema:\n", " printSchemaTypeDetail(propertyIndent, assetResponse.get('schemaType'), serverName, serverPlatformName, serverPlatformURL, serviceURLMarker, userId, printOrigin)\n", - " certificationCount = assetResponse.get('certificationCount')\n", - " commentCount = assetResponse.get('commentCount')\n", - " if commentCount > 0:\n", - " printComments(propertyIndent, serverName, serverPlatformName, serverPlatformURL, serviceURLMarker, userId, assetGUID, printOrigin)\n", - " connectionCount = assetResponse.get('connectionCount')\n", - " externalIdentifierCount = assetResponse.get('externalIdentifierCount')\n", - " externalReferencesCount = assetResponse.get('externalReferencesCount')\n", - " informalTagCount = assetResponse.get('informalTagCount')\n", - " licenseCount = assetResponse.get('licenseCount')\n", - " likeCount = assetResponse.get('likeCount')\n", - " keywordCount = assetResponse.get('keywordCount')\n", - " knownLocationsCount = assetResponse.get('knownLocationsCount')\n", - " noteLogsCount = assetResponse.get('noteLogsCount')\n", - " ratingsCount = assetResponse.get('ratingsCount')\n", - " relatedAssetCount = assetResponse.get('relatedAssetCount')\n", - " if relatedAssetCount > 0:\n", - " printRelatedAssets(propertyIndent, serverName, serverPlatformName, serverPlatformURL, serviceURLMarker, userId, assetGUID)\n", - " relatedMediaReferenceCount = assetResponse.get('relatedMediaReferenceCount')\n", " \n", " \n", "def printCommentList(responseObjects):\n", @@ -5199,7 +5199,10 @@ "cell_type": "code", "execution_count": null, "metadata": { - "collapsed": false + "collapsed": false, + "jupyter": { + "outputs_hidden": false + } }, "outputs": [], "source": [ @@ -5223,7 +5226,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -5237,7 +5240,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.3" + "version": "3.11.9" } }, "nbformat": 4, diff --git a/coco-jupyter-labs/common/config_coco_core.py b/coco-jupyter-labs/common/config_coco_core.py new file mode 100644 index 00000000..034b2507 --- /dev/null +++ b/coco-jupyter-labs/common/config_coco_core.py @@ -0,0 +1,298 @@ +#!/usr/bin/env python3 +""" +SPDX-License-Identifier: Apache-2.0 +Copyright Contributors to the ODPi Egeria project. + + + +Egeria Coco Pharmaceutical demonstration labs. + +This script configures and initiates the Egeria OMAG Servers deployed on the Core Platform. +It is automatically run whenever the Coco Lab Compose script is started. + +""" + + +import json +import argparse +from datetime import datetime + +from globals import (cocoMDS2Name, cocoMDS3Name, corePlatformURL, cocoCohort, devCohort, iotCohort, max_paging_size, adminUserId, + cocoMDS5Name, cocoMDS6Name) +from pyegeria import CoreServerConfig, Platform +from pyegeria import ( + print_exception_response, +) + + +def config_coco_core(url: str, userid: str): + disable_ssl_warnings = True + print("Configuring and activating the Datalake") + platform_url = url + admin_user = userid + + # event_bus_config = { + # "producer": { + # "bootstrap.servers": "{{kafkaEndpoint}}" + # }, + # "consumer": { + # "bootstrap.servers": "{{kafkaEndpoint}}" + # } + # } + event_bus_config = { + "producer": { + "bootstrap.servers": "host.docker.internal:7192" + }, + "consumer": { + "bootstrap.servers": "host.docker.internal:7192" + } + } + + security_connection_body = { + "class": "Connection", + "connectorType": { + "class": "ConnectorType", + "connectorProviderClassName": + "org.odpi.openmetadata.metadatasecurity.samples.CocoPharmaServerSecurityProvider" + } + } + + # + # Configure MDS2 + # + mdr_server = cocoMDS2Name + mdr_server_user_id = "cocoMDS2npa" + mdr_server_password = "cocoMDS2passw0rd" + metadataCollectionId = f"{mdr_server}-e915f2fa-aa3g-4396-8bde-bcd65e642b1d" + metadataCollectionName = "Governance Catalog" + + + try: + o_client = CoreServerConfig(mdr_server, platform_url, admin_user) + + o_client.set_basic_server_properties("Governance Server", + "Coco Pharmaceuticals", + platform_url, + mdr_server_user_id, mdr_server_password, + max_paging_size) + + o_client.set_event_bus(event_bus_config) + o_client.set_server_security_connection(security_connection_body) + o_client.add_default_log_destinations() + + # o_client.set_in_mem_local_repository() + o_client.set_xtdb_local_kv_repository() + + o_client.set_local_metadata_collection_id(metadataCollectionId) + o_client.set_local_metadata_collection_name(metadataCollectionName) + + o_client.add_cohort_registration(cocoCohort) + o_client.add_cohort_registration(devCohort) + o_client.add_cohort_registration(iotCohort) + + # o_client.configure_access_service("asset-catalog", {}) + o_client.configure_access_service("asset-consumer", {}) + + o_client.configure_access_service("asset-owner", {}) + o_client.configure_access_service("community-profile", + {"KarmaPointPlateau": "500"}) + # o_client.configure_access_service("glossary-view", {}) + # o_client.configure_access_service("subject-area", {}) + o_client.configure_access_service("governance-engine", {}) + o_client.configure_access_service("governance-server", {}) + o_client.configure_access_service("governance-program", {}) + # o_client.configure_access_service("data-privacy", {}) + o_client.configure_access_service("digital-architecture", {}) + o_client.configure_access_service("security-manager", {}) + o_client.configure_access_service("asset-lineage", {}) + o_client.configure_access_service("it-infrastructure", {}) + o_client.configure_access_service("project-management", {}) + print(f"Activating {mdr_server}") + p_client = Platform(mdr_server, platform_url, admin_user) + p_client.activate_server_stored_config() + print(f"{mdr_server} activated") + except Exception as e: + print_exception_response(e) + + # + # Configure MDS3 + # + mdr_server = cocoMDS3Name + mdr_server_user_id = "cocoMDS3npa" + mdr_server_password = "cocoMD32passw0rd" + metadataCollectionId = f"{mdr_server}-e915f2fa-aa3g-4396-8bde-bcd65e642b1d" + metadataCollectionName = "Research Catalog" + + try: + o_client = CoreServerConfig(mdr_server, platform_url, admin_user) + + o_client.set_basic_server_properties("Research Server", + "Coco Pharmaceuticals", + platform_url, + mdr_server_user_id, mdr_server_password, + max_paging_size) + + o_client.set_event_bus(event_bus_config) + o_client.set_server_security_connection(security_connection_body) + o_client.add_default_log_destinations() + + # o_client.set_in_mem_local_repository() + o_client.set_xtdb_local_kv_repository() + + o_client.set_local_metadata_collection_id(metadataCollectionId) + o_client.set_local_metadata_collection_name(metadataCollectionName) + + o_client.add_cohort_registration(cocoCohort) + access_service_options = { + "SupportedZones": ["personal-files", "clinical-trials", "research", "data-lake", "trash-can"] + } + + + o_client.configure_access_service("asset-consumer", access_service_options) + + access_service_options["DefaultZones"] = ["personal-files"] + + o_client.configure_access_service("asset-owner", access_service_options) + o_client.configure_access_service("community-profile", + {"KarmaPointPlateau": "500"}) + + o_client.configure_access_service("asset-manager", access_service_options) + o_client.configure_access_service("governance-engine", access_service_options) + # o_client.configure_access_service("governance-server", {}) + # o_client.configure_access_service("governance-program", {}) + + # o_client.configure_access_service("digital-architecture", {}) + # o_client.configure_access_service("security-manager", {}) + # o_client.configure_access_service("asset-lineage", {}) + # o_client.configure_access_service("it-infrastructure", {}) + o_client.configure_access_service("project-management", access_service_options) + print(f"Activating {mdr_server}") + p_client = Platform(mdr_server, platform_url, admin_user) + p_client.activate_server_stored_config() + print(f"{mdr_server} activated") + except Exception as e: + print_exception_response(e) + # + # Configure MDS5 + # + disable_ssl_warnings = True + + mdr_server = cocoMDS5Name + mdr_server_user_id = "cocoMDS5npa" + mdr_server_password = "cocoMDS5passw0rd" + metadataCollectionId = f"{mdr_server}-e915f2fa-aa3g-4396-8bde-bcd65e642b1d" + metadataCollectionName = "Business Systems Catalog" + + print("Configuring " + mdr_server + "...") + try: + o_client = CoreServerConfig(mdr_server, platform_url, admin_user) + + o_client.set_basic_server_properties("Business Systems", + "Coco Pharmaceuticals", + platform_url, + mdr_server_user_id, mdr_server_password, + max_paging_size) + + o_client.set_event_bus(event_bus_config) + o_client.set_server_security_connection(security_connection_body) + o_client.add_default_log_destinations() + + # o_client.set_in_mem_local_repository() + o_client.set_xtdb_local_kv_repository() + + o_client.set_local_metadata_collection_id(metadataCollectionId) + o_client.set_local_metadata_collection_name(metadataCollectionName) + + o_client.add_cohort_registration(cocoCohort) + + proxy_details = ("org.odpi.openmetadata.adapters.repositoryservices.readonly.repositoryconnector." + + "ReadOnlyOMRSRepositoryConnectorProvider") + o_client.set_repository_proxy_details(proxy_details) + print(f"Activating {mdr_server}") + p_client = Platform(mdr_server, platform_url, admin_user) + p_client.activate_server_stored_config() + print(f"{mdr_server} activated") + except Exception as e: + print_exception_response(e) + + # + # Configure MDS6 + # + disable_ssl_warnings = True + + mdr_server = cocoMDS6Name + platform_url = corePlatformURL + admin_user = "garygeeke" + mdr_server_user_id = "cocoMDS6npa" + mdr_server_password = "cocoMDS6passw0rd" + metadataCollectionId = f"{mdr_server}-e915f2fa-aa3g-4396-8bde-bcd65e642b1d" + metadataCollectionName = "Manufacturing Catalog" + + print("Configuring " + mdr_server + "...") + + try: + o_client = CoreServerConfig(mdr_server, platform_url, admin_user) + + o_client.set_basic_server_properties("Manufacturing", + "Coco Pharmaceuticals", + platform_url, + mdr_server_user_id, mdr_server_password, + max_paging_size) + + o_client.set_event_bus(event_bus_config) + + o_client.set_server_security_connection(security_connection_body) + o_client.add_default_log_destinations() + + # o_client.set_in_mem_local_repository() + o_client.set_xtdb_local_kv_repository() + + o_client.set_local_metadata_collection_id(metadataCollectionId) + o_client.set_local_metadata_collection_name(metadataCollectionName) + + o_client.add_cohort_registration(cocoCohort) + o_client.add_cohort_registration(iotCohort) + + access_service_options = { + "SupportedZones": ["manufacturing"], + "DefaultZones": ["manufacturing"] + } + + # o_client.configure_access_service("asset-catalog", access_service_options) + o_client.configure_access_service("asset-consumer", access_service_options) + o_client.configure_access_service("asset-owner", access_service_options) + o_client.configure_access_service("community-profile", + {"KarmaPointPlateau": "500"}) + # o_client.configure_access_service("glossary-view", {}) + o_client.configure_access_service("data-science", access_service_options) + # o_client.configure_access_service("subject-area", {}) + o_client.configure_access_service("asset-manager", access_service_options) + o_client.configure_access_service("governance-engine", access_service_options) + o_client.configure_access_service("governance-server", access_service_options) + o_client.configure_access_service("asset-owner", access_service_options) + # o_client.configure_access_service("data-engine", access_service_options) + o_client.configure_access_service("data-manager", access_service_options) + o_client.configure_access_service("it-infrastructure", access_service_options) + o_client.configure_access_service("project-management", access_service_options) + + print(f"Activating {mdr_server}") + p_client = Platform(mdr_server, platform_url, admin_user) + p_client.activate_server_stored_config() + print(f"{mdr_server} activated") + except Exception as e: + print_exception_response(e) + +def main(): + parser = argparse.ArgumentParser() + + parser.add_argument("--url", help="URL Platform to connect to") + parser.add_argument("--userid", help="User Id") + args = parser.parse_args() + + url = args.url if args.url is not None else corePlatformURL + userid = args.userid if args.userid is not None else adminUserId + + config_coco_core(url, userid) + +if __name__ == "__main__": + main() diff --git a/coco-jupyter-labs/common/config_coco_datalake.py b/coco-jupyter-labs/common/config_coco_datalake.py new file mode 100644 index 00000000..5e6e9ed4 --- /dev/null +++ b/coco-jupyter-labs/common/config_coco_datalake.py @@ -0,0 +1,415 @@ +#!/usr/bin/env python3 +""" +SPDX-License-Identifier: Apache-2.0 +Copyright Contributors to the ODPi Egeria project. + + + +Egeria Coco Pharmaceutical demonstration labs. + +This script configures and initiates the Egeria OMAG Servers deployed on the Datalake Platform. +It is automatically run whenever the Coco Lab Compose script is started. + +""" + +import argparse + +from pyegeria import CoreServerConfig, Platform, FullServerConfig +from pyegeria import ( + print_exception_response, +) + +from globals import (corePlatformURL, cocoCohort, max_paging_size, cocoMDS1Name, + cocoMDS4Name, dataLakePlatformURL, fileSystemRoot, adminUserId) + + +def config_coco_datalake(url:str, userid:str): + print("Configuring and starting the Data Lake") + disable_ssl_warnings = True + platform_url = url + admin_user = userid + + # Change the kafka configuration to reflect the distinguished kafka setting for the labs + # event_bus_config = { + # "producer": { + # "bootstrap.servers": "{{kafkaEndpoint}}" + # }, + # "consumer": { + # "bootstrap.servers": "{{kafkaEndpoint}}" + # } + # } + event_bus_config = { + "producer": { + "bootstrap.servers": "host.docker.internal:7192" + }, + "consumer": { + "bootstrap.servers": "host.docker.internal:7192" + } + } + + security_connection_body = { + "class": "Connection", + "connectorType": { + "class": "ConnectorType", + "connectorProviderClassName": + "org.odpi.openmetadata.metadatasecurity.samples.CocoPharmaServerSecurityProvider" + } + } + + # + # Configure MDS1 + # + + mdr_server = cocoMDS1Name + mdr_server_user_id = "cocoMDS1npa" + mdr_server_password = "cocoMDS1passw0rd" + metadataCollectionId = f"{mdr_server}-e915f2fa-aa3g-4396-8bde-bcd65e642b1d" + metadataCollectionName = "Data Lake Operations" + + print("Configuring " + mdr_server + "...") + try: + o_client = CoreServerConfig(mdr_server, platform_url, admin_user) + + o_client.set_basic_server_properties(metadataCollectionName, + "Coco Pharmaceuticals", + platform_url, + mdr_server_user_id, mdr_server_password, + max_paging_size) + + o_client.set_event_bus(event_bus_config) + o_client.set_server_security_connection(security_connection_body) + o_client.add_default_log_destinations() + + # o_client.set_in_mem_local_repository() + o_client.set_xtdb_local_kv_repository() + + o_client.set_local_metadata_collection_id(metadataCollectionId) + o_client.set_local_metadata_collection_name(metadataCollectionName) + + o_client.add_cohort_registration(cocoCohort) + access_service_options = { + "SupportedZones": ["quarantine", "clinical-trials", "research", "data-lake", "trash-can"] + } + + # o_client.configure_access_service("asset-catalog", {}) + o_client.configure_access_service("asset-consumer", {}) + + access_service_options["DefaultZones"] = ["quarantine"] + access_service_options["PublishZones"] = ["data-lake"] + + # print(f"Access Service Options: {access_service_options}") + + o_client.configure_access_service("asset-manager", access_service_options) + o_client.configure_access_service("asset-owner", access_service_options) + o_client.configure_access_service("community-profile", + {"KarmaPointPlateau": "500"}) + # o_client.configure_access_service("glossary-view", {}) + # o_client.configure_access_service("data-engine", access_service_options) + o_client.configure_access_service("data-manager", access_service_options) + o_client.configure_access_service("digital-architecture", access_service_options) + o_client.configure_access_service("governance-engine", access_service_options) + o_client.configure_access_service("governance-server", access_service_options) + o_client.configure_access_service("asset-lineage", access_service_options) + + print(f"Activating {mdr_server}") + p_client = Platform(mdr_server, platform_url, admin_user) + p_client.activate_server_stored_config() + print(mdr_server + " activated") + + except Exception as e: + print_exception_response(e) + + # + # Configure MDS4 + # + mdr_server = cocoMDS4Name + mdr_server_user_id = "cocoMDS4npa" + mdr_server_password = "cocoMDS4passw0rd" + metadataCollectionId = f"{mdr_server}-e915f2fa-aa3g-4396-8bde-bcd65e642b1d" + metadataCollectionName = "Data Lake Users" + print("Configuring " + mdr_server + "...") + + try: + + o_client = CoreServerConfig(mdr_server, platform_url, admin_user) + + o_client.set_basic_server_properties("Data Lake Users", + "Coco Pharmaceuticals", + platform_url, + mdr_server_user_id, mdr_server_password, + max_paging_size) + + + o_client.set_event_bus(event_bus_config) + o_client.set_server_security_connection(security_connection_body) + o_client.add_default_log_destinations() + + # Note: no metadata repository or collection configuration in this server. + + o_client.add_cohort_registration(cocoCohort) + + accessServiceOptions = { + "SupportedZones": ["data-lake"] + } + + # o_client.configure_access_service("asset-catalog", accessServiceOptions) + o_client.configure_access_service("asset-consumer", accessServiceOptions) + o_client.configure_access_service("asset-owner", {}) + o_client.configure_access_service("community-profile", + {"KarmaPointPlateau": "500"}) + # o_client.configure_access_service("glossary-view", {}) + o_client.configure_access_service("data-science", accessServiceOptions) + + print(f"Activating {mdr_server}") + p_client = Platform(mdr_server, platform_url, admin_user) + p_client.activate_server_stored_config() + + print(f"{mdr_server} activated") + + except Exception as e: + print_exception_response(e) + + # + # Configure exchangeDL01 + # + + daemon_server_name = "exchangeDL01" + daemon_server_platform = platform_url + daemon_server_user_id = "exchangeDL01npa" + daemon_server_password = "exchangeDL01passw0rd" + + mdr_server = "cocoMDS1" + folder_connector_name = "DropFootClinicalTrialResultsFolderMonitor" + folder_connector_user_id = "monitorDL01npa" + folder_connector_source_name = "DropFootClinicalTrialResults" + folder_connector_folder = fileSystemRoot + '/data-lake/research/clinical-trials/drop-foot/weekly-measurements' + folder_connector_connection = { + "class": "Connection", + "connectorType": + { + "class": "ConnectorType", + "connectorProviderClassName": + "org.odpi.openmetadata.adapters.connectors.integration.basicfiles.DataFolderMonitorIntegrationProvider" + }, + "endpoint": + { + "class": "Endpoint", + "address": folder_connector_folder + } + } + + integration_group_name = "Onboarding" + + print("Configuring " + daemon_server_name) + + try: + f_client = FullServerConfig(daemon_server_name, daemon_server_platform, admin_user) + + f_client.set_basic_server_properties("Supports exchange of metadata with third party technologies", + "Coco Pharmaceuticals", + daemon_server_platform, + daemon_server_user_id, daemon_server_password, + max_paging_size) + + f_client.set_server_security_connection(security_connection_body) + f_client.add_default_log_destinations() + + connector_configs = [ + { + "class": "IntegrationConnectorConfig", + "connectorName": folder_connector_name, + "connectorUserId": folder_connector_user_id, + "connection": folder_connector_connection, + "metadataSourceQualifiedName": folder_connector_source_name, + "refreshTimeInterval": 10, + "usesBlockingCalls": "false" + } + ] + + f_client.config_integration_service(mdr_server, platform_url, "files-integrator", + {}, connector_configs) + + f_client.config_integration_group(mdr_server, daemon_server_platform, + integration_group_name) + print(f"Activating {daemon_server_name}") + p_client = Platform(daemon_server_name, daemon_server_platform, adminUserId) + p_client.activate_server_stored_config() + + print(f"{daemon_server_name} activated") + + except Exception as e: + print_exception_response(e) + + # + # Configure governDL01 + # + engine_server = "governDL01" + engine_server_platform = platform_url + + engine_server_user_id = "governDL01npa" + engine_server_password = "governDL01passw0rd" + mdr_server = "cocoMDS1" + mdr_engine_server_platform = dataLakePlatformURL + + print("Configuring " + engine_server) + + try: + o_client = CoreServerConfig(engine_server, engine_server_platform, admin_user) + + o_client.set_basic_server_properties("An Engine Host to run governance actions for Coco Pharmaceuticals", + "Coco Pharmaceuticals", + engine_server_platform, + engine_server_user_id, engine_server_password, + max_paging_size) + + o_client.set_server_security_connection(security_connection_body) + + o_client.set_engine_definitions_client_config(mdr_server, mdr_engine_server_platform) + + engine_list_body = [ + { + "class": "EngineConfig", + "engineQualifiedName": "AssetDiscovery", + "engineUserId": "findItDL01npa" + }, + { + "class": "EngineConfig", + "engineQualifiedName": "AssetQuality", + "engineUserId": "findItDL01npa" + } + ] + + o_client.set_engine_list(engine_list_body) + + # config = o_client.get_stored_configuration() + # print(f"The server stored configuration is {json.dumps(config, indent=4)}") + print(f"Activating {engine_server}") + p_client = Platform(engine_server, engine_server_platform, admin_user) + p_client.activate_server_stored_config() + print(f"{engine_server} activated") + except Exception as e: + print_exception_response(e) + + # + # Configure cocoView1 + # + + view_server = "cocoView1" + view_server_user_id = "cocoView1npa" + view_server_password = "cocoView1passw0rd" + view_server_type = "View Server" + remote_platform_url = corePlatformURL + remote_server_name = cocoMDS1Name + + print("Configuring " + view_server) + try: + f_client = FullServerConfig(view_server, platform_url, admin_user) + + f_client.set_server_user_id(view_server_user_id) + f_client.set_server_user_password(view_server_password) + f_client.set_organization_name("Coco Pharmaceuticals") + + f_client.set_server_description("Coco View Server") + f_client.set_server_url_root(platform_url) + f_client.set_event_bus(event_bus_config) + f_client.set_server_security_connection(security_connection_body) + + f_client.add_default_log_destinations() + + f_client.config_all_view_services(remote_server_name, platform_url) + print(f"Activating {view_server}") + p_client = Platform(view_server, platform_url, admin_user) + p_client.activate_server_stored_config() + + print(f"{view_server} activated") + + except Exception as e: + print_exception_response(e) + + # + # Configure cocoOLS1 + # + + lineageServerName = "cocoOLS1" + lineageServerPlatform = dataLakePlatformURL + + mdrServerName = "cocoMDS1" + mdrServerUserId = "cocoMDS1npa" + mdrServerPassword = "secret" + mdrServerPlatform = dataLakePlatformURL + + print("Configuring " + lineageServerName) + + requestBody = { + "class": "OpenLineageConfig", + "openLineageDescription": "Open Lineage Service is used for the storage and querying of lineage", + "lineageGraphConnection": { + "class": "Connection", + "displayName": "Lineage Graph Connection", + "description": "Used for storing lineage in the Open Metadata format", + "connectorType": { + "class": "ConnectorType", + "connectorProviderClassName": "org.odpi.openmetadata.openconnectors.governancedaemonconnectors.lineagewarehouseconnectors.janusconnector.graph.LineageGraphConnectorProvider" + }, + "configurationProperties": { + "gremlin.graph": "org.janusgraph.core.JanusGraphFactory", + "storage.backend": "berkeleyje", + "storage.directory": "data/servers/" + lineageServerName + "/lineage-repository/berkeley", + "index.search.backend": "lucene", + "index.search.directory": "data/servers/" + lineageServerName + "/lineage-repository/searchindex" + } + }, + "accessServiceConfig": { + "serverName": mdrServerName, + "serverPlatformUrlRoot": mdrServerPlatform, + "user": mdrServerUserId, + "password": mdrServerPassword + }, + "backgroundJobs": [ + { + "jobName": "LineageGraphJob", + "jobInterval": 120, + "jobEnabled": "false" + }, + { + "jobName": "AssetLineageUpdateJob", + "jobInterval": 120, + "jobEnabled": "false", + "jobDefaultValue": "2021-12-03T10:15:30" + } + ] + } + + + try: + f_client = FullServerConfig(lineageServerName, lineageServerPlatform, admin_user) + + f_client.set_server_description("Coco View Server") + f_client.set_server_url_root(lineageServerPlatform) + f_client.set_event_bus(event_bus_config) + f_client.add_default_log_destinations() + + f_client.set_lineage_warehouse_services(requestBody, lineageServerName) + print(f"Activating {lineageServerName}") + p_client = Platform(lineageServerName, lineageServerPlatform, admin_user) + p_client.activate_server_stored_config() + + print(f"{lineageServerName} activated") + + except Exception as e: + print_exception_response(e) + +def main(): + parser = argparse.ArgumentParser() + + parser.add_argument("--url", help="URL Platform to connect to") + parser.add_argument("--userid", help="User Id") + args = parser.parse_args() + + url = args.url if args.url is not None else dataLakePlatformURL + userid = args.userid if args.userid is not None else adminUserId + + config_coco_datalake(url, userid) + +if __name__ == "__main__": + main() diff --git a/coco-jupyter-labs/common/config_coco_development.py b/coco-jupyter-labs/common/config_coco_development.py new file mode 100755 index 00000000..94a39a3d --- /dev/null +++ b/coco-jupyter-labs/common/config_coco_development.py @@ -0,0 +1,368 @@ +""" +SPDX-License-Identifier: Apache-2.0 +Copyright Contributors to the ODPi Egeria project. + + + +Egeria Coco Pharmaceutical demonstration labs. + +This script configures and initiates the Egeria OMAG Servers deployed on the Development Platform. +It is automatically run whenever the Coco Lab Compose script is started. + +""" + +import argparse + +from globals import (cocoMDSxName, devPlatformURL, cocoCohort, iotCohort, max_paging_size, dataLakePlatformURL, + fileSystemRoot, adminUserId) +from pyegeria import CoreServerConfig, Platform, FullServerConfig +from pyegeria import ( + print_exception_response, +) + + +def config_coco_development(url: str, userid: str): + disable_ssl_warnings = True + + mdr_server = cocoMDSxName + platform_url = devPlatformURL + admin_user = "garygeeke" + mdr_server_user_id = "cocoMDSxnpa" + mdr_server_password = "cocoMDSxpassw0rd" + metadataCollectionId = f"{mdr_server}-e915f2fa-aa3g-4396-8bde-bcd65e642b1d" + metadataCollectionName = "Development Catalog" + + print("Configuring " + mdr_server + "...") + + try: + o_client = CoreServerConfig(mdr_server, platform_url, admin_user) + + o_client.set_basic_server_properties("Development Server", + "Coco Pharmaceuticals", + platform_url, + mdr_server_user_id, mdr_server_password, + max_paging_size) + + event_bus_config = { + "producer": { + "bootstrap.servers": "host.docker.internal:7192" + }, + "consumer": { + "bootstrap.servers": "host.docker.internal:7192" + } + } + + # security_connection_body = { + # "class": "Connection", + # "connectorType": { + # "class": "ConnectorType", + # "connectorProviderClassName": + # "org.odpi.openmetadata.metadatasecurity.samples.CocoPharmaServerSecurityProvider" + # } + # } + + security_connection_body = { + "class": "Connection", + "connectorType": { + "class": "ConnectorType", + "connectorProviderClassName": + "org.odpi.openmetadata.metadatasecurity.samples.CocoPharmaServerSecurityProvider" + } + } + + o_client.set_event_bus(event_bus_config) + o_client.set_server_security_connection(security_connection_body) + o_client.add_default_log_destinations() + # o_client.set_in_mem_local_repository() + o_client.set_xtdb_local_kv_repository() + + o_client.set_local_metadata_collection_id(metadataCollectionId) + o_client.set_local_metadata_collection_name(metadataCollectionName) + + o_client.add_cohort_registration(cocoCohort) + o_client.add_cohort_registration(iotCohort) + + print(f"Configuring {mdr_server} Access Services (OMAS)....") + + access_service_options = { + "SupportedZones": ["sdlc", "quarantine", "clinical-trials", "research", "data-lake", "trash-can"], + "DefaultZones": ["sdlc", "quarantine"] + } + + # o_client.configure_access_service("asset-catalog", access_service_options) + o_client.configure_access_service("asset-consumer", access_service_options) + o_client.configure_access_service("asset-owner", access_service_options) + o_client.configure_access_service("community-profile", + {"KarmaPointPlateau": "500"}) + # o_client.configure_access_service("glossary-view", {}) + o_client.configure_access_service("data-science", access_service_options) + # o_client.configure_access_service("subject-area", {}) + o_client.configure_access_service("asset-manager", access_service_options) + o_client.configure_access_service("governance-engine", access_service_options) + o_client.configure_access_service("governance-server", access_service_options) + # o_client.configure_access_service("data-manager", access_service_options) + o_client.configure_access_service("it-infrastructure", access_service_options) + o_client.configure_access_service("project-management", access_service_options) + o_client.configure_access_service("software-developer", access_service_options) + # o_client.configure_access_service("devops", access_service_options) + o_client.configure_access_service("digital-architecture", access_service_options) + o_client.configure_access_service("design-model", access_service_options) + + print(f"Configuring {mdr_server}") + p_client = Platform(mdr_server, platform_url, admin_user) + p_client.activate_server_stored_config() + + print(f"\n\n\tActivation of {mdr_server} is complete.") + + except Exception as e: + print_exception_response(e) + + # + # monitorDev01 + # + daemon_server_name = "monitorDev01" + daemon_server_platform = devPlatformURL + daemon_server_user_id = "erinoverview" + daemon_server_password = "erinoverviewpassw0rd" + + mdr_server = "cocoMDSx" + platform_url = devPlatformURL + + print("Configuring " + daemon_server_name + "...") + + try: + f_client = FullServerConfig(daemon_server_name, daemon_server_platform, admin_user) + + f_client.set_basic_server_properties("Integration daemon supporting the development team", + "Coco Pharmaceuticals", + daemon_server_platform, + daemon_server_user_id, daemon_server_password, + max_paging_size) + + security_connection_body = { + "class": "Connection", + "connectorType": { + "class": "ConnectorType", + "connectorProviderClassName": + "org.odpi.openmetadata.metadatasecurity.samples.CocoPharmaServerSecurityProvider" + } + } + f_client.set_server_security_connection(security_connection_body) + f_client.add_default_log_destinations() + + print(f"Initial configuration of {daemon_server_name} is complete.") + + except Exception as e: + print_exception_response(e) + + + # + # monitorGov01 + # + + daemon_server_name = "monitorGov01" + daemon_server_platform = dataLakePlatformURL + daemon_server_user_id = "exchangeDL01npa" + daemon_server_password = "exchangeDL01passw0rd" + + mdr_server = "cocoMDS1" + mdr_platform_url = dataLakePlatformURL + admin_user = "garygeeke" + + KafkaReceiverConnectorName = "KafkaOpenLineageEventReceiver" + KafkaReceiverConnectorUserId = "onboardDL01npa" + KafkaReceiverConnectorSourceName = "Apache Kafka" + KafkaReceiverConnectorConnection = { + "class": "VirtualConnection", + "connectorType": + { + "class": "ConnectorType", + "connectorProviderClassName": "org.odpi.openmetadata.adapters.connectors.integration.openlineage.OpenLineageEventReceiverIntegrationProvider" + }, + "embeddedConnections": + [ + { + "class": "EmbeddedConnection", + "embeddedConnection": + { + "class": "Connection", + "connectorType": + { + "class": "ConnectorType", + "connectorProviderClassName": "org.odpi.openmetadata.adapters.eventbus.topic.kafka.KafkaOpenMetadataTopicProvider", + }, + "endpoint": + { + "class": "Endpoint", + "address": "openlineage.topic" + }, + "configurationProperties": + { + "producer": + { + "bootstrap.servers": "host.docker.internal:7192" + }, + "local.server.id": "f234e808-2d0c-4d88-83df-275eee20c1b7", + "consumer": + { + "bootstrap.servers": "host.docker.internal:7192" + } + } + } + } + ] + } + + GovernanceActionConnectorName = "GovernanceActionOpenLineageCreator" + GovernanceActionConnectorUserId = "onboardDL01npa" + GovernanceActionConnectorSourceName = "Egeria" + GovernanceActionConnectorConnection = { + "class": "Connection", + "connectorType": + { + "class": "ConnectorType", + "connectorProviderClassName": "org.odpi.openmetadata.adapters.connectors.integration.openlineage.GovernanceActionOpenLineageIntegrationProvider" + }, + } + + APILoggerConnectorName = "APIBasedOpenLineageLogStore" + APILoggerConnectorUserId = "onboardDL01npa" + APILoggerConnectorSourceName = "Egeria" + APILoggerConnectorConnection = { + "class": "Connection", + "connectorType": + { + "class": "ConnectorType", + "connectorProviderClassName": "org.odpi.openmetadata.adapters.connectors.integration.openlineage.APIBasedOpenLineageLogStoreProvider" + }, + "endpoint": + { + "class": "Endpoint", + "address": "http://localhost:5000/api/v1/lineage" + } + } + + FileLoggerConnectorName = "FileBasedOpenLineageLogStore" + FileLoggerConnectorUserId = "onboardDL01npa" + FileLoggerConnectorSourceName = "Egeria" + FileLoggerConnectorConnection = { + "class": "Connection", + "connectorType": + { + "class": "ConnectorType", + "connectorProviderClassName": "org.odpi.openmetadata.adapters.connectors.integration.openlineage.FileBasedOpenLineageLogStoreProvider" + }, + "endpoint": + { + "class": "Endpoint", + "address": fileSystemRoot + '/openlineage.log' + } + } + + CataloguerConnectorName = "OpenLineageCataloguer" + CataloguerConnectorUserId = "onboardDL01npa" + CataloguerConnectorSourceName = "OpenLineageSources" + CataloguerConnectorConnection = { + "class": "Connection", + "connectorType": + { + "class": "ConnectorType", + "connectorProviderClassName": "org.odpi.openmetadata.adapters.connectors.integration.openlineage.OpenLineageCataloguerIntegrationProvider" + } + } + + print("Configuring " + daemon_server_name ) + + connectorConfigs = [ + { + "class": "IntegrationConnectorConfig", + "connectorName": KafkaReceiverConnectorName, + "connectorUserId": KafkaReceiverConnectorUserId, + "connection": KafkaReceiverConnectorConnection, + "metadataSourceQualifiedName": KafkaReceiverConnectorSourceName, + "refreshTimeInterval": 10, + "usesBlockingCalls": "false" + }, + { + "class": "IntegrationConnectorConfig", + "connectorName": GovernanceActionConnectorName, + "connectorUserId": GovernanceActionConnectorUserId, + "connection": GovernanceActionConnectorConnection, + "metadataSourceQualifiedName": GovernanceActionConnectorSourceName, + "refreshTimeInterval": 10, + "usesBlockingCalls": "false" + }, + { + "class": "IntegrationConnectorConfig", + "connectorName": APILoggerConnectorName, + "connectorUserId": APILoggerConnectorUserId, + "connection": APILoggerConnectorConnection, + "metadataSourceQualifiedName": APILoggerConnectorSourceName, + "refreshTimeInterval": 10, + "usesBlockingCalls": "false" + }, + { + "class": "IntegrationConnectorConfig", + "connectorName": FileLoggerConnectorName, + "connectorUserId": FileLoggerConnectorUserId, + "connection": FileLoggerConnectorConnection, + "metadataSourceQualifiedName": FileLoggerConnectorSourceName, + "refreshTimeInterval": 10, + "usesBlockingCalls": "false" + }, + { + "class": "IntegrationConnectorConfig", + "connectorName": CataloguerConnectorName, + "connectorUserId": CataloguerConnectorUserId, + "connection": CataloguerConnectorConnection, + "metadataSourceQualifiedName": CataloguerConnectorSourceName, + "refreshTimeInterval": 10, + "usesBlockingCalls": "false" + }] + + print("\nDone.") + + try: + f_client = FullServerConfig(daemon_server_name, daemon_server_platform, admin_user) + + f_client.set_basic_server_properties("An Engine Host to run governance actions for Coco Pharmaceuticals", + "Coco Pharmaceuticals", + daemon_server_platform, + daemon_server_user_id, daemon_server_password, + max_paging_size) + + security_connection_body = { + "class": "Connection", + "connectorType": { + "class": "ConnectorType", + "connectorProviderClassName": + "org.odpi.openmetadata.metadatasecurity.samples.CocoPharmaServerSecurityProvider" + } + } + f_client.set_server_security_connection(security_connection_body) + f_client.add_default_log_destinations() + + f_client.config_integration_service(mdr_server, mdr_platform_url, + "lineage-integrator", {}, connectorConfigs) + + print(f"Activating {daemon_server_name}") + p_client = Platform(daemon_server_name, daemon_server_platform, admin_user) + p_client.activate_server_stored_config() + print(f"Activation of {daemon_server_name} complete") + except Exception as e: + print_exception_response(e) + +def main(): + parser = argparse.ArgumentParser() + + parser.add_argument("--url", help="URL Platform to connect to") + parser.add_argument("--userid", help="User Id") + args = parser.parse_args() + + url = args.url if args.url is not None else devPlatformURL + userid = args.userid if args.userid is not None else adminUserId + + config_coco_development(url, userid) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/common/environment-check.ipynb b/coco-jupyter-labs/common/environment-check.ipynb similarity index 100% rename from common/environment-check.ipynb rename to coco-jupyter-labs/common/environment-check.ipynb diff --git a/common/globals.ipynb b/coco-jupyter-labs/common/globals.ipynb old mode 100755 new mode 100644 similarity index 96% rename from common/globals.ipynb rename to coco-jupyter-labs/common/globals.ipynb index c6e8ca5b..1744c494 --- a/common/globals.ipynb +++ b/coco-jupyter-labs/common/globals.ipynb @@ -40,7 +40,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, @@ -54,7 +54,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.3" + "version": "3.11.9" } }, "nbformat": 4, diff --git a/coco-jupyter-labs/common/globals.py b/coco-jupyter-labs/common/globals.py new file mode 100755 index 00000000..116ac561 --- /dev/null +++ b/coco-jupyter-labs/common/globals.py @@ -0,0 +1,153 @@ +# +# This file contains the common functions and definitions used in all Egeria Hands on +# Lab Notebooks. Typically these will be system-wide functions and common environment settings +# + + +# +# Disable certificate checking for local development use with self-signed certificate +# + +import urllib3 +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + +import os +os.environ['CURL_CA_BUNDLE'] = '' + +# +# This is the location of the file system that is used in the lab. It defaults to a "data" subfolder in +# the platform's home folder. Here you will finc the runtime files of the platforms and the data files +# used in various labs. The default value can be overridden using the "fileSystemRoot" environment variable. +# +fileSystemRoot = os.environ.get('fileSystemRoot', 'data') +# +# This is directory where the sample data from the Egeria distribution has been unpacked in. The default value +# describes its location in an IntelliJ workspace. The default value can be overridden using the "egeriaDistributionRoot" +# environment variable. +# +egeriaSampleDataRoot = os.environ.get('egeriaSampleDataRoot', '') +# +# Flag to enable debug, this cases extra information to be printed including rest calls request and response details +# Switching this flag to True produces a very large amount of output and is not recommended. +# A targeted use of this flag is recommended, set this before and reset this after the code you would like to produce debug +# +isDebug = False +disable_ssl_warnings = True +max_paging_size = 1200 +# +# Definitions of the Coco Pharmaceuticals Environment +# + +# These are the three main platforms used to run Egeria's OMAG Servers +corePlatformURL = os.environ.get('corePlatformURL', 'https://host.docker.internal:7443') +corePlatformName = "Core Platform" + +dataLakePlatformURL = os.environ.get('dataLakePlatformURL', 'https://host.docker.internal:7444') +dataLakePlatformName = "Data Lake Platform" + +devPlatformURL = os.environ.get('devPlatformURL', 'https://host.docker.internal:7445') +devPlatformName = "Dev Platform" + + +# The OMAG Server Platforms host one to many OMAG Servers. An OMAG Server could be +# a metadata server or a specialized governance server. Its behavior is determined +# by a configuration document that defines which OMAG services are activated. +# All OMAG Server Platforms support the administration commands to define a server's +# configuration document. It is also possible to create configuration documents +# through admin calls to one OMAG Server Platform and then deploy them to the +# OMAG Server Platform where they are to run. In the Egeria hands on lab, the +# OMAG Server configuration is created on the dev platform and deployed to the +# production platforms as needed. + +adminPlatformURL = devPlatformURL + +# Gary Geeke is the IT Administration Lead at Coco Pharmaceuticals. +# He does all of the configuration for the OMAG Servers. Other users are introduced and make +# calls to the server as required + +adminUserId = "garygeeke" +petersUserId = "peterprofile" +erinsUserId = "erinoverview" +calliesUserId = "calliequartile" +faithsUserId = "faithbroker" +harrysUserId = "harryhopeful" + +# These are the names of the metadata servers used by Coco Pharmaceuticals. Each metadata +# server runs as an OMAG Server on one of the OMAG Server Platforms + +cocoMDS1PlatformURL = dataLakePlatformURL +cocoMDS1PlatformName = dataLakePlatformName +cocoMDS1Name = "cocoMDS1" + +cocoMDS2PlatformURL = corePlatformURL +cocoMDS2PlatformName = corePlatformName +cocoMDS2Name = "cocoMDS2" + +cocoMDS3PlatformURL = corePlatformURL +cocoMDS3PlatformName = corePlatformName +cocoMDS3Name = "cocoMDS3" + +cocoMDS4PlatformURL = dataLakePlatformURL +cocoMDS4PlatformName = dataLakePlatformName +cocoMDS4Name = "cocoMDS4" + +cocoMDS5PlatformURL = corePlatformURL +cocoMDS5PlatformName = corePlatformName +cocoMDS5Name = "cocoMDS5" + +cocoMDS6PlatformURL = corePlatformURL +cocoMDS6PlatformName = corePlatformName +cocoMDS6Name = "cocoMDS6" + +cocoMDSxPlatformURL = devPlatformURL +cocoMDSxPlatformName = devPlatformName +cocoMDSxName = "cocoMDSx" + +cocoOLS1PlatformName = dataLakePlatformName +cocoOLS1PlafformURL = dataLakePlatformURL +cocoOLS1Name = "cocoOLS1" + +# The open metadata servers are linked together through the following open metadata cohorts. +# The servers linked via a cohort can exchange open metadata either through federated +# queries or metadata replication. + +cocoCohort = "cocoCohort" +devCohort = "devCohort" +iotCohort = "iotCohort" +ctsCohort = "ctsCohort" + +# This is the view server that runs the services that support Egeria's user interface. + +cocoView1PlatformURL = dataLakePlatformURL +cocoView1PlatformName = dataLakePlatformName +cocoView1Name = "cocoView1" + +# These are the names of the governance servers used in Coco Pharmaceuticals' data lake. +# Each governance server runs as an OMAG Server on the Data Lake OMAG Server Platform. +# They also connect to a metadata server to retrieve their configuration and store their +# results. + +governDL01PlatformURL = dataLakePlatformURL +governDL01PlatformName = dataLakePlatformName +governDL01Name = "governDL01" +governDL01ServerType = "Engine Host" +governDL01MDS = "cocoMDS1" + +exchangeDL01PlatformURL = dataLakePlatformURL +exchangeDL01PlatformName = dataLakePlatformName +exchangeDL01Name = "exchangeDL01" +exchangeDL01ServerType = "Integration Daemon" +exchangeDL01MDS = "cocoMDS1" + +monitorGov01PlatformURL = dataLakePlatformURL +monitorGov01PlatformName = dataLakePlatformName +monitorGov01Name = "monitorGov01" +monitorGov01ServerType = "Integration Daemon" +monitorGov01MDS = "cocoMDS1" + +monitorDev01PlatformURL = devPlatformURL +monitorDev01PlatformName = devPlatformName +monitorDev01Name = "monitorDev01" +monitorDev01ServerType = "Integration Daemon" +monitorDev01MDS = "cocoMDSx" + diff --git a/conformance-testing-labs/analyze-performance-results.ipynb b/coco-jupyter-labs/conformance-testing-labs/analyze-performance-results.ipynb similarity index 100% rename from conformance-testing-labs/analyze-performance-results.ipynb rename to coco-jupyter-labs/conformance-testing-labs/analyze-performance-results.ipynb diff --git a/conformance-testing-labs/run-conformance-test-suite.ipynb b/coco-jupyter-labs/conformance-testing-labs/run-conformance-test-suite.ipynb similarity index 100% rename from conformance-testing-labs/run-conformance-test-suite.ipynb rename to coco-jupyter-labs/conformance-testing-labs/run-conformance-test-suite.ipynb diff --git a/conformance-testing-labs/run-performance-test-suite.ipynb b/coco-jupyter-labs/conformance-testing-labs/run-performance-test-suite.ipynb similarity index 100% rename from conformance-testing-labs/run-performance-test-suite.ipynb rename to coco-jupyter-labs/conformance-testing-labs/run-performance-test-suite.ipynb diff --git a/conformance-testing-labs/run-ui-test-server.ipynb b/coco-jupyter-labs/conformance-testing-labs/run-ui-test-server.ipynb similarity index 100% rename from conformance-testing-labs/run-ui-test-server.ipynb rename to coco-jupyter-labs/conformance-testing-labs/run-ui-test-server.ipynb diff --git a/development-labs/governance-server-operation.ipynb b/coco-jupyter-labs/development-labs/governance-server-operation.ipynb similarity index 100% rename from development-labs/governance-server-operation.ipynb rename to coco-jupyter-labs/development-labs/governance-server-operation.ipynb diff --git a/development-labs/metadata-representations.ipynb b/coco-jupyter-labs/development-labs/metadata-representations.ipynb similarity index 100% rename from development-labs/metadata-representations.ipynb rename to coco-jupyter-labs/development-labs/metadata-representations.ipynb diff --git a/development-labs/simple-cohort-demo.ipynb b/coco-jupyter-labs/development-labs/simple-cohort-demo.ipynb similarity index 100% rename from development-labs/simple-cohort-demo.ipynb rename to coco-jupyter-labs/development-labs/simple-cohort-demo.ipynb diff --git a/egeria-server-config.ipynb b/coco-jupyter-labs/egeria-server-config.ipynb similarity index 100% rename from egeria-server-config.ipynb rename to coco-jupyter-labs/egeria-server-config.ipynb diff --git a/egeria-server-start.ipynb b/coco-jupyter-labs/egeria-server-start.ipynb similarity index 100% rename from egeria-server-start.ipynb rename to coco-jupyter-labs/egeria-server-start.ipynb diff --git a/images/CTS-Cohort.png b/coco-jupyter-labs/images/CTS-Cohort.png similarity index 100% rename from images/CTS-Cohort.png rename to coco-jupyter-labs/images/CTS-Cohort.png diff --git a/images/asset-zones-for-building-catalog.png b/coco-jupyter-labs/images/asset-zones-for-building-catalog.png similarity index 100% rename from images/asset-zones-for-building-catalog.png rename to coco-jupyter-labs/images/asset-zones-for-building-catalog.png diff --git a/images/automated-curation-scenario-1.png b/coco-jupyter-labs/images/automated-curation-scenario-1.png similarity index 100% rename from images/automated-curation-scenario-1.png rename to coco-jupyter-labs/images/automated-curation-scenario-1.png diff --git a/images/automated-curation-scenario-2.png b/coco-jupyter-labs/images/automated-curation-scenario-2.png similarity index 100% rename from images/automated-curation-scenario-2.png rename to coco-jupyter-labs/images/automated-curation-scenario-2.png diff --git a/images/automated-curation-scenario-3.png b/coco-jupyter-labs/images/automated-curation-scenario-3.png similarity index 100% rename from images/automated-curation-scenario-3.png rename to coco-jupyter-labs/images/automated-curation-scenario-3.png diff --git a/images/automated-curation-scenario-3b.png b/coco-jupyter-labs/images/automated-curation-scenario-3b.png similarity index 100% rename from images/automated-curation-scenario-3b.png rename to coco-jupyter-labs/images/automated-curation-scenario-3b.png diff --git a/images/automated-curation-scenario-4.png b/coco-jupyter-labs/images/automated-curation-scenario-4.png similarity index 100% rename from images/automated-curation-scenario-4.png rename to coco-jupyter-labs/images/automated-curation-scenario-4.png diff --git a/images/automated-curation-scenario-5.png b/coco-jupyter-labs/images/automated-curation-scenario-5.png similarity index 100% rename from images/automated-curation-scenario-5.png rename to coco-jupyter-labs/images/automated-curation-scenario-5.png diff --git a/images/automated-curation-scenario-6.png b/coco-jupyter-labs/images/automated-curation-scenario-6.png similarity index 100% rename from images/automated-curation-scenario-6.png rename to coco-jupyter-labs/images/automated-curation-scenario-6.png diff --git a/images/automated-curation-scenario-7.png b/coco-jupyter-labs/images/automated-curation-scenario-7.png similarity index 100% rename from images/automated-curation-scenario-7.png rename to coco-jupyter-labs/images/automated-curation-scenario-7.png diff --git a/images/automated-curation-scenario.png b/coco-jupyter-labs/images/automated-curation-scenario.png similarity index 100% rename from images/automated-curation-scenario.png rename to coco-jupyter-labs/images/automated-curation-scenario.png diff --git a/images/callie-quartile.png b/coco-jupyter-labs/images/callie-quartile.png similarity index 100% rename from images/callie-quartile.png rename to coco-jupyter-labs/images/callie-quartile.png diff --git a/images/coco-pharmaceuticals-systems-cohorts.png b/coco-jupyter-labs/images/coco-pharmaceuticals-systems-cohorts.png similarity index 100% rename from images/coco-pharmaceuticals-systems-cohorts.png rename to coco-jupyter-labs/images/coco-pharmaceuticals-systems-cohorts.png diff --git a/images/coco-pharmaceuticals-systems-omag-server-platforms-engine-host.png b/coco-jupyter-labs/images/coco-pharmaceuticals-systems-omag-server-platforms-engine-host.png similarity index 100% rename from images/coco-pharmaceuticals-systems-omag-server-platforms-engine-host.png rename to coco-jupyter-labs/images/coco-pharmaceuticals-systems-omag-server-platforms-engine-host.png diff --git a/images/coco-pharmaceuticals-systems-omag-server-platforms-integration-daemon.png b/coco-jupyter-labs/images/coco-pharmaceuticals-systems-omag-server-platforms-integration-daemon.png similarity index 100% rename from images/coco-pharmaceuticals-systems-omag-server-platforms-integration-daemon.png rename to coco-jupyter-labs/images/coco-pharmaceuticals-systems-omag-server-platforms-integration-daemon.png diff --git a/images/coco-pharmaceuticals-systems-omag-server-platforms-metadata-server.png b/coco-jupyter-labs/images/coco-pharmaceuticals-systems-omag-server-platforms-metadata-server.png similarity index 100% rename from images/coco-pharmaceuticals-systems-omag-server-platforms-metadata-server.png rename to coco-jupyter-labs/images/coco-pharmaceuticals-systems-omag-server-platforms-metadata-server.png diff --git a/images/coco-pharmaceuticals-systems-omag-server-platforms-view-server.png b/coco-jupyter-labs/images/coco-pharmaceuticals-systems-omag-server-platforms-view-server.png similarity index 100% rename from images/coco-pharmaceuticals-systems-omag-server-platforms-view-server.png rename to coco-jupyter-labs/images/coco-pharmaceuticals-systems-omag-server-platforms-view-server.png diff --git a/images/coco-pharmaceuticals-systems-omag-server-platforms.png b/coco-jupyter-labs/images/coco-pharmaceuticals-systems-omag-server-platforms.png similarity index 100% rename from images/coco-pharmaceuticals-systems-omag-server-platforms.png rename to coco-jupyter-labs/images/coco-pharmaceuticals-systems-omag-server-platforms.png diff --git a/images/coco-pharmaceuticals-systems.drawio b/coco-jupyter-labs/images/coco-pharmaceuticals-systems.drawio similarity index 100% rename from images/coco-pharmaceuticals-systems.drawio rename to coco-jupyter-labs/images/coco-pharmaceuticals-systems.drawio diff --git a/images/creating-configuration-documents.png b/coco-jupyter-labs/images/creating-configuration-documents.png similarity index 100% rename from images/creating-configuration-documents.png rename to coco-jupyter-labs/images/creating-configuration-documents.png diff --git a/images/database-type-hierarchy.png b/coco-jupyter-labs/images/database-type-hierarchy.png similarity index 100% rename from images/database-type-hierarchy.png rename to coco-jupyter-labs/images/database-type-hierarchy.png diff --git a/images/deploying-configuration-documents.png b/coco-jupyter-labs/images/deploying-configuration-documents.png similarity index 100% rename from images/deploying-configuration-documents.png rename to coco-jupyter-labs/images/deploying-configuration-documents.png diff --git a/images/design-model-to-archive-load.png b/coco-jupyter-labs/images/design-model-to-archive-load.png similarity index 100% rename from images/design-model-to-archive-load.png rename to coco-jupyter-labs/images/design-model-to-archive-load.png diff --git a/images/discovery-engine-configuration.png b/coco-jupyter-labs/images/discovery-engine-configuration.png similarity index 100% rename from images/discovery-engine-configuration.png rename to coco-jupyter-labs/images/discovery-engine-configuration.png diff --git a/images/discovery-servers.png b/coco-jupyter-labs/images/discovery-servers.png similarity index 100% rename from images/discovery-servers.png rename to coco-jupyter-labs/images/discovery-servers.png diff --git a/images/distributed-asset-management-lab.drawio b/coco-jupyter-labs/images/distributed-asset-management-lab.drawio similarity index 100% rename from images/distributed-asset-management-lab.drawio rename to coco-jupyter-labs/images/distributed-asset-management-lab.drawio diff --git a/images/distributed-asset-with-comments.png b/coco-jupyter-labs/images/distributed-asset-with-comments.png similarity index 100% rename from images/distributed-asset-with-comments.png rename to coco-jupyter-labs/images/distributed-asset-with-comments.png diff --git a/images/distributed-discovery-engines.png b/coco-jupyter-labs/images/distributed-discovery-engines.png similarity index 100% rename from images/distributed-discovery-engines.png rename to coco-jupyter-labs/images/distributed-discovery-engines.png diff --git a/images/distributed-engine-services-config.png b/coco-jupyter-labs/images/distributed-engine-services-config.png similarity index 100% rename from images/distributed-engine-services-config.png rename to coco-jupyter-labs/images/distributed-engine-services-config.png diff --git a/images/egeria-as-tool-integrator.png b/coco-jupyter-labs/images/egeria-as-tool-integrator.png similarity index 100% rename from images/egeria-as-tool-integrator.png rename to coco-jupyter-labs/images/egeria-as-tool-integrator.png diff --git a/images/egeria-ui-asset-catalog-archive-search-results.png b/coco-jupyter-labs/images/egeria-ui-asset-catalog-archive-search-results.png similarity index 100% rename from images/egeria-ui-asset-catalog-archive-search-results.png rename to coco-jupyter-labs/images/egeria-ui-asset-catalog-archive-search-results.png diff --git a/images/egeria-ui-asset-catalog-no-rows.png b/coco-jupyter-labs/images/egeria-ui-asset-catalog-no-rows.png similarity index 100% rename from images/egeria-ui-asset-catalog-no-rows.png rename to coco-jupyter-labs/images/egeria-ui-asset-catalog-no-rows.png diff --git a/images/egeria-ui-asset-catalog-search-field-input.png b/coco-jupyter-labs/images/egeria-ui-asset-catalog-search-field-input.png similarity index 100% rename from images/egeria-ui-asset-catalog-search-field-input.png rename to coco-jupyter-labs/images/egeria-ui-asset-catalog-search-field-input.png diff --git a/images/egeria-ui-asset-end-to-end-lineage.png b/coco-jupyter-labs/images/egeria-ui-asset-end-to-end-lineage.png similarity index 100% rename from images/egeria-ui-asset-end-to-end-lineage.png rename to coco-jupyter-labs/images/egeria-ui-asset-end-to-end-lineage.png diff --git a/images/egeria-ui-asset-list.png b/coco-jupyter-labs/images/egeria-ui-asset-list.png similarity index 100% rename from images/egeria-ui-asset-list.png rename to coco-jupyter-labs/images/egeria-ui-asset-list.png diff --git a/images/egeria-ui-asset-search.png b/coco-jupyter-labs/images/egeria-ui-asset-search.png similarity index 100% rename from images/egeria-ui-asset-search.png rename to coco-jupyter-labs/images/egeria-ui-asset-search.png diff --git a/images/egeria-ui-end-to-end-lineage-graph.png b/coco-jupyter-labs/images/egeria-ui-end-to-end-lineage-graph.png similarity index 100% rename from images/egeria-ui-end-to-end-lineage-graph.png rename to coco-jupyter-labs/images/egeria-ui-end-to-end-lineage-graph.png diff --git a/images/egeria-ui-erin-logon.png b/coco-jupyter-labs/images/egeria-ui-erin-logon.png similarity index 100% rename from images/egeria-ui-erin-logon.png rename to coco-jupyter-labs/images/egeria-ui-erin-logon.png diff --git a/images/egeria-ui-login.png b/coco-jupyter-labs/images/egeria-ui-login.png similarity index 100% rename from images/egeria-ui-login.png rename to coco-jupyter-labs/images/egeria-ui-login.png diff --git a/images/egeria-ui-nav-bar.png b/coco-jupyter-labs/images/egeria-ui-nav-bar.png similarity index 100% rename from images/egeria-ui-nav-bar.png rename to coco-jupyter-labs/images/egeria-ui-nav-bar.png diff --git a/images/engine-host.png b/coco-jupyter-labs/images/engine-host.png similarity index 100% rename from images/engine-host.png rename to coco-jupyter-labs/images/engine-host.png diff --git a/images/exchangeDL01-config.png b/coco-jupyter-labs/images/exchangeDL01-config.png similarity index 100% rename from images/exchangeDL01-config.png rename to coco-jupyter-labs/images/exchangeDL01-config.png diff --git a/images/file-asset-hierarchy.png b/coco-jupyter-labs/images/file-asset-hierarchy.png similarity index 100% rename from images/file-asset-hierarchy.png rename to coco-jupyter-labs/images/file-asset-hierarchy.png diff --git a/images/glossary-revision-history.png b/coco-jupyter-labs/images/glossary-revision-history.png similarity index 100% rename from images/glossary-revision-history.png rename to coco-jupyter-labs/images/glossary-revision-history.png diff --git a/images/glossary-structure.png b/coco-jupyter-labs/images/glossary-structure.png similarity index 100% rename from images/glossary-structure.png rename to coco-jupyter-labs/images/glossary-structure.png diff --git a/images/governance-action-process-flow.png b/coco-jupyter-labs/images/governance-action-process-flow.png similarity index 100% rename from images/governance-action-process-flow.png rename to coco-jupyter-labs/images/governance-action-process-flow.png diff --git a/images/governance-action-process-implementation.png b/coco-jupyter-labs/images/governance-action-process-implementation.png similarity index 100% rename from images/governance-action-process-implementation.png rename to coco-jupyter-labs/images/governance-action-process-implementation.png diff --git a/images/governance-action-request-type.png b/coco-jupyter-labs/images/governance-action-request-type.png similarity index 100% rename from images/governance-action-request-type.png rename to coco-jupyter-labs/images/governance-action-request-type.png diff --git a/images/governance-engine-definition.png b/coco-jupyter-labs/images/governance-engine-definition.png similarity index 100% rename from images/governance-engine-definition.png rename to coco-jupyter-labs/images/governance-engine-definition.png diff --git a/images/governance-engine-lab-platform-part.png b/coco-jupyter-labs/images/governance-engine-lab-platform-part.png similarity index 100% rename from images/governance-engine-lab-platform-part.png rename to coco-jupyter-labs/images/governance-engine-lab-platform-part.png diff --git a/images/governance-engine-lab-platform.png b/coco-jupyter-labs/images/governance-engine-lab-platform.png similarity index 100% rename from images/governance-engine-lab-platform.png rename to coco-jupyter-labs/images/governance-engine-lab-platform.png diff --git a/images/governance-engine-lab-servers-part-2.png b/coco-jupyter-labs/images/governance-engine-lab-servers-part-2.png similarity index 100% rename from images/governance-engine-lab-servers-part-2.png rename to coco-jupyter-labs/images/governance-engine-lab-servers-part-2.png diff --git a/images/governance-engine-lab-servers-part.png b/coco-jupyter-labs/images/governance-engine-lab-servers-part.png similarity index 100% rename from images/governance-engine-lab-servers-part.png rename to coco-jupyter-labs/images/governance-engine-lab-servers-part.png diff --git a/images/governance-engine-lab-servers.png b/coco-jupyter-labs/images/governance-engine-lab-servers.png similarity index 100% rename from images/governance-engine-lab-servers.png rename to coco-jupyter-labs/images/governance-engine-lab-servers.png diff --git a/images/governance-servers-for-automated-onboarding.png b/coco-jupyter-labs/images/governance-servers-for-automated-onboarding.png similarity index 100% rename from images/governance-servers-for-automated-onboarding.png rename to coco-jupyter-labs/images/governance-servers-for-automated-onboarding.png diff --git a/images/governance-servers-running-automated-onboarding.png b/coco-jupyter-labs/images/governance-servers-running-automated-onboarding.png similarity index 100% rename from images/governance-servers-running-automated-onboarding.png rename to coco-jupyter-labs/images/governance-servers-running-automated-onboarding.png diff --git a/images/integration-daemon.png b/coco-jupyter-labs/images/integration-daemon.png similarity index 100% rename from images/integration-daemon.png rename to coco-jupyter-labs/images/integration-daemon.png diff --git a/images/jupyter-notebook-play-button.png b/coco-jupyter-labs/images/jupyter-notebook-play-button.png similarity index 100% rename from images/jupyter-notebook-play-button.png rename to coco-jupyter-labs/images/jupyter-notebook-play-button.png diff --git a/images/kafka-server-properties.png b/coco-jupyter-labs/images/kafka-server-properties.png similarity index 100% rename from images/kafka-server-properties.png rename to coco-jupyter-labs/images/kafka-server-properties.png diff --git a/images/omag-server-platform-ready.png b/coco-jupyter-labs/images/omag-server-platform-ready.png similarity index 100% rename from images/omag-server-platform-ready.png rename to coco-jupyter-labs/images/omag-server-platform-ready.png diff --git a/images/open-lineage-server.png b/coco-jupyter-labs/images/open-lineage-server.png similarity index 100% rename from images/open-lineage-server.png rename to coco-jupyter-labs/images/open-lineage-server.png diff --git a/images/open-lineage-service-lab-assets.png b/coco-jupyter-labs/images/open-lineage-service-lab-assets.png similarity index 100% rename from images/open-lineage-service-lab-assets.png rename to coco-jupyter-labs/images/open-lineage-service-lab-assets.png diff --git a/images/open-lineage-services.drawio b/coco-jupyter-labs/images/open-lineage-services.drawio similarity index 100% rename from images/open-lineage-services.drawio rename to coco-jupyter-labs/images/open-lineage-services.drawio diff --git a/images/peter-and-erin.png b/coco-jupyter-labs/images/peter-and-erin.png similarity index 100% rename from images/peter-and-erin.png rename to coco-jupyter-labs/images/peter-and-erin.png diff --git a/images/peter-profile.png b/coco-jupyter-labs/images/peter-profile.png similarity index 100% rename from images/peter-profile.png rename to coco-jupyter-labs/images/peter-profile.png diff --git a/images/simple-api-catalog.png b/coco-jupyter-labs/images/simple-api-catalog.png similarity index 100% rename from images/simple-api-catalog.png rename to coco-jupyter-labs/images/simple-api-catalog.png diff --git a/images/simple-catalog-demo-asset-properties.png b/coco-jupyter-labs/images/simple-catalog-demo-asset-properties.png similarity index 100% rename from images/simple-catalog-demo-asset-properties.png rename to coco-jupyter-labs/images/simple-catalog-demo-asset-properties.png diff --git a/images/simple-catalog-demo-connected-catalogs.png b/coco-jupyter-labs/images/simple-catalog-demo-connected-catalogs.png similarity index 100% rename from images/simple-catalog-demo-connected-catalogs.png rename to coco-jupyter-labs/images/simple-catalog-demo-connected-catalogs.png diff --git a/images/simple-catalog-demo-data-map.png b/coco-jupyter-labs/images/simple-catalog-demo-data-map.png similarity index 100% rename from images/simple-catalog-demo-data-map.png rename to coco-jupyter-labs/images/simple-catalog-demo-data-map.png diff --git a/images/simple-catalog-demo-independent-catalogs.png b/coco-jupyter-labs/images/simple-catalog-demo-independent-catalogs.png similarity index 100% rename from images/simple-catalog-demo-independent-catalogs.png rename to coco-jupyter-labs/images/simple-catalog-demo-independent-catalogs.png diff --git a/images/simple-catalog-demo.png b/coco-jupyter-labs/images/simple-catalog-demo.png similarity index 100% rename from images/simple-catalog-demo.png rename to coco-jupyter-labs/images/simple-catalog-demo.png diff --git a/images/simple-catalog-metadata-origin-comment.png b/coco-jupyter-labs/images/simple-catalog-metadata-origin-comment.png similarity index 100% rename from images/simple-catalog-metadata-origin-comment.png rename to coco-jupyter-labs/images/simple-catalog-metadata-origin-comment.png diff --git a/images/simple-catalog-metadata-origin.png b/coco-jupyter-labs/images/simple-catalog-metadata-origin.png similarity index 100% rename from images/simple-catalog-metadata-origin.png rename to coco-jupyter-labs/images/simple-catalog-metadata-origin.png diff --git a/images/simple-catalog.png b/coco-jupyter-labs/images/simple-catalog.png similarity index 100% rename from images/simple-catalog.png rename to coco-jupyter-labs/images/simple-catalog.png diff --git a/images/simple-data-catalog.png b/coco-jupyter-labs/images/simple-data-catalog.png similarity index 100% rename from images/simple-data-catalog.png rename to coco-jupyter-labs/images/simple-data-catalog.png diff --git a/images/simple-event-catalog.png b/coco-jupyter-labs/images/simple-event-catalog.png similarity index 100% rename from images/simple-event-catalog.png rename to coco-jupyter-labs/images/simple-event-catalog.png diff --git a/information-architecture/working-with-standard-models.ipynb b/coco-jupyter-labs/information-architecture/working-with-standard-models.ipynb similarity index 100% rename from information-architecture/working-with-standard-models.ipynb rename to coco-jupyter-labs/information-architecture/working-with-standard-models.ipynb diff --git a/labs-in-development/README.md b/coco-jupyter-labs/labs-in-development/README.md similarity index 100% rename from labs-in-development/README.md rename to coco-jupyter-labs/labs-in-development/README.md diff --git a/labs-in-development/jdbc-connector.ipynb b/coco-jupyter-labs/labs-in-development/jdbc-connector.ipynb similarity index 100% rename from labs-in-development/jdbc-connector.ipynb rename to coco-jupyter-labs/labs-in-development/jdbc-connector.ipynb diff --git a/overview/egeria-demo.ipynb b/coco-jupyter-labs/overview/egeria-demo.ipynb similarity index 100% rename from overview/egeria-demo.ipynb rename to coco-jupyter-labs/overview/egeria-demo.ipynb diff --git a/read-me-first.ipynb b/coco-jupyter-labs/read-me-first.ipynb old mode 100755 new mode 100644 similarity index 96% rename from read-me-first.ipynb rename to coco-jupyter-labs/read-me-first.ipynb index 532d8090..c7b8ea67 --- a/read-me-first.ipynb +++ b/coco-jupyter-labs/read-me-first.ipynb @@ -54,7 +54,7 @@ "\n", "## Starting up the Egeria platforms\n", "\n", - "Egeria's hands-on labs assume that there are three OMAG Server Platforms running, along with Apache Zookeeper and Apache Kafka. Click the play button again to move to the code below, and then again to run the code below. This will check if the platforms are running. \n", + "Egeria's hands-on labs assume that there are three OMAG Server Platforms running, along with Apache Kafka. Click the play button again to move to the code below, and then again to run the code below. This will check if the platforms are running. \n", "\n", "You are looking to confirm that `CorePlatform`, `DataLakePlatform` and `Dev Platform` are running.\n", "It has completed its check when you see `Done.`." @@ -78,7 +78,9 @@ "If you see that all platforms are running then you are ready to begin. If they are not running then\n", "there are a number of choices on how to start them. Follow [this link to set up and run the platforms](https://egeria-project.org/education/open-metadata-labs/overview/).\n", "\n", - "## Configuring the servers (one time task)\n", + "## Configuring the servers (one time task) - **Only needed if you have not configured with the Docker Compose script**.\n", + "\n", + "**If you have deployed the Egeria Coco Hands-On Labs with the Docker Compose script you are ready to go.** If not, then please configure the environment with the following instructions.\n", "\n", "Once the platforms are running, it is necessary to configure the Egeria servers that run on the platforms.\n", "If the `%run common/environment-check.ipynb` command showed that the servers are not configured,\n", diff --git a/ui-labs/ui-asset-search.ipynb b/coco-jupyter-labs/ui-labs/ui-asset-search.ipynb similarity index 100% rename from ui-labs/ui-asset-search.ipynb rename to coco-jupyter-labs/ui-labs/ui-asset-search.ipynb diff --git a/coco-lab-docker-compose/Coco-Pharma Docker Compose.iml b/coco-lab-docker-compose/Coco-Pharma Docker Compose.iml new file mode 100644 index 00000000..c90834f2 --- /dev/null +++ b/coco-lab-docker-compose/Coco-Pharma Docker Compose.iml @@ -0,0 +1,11 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/coco-lab-docker-compose/Dockerfile b/coco-lab-docker-compose/Dockerfile new file mode 100644 index 00000000..243e2108 --- /dev/null +++ b/coco-lab-docker-compose/Dockerfile @@ -0,0 +1,34 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright Contributors to the Egeria project. + +# This build script corrects some permission issues needed to run +# on some enterprise k8s environments. see https://github.com/odpi/egeria-jupyter-notebooks/issues/9 + +# The published image tag is taken from the numerical version of +# our base image, and appended with the contents of .tag-append (file) +FROM quay.io/jupyter/minimal-notebook + +#COPY ../config_coco_core.py /config_coco.py +#RUN chmod +x /config_coco.py + +RUN pip install --no-cache-dir 'pyegeria>=0.7.35' +RUN pip install --no-cache-dir 'rich' + +# && \ +# fix-permissions "${CONDA_DIR}" && \ +# fix-permissions "/home/${NB_USER}" \ + +#ENTRYPOINT ["jupyter","notebook"] +# +#CMD ["python3", "/config_coco_core.py"] +#USER root + +# Needed to dynamically add the selected user on startup - see link below +#RUN chmod g+w /etc/passwd +# +#RUN chown -R $NB_UID:$NB_GID $HOME +# +## https://cloud.redhat.com/blog/jupyter-on-openshift-part-6-running-as-an-assigned-user-id +#RUN chgrp -Rf root /home/$NB_USER && chmod -Rf g+w /home/$NB_USER && chgrp -Rf root /opt/conda && chmod -Rf g+w /opt/conda +# +#USER 1000 diff --git a/coco-lab-docker-compose/README.md b/coco-lab-docker-compose/README.md new file mode 100644 index 00000000..e6b99cb6 --- /dev/null +++ b/coco-lab-docker-compose/README.md @@ -0,0 +1,49 @@ + + + +# Open Metadata Labs - Using Docker Compose + +The open metadata labs provide an interactive environment that allows you to +experiment with different capabilities of Egeria. More information about the labs can be found at: +[Overview of the Labs](https://egeria-project.org/education/open-metadata-labs/overview/). +The labs are written using Python Jupyter notebooks that +run in a Jupyter Server. The interactive exercises in the notebooks call python functions +that communicate with Egeria. An Apache Kafka server is used by Egeria for communications. + +One way to easily deploy a running +Open Metadata Labs environment is by using the Docker Compose script contained in this directory. + +A docker compose script, coco-lab-setup.yaml uses docker compose to deploy, configure and run a complete working +environment that includes: + +* Three Egeria OMAG Server Platforms (Core, Datalake, and Development) +* Kafka +* Jupyter Server that is used to run the lab exercises + + +# Getting Started + +To get started, you need a computer with Docker installed and configured. Our experience is with running Docker on Mac and +Linux machines, Windows machines should also work (reach out if you run into issues). Docker can be installed from +[Docker](https://docker.com). Compatible alternatives to Docker Compose exist but have not yet been validated by us. + +The steps to get started are: +1) If not already installed in your environment, install and start Docker and Docker Compose. If you are running on a local machine, then it is often simplest to install `Docker Desktop`. +2) Download the github repo. You can download and unpack a zip file of the repo or clone the repo if you have **git** set up: + a) download the zipfile by navigating to `https://github.com/odpi/egeria-coco-labs` and selecting `Download ZIP` from the green Code drop down. + b) clone the repo - `git clone https://github.com/odpi/egeria-coco-labs.git` +3) Change directory to egeria-jupyter-notebooks/coco-lab-docker-compose and start up the environment by entering `docker compose -f coco-lab-setup.yaml up --build` +This should download and install all the components, start them up and then configure the Egeria servers for the Coco labs. Once this is all done (may take a few minutes), you +can open a browser to http://localhost:7888. If it asks for a password or token, type `egeria`. You should now be in a Jupyter notebook environment. Open the `read-me-first` notebook. + +Congratulations! You should be up and running! + + + + + + + +---- +License: [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/), +Copyright Contributors to the ODPi Egeria project. diff --git a/coco-lab-docker-compose/before-notebook.d/config-coco.sh b/coco-lab-docker-compose/before-notebook.d/config-coco.sh new file mode 100755 index 00000000..432b1d0b --- /dev/null +++ b/coco-lab-docker-compose/before-notebook.d/config-coco.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# +# SPDX-License-Identifier: Apache-2.0 +# Copyright Contributors to the ODPi Egeria project. +# +# Coco Pharmaceuticals Lab Docker Compose configuration +# This shell script is automatically invoked when the Jupyter Container is initiated as part of the Docker Compose +# configuration.The script configures and activates the Egeria OMAG servers on each of the three Egeria OMAG Server +# Platforms that are part of the sample Coco Pharmaceuticals deployment environment. + +#rm -rf ./__pychache__ +export PYTHONDONTWRITEBYTECODE=1 +python3 /home/jovyan/work/common/config_coco_core.py +python3 /home/jovyan/work/common/config_coco_datalake.py +python3 /home/jovyan/work/common/config_coco_development.py +echo "Launching Jupyter notebook server.." +#exec jupyter notebook "$@" +#python3 config_cocoMDS2.py diff --git a/coco-lab-docker-compose/coco-lab-setup.yaml b/coco-lab-docker-compose/coco-lab-setup.yaml new file mode 100644 index 00000000..a5cfc0c2 --- /dev/null +++ b/coco-lab-docker-compose/coco-lab-setup.yaml @@ -0,0 +1,130 @@ +--- +# SPDX-License-Identifier: Apache-2.0 +# Copyright Contributors to the Egeria project + + +# To run +# * Ensure Docker is installed and running +# * Start Egeria Platform stack using +# 'docker compose -f ./coco-lab-setyp.yaml up --build' +# +# Assumptions: +# * The following ports are used: +# * 7443, 7444, 7445 are used by the egeria-core, egeria-datalake, and egeria-dev Egeria OMAG Platforms. +# * 7888 is used by Jupyter +# * 7192 is used by Kafka +# * by default, the repository data is stored locally within the container using XTDB as the storage provider. +# * by default, strict SSL validation is turned off +# + +services: + jupyter-hub: + depends_on: + egeria-core: + condition: service_healthy + egeria-datalake: + condition: service_healthy + egeria-dev: + condition: service_healthy + + image: quay.io/jupyter/minimal-notebook + container_name: jupyter + ports: + - 7888:7888 + environment: + JUPYTER_ENABLE_LAB: "yes" + JUPYTER_TOKEN: "egeria" + JUPYTER_PORT: 7888 + build: + dockerfile: Dockerfile + + volumes: + - ../coco-jupyter-labs/:/home/jovyan + - ./before-notebook.d:/usr/local/bin/before-notebook.d + # - ./coco-lab-docker-compose:/home/jovyan/scripts + + kafka: + image: 'bitnami/kafka:latest' + ports: + - '7192:7192' + + environment: + - KAFKA_CFG_NODE_ID=0 + - KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true + - KAFKA_CFG_PROCESS_ROLES=controller,broker + - KAFKA_CFG_LISTENERS=PLAINTEXT://:7192,CONTROLLER://:7193 + - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT + - KAFKA_CFG_CONTROLLER_QUORUM_VOTERS=0@kafka:7193 + - KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER + + egeria-core: + depends_on: + - kafka + image: 'docker.io/odpi/egeria-platform:stable' + ports: + - '7443:7443' + + environment: + - XTDB_ENABLE_BYTEUTILS_SHA1=True + - XTDB_DISABLE_LIBCRYPTO=True + - startup.server.list= + - server.port=7443 + healthcheck: + test: curl -k -X GET "https://localhost:7443/open-metadata/platform-services/users/garygeeke/server-platform/origin" || exit 1 + interval: 20s + timeout: 10s + retries: 3 + start_period: 10s + + egeria-datalake: + depends_on: + - kafka + + image: 'docker.io/odpi/egeria-platform:stable' + ports: + - '7444:7444' + + environment: + - XTDB_ENABLE_BYTEUTILS_SHA1=True + - XTDB_DISABLE_LIBCRYPTO=True + - startup.server.list= + - server.port=7444 + healthcheck: + test: curl -k -X GET "https://localhost:7444/open-metadata/platform-services/users/garygeeke/server-platform/origin" || exit 1 + interval: 20s + timeout: 10s + retries: 3 + start_period: 10s + + + egeria-dev: + depends_on: + - kafka + + image: 'docker.io/odpi/egeria-platform:stable' + ports: + - '7445:7445' + environment: + - XTDB_ENABLE_BYTEUTILS_SHA1=True + - XTDB_DISABLE_LIBCRYPTO=True + - startup.server.list= + - server.port=7445 + healthcheck: + test: curl -k -X GET "https://localhost:7445/open-metadata/platform-services/users/garygeeke/server-platform/origin" || exit 1 + interval: 20s + timeout: 10s + retries: 3 + start_period: 10s + + + +# +# Change external to true and create volumes manually if you wish to persist between runs +# +volumes: + zookeeper-data: + external: false + kafka-data: + external: false + db_home: + external: false diff --git a/coco-lab-docker-compose/core_application.properties b/coco-lab-docker-compose/core_application.properties new file mode 100755 index 00000000..6c903116 --- /dev/null +++ b/coco-lab-docker-compose/core_application.properties @@ -0,0 +1,227 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright Contributors to the ODPi Egeria project. + +# ======================================================================================================== +# The application.properties file is used to configure the OMAG Server Platform which is implemented as +# a spring boot application. + +############################################### +### Default port for the OMAG Server Platform +################################################ +server.port=7443 + +############################################### +### Set up the configuration document store the OMAG Server Platform +### The values below will set up the clear text config document store +### (the default is the encrypted config document store). +################################################ +platform.configstore.provider=org.odpi.openmetadata.adapters.adminservices.configurationstore.file.FileBasedServerConfigStoreProvider +platform.configstore.endpoint=data/servers/{0}/config/{0}.config + +############################################### +### Set up the platform metadata security connector that provides authorization +### for platform administration, server operations and diagnostic calls. +### By default, there is no platform metadata security connector. +### The values below are for a sample platform metadata security connector where the only userId +### that is permitted to use the administration and platform services is `garygeeke`. +############################################### +platform.security.provider=org.odpi.openmetadata.metadatasecurity.samples.CocoPharmaPlatformSecurityProvider +platform.security.name=Coco Pharmaceuticals Platform + +############################################### +### Set up the default configuration document for any new OMAG Server configurations. +############################################### +#platform.default.config.document=\ +# {\ +# "class": "OMAGServerConfig",\ +# "organizationName": "myOrg",\ +# "maxPageSize": 1200,\ +# "eventBusConfig": \ +# {\ +# "class": "EventBusConfig",\ +# "topicURLRoot": "egeria.omag",\ +# "configurationProperties":\ +# {\ +# "producer": {"bootstrap.servers": "{{kafkaEndpoint}}"},\ +# "consumer": {"bootstrap.servers": "{{kafkaEndpoint}}"}\ +# }\ +# }\ +# } + +################################################ +### Placeholder variables are added to field values in the configuration document +### when the server is being configured using double curly braces. They are replaced by +### the values specified in platform.placeholder.variables each time the server starts up. +### +### The "kafkaEndpoint" value is the Apache Kafka endpoint, and it is used in the active-metadata-store +### sample configuration. If your Apache Kafka broker is listening on a different endpoint +### and you want to use active-metadata-store, change this variable to your Apache Kafka's address. +################################################ +platform.placeholder.variables=\ + {\ + "kafkaEndpoint" : "host.docker.internal:7192",\ + "egeriaEndpoint" : "https://localhost:7443"\ + } + +################################################ +### startup servers configuration +################################################ +#userId used to start up the list of configured servers default is 'system' +startup.user=garygeeke +# Comma separated names of servers to be started. The server names should be unquoted. +startup.server.list=cocoMDS2, cocoMDS5, cocoMDS6, cocoMDS3 + + +################################################ +### SSL security. +# The keystore determines the information sent out by the server to identify itself. +# The truststore is where the certificates of trusted servers the platform is calling are located. +# (Note SSL certificate checking is performed on client-side only.) +################################################ +server.ssl.key-store=keystore.p12 +server.ssl.key-store-password=egeria +server.ssl.keyStoreType=PKCS12 +server.ssl.keyAlias=egeriaserverchassis + +server.ssl.trust-store=truststore.p12 +server.ssl.trust-store-password=egeria + +# WARNING! setting 'strict.ssl=false' allows java clients to open https connections without checking the validity of +# certificates from the servers it is calling. +# Alternate you can import self-signed certificates into java truststore or set up a truststore only for this app +# by adding the store into server.ssl.trust-store parameter +strict.ssl=true + + +################################################ +# User security +################################################ + +# Authentication source (possible values: demo, ldap, ad) +authentication.source=demo +# Authentication mode (possible values: session,token,redis) +authentication.mode=token + +#token timeout in minutes +token.timeout=15 +token.absolute.timeout=720 +token.secret=doNotTell + +#LDAP authentication + +ldap.domain= +ldap.user.search.base=ou=people,dc=egeria,dc=com +ldap.user.search.filter=uid={0} +ldap.group.search.base=ou=Groups,dc=egeria,dc=com +ldap.group.search.filter=member={0} +ldap.url=ldap://localhost:389 +ldap.group.role.attribute= +ldap.npa.dn= +ldap.npa.password= +#ldap.user.dn.patterns patterns is a list of values separated by ";" as comma is used in the ldap pattern +ldap.user.dn.patterns= + +# Redis configuration +#redis.host=localhost +#redis.port=6379 + +################################################ +### Additional demo users configuration for when authentication.source=demo +### This file is located in the resources folder of the user-authn module and built into its runtime jar +################################################ +spring.config.import=classpath:demo-users.yml + +################################################ +### Comma separated list of header names to extract from incoming HTTP requests and add to thread local. +### The default value is null and adds no headers. +### Setting the list to * means all headers are captured. +### Otherwise, list the header name in use. +################################################ +authn.header.name.list= + +################################################ +### CORS +################################################ +# Comma-separated list of origins. +# Example configuration below is for setting up local development environment where egeria-ui is hosted on one of the two urls. +# cors.allowed-origins=http://localhost,http://localhost:8081 +cors.allowed-origins=* + +################################################ +# landing page (/api/public/app/info) +################################################ +app.description=Have a question? || Get in touch via our Slack community https://slack.lfai.foundation/ @@What is Open Metadata? || Find out more on our website https://egeria-project.org/ @@Have more cool ideas? || Feel free to let us know your ideas so we can make it better. +app.title=Egeria Open Metadata | Find the right data with governance + +# ############################################################## +# Component visibility for Role based access ################### +# ############################################################## +# How it works? +# +# The roles are defined in external authentication source (provider) configured with `authentication.source`. +# For demo purposes, we are providing simple file based authentication provider. See demo-users.yml +# The matrix controlling what components are allowed on the UI views for specific role is defined in the 'role.visibleComponents' prefixed properties as follows" +# +# role.visibleComponents.{ROLE-1}={component-name-1} +# role.visibleComponents.{ROLE-2}={component-name-1},{component-name-2} +# +# This will configure the application to show the component named 'component-name-1' for all uses assigned to 'ROLE-1' +# In the same way, users that have assigned 'ROLE-2' can see more 'component-name-1' and 'component-name-2'. +# It is also possible to use wildcard '*' to enable full visibility of all components to users in the given role. +# +# Complete list of components names that can be used: +# +# about +# asset-catalog +# asset-details +# asset-details-print +# glossary +# repository-explorer +# type-explorer +# asset-lineage +# asset-lineage-print +# end-to-end +# ultimate-source +# ultimate-destination +# vertical-lineage +# +# Below is the default configuration for the two COCO_PHARMA roles we use for demo: + +role.visibleComponents.COCO_PHARMA_USER=about,asset-catalog,asset-details,asset-details-print,asset-lineage,asset-lineage-print,end-to-end,ultimate-source,ultimate-destination,vertical-lineage,glossary,repository-explorer +role.visibleComponents.COCO_PHARMA_ADMIN=* + +################################################ +### Which java packages should be scanned to locate the Spring resource definitions that define the REST APIs? +################################################ +scan.packages=org.odpi.openmetadata.* + +################################################ +### Logging +################################################ +logging.level.root=OFF +logging.level.org.springframework=ERROR +logging.level.org.springframework.boot.web.embedded.tomcat=INFO +logging.level.org.odpi.openmetadata.platformchassis.springboot=INFO +#tracing REST calls +#logging.level.org.odpi.openmetadata.commonservices.ffdc.RESTCallLogger=DEBUG + +################################################ +### Swagger Docs +################################################ +springdoc.version='@springdoc.version@' +springdoc.api-docs.enabled=true +springdoc.api-docs.path=/v3/api-docs +springdoc.swagger-ui.path=/swagger-ui.html +springdoc.swagger-ui.displayRequestDuration=true +springdoc.swagger-ui.tagsSorter=alpha +springdoc.swagger-ui.operationsSorter=alpha +springdoc.swagger-ui.docExpansion=none + +################################################ +### Spring Boot Actuator +################################################ +# Endpoints web configuration +#management.endpoints.web.exposure.include=* +management.health.cassandra.enabled=false +management.health.redis.enabled=false +management.health.ldap.enabled=false \ No newline at end of file diff --git a/coco-lab-docker-compose/datalake_application.properties b/coco-lab-docker-compose/datalake_application.properties new file mode 100755 index 00000000..9919350e --- /dev/null +++ b/coco-lab-docker-compose/datalake_application.properties @@ -0,0 +1,226 @@ +core_application.properties# SPDX-License-Identifier: Apache-2.0 +# Copyright Contributors to the ODPi Egeria project. + +# ======================================================================================================== +# The application.properties file is used to configure the OMAG Server Platform which is implemented as +# a spring boot application. + +############################################### +### Default port for the OMAG Server Platform +################################################ +server.port=7444 + +############################################### +### Set up the configuration document store the OMAG Server Platform +### The values below will set up the clear text config document store +### (the default is the encrypted config document store). +################################################ +platform.configstore.provider=org.odpi.openmetadata.adapters.adminservices.configurationstore.file.FileBasedServerConfigStoreProvider +platform.configstore.endpoint=data/servers/{0}/config/{0}.config + +############################################### +### Set up the platform metadata security connector that provides authorization +### for platform administration, server operations and diagnostic calls. +### By default, there is no platform metadata security connector. +### The values below are for a sample platform metadata security connector where the only userId +### that is permitted to use the administration and platform services is `garygeeke`. +############################################### +platform.security.provider=org.odpi.openmetadata.metadatasecurity.samples.CocoPharmaPlatformSecurityProvider +platform.security.name=Coco Pharmaceuticals Platform + +############################################### +### Set up the default configuration document for any new OMAG Server configurations. +############################################### +#platform.default.config.document=\ +# {\ +# "class": "OMAGServerConfig",\ +# "organizationName": "myOrg",\ +# "maxPageSize": 1200,\ +# "eventBusConfig": \ +# {\ +# "class": "EventBusConfig",\ +# "topicURLRoot": "egeria.omag",\ +# "configurationProperties":\ +# {\ +# "producer": {"bootstrap.servers": "{{kafkaEndpoint}}"},\ +# "consumer": {"bootstrap.servers": "{{kafkaEndpoint}}"}\ +# }\ +# }\ +# } + +################################################ +### Placeholder variables are added to field values in the configuration document +### when the server is being configured using double curly braces. They are replaced by +### the values specified in platform.placeholder.variables each time the server starts up. +### +### The "kafkaEndpoint" value is the Apache Kafka endpoint, and it is used in the active-metadata-store +### sample configuration. If your Apache Kafka broker is listening on a different endpoint +### and you want to use active-metadata-store, change this variable to your Apache Kafka's address. +################################################ +platform.placeholder.variables=\ + {\ + "kafkaEndpoint" : "host.docker.internal:7192",\ + "egeriaEndpoint" : "https://localhost:7444"\ + } + +################################################ +### startup servers configuration +################################################ +#userId used to start up the list of configured servers default is 'system' +startup.user=garygeeke +# Comma separated names of servers to be started. The server names should be unquoted. +startup.server.lost=exchangeDL01, governDL01, cocoView1, cocoMDS4, cocoMDS1 + +################################################ +### SSL security. +# The keystore determines the information sent out by the server to identify itself. +# The truststore is where the certificates of trusted servers the platform is calling are located. +# (Note SSL certificate checking is performed on client-side only.) +################################################ +server.ssl.key-store=keystore.p12 +server.ssl.key-store-password=egeria +server.ssl.keyStoreType=PKCS12 +server.ssl.keyAlias=egeriaserverchassis + +server.ssl.trust-store=truststore.p12 +server.ssl.trust-store-password=egeria + +# WARNING! setting 'strict.ssl=false' allows java clients to open https connections without checking the validity of +# certificates from the servers it is calling. +# Alternate you can import self-signed certificates into java truststore or set up a truststore only for this app +# by adding the store into server.ssl.trust-store parameter +strict.ssl=true + + +################################################ +# User security +################################################ + +# Authentication source (possible values: demo, ldap, ad) +authentication.source=demo +# Authentication mode (possible values: session,token,redis) +authentication.mode=token + +#token timeout in minutes +token.timeout=15 +token.absolute.timeout=720 +token.secret=doNotTell + +#LDAP authentication + +ldap.domain= +ldap.user.search.base=ou=people,dc=egeria,dc=com +ldap.user.search.filter=uid={0} +ldap.group.search.base=ou=Groups,dc=egeria,dc=com +ldap.group.search.filter=member={0} +ldap.url=ldap://localhost:389 +ldap.group.role.attribute= +ldap.npa.dn= +ldap.npa.password= +#ldap.user.dn.patterns patterns is a list of values separated by ";" as comma is used in the ldap pattern +ldap.user.dn.patterns= + +# Redis configuration +#redis.host=localhost +#redis.port=6379 + +################################################ +### Additional demo users configuration for when authentication.source=demo +### This file is located in the resources folder of the user-authn module and built into its runtime jar +################################################ +spring.config.import=classpath:demo-users.yml + +################################################ +### Comma separated list of header names to extract from incoming HTTP requests and add to thread local. +### The default value is null and adds no headers. +### Setting the list to * means all headers are captured. +### Otherwise, list the header name in use. +################################################ +authn.header.name.list= + +################################################ +### CORS +################################################ +# Comma-separated list of origins. +# Example configuration below is for setting up local development environment where egeria-ui is hosted on one of the two urls. +# cors.allowed-origins=http://localhost,http://localhost:8081 +cors.allowed-origins=* + +################################################ +# landing page (/api/public/app/info) +################################################ +app.description=Have a question? || Get in touch via our Slack community https://slack.lfai.foundation/ @@What is Open Metadata? || Find out more on our website https://egeria-project.org/ @@Have more cool ideas? || Feel free to let us know your ideas so we can make it better. +app.title=Egeria Open Metadata | Find the right data with governance + +# ############################################################## +# Component visibility for Role based access ################### +# ############################################################## +# How it works? +# +# The roles are defined in external authentication source (provider) configured with `authentication.source`. +# For demo purposes, we are providing simple file based authentication provider. See demo-users.yml +# The matrix controlling what components are allowed on the UI views for specific role is defined in the 'role.visibleComponents' prefixed properties as follows" +# +# role.visibleComponents.{ROLE-1}={component-name-1} +# role.visibleComponents.{ROLE-2}={component-name-1},{component-name-2} +# +# This will configure the application to show the component named 'component-name-1' for all uses assigned to 'ROLE-1' +# In the same way, users that have assigned 'ROLE-2' can see more 'component-name-1' and 'component-name-2'. +# It is also possible to use wildcard '*' to enable full visibility of all components to users in the given role. +# +# Complete list of components names that can be used: +# +# about +# asset-catalog +# asset-details +# asset-details-print +# glossary +# repository-explorer +# type-explorer +# asset-lineage +# asset-lineage-print +# end-to-end +# ultimate-source +# ultimate-destination +# vertical-lineage +# +# Below is the default configuration for the two COCO_PHARMA roles we use for demo: + +role.visibleComponents.COCO_PHARMA_USER=about,asset-catalog,asset-details,asset-details-print,asset-lineage,asset-lineage-print,end-to-end,ultimate-source,ultimate-destination,vertical-lineage,glossary,repository-explorer +role.visibleComponents.COCO_PHARMA_ADMIN=* + +################################################ +### Which java packages should be scanned to locate the Spring resource definitions that define the REST APIs? +################################################ +scan.packages=org.odpi.openmetadata.* + +################################################ +### Logging +################################################ +logging.level.root=OFF +logging.level.org.springframework=ERROR +logging.level.org.springframework.boot.web.embedded.tomcat=INFO +logging.level.org.odpi.openmetadata.platformchassis.springboot=INFO +#tracing REST calls +#logging.level.org.odpi.openmetadata.commonservices.ffdc.RESTCallLogger=DEBUG + +################################################ +### Swagger Docs +################################################ +springdoc.version='@springdoc.version@' +springdoc.api-docs.enabled=true +springdoc.api-docs.path=/v3/api-docs +springdoc.swagger-ui.path=/swagger-ui.html +springdoc.swagger-ui.displayRequestDuration=true +springdoc.swagger-ui.tagsSorter=alpha +springdoc.swagger-ui.operationsSorter=alpha +springdoc.swagger-ui.docExpansion=none + +################################################ +### Spring Boot Actuator +################################################ +# Endpoints web configuration +#management.endpoints.web.exposure.include=* +management.health.cassandra.enabled=false +management.health.redis.enabled=false +management.health.ldap.enabled=false \ No newline at end of file diff --git a/coco-lab-docker-compose/dev_application.properties b/coco-lab-docker-compose/dev_application.properties new file mode 100755 index 00000000..b8450707 --- /dev/null +++ b/coco-lab-docker-compose/dev_application.properties @@ -0,0 +1,226 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright Contributors to the ODPi Egeria project. + +# ======================================================================================================== +# The application.properties file is used to configure the OMAG Server Platform which is implemented as +# a spring boot application. + +############################################### +### Default port for the OMAG Server Platform +################################################ +server.port=7445 + +############################################### +### Set up the configuration document store the OMAG Server Platform +### The values below will set up the clear text config document store +### (the default is the encrypted config document store). +################################################ +platform.configstore.provider=org.odpi.openmetadata.adapters.adminservices.configurationstore.file.FileBasedServerConfigStoreProvider +platform.configstore.endpoint=data/servers/{0}/config/{0}.config + +############################################### +### Set up the platform metadata security connector that provides authorization +### for platform administration, server operations and diagnostic calls. +### By default, there is no platform metadata security connector. +### The values below are for a sample platform metadata security connector where the only userId +### that is permitted to use the administration and platform services is `garygeeke`. +############################################### +platform.security.provider=org.odpi.openmetadata.metadatasecurity.samples.CocoPharmaPlatformSecurityProvider +platform.security.name=Coco Pharmaceuticals Platform + +############################################### +### Set up the default configuration document for any new OMAG Server configurations. +############################################### +#platform.default.config.document=\ +# {\ +# "class": "OMAGServerConfig",\ +# "organizationName": "myOrg",\ +# "maxPageSize": 1200,\ +# "eventBusConfig": \ +# {\ +# "class": "EventBusConfig",\ +# "topicURLRoot": "egeria.omag",\ +# "configurationProperties":\ +# {\ +# "producer": {"bootstrap.servers": "{{kafkaEndpoint}}"},\ +# "consumer": {"bootstrap.servers": "{{kafkaEndpoint}}"}\ +# }\ +# }\ +# } + +################################################ +### Placeholder variables are added to field values in the configuration document +### when the server is being configured using double curly braces. They are replaced by +### the values specified in platform.placeholder.variables each time the server starts up. +### +### The "kafkaEndpoint" value is the Apache Kafka endpoint, and it is used in the active-metadata-store +### sample configuration. If your Apache Kafka broker is listening on a different endpoint +### and you want to use active-metadata-store, change this variable to your Apache Kafka's address. +################################################ +platform.placeholder.variables=\ + {\ + "kafkaEndpoint" : "host.docker.internal:7192",\ + "egeriaEndpoint" : "https://localhost:7445"\ + } + +################################################ +### startup servers configuration +################################################ +#userId used to start up the list of configured servers default is 'system' +startup.user=garygeeke +# Comma separated names of servers to be started. The server names should be unquoted. +startup.server.lost=cocoMDSx + +################################################ +### SSL security. +# The keystore determines the information sent out by the server to identify itself. +# The truststore is where the certificates of trusted servers the platform is calling are located. +# (Note SSL certificate checking is performed on client-side only.) +################################################ +server.ssl.key-store=keystore.p12 +server.ssl.key-store-password=egeria +server.ssl.keyStoreType=PKCS12 +server.ssl.keyAlias=egeriaserverchassis + +server.ssl.trust-store=truststore.p12 +server.ssl.trust-store-password=egeria + +# WARNING! setting 'strict.ssl=false' allows java clients to open https connections without checking the validity of +# certificates from the servers it is calling. +# Alternate you can import self-signed certificates into java truststore or set up a truststore only for this app +# by adding the store into server.ssl.trust-store parameter +strict.ssl=true + + +################################################ +# User security +################################################ + +# Authentication source (possible values: demo, ldap, ad) +authentication.source=demo +# Authentication mode (possible values: session,token,redis) +authentication.mode=token + +#token timeout in minutes +token.timeout=15 +token.absolute.timeout=720 +token.secret=doNotTell + +#LDAP authentication + +ldap.domain= +ldap.user.search.base=ou=people,dc=egeria,dc=com +ldap.user.search.filter=uid={0} +ldap.group.search.base=ou=Groups,dc=egeria,dc=com +ldap.group.search.filter=member={0} +ldap.url=ldap://localhost:389 +ldap.group.role.attribute= +ldap.npa.dn= +ldap.npa.password= +#ldap.user.dn.patterns patterns is a list of values separated by ";" as comma is used in the ldap pattern +ldap.user.dn.patterns= + +# Redis configuration +#redis.host=localhost +#redis.port=6379 + +################################################ +### Additional demo users configuration for when authentication.source=demo +### This file is located in the resources folder of the user-authn module and built into its runtime jar +################################################ +spring.config.import=classpath:demo-users.yml + +################################################ +### Comma separated list of header names to extract from incoming HTTP requests and add to thread local. +### The default value is null and adds no headers. +### Setting the list to * means all headers are captured. +### Otherwise, list the header name in use. +################################################ +authn.header.name.list= + +################################################ +### CORS +################################################ +# Comma-separated list of origins. +# Example configuration below is for setting up local development environment where egeria-ui is hosted on one of the two urls. +# cors.allowed-origins=http://localhost,http://localhost:8081 +cors.allowed-origins=* + +################################################ +# landing page (/api/public/app/info) +################################################ +app.description=Have a question? || Get in touch via our Slack community https://slack.lfai.foundation/ @@What is Open Metadata? || Find out more on our website https://egeria-project.org/ @@Have more cool ideas? || Feel free to let us know your ideas so we can make it better. +app.title=Egeria Open Metadata | Find the right data with governance + +# ############################################################## +# Component visibility for Role based access ################### +# ############################################################## +# How it works? +# +# The roles are defined in external authentication source (provider) configured with `authentication.source`. +# For demo purposes, we are providing simple file based authentication provider. See demo-users.yml +# The matrix controlling what components are allowed on the UI views for specific role is defined in the 'role.visibleComponents' prefixed properties as follows" +# +# role.visibleComponents.{ROLE-1}={component-name-1} +# role.visibleComponents.{ROLE-2}={component-name-1},{component-name-2} +# +# This will configure the application to show the component named 'component-name-1' for all uses assigned to 'ROLE-1' +# In the same way, users that have assigned 'ROLE-2' can see more 'component-name-1' and 'component-name-2'. +# It is also possible to use wildcard '*' to enable full visibility of all components to users in the given role. +# +# Complete list of components names that can be used: +# +# about +# asset-catalog +# asset-details +# asset-details-print +# glossary +# repository-explorer +# type-explorer +# asset-lineage +# asset-lineage-print +# end-to-end +# ultimate-source +# ultimate-destination +# vertical-lineage +# +# Below is the default configuration for the two COCO_PHARMA roles we use for demo: + +role.visibleComponents.COCO_PHARMA_USER=about,asset-catalog,asset-details,asset-details-print,asset-lineage,asset-lineage-print,end-to-end,ultimate-source,ultimate-destination,vertical-lineage,glossary,repository-explorer +role.visibleComponents.COCO_PHARMA_ADMIN=* + +################################################ +### Which java packages should be scanned to locate the Spring resource definitions that define the REST APIs? +################################################ +scan.packages=org.odpi.openmetadata.* + +################################################ +### Logging +################################################ +logging.level.root=OFF +logging.level.org.springframework=ERROR +logging.level.org.springframework.boot.web.embedded.tomcat=INFO +logging.level.org.odpi.openmetadata.platformchassis.springboot=INFO +#tracing REST calls +#logging.level.org.odpi.openmetadata.commonservices.ffdc.RESTCallLogger=DEBUG + +################################################ +### Swagger Docs +################################################ +springdoc.version='@springdoc.version@' +springdoc.api-docs.enabled=true +springdoc.api-docs.path=/v3/api-docs +springdoc.swagger-ui.path=/swagger-ui.html +springdoc.swagger-ui.displayRequestDuration=true +springdoc.swagger-ui.tagsSorter=alpha +springdoc.swagger-ui.operationsSorter=alpha +springdoc.swagger-ui.docExpansion=none + +################################################ +### Spring Boot Actuator +################################################ +# Endpoints web configuration +#management.endpoints.web.exposure.include=* +management.health.cassandra.enabled=false +management.health.redis.enabled=false +management.health.ldap.enabled=false \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 1336ac71..86dfca4f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,4 +3,7 @@ # Add any additional python modules here. Install uses 'conda' pandas -numpy>=1.22.0 # not directly required, pinned by Snyk to avoid a vulnerability \ No newline at end of file +numpy>=1.22.0 # not directly required, pinned by Snyk to avoid a vulnerability +pyegeria +rich +asyncio \ No newline at end of file