⚗️(project) add draft Apache Superset integration

Makefile

@@ -3,7 +3,7 @@
 DOCKER_UID           = $(shell id -u)
 DOCKER_GID           = $(shell id -g)
 DOCKER_USER          = $(DOCKER_UID):$(DOCKER_GID)
-COMPOSE              = DOCKER_USER=$(DOCKER_USER) docker-compose
+COMPOSE              = DOCKER_USER=$(DOCKER_USER) docker compose -f docker-compose.yml -f docker-compose-superset.yml
 COMPOSE_RUN          = $(COMPOSE) run --rm
 
 # -- Node
@@ -50,7 +50,8 @@ bootstrap: \
 	build \
 	compile \
 	fixtures \
-	hooks
+	hooks \
+	init-superset
 bootstrap: ## bootstrap the application
 .PHONY: bootstrap
 
@@ -86,7 +87,7 @@ fixtures: ## Load test data (for development)
 	@$(WAIT_MYSQL)
 	zcat ./fixtures/elasticsearch/lrs.json.gz | \
 	  $(COMPOSE_RUN) patch_statements_date | \
-	  $(COMPOSE_RUN) -T ralph push -b es --es-index statements-fixtures  && \
+	  $(COMPOSE_RUN) -T ralph push -b es && \
 	  $(COMPOSE_RUN) users-permissions sh /scripts/users-permissions.sh
 .PHONY: fixtures
 
@@ -98,6 +99,10 @@ hooks: ## run post-deployment hooks
 	@$(COMPOSE_RUN) hooks ./post-deploy
 .PHONY: hooks
 
+init-superset: ## initialize superset
+	@$(COMPOSE) run superset-init
+.PHONY: init-superset
+
 lint: ## lint Jsonnet sources and libraries
 	bin/jsonnet-lint $(sources) $(libraries)
 .PHONY: lint
@@ -106,6 +111,10 @@ logs: ## display grafana logs (follow mode)
 	@$(COMPOSE) logs -f grafana
 .PHONY: logs
 
+logs-superset: ## display superset logs (follow mode)
+	@$(COMPOSE) logs -f superset
+.PHONY: logs-superset
+
 plugins: ## download, build and install plugins
 	@$(YARN) build
 .PHONY: plugins
@@ -131,6 +140,12 @@ run: ## start the development server
 	@$(WAIT_GRAFANA)
 .PHONY: run
 
+run-superset: \
+	run
+run-superset: ## start superset server
+	@$(COMPOSE) up -d superset superset-worker superset-worker-beat
+.PHONY: run-superset
+
 status: ## an alias for "docker-compose ps"
 	@$(COMPOSE) ps
 .PHONY: status

bin/compose

@@ -5,5 +5,7 @@ declare DOCKER_GID="$(id -g)"
 
 DOCKER_UID=${DOCKER_UID} \
   DOCKER_GID=${DOCKER_GID} \
-  docker-compose \
+  docker compose \
+  -f docker-compose.yml \
+  -f docker-compose-superset.yml \
   "$@"

docker-compose-superset.yml

@@ -0,0 +1,90 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+x-superset-image: &superset-image apache/superset:${TAG:-latest-dev}
+x-superset-depends-on: &superset-depends-on
+  - db
+  - redis
+x-superset-volumes: &superset-volumes
+  # /app/pythonpath_docker will be appended to the PYTHONPATH in the final container
+  - ./superset:/app/docker
+  - superset_home:/app/superset_home
+
+version: "3.7"
+services:
+  redis:
+    image: redis:latest
+    container_name: superset_cache
+    restart: unless-stopped
+    volumes:
+      - redis:/data
+
+  db:
+    env_file: superset/.env-non-dev
+    image: postgres:10
+    container_name: superset_db
+    restart: unless-stopped
+    volumes:
+      - db_home:/var/lib/postgresql/data
+
+  superset:
+    env_file: superset/.env-non-dev
+    image: *superset-image
+    container_name: superset_app
+    command: ["/app/docker/docker-bootstrap.sh", "app-gunicorn"]
+    user: "root"
+    restart: unless-stopped
+    ports:
+      - 8088:8088
+    depends_on: *superset-depends-on
+    volumes: *superset-volumes
+
+  superset-init:
+    image: *superset-image
+    container_name: superset_init
+    command: ["/app/docker/docker-init.sh"]
+    env_file: superset/.env-non-dev
+    depends_on: *superset-depends-on
+    user: "root"
+    volumes: *superset-volumes
+
+  superset-worker:
+    image: *superset-image
+    container_name: superset_worker
+    command: ["/app/docker/docker-bootstrap.sh", "worker"]
+    env_file: superset/.env-non-dev
+    restart: unless-stopped
+    depends_on: *superset-depends-on
+    user: "root"
+    volumes: *superset-volumes
+
+  superset-worker-beat:
+    image: *superset-image
+    container_name: superset_worker_beat
+    command: ["/app/docker/docker-bootstrap.sh", "beat"]
+    env_file: superset/.env-non-dev
+    restart: unless-stopped
+    depends_on: *superset-depends-on
+    user: "root"
+    volumes: *superset-volumes
+
+volumes:
+  superset_home:
+    external: false
+  db_home:
+    external: false
+  redis:
+    external: false

docker-compose.yml

@@ -77,8 +77,8 @@ services:
     user: "${DOCKER_USER:-1000}"
     entrypoint: ["ralph"]
     environment:
-      - RALPH_ES_HOSTS=http://elasticsearch:9200
-      - RALPH_ES_INDEX=statements
+      - RALPH_BACKENDS__DATABASE__ES__HOSTS=http://elasticsearch:9200
+      - RALPH_BACKENDS__DATABASE__ES__INDEX=statements
 
   elasticsearch:
     image: elasticsearch:8.2.0

superset/.env

@@ -0,0 +1,46 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+COMPOSE_PROJECT_NAME=superset
+
+# database configurations (do not modify)
+DATABASE_DB=superset
+DATABASE_HOST=db
+DATABASE_PASSWORD=superset
+DATABASE_USER=superset
+
+# database engine specific environment variables
+# change the below if you prefers another database engine
+DATABASE_PORT=5432
+DATABASE_DIALECT=postgresql
+POSTGRES_DB=superset
+POSTGRES_USER=superset
+POSTGRES_PASSWORD=superset
+#MYSQL_DATABASE=superset
+#MYSQL_USER=superset
+#MYSQL_PASSWORD=superset
+#MYSQL_RANDOM_ROOT_PASSWORD=yes
+
+# Add the mapped in /app/pythonpath_docker which allows devs to override stuff
+PYTHONPATH=/app/pythonpath:/app/docker/pythonpath_dev
+REDIS_HOST=redis
+REDIS_PORT=6379
+
+FLASK_ENV=development
+SUPERSET_ENV=development
+SUPERSET_LOAD_EXAMPLES=yes
+CYPRESS_CONFIG=false
+SUPERSET_PORT=8088

superset/.env-non-dev

@@ -0,0 +1,46 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+COMPOSE_PROJECT_NAME=superset
+
+# database configurations (do not modify)
+DATABASE_DB=superset
+DATABASE_HOST=db
+DATABASE_PASSWORD=superset
+DATABASE_USER=superset
+
+# database engine specific environment variables
+# change the below if you prefers another database engine
+DATABASE_PORT=5432
+DATABASE_DIALECT=postgresql
+POSTGRES_DB=superset
+POSTGRES_USER=superset
+POSTGRES_PASSWORD=superset
+#MYSQL_DATABASE=superset
+#MYSQL_USER=superset
+#MYSQL_PASSWORD=superset
+#MYSQL_RANDOM_ROOT_PASSWORD=yes
+
+# Add the mapped in /app/pythonpath_docker which allows devs to override stuff
+PYTHONPATH=/app/pythonpath:/app/docker/pythonpath_dev
+REDIS_HOST=redis
+REDIS_PORT=6379
+
+FLASK_ENV=production
+SUPERSET_ENV=production
+SUPERSET_LOAD_EXAMPLES=yes
+CYPRESS_CONFIG=false
+SUPERSET_PORT=8088

superset/README.md

@@ -0,0 +1,75 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# Getting Started with Superset using Docker
+
+Docker is an easy way to get started with Superset.
+
+## Prerequisites
+
+1. Docker! [link](https://www.docker.com/get-started)
+2. Docker-compose [link](https://docs.docker.com/compose/install/)
+
+## Configuration
+
+The `/app/pythonpath` folder is mounted from [`./docker/pythonpath_dev`](./pythonpath_dev)
+which contains a base configuration [`./docker/pythonpath_dev/superset_config.py`](./pythonpath_dev/superset_config.py)
+intended for use with local development.
+
+### Local overrides
+
+In order to override configuration settings locally, simply make a copy of [`./docker/pythonpath_dev/superset_config_local.example`](./pythonpath_dev/superset_config_local.example)
+into `./docker/pythonpath_dev/superset_config_docker.py` (git ignored) and fill in your overrides.
+
+### Local packages
+
+If you want to add Python packages in order to test things like databases locally, you can simply add a local requirements.txt (`./docker/requirements-local.txt`)
+and rebuild your Docker stack.
+
+Steps:
+
+1. Create `./docker/requirements-local.txt`
+2. Add your new packages
+3. Rebuild docker-compose
+    1. `docker-compose down -v`
+    2. `docker-compose up`
+
+## Initializing Database
+
+The database will initialize itself upon startup via the init container ([`superset-init`](./docker-init.sh)). This may take a minute.
+
+## Normal Operation
+
+To run the container, simply run: `docker-compose up`
+
+After waiting several minutes for Superset initialization to finish, you can open a browser and view [`http://localhost:8088`](http://localhost:8088)
+to start your journey.
+
+## Developing
+
+While running, the container server will reload on modification of the Superset Python and JavaScript source code.
+Don't forget to reload the page to take the new frontend into account though.
+
+## Production
+
+It is possible to run Superset in non-development mode by using [`docker-compose-non-dev.yml`](../docker-compose-non-dev.yml). This file excludes the volumes needed for development and uses [`./docker/.env-non-dev`](./.env-non-dev) which sets the variable `SUPERSET_ENV` to `production`.
+
+## Resource Constraints
+
+If you are attempting to build on macOS and it exits with 137 you need to increase your Docker resources. See instructions [here](https://docs.docker.com/docker-for-mac/#advanced) (search for memory)

superset/docker-bootstrap.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -eo pipefail
+
+REQUIREMENTS_LOCAL="/app/docker/requirements-local.txt"
+# If Cypress run – overwrite the password for admin and export env variables
+if [ "$CYPRESS_CONFIG" == "true" ]; then
+    export SUPERSET_CONFIG=tests.integration_tests.superset_test_config
+    export SUPERSET_TESTENV=true
+    export ENABLE_REACT_CRUD_VIEWS=true
+    export SUPERSET__SQLALCHEMY_DATABASE_URI=postgresql+psycopg2://superset:superset@db:5432/superset
+fi
+#
+# Make sure we have dev requirements installed
+#
+if [ -f "${REQUIREMENTS_LOCAL}" ]; then
+  echo "Installing local overrides at ${REQUIREMENTS_LOCAL}"
+  pip install -r "${REQUIREMENTS_LOCAL}"
+else
+  echo "Skipping local overrides"
+fi
+
+if [[ "${1}" == "worker" ]]; then
+  echo "Starting Celery worker..."
+  celery --app=superset.tasks.celery_app:app worker -Ofair -l INFO
+elif [[ "${1}" == "beat" ]]; then
+  echo "Starting Celery beat..."
+  celery --app=superset.tasks.celery_app:app beat --pidfile /tmp/celerybeat.pid -l INFO -s "${SUPERSET_HOME}"/celerybeat-schedule
+elif [[ "${1}" == "app" ]]; then
+  echo "Starting web app..."
+  flask run -p 8088 --with-threads --reload --debugger --host=0.0.0.0
+elif [[ "${1}" == "app-gunicorn" ]]; then
+  echo "Starting web app..."
+  /usr/bin/run-server.sh
+fi