Merge pull request #79 from LamaAni/add_docker_example

Add docker image example

examples/docker/Dockerfile

@@ -0,0 +1,15 @@
+FROM apache/airflow
+
+RUN pip3 install --user airflow-kubernetes-job-operator
+
+COPY --chown=airflow webserver_config.py .
+COPY --chown=airflow start_airflow .
+
+RUN chmod +x ./start_airflow
+
+# Changed the port so not to collide with local
+ENV AIRFLOW__WEBSERVER__WEB_SERVER_PORT=8888
+ENV AIRFLOW__CORE__LOAD_EXAMPLES=False
+
+ENTRYPOINT []
+CMD [ "./start_airflow" ]

examples/docker/README.md

@@ -0,0 +1,6 @@
+# Docker image example
+
+This docker image allows for the execution of the dags collection in test. It uses the local
+machines .kube/config to connect and runs on host network.
+
+The airflow is served to http://localhost:8888

examples/docker/run_in_docker

@@ -0,0 +1,16 @@
+#!/bin/bash
+type realpath &>/dev/null
+if [ $? -ne 0 ]; then
+    function realpath() {
+        [[ $1 = /* ]] && echo "$1" || echo "$PWD/${1#./}"
+    }
+fi
+
+: "${RUN_PATH:="$(realpath "$(dirname "${BASH_SOURCE[0]}")")"}"
+: "${IMAGE_TAG:="airflow_kubernetes_job_operator_example_image"}"
+: "${DAGS_PATH:="$(realpath "$RUN_PATH/../../tests/dags")"}"
+
+docker build -t "$IMAGE_TAG" "$RUN_PATH" &&
+    docker run -it --rm --network host \
+        -v "$HOME/.kube/config:/home/airflow/.kube/config" \
+        -v "$DAGS_PATH:/opt/airflow/dags" "$IMAGE_TAG"

examples/docker/start_airflow

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+if [ "$#" -eq 0 ]; then
+    echo "----------------------"
+    echo "Running python version:"
+    python --version
+    echo "----------------------"
+    echo "Running airflow version:"
+    airflow version
+    echo "----------------------"
+    airflow db init || exit $?
+
+    airflow scheduler &
+    SCHEDULER_PID="$!"
+    airflow webserver
+    kill "$SCHEDULER_PID"
+else
+    airflow "$@"
+fi

examples/docker/webserver_config.py

@@ -0,0 +1,129 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Default configuration for the Airflow webserver."""
+from __future__ import annotations
+
+import os
+
+from airflow.www.fab_security.manager import AUTH_DB
+
+# from airflow.www.fab_security.manager import AUTH_LDAP
+# from airflow.www.fab_security.manager import AUTH_OAUTH
+# from airflow.www.fab_security.manager import AUTH_OID
+# from airflow.www.fab_security.manager import AUTH_REMOTE_USER
+
+basedir = os.path.abspath(os.path.dirname(__file__))
+
+# Flask-WTF flag for CSRF
+WTF_CSRF_ENABLED = True
+
+# ----------------------------------------------------
+# AUTHENTICATION CONFIG
+# ----------------------------------------------------
+# For details on how to set up each of the following authentication, see
+# http://flask-appbuilder.readthedocs.io/en/latest/security.html# authentication-methods
+# for details.
+
+# The authentication type
+# AUTH_OID : Is for OpenID
+# AUTH_DB : Is for database
+# AUTH_LDAP : Is for LDAP
+# AUTH_REMOTE_USER : Is for using REMOTE_USER from web server
+# AUTH_OAUTH : Is for OAuth
+AUTH_TYPE = AUTH_DB
+
+# Uncomment to setup Full admin role name
+AUTH_ROLE_ADMIN = "Admin"
+
+# Uncomment and set to desired role to enable access without authentication
+AUTH_ROLE_PUBLIC = "Admin"
+
+# Will allow user self registration
+AUTH_USER_REGISTRATION = False
+
+# The recaptcha it's automatically enabled for user self registration is active and the keys are necessary
+# RECAPTCHA_PRIVATE_KEY = PRIVATE_KEY
+# RECAPTCHA_PUBLIC_KEY = PUBLIC_KEY
+
+# Config for Flask-Mail necessary for user self registration
+# MAIL_SERVER = 'smtp.gmail.com'
+# MAIL_USE_TLS = True
+# MAIL_USERNAME = '[email protected]'
+# MAIL_PASSWORD = 'passwordformail'
+# MAIL_DEFAULT_SENDER = '[email protected]'
+
+# The default user self registration role
+# AUTH_USER_REGISTRATION_ROLE = "Public"
+
+# When using OAuth Auth, uncomment to setup provider(s) info
+# Google OAuth example:
+# OAUTH_PROVIDERS = [{
+#   'name':'google',
+#     'token_key':'access_token',
+#     'icon':'fa-google',
+#         'remote_app': {
+#             'api_base_url':'https://www.googleapis.com/oauth2/v2/',
+#             'client_kwargs':{
+#                 'scope': 'email profile'
+#             },
+#             'access_token_url':'https://accounts.google.com/o/oauth2/token',
+#             'authorize_url':'https://accounts.google.com/o/oauth2/auth',
+#             'request_token_url': None,
+#             'client_id': GOOGLE_KEY,
+#             'client_secret': GOOGLE_SECRET_KEY,
+#         }
+# }]
+
+# When using LDAP Auth, setup the ldap server
+# AUTH_LDAP_SERVER = "ldap://ldapserver.new"
+
+# When using OpenID Auth, uncomment to setup OpenID providers.
+# example for OpenID authentication
+# OPENID_PROVIDERS = [
+#    { 'name': 'Yahoo', 'url': 'https://me.yahoo.com' },
+#    { 'name': 'AOL', 'url': 'http://openid.aol.com/<username>' },
+#    { 'name': 'Flickr', 'url': 'http://www.flickr.com/<username>' },
+#    { 'name': 'MyOpenID', 'url': 'https://www.myopenid.com' }]
+
+# ----------------------------------------------------
+# Theme CONFIG
+# ----------------------------------------------------
+# Flask App Builder comes up with a number of predefined themes
+# that you can use for Apache Airflow.
+# http://flask-appbuilder.readthedocs.io/en/latest/customizing.html#changing-themes
+# Please make sure to remove "navbar_color" configuration from airflow.cfg
+# in order to fully utilize the theme. (or use that property in conjunction with theme)
+# APP_THEME = "bootstrap-theme.css"  # default bootstrap
+# APP_THEME = "amelia.css"
+# APP_THEME = "cerulean.css"
+# APP_THEME = "cosmo.css"
+# APP_THEME = "cyborg.css"
+# APP_THEME = "darkly.css"
+# APP_THEME = "flatly.css"
+# APP_THEME = "journal.css"
+# APP_THEME = "lumen.css"
+# APP_THEME = "paper.css"
+# APP_THEME = "readable.css"
+# APP_THEME = "sandstone.css"
+# APP_THEME = "simplex.css"
+# APP_THEME = "slate.css"
+# APP_THEME = "solar.css"
+# APP_THEME = "spacelab.css"
+# APP_THEME = "superhero.css"
+# APP_THEME = "united.css"
+# APP_THEME = "yeti.css"