Fix/optimization (#98)

Fix/optimization Added: Caching (RedisCache or SimpleCache) The follow routes are now cached: "/" "/index" "/incidents/<incident_id>" "/history" "/availability" Debugging expanded: If the RedisCache is used, then the connection to the redis server is checked before creating an app instance, if the check fails, then the app will use SimpleDimple Benchmark test: $ flask --debug run --host=0.0.0.0 $ ab -n 3000 -c 100 http://127.0.0.0:5000/ Benchmarking 127.0.0.0 (be patient) Completed 300 requests Completed 600 requests Completed 900 requests Completed 1200 requests Completed 1500 requests Completed 1800 requests Completed 2100 requests Completed 2400 requests Completed 2700 requests Completed 3000 requests Finished 3000 requests Server Software: Werkzeug/3.0.1 Server Hostname: 127.0.0.0 Server Port: 5000 Document Path: / Document Length: 101946 bytes Concurrency Level: 100 Time taken for tests: 4.010 seconds Complete requests: 3000 Failed requests: 0 Total transferred: 306369000 bytes HTML transferred: 305838000 bytes Requests per second: 748.07 [#/sec] (mean) Time per request: 133.678 [ms] (mean) Time per request: 1.337 [ms] (mean, across all concurrent requests) Transfer rate: 74604.17 [Kbytes/sec] received Connection Times (ms) min mean[+/-sd] median max Connect: 0 0 0.5 0 4 Processing: 24 127 12.5 126 158 Waiting: 9 115 12.5 114 146 Total: 28 127 12.1 126 158 Percentage of the requests served within a certain time (ms) 50% 126 66% 132 75% 135 80% 136 90% 139 95% 141 98% 143 99% 144 100% 158 (longest request) Reviewed-by: Olha Kashyrina Reviewed-by: Vladimir Vshivkov Reviewed-by: Vladimir Hasko <[email protected]> Reviewed-by: Ilia Bakhterev Reviewed-by: Anton Sidelnikov
stackmon · Feb 7, 2024 · fa3bf67 · fa3bf67
1 parent 97eb869
commit fa3bf67
Show file tree

Hide file tree

Showing 8 changed files with 173 additions and 13 deletions.
diff --git a/README.md b/README.md
@@ -11,6 +11,23 @@ flask --debug run
 
 flask db stamp head # to start the upgrade db models
 ```
+### Using redis for caching and storing sessions
+It needs to run with redis;  
+Environment variables can be configured using the "SDB" prefix.
+```
+"SDB_CACHE_TYPE"
+"SDB_CACHE_KEY_PREFIX"
+"SDB_CACHE_REDIS_HOST"
+"SDB_CACHE_REDIS_PORT"
+"SDB_CACHE_REDIS_URL"
+"SDB_CACHE_REDIS_PASSWORD"
+"SDB_CACHE_DEFAULT_TIMEOUT"
+# session variables:
+"SDB_SESSION_TYPE"
+"SDB_REDIS_PASS"
+"SDB_REDIS_PORT"
+"SDB_REDIS_HOST"
+```
 
 ## Bootstraping
 

diff --git a/app/__init__.py b/app/__init__.py
@@ -23,17 +23,65 @@
 
 from flask_migrate import Migrate
 
+from flask_session import Session
+
 from flask_smorest import Api
 
 from flask_sqlalchemy import SQLAlchemy
 
+import redis
+
 import yaml
 
 
 db = SQLAlchemy()
 migrate = Migrate()
 oauth = OAuth()
-cache = Cache(config={"CACHE_TYPE": "SimpleCache"})
+session = Session()
+
+# Cache settings
+cache_config_options = [
+    "CACHE_TYPE",
+    "CACHE_KEY_PREFIX",
+    "CACHE_REDIS_HOST",
+    "CACHE_REDIS_PORT",
+    "CACHE_REDIS_URL",
+    "CACHE_REDIS_PASSWORD",
+    "CACHE_DEFAULT_TIMEOUT",
+]
+cache_config = {
+    option: os.getenv(
+        f"SDB_{option}", DefaultConfiguration.__dict__.get(option))
+    for option in cache_config_options
+}
+
+
+def check_redis_connection(cache_config):
+    if (
+        "CACHE_TYPE" in cache_config
+        and cache_config["CACHE_TYPE"] == "RedisCache"
+    ):
+        try:
+            r = redis.StrictRedis(
+                host=cache_config.get("CACHE_REDIS_HOST", "localhost"),
+                port=cache_config.get("CACHE_REDIS_PORT", 6379),
+                password=cache_config.get("CACHE_REDIS_PASSWORD"),
+            )
+            r.ping()
+            return True
+        except redis.ConnectionError:
+            return False
+    return False
+
+
+if (
+    "CACHE_TYPE" in cache_config.keys()
+    and cache_config["CACHE_TYPE"] == "RedisCache"
+    and check_redis_connection(cache_config)
+):
+    cache = Cache(config=cache_config)
+else:
+    cache = Cache(config={"CACHE_TYPE": "SimpleCache"})
 
 
 def create_app(test_config=None):
@@ -44,6 +92,18 @@ def create_app(test_config=None):
     api = Api(app)  # noqa
 
     app.config.from_prefixed_env(prefix="SDB")
+    if (
+        "SESSION_TYPE" in app.config
+        and app.config["SESSION_TYPE"] == "redis"
+    ):
+        app.config["SESSION_REDIS"] = redis.StrictRedis(
+            host=os.getenv("SDB_REDIS_HOST",
+                           DefaultConfiguration.__dict__.get("REDIS_HOST")),
+            port=os.getenv("SDB_REDIS_PORT",
+                           DefaultConfiguration.__dict__.get("REDIS_PORT")),
+            password=os.getenv("SDB_REDIS_PASS",
+                               DefaultConfiguration.__dict__.get("REDIS_PASS"))
+        )
 
     if test_config is None:
         # load the instance config, if it exists, when not testing
@@ -73,9 +133,12 @@ def create_app(test_config=None):
         pass
 
     cache.init_app(app)
+    app.logger.debug(f"CACHE_TYPE: {cache.config['CACHE_TYPE']}")
     db.init_app(app)
     migrate.init_app(app, db)
     oauth.init_app(app, cache=cache)
+    if "SESSION_TYPE" in app.config:
+        session.init_app(app)
 
     if (
         "GITHUB_CLIENT_ID" in app.config
@@ -94,6 +157,30 @@ def create_app(test_config=None):
             client_kwargs={"scope": "user"},
             userinfo_endpoint="https://api.github.com/user",
         )
+
+    # testing caching and redis connection
+    if app.debug:
+        cache_dict = app.extensions["cache"]
+        cache_obj = list(cache_dict.values())[0]
+        cache_obj.set("test_key", "test_value")
+        value = cache_obj.get("test_key")
+        if value == "test_value":
+            app.logger.debug(f"Caching works, test_key has value: '{value}'")
+            cache_obj.delete("test_key")
+        # checking connection to redis
+        if (
+            ("SESSION_TYPE" in app.config
+             and app.config["SESSION_TYPE"] == "redis")
+            or ("CACHE_TYPE" in cache_config
+                and cache_config["CACHE_TYPE"] == "RedisCache")
+        ):
+            if check_redis_connection(cache_config):
+                app.logger.debug("Connection to redis was successful")
+                cache.init_app(app)
+            else:
+                app.logger.error("Error connecting to redis")
+    # end of testing caching and redis connection
+
     if (
         "OPENID_ISSUER_URL" in app.config
         and "OPENID_CLIENT_ID" in app.config
@@ -125,5 +212,4 @@ def create_app(test_config=None):
     with app.app_context():
         # Ensure there is some DB when we start the app
         db.create_all()
-
     return app
diff --git a/app/api/routes.py b/app/api/routes.py
@@ -13,6 +13,7 @@
 from datetime import datetime
 
 from app import authorization
+from app import cache
 from app.api import bp
 from app.api.schemas.components import ComponentSchema
 from app.api.schemas.components import ComponentSearchQueryArgs
@@ -156,10 +157,18 @@ def handling_incidents(
         current_app.logger.error("Unexpected ERROR")
 
 
+def get_component_from_cache(cache_key):
+    cached_value = cache.get(cache_key)
+    if cached_value:
+        current_app.logger.debug(f"Cache hit for key: '{cache_key}'")
+        return cached_value
+    return None
+
+
 @bp.route("/v1/component_status", methods=["GET", "POST"])
 class ApiComponentStatus(MethodView):
     @bp.arguments(ComponentSearchQueryArgs, location="query")
-    @bp.response(200, ComponentSchema(many=True))
+    @bp.response(200)
     def get(self, search_args):
         """Get components
 
@@ -179,17 +188,37 @@ def get(self, search_args):
         name = search_args.get("name", "")
         attribute_name = search_args.get("attribute_name", None)
         attribute_value = search_args.get("attribute_value", None)
-        attribute = {attribute_name: attribute_value}
+        if attribute_name and attribute_value:
+            attribute = {attribute_name: attribute_value}
+        else:
+            attribute = None
+        component_schema = ComponentSchema()
+        cache_key = (f"component_status:{name if name else 'all'}"
+                     f"{attribute if attribute else ''}"
+        )
+
+        cached_component = get_component_from_cache(cache_key)
+        if cached_component:
+            return [cached_component]
+
         if attribute_name is not None and attribute_value is not None:
             target_component = Component.find_by_name_and_attributes(
                 name, attribute
             )
             if target_component is None:
                 abort(404, message="Component does not exist")
-            return [target_component]
-        return db.session.scalars(
+            serialized_component = component_schema.dump(target_component)
+            cache.set(cache_key, serialized_component)
+            return [serialized_component]
+
+        components = db.session.scalars(
             db.select(Component).filter(Component.name.startswith(name))
         ).all()
+        if components is None:
+            abort(404, message="Component(s) does not (do not) exist")
+        serialized_components = component_schema.dump(components, many=True)
+        cache.set(cache_key, serialized_components)
+        return [serialized_components]
 
     @bp.arguments(ComponentStatusArgsSchema)
     @auth.login_required

diff --git a/app/default_settings.py b/app/default_settings.py
@@ -29,6 +29,14 @@ class DefaultConfiguration:
     SECRET_KEY = "dev"
     SQLALCHEMY_ECHO = False
     JSON_SORT_KEYS = True
+    # session settings
+    SESSION_KEY_PREFIX = 'sdb_session:'
+    SESSION_PERMANENT = False
+    SESSION_USE_SIGNER = True
+    # cache settings
+    CACHE_KEY_PREFIX = "sdb_cache:"
+    CACHE_DEFAULT_TIMEOUT = 30
+    CACHE_TYPE = "SimpleCache"
 
     # Incident impacts map
     # key - integer to identify impact and compare "severity"

diff --git a/app/tests/unit/test_web.py b/app/tests/unit/test_web.py
@@ -24,7 +24,9 @@
 class TestBase(TestCase):
 
     test_config = dict(
-        TESTING=True, SQLALCHEMY_DATABASE_URI="sqlite:///:memory:"
+        TESTING=True,
+        SQLALCHEMY_DATABASE_URI="sqlite:///:memory:",
+        CACHE_TYPE="SimpleCache"
     )
 
     def setUp(self):
@@ -41,8 +43,7 @@ def tearDown(self):
 
 class TestWeb(TestBase):
     def setUp(self):
-        super().setUp()
-
+        super(TestWeb, self).setUp()
         self.client = self.app.test_client()
 
         with self.app.app_context():
@@ -67,14 +68,14 @@ def setUp(self):
             db.session.commit()
             self.incident_id = inc1.id
 
-    def test_get_root(self):
+    def test_01_get_root(self):
         res = self.client.get("/")
         self.assertEqual(200, res.status_code)
 
-    def test_get_incidents_no_auth(self):
+    def test_02_get_incidents_no_auth(self):
         res = self.client.get("/incidents")
         self.assertEqual(401, res.status_code)
 
-    def test_get_incident(self):
+    def test_03_get_incident(self):
         res = self.client.get(f"/incidents/{self.incident_id}")
         self.assertEqual(200, res.status_code)
diff --git a/app/web/routes.py b/app/web/routes.py
@@ -14,6 +14,7 @@
 from datetime import timezone
 
 from app import authorization
+from app import cache
 from app import oauth
 from app.models import Component
 from app.models import ComponentAttribute
@@ -31,12 +32,16 @@
 from flask import flash
 from flask import redirect
 from flask import render_template
+from flask import request
 from flask import session
 from flask import url_for
 
 
 @bp.route("/", methods=["GET"])
 @bp.route("/index", methods=["GET"])
+@cache.cached(unless=lambda: "user" in session,
+              key_prefix="/index"
+)
 def index():
     return render_template(
         "index.html",
@@ -151,9 +156,16 @@ def new_incident(current_user):
 
 
 @bp.route("/incidents/<incident_id>", methods=["GET", "POST"])
+@cache.cached(
+    unless=lambda: "user" in session,
+    key_prefix=lambda: f"{request.path}",
+)
 def incident(incident_id):
     """Manage incident by ID"""
     incident = Incident.get_by_id(incident_id)
+    if not incident:
+        abort(404)
+
     form = None
     if "user" in session:
         form = IncidentUpdateForm(id)
@@ -228,6 +240,7 @@ def separate_incident(current_user, incident_id, component_id):
 
 
 @bp.route("/history", methods=["GET"])
+@cache.cached(unless=lambda: "user" in session)
 def history():
     return render_template(
         "history.html",
@@ -237,6 +250,10 @@ def history():
 
 
 @bp.route("/availability", methods=["GET"])
+@cache.cached(
+    unless=lambda: "user" in session,
+    timeout=300,
+)
 def sla():
     time_now = datetime.now()
     months = [time_now + relativedelta(months=-mon) for mon in range(6)]

diff --git a/requirements.txt b/requirements.txt
@@ -3,6 +3,7 @@ Flask-Caching>=2.1.0 # BSD
 Flask-SQLAlchemy>=3.1.1 # BSD-3-Clause
 Flask-Migrate>4.0 # MIT
 Flask-Wtf>=1.2.1 # BSD-3-Clause
+Flask-Session>=0.6.0 # BSD
 authlib>=1.2 # BSD-3-Clause
 # Requests is an unlisted dependency of authlib
 requests>=2.28 # Apache-2.0
@@ -12,4 +13,5 @@ marshmallow>=3.19.0 #MIT License
 PyJWT>=2.6.0 #MIT License
 flask_httpauth>= 4.8.0 #MIT License
 feedgen>=0.9.0 #BSD License
+redis>=5.0.1 #MIT License
 
diff --git a/zuul.yaml b/zuul.yaml
@@ -27,7 +27,7 @@
     check:
       jobs:
         - otc-tox-pep8
-        - otc-tox-py38
+        - otc-tox-py310
         - stackmon-status-dashboard-build-image
     check-post:
       jobs: