Add scheduler misfire gracetime (#199)

* Add scheduler misfire gracetime Signed-off-by: M. David Bennett <[email protected]> * Retry should stop when successful =/. Also added random sleeps. Signed-off-by: M. David Bennett <[email protected]> * OMG pylint, really? Signed-off-by: M. David Bennett <[email protected]> * Embiggen the random num Signed-off-by: M. David Bennett <[email protected]>
objectrocket · Sep 16, 2016 · 6f661e4 · 6f661e4
1 parent 6e2c64b
commit 6f661e4
Show file tree

Hide file tree

Showing 3 changed files with 18 additions and 6 deletions.
diff --git a/contrib/opsy.ini b/contrib/opsy.ini
@@ -10,6 +10,7 @@ cache = sqlite:////tmp/opsy-cache.db
 [opsy]
 enabled_plugins = monitoring
 log_file = /tmp/opsy.log
+scheduler_grace_time = 10
 
 [monitoring]
 uchiwa_url = http://url.to.uchiwa.com/

diff --git a/opsy/app.py b/opsy/app.py
@@ -57,7 +57,12 @@ def create_logging(app):
 
 
 def create_scheduler(app, scheduler_class=BlockingScheduler):
-    scheduler = scheduler_class()
+    opsy_config = app.config.get('opsy')
+    grace_time = int(opsy_config.get('scheduler_grace_time', 10))
+    job_defaults = {
+        'misfire_grace_time': grace_time
+    }
+    scheduler = scheduler_class(job_defaults=job_defaults)
     for plugin in load_plugins(app):
         plugin.register_scheduler_jobs(app)
     for args, kwargs in app.jobs:

diff --git a/opsy/plugins/monitoring/plugin.py b/opsy/plugins/monitoring/plugin.py
@@ -1,5 +1,7 @@
 import asyncio
-from datetime import datetime
+import random
+import time
+from datetime import datetime, timedelta
 from stevedore import driver
 from flask import current_app, render_template
 from sqlalchemy.exc import OperationalError
@@ -82,15 +84,17 @@ def update_cache(zone, config_file):
                                 del_object.delete()
                             db.session.bulk_save_objects(init_objects)
                             db.session.commit()
+                            current_app.logger.info('Cache updated for %s' % zone.name)
+                            break
                         except OperationalError as e:  # pylint: disable=invalid-name
                             if i == (3 - 1):
                                 raise
                             current_app.logger.info(
                                 'Retryable error in transaction on '
-                                'attempt %d for zone %s. %s: %s',
-                                i + 1, zone.name, e.__class__.__name__, e)
+                                'attempt %d for zone %s: %s',
+                                i + 1, zone.name, e.__class__.__name__)
                             db.session.rollback()  # pylint: disable=no-member
-                current_app.logger.info('Cache updated for %s' % zone.name)
+                            time.sleep(random.uniform(.5, 1.5))
         # Create the db object for the zones.
         with app.app_context():
             for zone in Zone.query.all():
@@ -101,14 +105,16 @@ def update_cache(zone, config_file):
                 db.session.bulk_save_objects(zone.create_poller_metadata(app))
             db.session.commit()
         for zone in self.zones:
+            next_run = datetime.now() + timedelta(
+                0, random.uniform(0, zone.interval))
             if run_once:
                 app.jobs.append([[update_cache],
                                  {'next_run_time': datetime.now(),
                                   'max_instances': 1,
                                   'args': [zone, app.config_file]}])
             else:
                 app.jobs.append([[update_cache, 'interval'],
-                                 {'next_run_time': datetime.now(),
+                                 {'next_run_time': next_run,
                                   'max_instances': 1,
                                   'seconds': zone.interval,
                                   'args': [zone, app.config_file]}])