Merge pull request #23 from cms-DQM/fix_cronjob_options

Fix cronjob options
cms-DQM · Jun 12, 2024 · bf979da · bf979da
2 parents 8a0b6d6 + a4b12a8
commit bf979da
Show file tree

Hide file tree

Showing 5 changed files with 45 additions and 43 deletions.
diff --git a/runregistry_backend/app.js b/runregistry_backend/app.js
@@ -38,7 +38,7 @@ console.error = getLogger("ERROR", console.error)
 
 // Logging for sanity
 const { database, host, port: db_port } = config[process.env.ENV];
-console.error(`Using database: ${database}@${host}:${db_port}`);
+console.info(`Using database: ${database}@${host}:${db_port}`);
 
 models.sequelize.sync({})
   .then(async () => {

diff --git a/runregistry_backend/config/config.js b/runregistry_backend/config/config.js
@@ -1,4 +1,4 @@
-// Common configuration used by all deployment modes
+// Common configuration used and overriden by all deployment modes.
 commonVars = {
   // Database config
   username: process.env.DB_USERNAME || 'postgres',
@@ -20,19 +20,26 @@ commonVars = {
     idle: 20000,
     acquire: 2000000,
   },
+  // Ignore run numbers lower than this one
+  MINIMUM_CMS_RUN_NUMBER: 100000,
   // DQMGUI
   WAITING_DQM_GUI_CONSTANT: 'waiting dqm gui',
   DQM_GUI_URL: 'https://cmsweb.cern.ch/dqm/offline/data/json/samples?match=',
   DQM_GUI_PING_CRON_ENABLED: true,
+  DQM_GUI_CHECK_EVERY_NTH_MINUTE: 60, // This needs to be <=60
   // OMS
   OMS_URL: `https://cmsoms.cern.ch/agg/api/v1`,
   OMS_GET_RUNS_CRON_ENABLED: true, // Get runs from OMS periodically or not
   OMS_SPECIFIC_RUN: (run_number) => `runs?filter[run_number]=${run_number}`,
   OMS_LUMISECTIONS: (run_number) => `lumisections?filter[run_number]=${run_number}&page[limit]=5000`,
+  // The default value here does not play a role, really, it's overridden by OMS_RUNS_PER_API_CALL
+  OMS_RUNS_ENDPOINT: (number_of_runs = 15) =>
+    `runs?sort=-last_update&page[limit]=${number_of_runs}`,
   CLIENT_ID: 'rr-api-client',
   CLIENT_SECRET: process.env.CLIENT_SECRET,
   OMS_AUDIENCE: 'cmsoms-prod',
-  RUNS_PER_API_CALL: 49,
+  OMS_RUNS_PER_API_CALL: 49,
+  OMS_API_CALL_EVERY_NTH_MINUTE: 30, // This needs to be <=60
   // Redis
   // redis://:<pass>@<host>:<port>
   REDIS_URL: `redis://${process.env.REDIS_PASSWORD ? ':' + process.env.REDIS_PASSWORD + '@' : ''}${process.env.REDIS_HOST || '127.0.0.1'}:${process.env.REDIS_PORT || 6379}`,
@@ -50,10 +57,8 @@ module.exports = {
     ...commonVars,
     API_URL: process.env.DOCKER_POSTGRES ? 'http://dev:9500' :
       'http://localhost:9500',
-    OMS_RUNS: (number_of_runs = 10) =>
-      `runs?sort=-last_update&page[limit]=${number_of_runs}`,
-    SECONDS_PER_API_CALL: 30,
-    SECONDS_PER_DQM_GUI_CHECK: 6000,
+    OMS_API_CALL_EVERY_NTH_MINUTE: 30,
+    DQM_GUI_CHECK_EVERY_NTH_MINUTE: 60,
     JSON_PROCESSING_ENABLED: false,
     OMS_GET_RUNS_CRON_ENABLED: false,
     DQM_GUI_PING_CRON_ENABLED: false
@@ -63,10 +68,8 @@ module.exports = {
     ...commonVars,
     API_URL: process.env.DOCKER_POSTGRES ? 'http://dev:9500' :
       'http://localhost:9500',
-    OMS_RUNS: (number_of_runs = 10) =>
-      `runs?sort=-last_update&page[limit]=${number_of_runs}`,
-    SECONDS_PER_API_CALL: 30,
-    SECONDS_PER_DQM_GUI_CHECK: 6000,
+    OMS_API_CALL_EVERY_NTH_MINUTE: 30,
+    DQM_GUI_CHECK_EVERY_NTH_MINUTE: 60,
     OMS_GET_RUNS_CRON_ENABLED: false,
     JSON_PROCESSING_ENABLED: false,
     DQM_GUI_PING_CRON_ENABLED: false
@@ -75,28 +78,22 @@ module.exports = {
   staging: {
     ...commonVars,
     API_URL: 'http://localhost:9500',
-    OMS_RUNS: (number_of_runs = 10) =>
-      `runs?sort=-last_update&page[limit]=${number_of_runs}`,
-    SECONDS_PER_API_CALL: 3600,
-    SECONDS_PER_DQM_GUI_CHECK: 3600,
+    OMS_API_CALL_EVERY_NTH_MINUTE: 5,
+    DQM_GUI_CHECK_EVERY_NTH_MINUTE: 10,
   },
   // Old "bare-metal" production
   production: {
     ...commonVars,
     API_URL: 'http://localhost:9500',
-    OMS_RUNS: (number_of_runs = 15) =>
-      `runs?sort=-last_update&page[limit]=${number_of_runs}`,
-    SECONDS_PER_API_CALL: 180,
-    SECONDS_PER_DQM_GUI_CHECK: 3600,
+    OMS_API_CALL_EVERY_NTH_MINUTE: 3,
+    DQM_GUI_CHECK_EVERY_NTH_MINUTE: 10,
   },
   // Dev kubernetes flavor which means no cronjobs, no JSON processing
   dev_kubernetes: {
     ...commonVars,
     API_URL: 'http://runregistry-backend:9500',
-    OMS_RUNS: (number_of_runs = 49) =>
-      `runs?sort=-last_update&page[limit]=${number_of_runs}`,
-    SECONDS_PER_API_CALL: 30,
-    SECONDS_PER_DQM_GUI_CHECK: 600,
+    OMS_API_CALL_EVERY_NTH_MINUTE: 30,
+    DQM_GUI_CHECK_EVERY_NTH_MINUTE: 60,
     OMS_GET_RUNS_CRON_ENABLED: false,
     JSON_PROCESSING_ENABLED: false,
     DQM_GUI_PING_CRON_ENABLED: false
@@ -105,19 +102,17 @@ module.exports = {
   staging_kubernetes: {
     ...commonVars,
     API_URL: 'http://runregistry-backend:9500',
-    OMS_RUNS: (number_of_runs = 15) =>
-      `runs?sort=-last_update&page[limit]=${number_of_runs}`,
-    SECONDS_PER_API_CALL: 60,
-    SECONDS_PER_DQM_GUI_CHECK: 600,
+    OMS_API_CALL_EVERY_NTH_MINUTE: 10,
+    OMS_RUNS_PER_API_CALL: 25,
+    DQM_GUI_CHECK_EVERY_NTH_MINUTE: 15,
   },
   // Production config for kubernetes
   prod_kubernetes: {
     ...commonVars,
     API_URL: 'http://runregistry-backend:9500',
-    OMS_RUNS: (number_of_runs = 15) =>
-      `runs?sort=-last_update&page[limit]=${number_of_runs}`,
-    SECONDS_PER_API_CALL: 180,
-    SECONDS_PER_DQM_GUI_CHECK: 3600,
+    OMS_RUNS_PER_API_CALL: 49,
+    OMS_API_CALL_EVERY_NTH_MINUTE: 2,
+    DQM_GUI_CHECK_EVERY_NTH_MINUTE: 10,
   },
 
   // The online components are also the rr_lumisection_whitelist

diff --git a/runregistry_backend/cron/1.get_runs.js b/runregistry_backend/cron/1.get_runs.js
@@ -6,11 +6,12 @@ const https = require('https');
 const config = require('../config/config');
 const {
   OMS_URL,
-  OMS_RUNS,
+  OMS_RUNS_ENDPOINT,
   OMS_GET_RUNS_CRON_ENABLED,
   API_URL,
-  RUNS_PER_API_CALL,
-  SECONDS_PER_API_CALL,
+  OMS_RUNS_PER_API_CALL,
+  OMS_API_CALL_EVERY_NTH_MINUTE,
+  MINIMUM_CMS_RUN_NUMBER
 } = config[process.env.ENV || 'development'];
 const { save_runs, update_runs } = require('./2.save_or_update_runs');
 
@@ -23,12 +24,12 @@ const instance = axios.create({
 
 // Will call itself recursively if all runs are new
 const fetch_runs = async (
-  fetch_amount = RUNS_PER_API_CALL,
+  fetch_amount = OMS_RUNS_PER_API_CALL,
   first_time = true
 ) => {
-  const oms_url = `${OMS_URL}/${OMS_RUNS(fetch_amount)}`;
-  // insert cookie that will authenticate OMS request:
+  const oms_url = `${OMS_URL}/${OMS_RUNS_ENDPOINT(fetch_amount)}`;
 
+  // insert cookie that will authenticate OMS request:
   if (first_time) {
     headers = {
       Authorization: `Bearer ${await getToken()}`,
@@ -40,7 +41,7 @@ const fetch_runs = async (
       setTimeout(resolve, 2000);
     });
   }
-
+  console.debug(`Fetching the ${fetch_amount} last updated OMS runs`)
   const oms_response = await instance.get(oms_url, {
     headers,
   });
@@ -57,7 +58,7 @@ const fetch_runs = async (
   let fetched_runs = first_time
     ? all_fetched_runs
     : all_fetched_runs.slice(fetch_amount / 2);
-
+  console.debug("Querying the last 50 updated RR runs")
   const { data: last_saved_runs } = await axios.get(
     `${API_URL}/runs_lastupdated_50`
   );
@@ -95,7 +96,7 @@ const fetch_runs = async (
 
 if (OMS_GET_RUNS_CRON_ENABLED === true) {
   const job = new CronJob(
-    `*/${SECONDS_PER_API_CALL} * * * * *`,
+    `*/${OMS_API_CALL_EVERY_NTH_MINUTE} * * * *`,
     handleErrors(fetch_runs, 'cron/1.get_runs.js # Error fetching new runs ')
   ).start();
 }
@@ -140,7 +141,12 @@ const calculate_runs_to_update = (fetched_runs, last_saved_runs) => {
   fetched_runs.forEach((fetched_run) => {
     // If the run_number is less than the minimum of the already saved runs, then it is one from the past, which needs to be updated. Else we compare timestamps
     if (fetched_run.run_number < min_run_number) {
-      runs_to_update.push(fetched_run);
+      if (fetched_run.run_number > MINIMUM_CMS_RUN_NUMBER) {
+        runs_to_update.push(fetched_run);
+
+      } else {
+        console.log(`Run number ${fetched_run.run_number} is lower than the threshold MINIMUM_CMS_RUN_NUMBER (${MINIMUM_CMS_RUN_NUMBER}), ignoring`)
+      }
     } else {
       // If the run_number is inside the existing already saved runs, then we check for the timestamp:
       last_saved_runs.forEach((existing_run) => {

diff --git a/runregistry_backend/cron/saving_updating_runs_lumisections_utils.js b/runregistry_backend/cron/saving_updating_runs_lumisections_utils.js
@@ -28,6 +28,7 @@ exports.get_OMS_lumisections = handleErrors(async (run_number) => {
   await new Promise((resolve) => {
     setTimeout(resolve, 2000);
   });
+  console.debug(`Requesting lumisections for run ${run_number} from OMS`)
   const oms_lumisection_url = `${OMS_URL}/${OMS_LUMISECTIONS(run_number)}`;
   // Keep fetching until totalresourcecount is # of lumisections
   const oms_lumisection_response = await instance

diff --git a/runregistry_backend/cron_datasets/2.ping_dqm_gui.js b/runregistry_backend/cron_datasets/2.ping_dqm_gui.js
@@ -24,7 +24,7 @@ const { handleErrors } = require('../utils/error_handlers');
 const {
   API_URL,
   DQM_GUI_URL,
-  SECONDS_PER_DQM_GUI_CHECK,
+  DQM_GUI_CHECK_EVERY_NTH_MINUTE,
   WAITING_DQM_GUI_CONSTANT,
   DQM_GUI_PING_CRON_ENABLED
 } = require('../config/config')[process.env.ENV || 'development'];
@@ -231,7 +231,7 @@ const ping_dqm_gui = async () => {
 // Cron job starts:
 if (DQM_GUI_PING_CRON_ENABLED === true) {
   const job = new CronJob(
-    `*/${SECONDS_PER_DQM_GUI_CHECK} * * * * *`,
+    `*/${DQM_GUI_CHECK_EVERY_NTH_MINUTE} * * * *`,
     handleErrors(ping_dqm_gui, 'cron_datasets/2.ping_dqm_gui.js # Error pinging DQM GUI')
   ).start();
 } else {