From d6bc70c0e5903cf2bdb7e96cc82f40ce9e79bb1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tim=20R=C3=BCb?= Date: Fri, 24 Apr 2020 18:35:44 +0200 Subject: [PATCH] add globals, fix table names --- .gitignore | 9 +++++++++ Database_Setup.md | 11 +++++++++++ globals.js | 9 +++++++++ scripts/createCalls.js | 21 +++++++++++---------- scripts/loader.js | 16 +++++++++------- system/AWS/batchSearchServiceAWS.js | 17 +++++++++-------- system/AWS/processingServiceAWS.js | 15 ++++++++------- system/batchSearch.js | 27 ++++++++++++++------------- system/batchSearchTest.js | 27 ++++++++++++++------------- system/processingTest.js | 25 +++++++++++++------------ 10 files changed, 107 insertions(+), 70 deletions(-) create mode 100644 .gitignore create mode 100644 Database_Setup.md create mode 100644 globals.js diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b98fdf3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +# ignore node modules +node_modules + +# keep csv in root directory, but ignore in sub directory +/**/*.csv +!/*.csv + +# DBeaver project +.project \ No newline at end of file diff --git a/Database_Setup.md b/Database_Setup.md new file mode 100644 index 0000000..35a7d8a --- /dev/null +++ b/Database_Setup.md @@ -0,0 +1,11 @@ +Install MariaDB docker +```bash +# creates a new mariadb database container +# and executes the provided sql +# ${PWD} should work on linux and powershell(Windows) +docker run --name hd-cache-db -e MYSQL_ROOT_PASSWORD=rootroot -v ${PWD}/hd_mock_replication_image.sql:/docker-entrypoint-initdb.d/sql_dump_file.sql -p 3306:3306 -d mariadb:latest + +# stop and delete the container +docker stop hd-cache-db +docker rm hd-cache-db +``` diff --git a/globals.js b/globals.js new file mode 100644 index 0000000..c5c5d32 --- /dev/null +++ b/globals.js @@ -0,0 +1,9 @@ +exports.dbConfig = { + host: 'localhost', + port: 33065, + database: 'hd_replication_db', + user: 'root', + passwor: 'rootroot', + debug: false, + multipleStatements: true +} \ No newline at end of file diff --git a/scripts/createCalls.js b/scripts/createCalls.js index e4b06c9..a4ee87e 100644 --- a/scripts/createCalls.js +++ b/scripts/createCalls.js @@ -17,6 +17,7 @@ See the License for the specific language governing permissions. // setup necessary libraries. let mysql = require('mysql'); +let { dbConfig } = require('../globals') let dbConnection; const TYPE = 'GAUSSIAN'; @@ -52,28 +53,28 @@ function shuffle(array) { */ function connectToDB() { - lambdaLog('DB-Connection lost. Start Reconnecting'); + console.log('DB-Connection lost. Start Reconnecting'); return new Promise(function(resolve, reject) { dbConnection = mysql.createConnection({ - host : 'TO_SET', - port : 3306, - database : 'TO_SET', - user : 'TO_SET', - password : 'TO_SET', - debug : false, - multipleStatements: true + host : dbConfig.host, + port : dbConfig.port, + database : dbConfig.database, + user : dbConfig.user, + password : dbConfig.password, + debug : dbConfig.debug, + multipleStatements: dbConfig.multipleStatements }); // Connect to the database. dbConnection.connect(function(err) { if (err) { - lambdaLog(JSON.stringify(err)); + console.log(JSON.stringify(err)); return reject(err); } - lambdaLog('Reconnecting done. DB-Connection established'); + console.log('Reconnecting done. DB-Connection established'); return resolve(true); }); }); diff --git a/scripts/loader.js b/scripts/loader.js index c1b2356..086985c 100644 --- a/scripts/loader.js +++ b/scripts/loader.js @@ -78,14 +78,16 @@ function connectToDB() { return new Promise(function(resolve, reject) { mysql = mysql ? mysql : require('mysql'); + let { dbConfig } = require('../globals'); + dbConnection = mysql.createConnection({ - host : '127.0.0.1', - port : 3306, - database : 'scdm', - user : 'root', - password : 'rootroot', - debug : false, - multipleStatements: true + host : dbConfig.host, + port : dbConfig.port, + database : dbConfig.database, + user : dbConfig.user, + password : dbConfig.password, + debug : dbConfig.debug, + multipleStatements: dbConfig.multipleStatements }); // Connect to the database. diff --git a/system/AWS/batchSearchServiceAWS.js b/system/AWS/batchSearchServiceAWS.js index 6f26652..b5d1055 100644 --- a/system/AWS/batchSearchServiceAWS.js +++ b/system/AWS/batchSearchServiceAWS.js @@ -678,7 +678,7 @@ function mimicEqual(left, right) { /** - * Queries the local database U_loc for the first g-many entries fitting + * Queries the local database u_loc for the first g-many entries fitting * a specified filter according to ascending sort. * * @param a: The specified filter as JSON. In detail: @@ -1005,17 +1005,18 @@ function connectToDB() { lambdaLog('DB-Connection lost. Start Reconnecting'); mysql2 = mysql2 ? mysql2 : require('mysql'); + let { dbConfig } = require('../../globals'); return new Promise(function(resolve, reject) { dbConnection2 = mysql2.createConnection({ - host : 'TO_SET', - port : 3306, - database : 'TO_SET', - user : 'TO_SET', - password : 'TO_SET', - debug : false, - multipleStatements: true + host : dbConfig.host, + port : dbConfig.port, + database : dbConfig.database, + user : dbConfig.user, + password : dbConfig.password, + debug : dbConfig.debug, + multipleStatements: dbConfig.multipleStatements }); // Connect to the database. diff --git a/system/AWS/processingServiceAWS.js b/system/AWS/processingServiceAWS.js index 62cfb70..081f3a2 100644 --- a/system/AWS/processingServiceAWS.js +++ b/system/AWS/processingServiceAWS.js @@ -959,17 +959,18 @@ function connectToDB() { lambdaLog('DB-Connection lost. Start Reconnecting'); mysql = mysql || require('mysql'); + let { dbConfig } = require('../../globals'); return new Promise(function(resolve, reject) { dbConnection = mysql.createConnection({ - host : 'TO_SET', - port : 3306, - database : 'TO_SET', - user : 'TO_SET', - password : 'TO_SET', - debug : false, - multipleStatements: true + host : dbConfig.host, + port : dbConfig.port, + database : dbConfig.database, + user : dbConfig.user, + password : dbConfig.password, + debug : dbConfig.debug, + multipleStatements: dbConfig.multipleStatements }); // Connect to the database. diff --git a/system/batchSearch.js b/system/batchSearch.js index fd59881..59f6927 100644 --- a/system/batchSearch.js +++ b/system/batchSearch.js @@ -599,7 +599,7 @@ return (function() { /** - * Queries the local database U_loc for the first g-many entries fitting + * Queries the local database u_loc for the first g-many entries fitting * a specified filter according to ascending sort. * * @param a: The specified filter as JSON. In detail: @@ -616,7 +616,7 @@ return (function() { function Q_loc(a, g, L_TRENCH) { // Specifies the attribute to use for the range filter. const field = L_TRENCH ? UNIQUE_IDENTIFICATOR : a.v; - let query = 'SELECT * FROM U_loc_' + MOCK_NAME + ' WHERE '; + let query = 'SELECT * FROM u_loc_' + MOCK_NAME + ' WHERE '; // If L_TRENCH is set, add an additional equalityFilter for the attribute a.v and the value in L_TRENCH. if(L_TRENCH) { query += mimicEqual(a.v, mysql2.escape(L_TRENCH)) + ' AND '; @@ -644,7 +644,7 @@ function Q_loc(a, g, L_TRENCH) { let query = ''; if(full) { // Adds a deleting query on the specified range. - query += 'DELETE FROM U_loc_' + MOCK_NAME + ' WHERE ' + mimicGreaterEqual(mysql2.escape(start), L_TRENCH ? UNIQUE_IDENTIFICATOR : field) + query += 'DELETE FROM u_loc_' + MOCK_NAME + ' WHERE ' + mimicGreaterEqual(mysql2.escape(start), L_TRENCH ? UNIQUE_IDENTIFICATOR : field) + ' AND ' + (L_TRENCH ? UNIQUE_IDENTIFICATOR : field) + ' < ' + mysql2.escape(end) + ';'; } // Adds the additional equality-filter if L_TRENCH is set. @@ -652,7 +652,7 @@ function Q_loc(a, g, L_TRENCH) { query = query.replace(/;$/, ' AND ' + mimicEqual(field, mysql2.escape(L_TRENCH)) + ';'); } // Creates the new insert query. will rewrite entries with the same unuqie identificator. - query += 'REPLACE INTO U_loc_' + MOCK_NAME + '(' + attributes.join(',') + ') VALUES' + query += 'REPLACE INTO u_loc_' + MOCK_NAME + '(' + attributes.join(',') + ') VALUES' for(let i=0; i < E.length; i++) { let E_values = []; for(let j=0; j < attributes.length; j++) { @@ -999,17 +999,18 @@ function Q_loc(a, g, L_TRENCH) { console.log('DB-Connection lost. Start Reconnecting'); mysql2 = mysql2 ? mysql2 : require('mysql'); + let { dbConfig } = require('../globals'); return new Promise(function(resolve, reject) { dbConnection2 = mysql2.createConnection({ - host : '127.0.0.1', - port : 3306, - database : 'scdm', - user : 'root', - password : 'rootroot', - debug : false, - multipleStatements: true + host : dbConfig.host, + port : dbConfig.port, + database : dbConfig.database, + user : dbConfig.user, + password : dbConfig.password, + debug : dbConfig.debug, + multipleStatements: dbConfig.multipleStatements }); // Connect to the database. @@ -1363,7 +1364,7 @@ function Q_loc(a, g, L_TRENCH) { console.log('Current limit: ' + limit); enter = false; // Extracts all entries to contain in the new, optimally sized splinter. - var followerQuery = 'SELECT ' + field + ', ' + UNIQUE_IDENTIFICATOR + ' FROM U_loc_' + MOCK_NAME + ' WHERE ' + var followerQuery = 'SELECT ' + field + ', ' + UNIQUE_IDENTIFICATOR + ' FROM u_loc_' + MOCK_NAME + ' WHERE ' if(lodis){ followerQuery += mimicEqual(field, mysql2.escape(lodis)) + ' AND '; } @@ -1385,7 +1386,7 @@ function Q_loc(a, g, L_TRENCH) { // Checks if the next shard has to be a shard over LODIS. if(!lodis && am === 0 && res.length === (G - P + 1) && res[0][field] === limit) { // Extract all entries matching the LODIS value. - var res2 = await u_locQuery('SELECT * FROM U_loc_' + MOCK_NAME + ' WHERE ' + mimicEqual(field,mysql2.escape(limit)) + ' ORDER BY ' + UNIQUE_IDENTIFICATOR + ' ASC;'); + var res2 = await u_locQuery('SELECT * FROM u_loc_' + MOCK_NAME + ' WHERE ' + mimicEqual(field,mysql2.escape(limit)) + ' ORDER BY ' + UNIQUE_IDENTIFICATOR + ' ASC;'); // Create splinter until all entries found for the LODIS value are covered. for(var i=0; i < res2.length; i += G - P) { splinter = { diff --git a/system/batchSearchTest.js b/system/batchSearchTest.js index 5e688e9..4a81008 100644 --- a/system/batchSearchTest.js +++ b/system/batchSearchTest.js @@ -598,7 +598,7 @@ See the License for the specific language governing permissions. /** - * Queries the local database U_loc for the first g-many entries fitting + * Queries the local database u_loc for the first g-many entries fitting * a specified filter according to ascending sort. * * @param a: The specified filter as JSON. In detail: @@ -615,7 +615,7 @@ See the License for the specific language governing permissions. function Q_loc(a, g, L_TRENCH) { // Specifies the attribute to use for the range filter. const field = L_TRENCH ? UNIQUE_IDENTIFICATOR : a.v; - let query = 'SELECT * FROM U_loc_' + MOCK_NAME + ' WHERE '; + let query = 'SELECT * FROM u_loc_' + MOCK_NAME + ' WHERE '; // If L_TRENCH is set, add an additional equalityFilter for the attribute a.v and the value in L_TRENCH. if(L_TRENCH) { query += mimicEqual(a.v, mysql2.escape(L_TRENCH)) + ' AND '; @@ -643,7 +643,7 @@ function Q_loc(a, g, L_TRENCH) { let query = ''; if(full) { // Adds a deleting query on the specified range. - query += 'DELETE FROM U_loc_' + MOCK_NAME + ' WHERE ' + mimicGreaterEqual(mysql2.escape(start), L_TRENCH ? UNIQUE_IDENTIFICATOR : field) + query += 'DELETE FROM u_loc_' + MOCK_NAME + ' WHERE ' + mimicGreaterEqual(mysql2.escape(start), L_TRENCH ? UNIQUE_IDENTIFICATOR : field) + ' AND ' + (L_TRENCH ? UNIQUE_IDENTIFICATOR : field) + ' < ' + mysql2.escape(end) + ';'; } // Adds the additional equality-filter if L_TRENCH is set. @@ -651,7 +651,7 @@ function Q_loc(a, g, L_TRENCH) { query = query.replace(/;$/, ' AND ' + mimicEqual(field, mysql2.escape(L_TRENCH)) + ';'); } // Creates the new insert query. will rewrite entries with the same unuqie identificator. - query += 'REPLACE INTO U_loc_' + MOCK_NAME + '(' + attributes.join(',') + ') VALUES' + query += 'REPLACE INTO u_loc_' + MOCK_NAME + '(' + attributes.join(',') + ') VALUES' for(let i=0; i < E.length; i++) { let E_values = []; for(let j=0; j < attributes.length; j++) { @@ -998,17 +998,18 @@ function Q_loc(a, g, L_TRENCH) { console.log('DB-Connection lost. Start Reconnecting'); mysql2 = mysql2 || require('mysql'); + let { dbConfig } = require('../globals'); return new Promise(function(resolve, reject) { dbConnection2 = mysql2.createConnection({ - host : '127.0.0.1', - port : 3306, - database : 'scdm', - user : 'root', - password : 'rootroot', - debug : false, - multipleStatements: true + host : dbConfig.host, + port : dbConfig.port, + database : dbConfig.database, + user : dbConfig.user, + password : dbConfig.password, + debug : dbConfig.debug, + multipleStatements: dbConfig.multipleStatements }); // Connect to the database. @@ -1362,7 +1363,7 @@ function Q_loc(a, g, L_TRENCH) { console.log('Current limit: ' + limit); enter = false; // Extracts all entries to contain in the new, optimally sized splinter. - var followerQuery = 'SELECT ' + field + ', ' + UNIQUE_IDENTIFICATOR + ' FROM U_loc_' + MOCK_NAME + ' WHERE ' + var followerQuery = 'SELECT ' + field + ', ' + UNIQUE_IDENTIFICATOR + ' FROM u_loc_' + MOCK_NAME + ' WHERE ' if(lodis){ followerQuery += mimicEqual(field, mysql2.escape(lodis)) + ' AND '; } @@ -1384,7 +1385,7 @@ function Q_loc(a, g, L_TRENCH) { // Checks if the next shard has to be a shard over LODIS. if(!lodis && am === 0 && res.length === (G - P + 1) && res[0][field] === limit) { // Extract all entries matching the LODIS value. - var res2 = await u_locQuery('SELECT * FROM U_loc_' + MOCK_NAME + ' WHERE ' + mimicEqual(field,mysql2.escape(limit)) + ' ORDER BY ' + UNIQUE_IDENTIFICATOR + ' ASC;'); + var res2 = await u_locQuery('SELECT * FROM u_loc_' + MOCK_NAME + ' WHERE ' + mimicEqual(field,mysql2.escape(limit)) + ' ORDER BY ' + UNIQUE_IDENTIFICATOR + ' ASC;'); // Create splinter until all entries found for the LODIS value are covered. for(var i=0; i < res2.length; i += G - P) { splinter = { diff --git a/system/processingTest.js b/system/processingTest.js index 559fdee..cc2cee5 100644 --- a/system/processingTest.js +++ b/system/processingTest.js @@ -899,19 +899,20 @@ function connectToDB() { console.log('DB-Connection lost. Start Reconnecting'); mysql = mysql || require('mysql'); + let { dbConfig } = require('../globals'); return new Promise(function(resolve, reject) { dbConnection = mysql.createConnection({ - host : '127.0.0.1', - port : 3306, - database : 'scdm', - user : 'root', - password : 'rootroot', - debug : false, - multipleStatements: true + host : dbConfig.host, + port : dbConfig.port, + database : dbConfig.database, + user : dbConfig.user, + password : dbConfig.password, + debug : dbConfig.debug, + multipleStatements: dbConfig.multipleStatements }); - + // Connect to the database. dbConnection.connect(function(err) { @@ -1122,10 +1123,10 @@ function fireUpdateQuery(navigator) { let u_locDelete = ''; // Creates a query to delete all entries in the queried range if the query to the foreign API was volume consistant. if(result.full) { - u_locDelete = 'DELETE FROM U_loc_' + MOCK_NAME + ' WHERE ' + mimicGreaterEqual(mysql.escape(r.start), r.field) + (r.end ? ' AND ' + r.field + ' < ' + mysql.escape(r.end) : '') + (navigator.lodisStart ? ' AND ' + mimicEqual(navigator.field, mysql.escape(navigator.start)): '') + ';'; + u_locDelete = 'DELETE FROM u_loc_' + MOCK_NAME + ' WHERE ' + mimicGreaterEqual(mysql.escape(r.start), r.field) + (r.end ? ' AND ' + r.field + ' < ' + mysql.escape(r.end) : '') + (navigator.lodisStart ? ' AND ' + mimicEqual(navigator.field, mysql.escape(navigator.start)): '') + ';'; } // Creates a query to insert the new entries into the local database. - let u_locInsert = 'REPLACE INTO U_loc_' + MOCK_NAME + '(' + COLUMNS_TO_SHOW.join(',') + ') VALUES'; + let u_locInsert = 'REPLACE INTO u_loc_' + MOCK_NAME + '(' + COLUMNS_TO_SHOW.join(',') + ') VALUES'; // Escapes all gathered values to avoid an accidential SQL-Injection. result.res.forEach(function(el) { @@ -1866,7 +1867,7 @@ function cont(ADpara) { dbRes = await simpleQuery('TRUNCATE TABLE u_loc_' + MOCK_NAME + ';'); dbRes = await simpleQuery('TRUNCATE TABLE hd_mock_' + MOCK_NAME + ';'); dbRes = await simpleQuery('TRUNCATE TABLE splinter;'); - dbRes = await simpleQuery('TRUNCATE TABLE maintenanceList;'); + dbRes = await simpleQuery('TRUNCATE TABLE maintenancelist;'); dbRes = await simpleQuery('TRUNCATE TABLE bank_' + MOCK_NAME + ';'); // Load entries into the database @@ -2164,7 +2165,7 @@ function prepareAD(c, first) { }); // Pass all splinter which are three days old or older into the maintenance list. - dbRes = await simpleQuery('REPLACE INTO maintenanceList SELECT * FROM splinter WHERE TIMESTAMP <= NOW() - INTERVAL 3 DAY;'); + dbRes = await simpleQuery('REPLACE INTO maintenancelist SELECT * FROM splinter WHERE TIMESTAMP <= NOW() - INTERVAL 3 DAY;'); return resolve(allCalls); });