Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Set global DB config & repair DB table names #1

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
# ignore node modules
node_modules

# keep csv in root directory, but ignore in sub directory
/**/*.csv
!/*.csv

# DBeaver project
.project
11 changes: 11 additions & 0 deletions Database_Setup.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
Install MariaDB docker
```bash
# creates a new mariadb database container
# and executes the provided sql
# ${PWD} should work on linux and powershell(Windows)
docker run --name hd-cache-db -e MYSQL_ROOT_PASSWORD=rootroot -v ${PWD}/hd_mock_replication_image.sql:/docker-entrypoint-initdb.d/sql_dump_file.sql -p 3306:3306 -d mariadb:latest

# stop and delete the container
docker stop hd-cache-db
docker rm hd-cache-db
```
9 changes: 9 additions & 0 deletions globals.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
exports.dbConfig = {
host: 'localhost',
port: 33065,
database: 'hd_replication_db',
user: 'root',
passwor: 'rootroot',
debug: false,
multipleStatements: true
}
21 changes: 11 additions & 10 deletions scripts/createCalls.js
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ See the License for the specific language governing permissions.

// setup necessary libraries.
let mysql = require('mysql');
let { dbConfig } = require('../globals')
let dbConnection;
const TYPE = 'GAUSSIAN';

Expand Down Expand Up @@ -52,28 +53,28 @@ function shuffle(array) {
*/
function connectToDB() {

lambdaLog('DB-Connection lost. Start Reconnecting');
console.log('DB-Connection lost. Start Reconnecting');

return new Promise(function(resolve, reject) {

dbConnection = mysql.createConnection({
host : 'TO_SET',
port : 3306,
database : 'TO_SET',
user : 'TO_SET',
password : 'TO_SET',
debug : false,
multipleStatements: true
host : dbConfig.host,
port : dbConfig.port,
database : dbConfig.database,
user : dbConfig.user,
password : dbConfig.password,
debug : dbConfig.debug,
multipleStatements: dbConfig.multipleStatements
});

// Connect to the database.
dbConnection.connect(function(err) {

if (err) {
lambdaLog(JSON.stringify(err));
console.log(JSON.stringify(err));
return reject(err);
}
lambdaLog('Reconnecting done. DB-Connection established');
console.log('Reconnecting done. DB-Connection established');
return resolve(true);
});
});
Expand Down
16 changes: 9 additions & 7 deletions scripts/loader.js
Original file line number Diff line number Diff line change
Expand Up @@ -78,14 +78,16 @@ function connectToDB() {
return new Promise(function(resolve, reject) {

mysql = mysql ? mysql : require('mysql');
let { dbConfig } = require('../globals');

dbConnection = mysql.createConnection({
host : '127.0.0.1',
port : 3306,
database : 'scdm',
user : 'root',
password : 'rootroot',
debug : false,
multipleStatements: true
host : dbConfig.host,
port : dbConfig.port,
database : dbConfig.database,
user : dbConfig.user,
password : dbConfig.password,
debug : dbConfig.debug,
multipleStatements: dbConfig.multipleStatements
});

// Connect to the database.
Expand Down
17 changes: 9 additions & 8 deletions system/AWS/batchSearchServiceAWS.js
Original file line number Diff line number Diff line change
Expand Up @@ -678,7 +678,7 @@ function mimicEqual(left, right) {


/**
* Queries the local database U_loc for the first g-many entries fitting
* Queries the local database u_loc for the first g-many entries fitting
* a specified filter according to ascending sort.
*
* @param a: The specified filter as JSON. In detail:
Expand Down Expand Up @@ -1005,17 +1005,18 @@ function connectToDB() {
lambdaLog('DB-Connection lost. Start Reconnecting');

mysql2 = mysql2 ? mysql2 : require('mysql');
let { dbConfig } = require('../../globals');

return new Promise(function(resolve, reject) {

dbConnection2 = mysql2.createConnection({
host : 'TO_SET',
port : 3306,
database : 'TO_SET',
user : 'TO_SET',
password : 'TO_SET',
debug : false,
multipleStatements: true
host : dbConfig.host,
port : dbConfig.port,
database : dbConfig.database,
user : dbConfig.user,
password : dbConfig.password,
debug : dbConfig.debug,
multipleStatements: dbConfig.multipleStatements
});

// Connect to the database.
Expand Down
15 changes: 8 additions & 7 deletions system/AWS/processingServiceAWS.js
Original file line number Diff line number Diff line change
Expand Up @@ -959,17 +959,18 @@ function connectToDB() {

lambdaLog('DB-Connection lost. Start Reconnecting');
mysql = mysql || require('mysql');
let { dbConfig } = require('../../globals');

return new Promise(function(resolve, reject) {

dbConnection = mysql.createConnection({
host : 'TO_SET',
port : 3306,
database : 'TO_SET',
user : 'TO_SET',
password : 'TO_SET',
debug : false,
multipleStatements: true
host : dbConfig.host,
port : dbConfig.port,
database : dbConfig.database,
user : dbConfig.user,
password : dbConfig.password,
debug : dbConfig.debug,
multipleStatements: dbConfig.multipleStatements
});

// Connect to the database.
Expand Down
27 changes: 14 additions & 13 deletions system/batchSearch.js
Original file line number Diff line number Diff line change
Expand Up @@ -599,7 +599,7 @@ return (function() {


/**
* Queries the local database U_loc for the first g-many entries fitting
* Queries the local database u_loc for the first g-many entries fitting
* a specified filter according to ascending sort.
*
* @param a: The specified filter as JSON. In detail:
Expand All @@ -616,7 +616,7 @@ return (function() {
function Q_loc(a, g, L_TRENCH) {
// Specifies the attribute to use for the range filter.
const field = L_TRENCH ? UNIQUE_IDENTIFICATOR : a.v;
let query = 'SELECT * FROM U_loc_' + MOCK_NAME + ' WHERE ';
let query = 'SELECT * FROM u_loc_' + MOCK_NAME + ' WHERE ';
// If L_TRENCH is set, add an additional equalityFilter for the attribute a.v and the value in L_TRENCH.
if(L_TRENCH) {
query += mimicEqual(a.v, mysql2.escape(L_TRENCH)) + ' AND ';
Expand Down Expand Up @@ -644,15 +644,15 @@ function Q_loc(a, g, L_TRENCH) {
let query = '';
if(full) {
// Adds a deleting query on the specified range.
query += 'DELETE FROM U_loc_' + MOCK_NAME + ' WHERE ' + mimicGreaterEqual(mysql2.escape(start), L_TRENCH ? UNIQUE_IDENTIFICATOR : field)
query += 'DELETE FROM u_loc_' + MOCK_NAME + ' WHERE ' + mimicGreaterEqual(mysql2.escape(start), L_TRENCH ? UNIQUE_IDENTIFICATOR : field)
+ ' AND ' + (L_TRENCH ? UNIQUE_IDENTIFICATOR : field) + ' < ' + mysql2.escape(end) + ';';
}
// Adds the additional equality-filter if L_TRENCH is set.
if(L_TRENCH) {
query = query.replace(/;$/, ' AND ' + mimicEqual(field, mysql2.escape(L_TRENCH)) + ';');
}
// Creates the new insert query. will rewrite entries with the same unuqie identificator.
query += 'REPLACE INTO U_loc_' + MOCK_NAME + '(' + attributes.join(',') + ') VALUES'
query += 'REPLACE INTO u_loc_' + MOCK_NAME + '(' + attributes.join(',') + ') VALUES'
for(let i=0; i < E.length; i++) {
let E_values = [];
for(let j=0; j < attributes.length; j++) {
Expand Down Expand Up @@ -999,17 +999,18 @@ function Q_loc(a, g, L_TRENCH) {
console.log('DB-Connection lost. Start Reconnecting');

mysql2 = mysql2 ? mysql2 : require('mysql');
let { dbConfig } = require('../globals');

return new Promise(function(resolve, reject) {

dbConnection2 = mysql2.createConnection({
host : '127.0.0.1',
port : 3306,
database : 'scdm',
user : 'root',
password : 'rootroot',
debug : false,
multipleStatements: true
host : dbConfig.host,
port : dbConfig.port,
database : dbConfig.database,
user : dbConfig.user,
password : dbConfig.password,
debug : dbConfig.debug,
multipleStatements: dbConfig.multipleStatements
});

// Connect to the database.
Expand Down Expand Up @@ -1363,7 +1364,7 @@ function Q_loc(a, g, L_TRENCH) {
console.log('Current limit: ' + limit);
enter = false;
// Extracts all entries to contain in the new, optimally sized splinter.
var followerQuery = 'SELECT ' + field + ', ' + UNIQUE_IDENTIFICATOR + ' FROM U_loc_' + MOCK_NAME + ' WHERE '
var followerQuery = 'SELECT ' + field + ', ' + UNIQUE_IDENTIFICATOR + ' FROM u_loc_' + MOCK_NAME + ' WHERE '
if(lodis){
followerQuery += mimicEqual(field, mysql2.escape(lodis)) + ' AND ';
}
Expand All @@ -1385,7 +1386,7 @@ function Q_loc(a, g, L_TRENCH) {
// Checks if the next shard has to be a shard over LODIS.
if(!lodis && am === 0 && res.length === (G - P + 1) && res[0][field] === limit) {
// Extract all entries matching the LODIS value.
var res2 = await u_locQuery('SELECT * FROM U_loc_' + MOCK_NAME + ' WHERE ' + mimicEqual(field,mysql2.escape(limit)) + ' ORDER BY ' + UNIQUE_IDENTIFICATOR + ' ASC;');
var res2 = await u_locQuery('SELECT * FROM u_loc_' + MOCK_NAME + ' WHERE ' + mimicEqual(field,mysql2.escape(limit)) + ' ORDER BY ' + UNIQUE_IDENTIFICATOR + ' ASC;');
// Create splinter until all entries found for the LODIS value are covered.
for(var i=0; i < res2.length; i += G - P) {
splinter = {
Expand Down
27 changes: 14 additions & 13 deletions system/batchSearchTest.js
Original file line number Diff line number Diff line change
Expand Up @@ -598,7 +598,7 @@ See the License for the specific language governing permissions.


/**
* Queries the local database U_loc for the first g-many entries fitting
* Queries the local database u_loc for the first g-many entries fitting
* a specified filter according to ascending sort.
*
* @param a: The specified filter as JSON. In detail:
Expand All @@ -615,7 +615,7 @@ See the License for the specific language governing permissions.
function Q_loc(a, g, L_TRENCH) {
// Specifies the attribute to use for the range filter.
const field = L_TRENCH ? UNIQUE_IDENTIFICATOR : a.v;
let query = 'SELECT * FROM U_loc_' + MOCK_NAME + ' WHERE ';
let query = 'SELECT * FROM u_loc_' + MOCK_NAME + ' WHERE ';
// If L_TRENCH is set, add an additional equalityFilter for the attribute a.v and the value in L_TRENCH.
if(L_TRENCH) {
query += mimicEqual(a.v, mysql2.escape(L_TRENCH)) + ' AND ';
Expand Down Expand Up @@ -643,15 +643,15 @@ function Q_loc(a, g, L_TRENCH) {
let query = '';
if(full) {
// Adds a deleting query on the specified range.
query += 'DELETE FROM U_loc_' + MOCK_NAME + ' WHERE ' + mimicGreaterEqual(mysql2.escape(start), L_TRENCH ? UNIQUE_IDENTIFICATOR : field)
query += 'DELETE FROM u_loc_' + MOCK_NAME + ' WHERE ' + mimicGreaterEqual(mysql2.escape(start), L_TRENCH ? UNIQUE_IDENTIFICATOR : field)
+ ' AND ' + (L_TRENCH ? UNIQUE_IDENTIFICATOR : field) + ' < ' + mysql2.escape(end) + ';';
}
// Adds the additional equality-filter if L_TRENCH is set.
if(L_TRENCH) {
query = query.replace(/;$/, ' AND ' + mimicEqual(field, mysql2.escape(L_TRENCH)) + ';');
}
// Creates the new insert query. will rewrite entries with the same unuqie identificator.
query += 'REPLACE INTO U_loc_' + MOCK_NAME + '(' + attributes.join(',') + ') VALUES'
query += 'REPLACE INTO u_loc_' + MOCK_NAME + '(' + attributes.join(',') + ') VALUES'
for(let i=0; i < E.length; i++) {
let E_values = [];
for(let j=0; j < attributes.length; j++) {
Expand Down Expand Up @@ -998,17 +998,18 @@ function Q_loc(a, g, L_TRENCH) {
console.log('DB-Connection lost. Start Reconnecting');

mysql2 = mysql2 || require('mysql');
let { dbConfig } = require('../globals');

return new Promise(function(resolve, reject) {

dbConnection2 = mysql2.createConnection({
host : '127.0.0.1',
port : 3306,
database : 'scdm',
user : 'root',
password : 'rootroot',
debug : false,
multipleStatements: true
host : dbConfig.host,
port : dbConfig.port,
database : dbConfig.database,
user : dbConfig.user,
password : dbConfig.password,
debug : dbConfig.debug,
multipleStatements: dbConfig.multipleStatements
});

// Connect to the database.
Expand Down Expand Up @@ -1362,7 +1363,7 @@ function Q_loc(a, g, L_TRENCH) {
console.log('Current limit: ' + limit);
enter = false;
// Extracts all entries to contain in the new, optimally sized splinter.
var followerQuery = 'SELECT ' + field + ', ' + UNIQUE_IDENTIFICATOR + ' FROM U_loc_' + MOCK_NAME + ' WHERE '
var followerQuery = 'SELECT ' + field + ', ' + UNIQUE_IDENTIFICATOR + ' FROM u_loc_' + MOCK_NAME + ' WHERE '
if(lodis){
followerQuery += mimicEqual(field, mysql2.escape(lodis)) + ' AND ';
}
Expand All @@ -1384,7 +1385,7 @@ function Q_loc(a, g, L_TRENCH) {
// Checks if the next shard has to be a shard over LODIS.
if(!lodis && am === 0 && res.length === (G - P + 1) && res[0][field] === limit) {
// Extract all entries matching the LODIS value.
var res2 = await u_locQuery('SELECT * FROM U_loc_' + MOCK_NAME + ' WHERE ' + mimicEqual(field,mysql2.escape(limit)) + ' ORDER BY ' + UNIQUE_IDENTIFICATOR + ' ASC;');
var res2 = await u_locQuery('SELECT * FROM u_loc_' + MOCK_NAME + ' WHERE ' + mimicEqual(field,mysql2.escape(limit)) + ' ORDER BY ' + UNIQUE_IDENTIFICATOR + ' ASC;');
// Create splinter until all entries found for the LODIS value are covered.
for(var i=0; i < res2.length; i += G - P) {
splinter = {
Expand Down
25 changes: 13 additions & 12 deletions system/processingTest.js
Original file line number Diff line number Diff line change
Expand Up @@ -899,19 +899,20 @@ function connectToDB() {

console.log('DB-Connection lost. Start Reconnecting');
mysql = mysql || require('mysql');
let { dbConfig } = require('../globals');

return new Promise(function(resolve, reject) {

dbConnection = mysql.createConnection({
host : '127.0.0.1',
port : 3306,
database : 'scdm',
user : 'root',
password : 'rootroot',
debug : false,
multipleStatements: true
host : dbConfig.host,
port : dbConfig.port,
database : dbConfig.database,
user : dbConfig.user,
password : dbConfig.password,
debug : dbConfig.debug,
multipleStatements: dbConfig.multipleStatements
});

// Connect to the database.
dbConnection.connect(function(err) {

Expand Down Expand Up @@ -1122,10 +1123,10 @@ function fireUpdateQuery(navigator) {
let u_locDelete = '';
// Creates a query to delete all entries in the queried range if the query to the foreign API was volume consistant.
if(result.full) {
u_locDelete = 'DELETE FROM U_loc_' + MOCK_NAME + ' WHERE ' + mimicGreaterEqual(mysql.escape(r.start), r.field) + (r.end ? ' AND ' + r.field + ' < ' + mysql.escape(r.end) : '') + (navigator.lodisStart ? ' AND ' + mimicEqual(navigator.field, mysql.escape(navigator.start)): '') + ';';
u_locDelete = 'DELETE FROM u_loc_' + MOCK_NAME + ' WHERE ' + mimicGreaterEqual(mysql.escape(r.start), r.field) + (r.end ? ' AND ' + r.field + ' < ' + mysql.escape(r.end) : '') + (navigator.lodisStart ? ' AND ' + mimicEqual(navigator.field, mysql.escape(navigator.start)): '') + ';';
}
// Creates a query to insert the new entries into the local database.
let u_locInsert = 'REPLACE INTO U_loc_' + MOCK_NAME + '(' + COLUMNS_TO_SHOW.join(',') + ') VALUES';
let u_locInsert = 'REPLACE INTO u_loc_' + MOCK_NAME + '(' + COLUMNS_TO_SHOW.join(',') + ') VALUES';

// Escapes all gathered values to avoid an accidential SQL-Injection.
result.res.forEach(function(el) {
Expand Down Expand Up @@ -1866,7 +1867,7 @@ function cont(ADpara) {
dbRes = await simpleQuery('TRUNCATE TABLE u_loc_' + MOCK_NAME + ';');
dbRes = await simpleQuery('TRUNCATE TABLE hd_mock_' + MOCK_NAME + ';');
dbRes = await simpleQuery('TRUNCATE TABLE splinter;');
dbRes = await simpleQuery('TRUNCATE TABLE maintenanceList;');
dbRes = await simpleQuery('TRUNCATE TABLE maintenancelist;');
dbRes = await simpleQuery('TRUNCATE TABLE bank_' + MOCK_NAME + ';');

// Load entries into the database
Expand Down Expand Up @@ -2164,7 +2165,7 @@ function prepareAD(c, first) {
});

// Pass all splinter which are three days old or older into the maintenance list.
dbRes = await simpleQuery('REPLACE INTO maintenanceList SELECT * FROM splinter WHERE TIMESTAMP <= NOW() - INTERVAL 3 DAY;');
dbRes = await simpleQuery('REPLACE INTO maintenancelist SELECT * FROM splinter WHERE TIMESTAMP <= NOW() - INTERVAL 3 DAY;');

return resolve(allCalls);
});
Expand Down