diff --git a/Dockerfile b/Dockerfile
index df6b10b..9aef307 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,5 +1,6 @@
FROM openjdk:8
-COPY ./out/production/ADD/ /tmp
+COPY ./target/Serengeti-1.2-SNAPSHOT-jar-with-dependencies.jar /tmp
WORKDIR /tmp
EXPOSE 1985
-ENTRYPOINT ["java","gl.ao.serengeti.Serengeti"]
\ No newline at end of file
+#ENTRYPOINT ["java","gl.ao.serengeti.Serengeti"]
+CMD java -jar Serengeti-1.2-SNAPSHOT-jar-with-dependencies.jar
diff --git a/data/.gitkeep b/data/.gitkeep
deleted file mode 100644
index e69de29..0000000
diff --git a/pom.xml b/pom.xml
index 2c23431..4ede208 100644
--- a/pom.xml
+++ b/pom.xml
@@ -77,6 +77,12 @@
junit-jupiter-api
5.7.0
+
+ org.apache.httpcomponents
+ httpclient
+ 4.5.12
+ test
+
diff --git a/src/main/java/gl/ao/serengeti/data/DatabaseObjectData.java b/src/main/java/gl/ao/serengeti/data/DatabaseObjectData.java
new file mode 100644
index 0000000..5d79b46
--- /dev/null
+++ b/src/main/java/gl/ao/serengeti/data/DatabaseObjectData.java
@@ -0,0 +1,19 @@
+package gl.ao.serengeti.data;
+
+public class DatabaseObjectData {
+
+ private byte[] data;
+
+ public DatabaseObjectData() {}
+ public DatabaseObjectData(byte[] data) {
+ this.data = data;
+ }
+
+ public byte[] getData() {
+ return data;
+ }
+
+ public void setData(byte[] data) {
+ this.data = data;
+ }
+}
diff --git a/src/main/java/gl/ao/serengeti/helpers/Globals.java b/src/main/java/gl/ao/serengeti/helpers/Globals.java
index 5032191..fecf22a 100644
--- a/src/main/java/gl/ao/serengeti/helpers/Globals.java
+++ b/src/main/java/gl/ao/serengeti/helpers/Globals.java
@@ -35,9 +35,6 @@ public class Globals {
public static String index_filename = "index.file";
public static int port_default = 1985;
- public static int port_communication = 19851;
-
- public static int max_cluster_nodes = 3;
/***
* Convert to Bytes
@@ -184,18 +181,13 @@ public static String getHost4Address() throws SocketException {
* @param path
*/
static public void deleteDirectory(File path) {
- if (path == null)
- return;
+ if (path == null) return;
if (path.exists()) {
for(File f : Objects.requireNonNull(path.listFiles())) {
- if(f.isDirectory()) {
- deleteDirectory(f);
- f.delete();
- } else {
- f.delete();
- }
+ if (f.isDirectory()) deleteDirectory(f);
+ f.delete();
}
- path.delete();
+ boolean delete = path.delete();
}
}
diff --git a/src/main/java/gl/ao/serengeti/network/Network.java b/src/main/java/gl/ao/serengeti/network/Network.java
index 091239c..09673ee 100644
--- a/src/main/java/gl/ao/serengeti/network/Network.java
+++ b/src/main/java/gl/ao/serengeti/network/Network.java
@@ -63,7 +63,9 @@ public void requestNetworkMetas() {
JSONObject jsonObject = node.getValue();
try {
- HttpURLConnection con = (HttpURLConnection) new URL("http://" + jsonObject.getString("ip") + ":" + Globals.port_default + "/meta").openConnection();
+ HttpURLConnection con = (HttpURLConnection) new URL(
+ String.format("http://%s:%d/meta", jsonObject.getString("ip"), Globals.port_default))
+ .openConnection();
con.setRequestMethod("GET");
con.getDoOutput();
con.setConnectTimeout(networkTimeout);
@@ -87,7 +89,7 @@ public void requestNetworkMetas() {
JSONArray jsonArr = (JSONArray) jsonMeta.get(db);
if (!Serengeti.storage.databaseExists(db)) {
- System.out.println("Startup: Creating missing database '"+db+"'");
+ System.out.printf("Startup: Creating missing database '%s'%n", db);
Serengeti.storage.createDatabase(db, true);
changesFound++;
}
@@ -95,17 +97,21 @@ public void requestNetworkMetas() {
for (int i = 0; i < jsonArr.length(); i++) {
String table = jsonArr.getString(i);
if (!Serengeti.storage.tableExists(db, table)) {
- System.out.println("Startup: Creating missing table '"+table+"' for database '"+db+"'");
+ System.out.printf("Startup: Creating missing table '%s' for database '%s'%n", table, db);
Serengeti.storage.createTable(db, table, true);
changesFound++;
- String row_replicas = Serengeti.network.communicateQueryLogSingleNode( jsonObject.getString("id"), jsonObject.getString("ip"), new JSONObject(){{
- put("type", "SendTableReplicaToNode");
- put("db", db);
- put("table", table);
- put("node_id", Serengeti.server.server_constants.id);
- put("node_ip", Serengeti.network.myIP);
- }}.toString() );
+ String row_replicas = Serengeti.network.communicateQueryLogSingleNode(
+ jsonObject.getString("id"),
+ jsonObject.getString("ip"),
+ new JSONObject() {{
+ put("type", "SendTableReplicaToNode");
+ put("db", db);
+ put("table", table);
+ put("node_id", Serengeti.server.server_constants.id);
+ put("node_ip", Serengeti.network.myIP);
+ }}.toString()
+ );
try {
JSONObject jsonRowsReplica = new JSONObject(row_replicas);
@@ -113,9 +119,11 @@ public void requestNetworkMetas() {
while (jkeys.hasNext()) {
String jrow_id = jkeys.next();
JSONObject _json = new JSONObject(jsonRowsReplica.getString(jrow_id));
- Storage.tableReplicaObjects.get(db+"#"+table).insertOrReplace(jrow_id, _json);
+ Storage.tableReplicaObjects
+ .get(db+"#"+table)
+ .insertOrReplace(jrow_id, _json);
}
- } catch (Exception e) {}
+ } catch (Exception ignore) {}
}
}
}
@@ -125,7 +133,6 @@ public void requestNetworkMetas() {
}
-
} catch (Exception e) {
e.printStackTrace();
}
@@ -133,7 +140,7 @@ public void requestNetworkMetas() {
}
}
online = true;
- System.out.println("Startup: Completed with "+changesFound+" changes found");
+ System.out.printf("Startup: Completed with %d changes found%n", changesFound);
Serengeti.server.serve();
}).start();
}
@@ -207,9 +214,8 @@ public void getNetworkIPsPorts() {
BufferedReader in = new BufferedReader(new InputStreamReader(con.getInputStream()));
String inputLine;
StringBuilder content = new StringBuilder();
- while ((inputLine = in.readLine()) != null) {
+ while ((inputLine = in.readLine()) != null)
content.append(inputLine);
- }
JSONObject jsonObj = new JSONObject(content.toString());
JSONObject nodeJSON = (JSONObject) jsonObj.get("this");
@@ -225,9 +231,8 @@ public void getNetworkIPsPorts() {
if (latencyRun) {
long elapsedTime = System.currentTimeMillis() - startTime;
- if (elapsedTime> latency) {
+ if (elapsedTime > latency)
latency = elapsedTime;
- }
}
@@ -327,7 +332,7 @@ public String communicateQueryLogSingleNode(String id, String ip, String jsonStr
StringBuilder response = new StringBuilder();
try {
- URL url2 = new URL("http://" + ip + ":" + Globals.port_default + "/post");
+ URL url2 = new URL(String.format("http://%s:%d/post", ip, Globals.port_default));
HttpURLConnection con2 = (HttpURLConnection) url2.openConnection();
con2.setRequestMethod("POST");
con2.setDoOutput(true);
@@ -335,15 +340,14 @@ public String communicateQueryLogSingleNode(String id, String ip, String jsonStr
con2.getOutputStream().write(jsonString.getBytes(StandardCharsets.UTF_8));
BufferedReader br = new BufferedReader(new InputStreamReader(con2.getInputStream()));
- for (String line = br.readLine(); line != null; line = br.readLine()) {
+ for (String line = br.readLine(); line != null; line = br.readLine())
response.append(line);
- }
return response.toString();
} catch (SocketException se) {
//System.out.println("Socket Exception (communicateQueryLogSingleNode): " + se.getMessage());
return "";
} catch (IOException ioe) {
- System.out.println("IOException: " + ioe.getMessage()+ " > Tried: Communicating to " + ip + ": " + jsonString );
+ System.out.printf("IOException: %s > Tried: Communicating to %s: %s%n", ioe.getMessage(), ip, jsonString);
return "";
} catch (Exception e) {
e.printStackTrace();
@@ -415,20 +419,13 @@ public JSONObject getRandomAvailableNode() {
if (an.size() > 0) {
// Should first remove `self` from the list
- Iterator it=an.entrySet().iterator();
- while(it.hasNext()) {
- Map.Entry item = (Map.Entry) it.next();
- if (item.getValue().get("ip").toString().equals(myIP)) {
- it.remove();
- }
- }
+ an.entrySet().removeIf(item -> item.getValue().get("ip").toString().equals(myIP));
if (an.size()==0) return null; // this can happen when there was only 1 node in the list and it was ourselves!
List list = new ArrayList();
- for (String key: an.keySet()) {
+ for (String key: an.keySet())
list.add(an.get(key));
- }
// Shuffle and send the first one back
Collections.shuffle(list);
@@ -463,10 +460,8 @@ public void getNetworkIPsOnly() {
System.out.println("Not Reachable: "+output);
}
-
-
} catch (Exception e) {
- System.out.println(ip);
+ System.out.println(Arrays.toString(ip));
e.printStackTrace();
}
}).start();
diff --git a/src/main/java/gl/ao/serengeti/query/QueryEngine.java b/src/main/java/gl/ao/serengeti/query/QueryEngine.java
index 2077e42..f4d7a2b 100644
--- a/src/main/java/gl/ao/serengeti/query/QueryEngine.java
+++ b/src/main/java/gl/ao/serengeti/query/QueryEngine.java
@@ -15,7 +15,7 @@ public class QueryEngine {
* @param query
* @return List
*/
- public static List query(String query) {
+ public static List query(String query) {
if (query.trim().equals("")) {
return null;
}
diff --git a/src/main/java/gl/ao/serengeti/schema/DatabaseObject.java b/src/main/java/gl/ao/serengeti/schema/DatabaseObject.java
index c90fc46..604b53b 100644
--- a/src/main/java/gl/ao/serengeti/schema/DatabaseObject.java
+++ b/src/main/java/gl/ao/serengeti/schema/DatabaseObject.java
@@ -1,5 +1,6 @@
package gl.ao.serengeti.schema;
+import gl.ao.serengeti.data.DatabaseObjectData;
import gl.ao.serengeti.helpers.Globals;
import java.io.*;
@@ -15,6 +16,7 @@ public class DatabaseObject implements Serializable {
public String name = "";
public List tables = new ArrayList<>();
+// public DatabaseObjectData tables2 = new DatabaseObjectData();
public DatabaseObject() {}
diff --git a/src/main/java/gl/ao/serengeti/schema/TableStorageObject.java b/src/main/java/gl/ao/serengeti/schema/TableStorageObject.java
index d714fbf..651245e 100644
--- a/src/main/java/gl/ao/serengeti/schema/TableStorageObject.java
+++ b/src/main/java/gl/ao/serengeti/schema/TableStorageObject.java
@@ -43,9 +43,8 @@ public String insert(String row_id, JSONObject json) {
rows.put(row_id, json.toString());
return row_id;
}
- public boolean update(String row_id, JSONObject json) {
+ public void update(String row_id, JSONObject json) {
rows.replace(row_id, json.toString());
- return true;
}
public boolean update(String update_key, String update_val, String where_col, String where_val) {
List results = select(where_col, where_val);
@@ -61,9 +60,8 @@ public boolean update(String update_key, String update_val, String where_col, St
return false;
}
- public boolean delete(String row_id) {
+ public void delete(String row_id) {
rows.remove(row_id);
- return true;
}
public boolean delete(String where_col, String where_val) {
List results = select(where_col, where_val);
@@ -78,7 +76,7 @@ public boolean delete(String where_col, String where_val) {
return false;
}
- public List select(String col, String val) {
+ public List select(String col, String val) {
List ret = new ArrayList<>();
for (String key: rows.keySet()) {
@@ -113,24 +111,20 @@ public byte[] returnDBObytes() {
return Globals.convertToBytes(this);
}
- public boolean saveToDisk() {
+ public void saveToDisk() {
ObjectOutputStream oos = null;
try {
FileOutputStream fos = new FileOutputStream(Globals.data_path + databaseName + "/" + tableName + "/" + Globals.storage_filename);
oos = new ObjectOutputStream(fos);
oos.writeObject(this);
- return true;
} catch (Exception e) {
e.printStackTrace();
} finally {
- if (oos != null) {
- try {
- oos.close();
- } catch (IOException ex) {
- ex.printStackTrace();
- }
+ if (oos != null) try {
+ oos.close();
+ } catch (IOException ex) {
+ ex.printStackTrace();
}
}
- return false;
}
}
diff --git a/src/main/java/gl/ao/serengeti/server/Server.java b/src/main/java/gl/ao/serengeti/server/Server.java
index b0674ed..ea1fbd2 100644
--- a/src/main/java/gl/ao/serengeti/server/Server.java
+++ b/src/main/java/gl/ao/serengeti/server/Server.java
@@ -36,7 +36,9 @@ public void init() {
try {
File root_directory = new File(Globals.data_path);
- if (! root_directory.exists()) root_directory.mkdir();
+ if (! root_directory.exists()) {
+ boolean mkdir = root_directory.mkdir();
+ }
server_constants_file = Paths.get(server_constants_file_location);
if (Files.exists(server_constants_file)) {
@@ -46,7 +48,7 @@ public void init() {
server_constants.id = UUID.randomUUID().toString();
saveServerConstants();
}
- } catch (Exception e) {}
+ } catch (Exception ignore) {}
}
private void saveServerConstants() {
@@ -77,17 +79,30 @@ public void serve() {
e.printStackTrace();
}
try {
- System.out.println("\nHTTP server started at http://" + Globals.getHost4Address() + ":1985/");
- System.out.println("Dashboard available at http://" + Globals.getHost4Address() + ":1985/dashboard");
- System.out.println("\nNode is 'online' and ready to contribute (took "+(System.currentTimeMillis()- Serengeti.startTime)+"ms to startup)");
+ System.out.printf("\nHTTP server started at http://%s:%d/%n",
+ Globals.getHost4Address(), Globals.port_default);
+ System.out.printf("Dashboard available at http://%s:%d/dashboard%n",
+ Globals.getHost4Address(), Globals.port_default);
+ System.out.printf("\nNode is 'online' and ready to contribute (took %dms to startup)%n",
+ System.currentTimeMillis() - Serengeti.startTime);
} catch (SocketException se) {
System.out.println("Could not start HTTP server started, IP lookup failed");
}
}
static InetAddress getMyIP() throws IOException {
- InetAddress IP = InetAddress.getLocalHost();
- return IP;
+ return InetAddress.getLocalHost();
+ }
+
+ private static ByteArrayOutputStream requestBodyToByteArray(HttpExchange t) throws IOException {
+ InputStream inputStream = t.getRequestBody();
+ ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
+ byte[] buffer = new byte[2048];
+ int read = 0;
+ while ((read = inputStream.read(buffer)) != -1) {
+ byteArrayOutputStream.write(buffer, 0, read);
+ }
+ return byteArrayOutputStream;
}
static class RootHandler implements HttpHandler {
@@ -196,7 +211,7 @@ public void handle(HttpExchange t) throws IOException {
}
t.sendResponseHeaders(200, Objects.requireNonNull(response).length());
OutputStream os = t.getResponseBody();
- os.write(response.getBytes());
+ os.write(Objects.requireNonNull(response).getBytes());
os.close();
}
}
@@ -204,8 +219,11 @@ static class DashboardHandler implements HttpHandler {
public void handle(HttpExchange t) throws IOException {
String response = null;
try {
- response = new Dashboard().IndexTemplate("http://"+t.getRequestHeaders().getFirst("Host"),
- t.getRequestURI().getPath());
+ response = new Dashboard()
+ .IndexTemplate(
+ "http://"+t.getRequestHeaders().getFirst("Host"),
+ t.getRequestURI().getPath()
+ );
} catch (URISyntaxException e) {
e.printStackTrace();
}
@@ -251,13 +269,7 @@ public void handle(HttpExchange t) throws IOException {
if (t.getRequestURI().toString().startsWith("/post?query")) {
//interactive!
- InputStream inputStream = t.getRequestBody();
- ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
- byte[] buffer = new byte[2048];
- int read = 0;
- while ((read = inputStream.read(buffer)) != -1) {
- byteArrayOutputStream.write(buffer, 0, read);
- }
+ ByteArrayOutputStream byteArrayOutputStream = requestBodyToByteArray(t);
JSONObject jsonObj = null;
try {
@@ -298,13 +310,7 @@ public void handle(HttpExchange t) throws IOException {
static Map getParameters(HttpExchange httpExchange) {
Map parameters = new HashMap<>();
try {
- InputStream inputStream = httpExchange.getRequestBody();
- ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
- byte[] buffer = new byte[2048];
- int read = 0;
- while ((read = inputStream.read(buffer)) != -1) {
- byteArrayOutputStream.write(buffer, 0, read);
- }
+ ByteArrayOutputStream byteArrayOutputStream = requestBodyToByteArray(httpExchange);
String[] keyValuePairs = byteArrayOutputStream.toString().split("&");
for (String keyValuePair : keyValuePairs) {
String[] keyValue = keyValuePair.split("=");
diff --git a/src/main/java/gl/ao/serengeti/storage/Storage.java b/src/main/java/gl/ao/serengeti/storage/Storage.java
index 14c5288..e9f088c 100644
--- a/src/main/java/gl/ao/serengeti/storage/Storage.java
+++ b/src/main/java/gl/ao/serengeti/storage/Storage.java
@@ -79,7 +79,7 @@ public void loadAllReplicaObjectsToMemory() {
* Get a List of existing Databases (use in-memory)
* @return List
*/
- public List getDatabases() {
+ public List getDatabases() {
return getDatabases(false);
}
@@ -90,12 +90,7 @@ public List getDatabases() {
*/
public List getDatabases(boolean getFromFileSystem) {
File dir = new File(Globals.data_path);
- File[] files = dir.listFiles(new FilenameFilter() {
- @Override
- public boolean accept(File dir, String name) {
- return name.endsWith(Globals.meta_extention);
- }
- });
+ File[] files = dir.listFiles((dir1, name) -> name.endsWith(Globals.meta_extention));
List ddbs = new ArrayList<>();
@@ -110,21 +105,16 @@ public boolean accept(File dir, String name) {
* Scan meta information and return a list of Databases and Tables included
* @return
*/
- public Map getDatabasesTablesMeta() {
+ public Map> getDatabasesTablesMeta() {
File dir = new File(Globals.data_path);
- File[] files = dir.listFiles(new FilenameFilter() {
- @Override
- public boolean accept(File dir, String name) {
- return name.endsWith(Globals.meta_extention);
- }
- });
+ File[] files = dir.listFiles((dir1, name) -> name.endsWith(Globals.meta_extention));
- Map ddbs = new HashMap<>();
+ Map> ddbs = new HashMap<>();
assert files != null;
for (File ddb : files) {
String dbName = ddb.getName().replace(Globals.meta_extention, "");
- List tables = getTables(dbName);
+ List tables = getTables(dbName);
ddbs.put(dbName, tables);
}
return ddbs;
@@ -148,7 +138,7 @@ public List select(String db, String table, String selectWhat, String co
if (tableExists(db, table)) {
List list = new ArrayList<>();
- Set uuids = new HashSet();
+ Set uuids = new HashSet();
JSONArray array = gl.ao.serengeti.Serengeti.network.communicateQueryLogAllNodes(new JSONObject() {{
put("type", "SelectRespond");
@@ -390,7 +380,7 @@ public boolean createDatabase(String db, boolean isReplicationAction) {
try {
DatabaseObject dbo = new DatabaseObject();
dbo.createNew(db, null);
- byte data[] = dbo.returnDBObytes();
+ byte[] data = dbo.returnDBObytes();
Path file = Paths.get(Globals.data_path + db + Globals.meta_extention);
Files.write(file, data);
@@ -488,9 +478,8 @@ public boolean tableExists(String db, String table) {
// DatabaseObject dbo = new DatabaseObject().loadExisting(file);
if (databases.containsKey(db)) {
DatabaseObject dbo = databases.get(db);
- if (dbo.tables != null && dbo.tables.size() > 0) {
+ if (dbo.tables != null && dbo.tables.size() > 0)
return dbo.tables.contains(table);
- }
}
return false;
}
@@ -551,13 +540,13 @@ public boolean dropTable(String db, String table, boolean isReplicationAction) {
* @param db
* @return List
*/
- public List getTables(String db) {
+ public List getTables(String db) {
// Path file = Paths.get(Globals.data_path + db + Globals.meta_extention);
// DatabaseObject dbo = new DatabaseObject().loadExisting(file);
DatabaseObject dbo = databases.get(db);
if (dbo.tables == null) return null;
else if (dbo.tables.size()>0) return dbo.tables;
- else return new ArrayList();
+ else return new ArrayList();
}
/***
@@ -644,7 +633,7 @@ public void deleteEverything() {
for (String db: dbs) {
try {
dropDatabase(db);
- } catch (Exception e) {}
+ } catch (Exception ignored) {}
}
}
diff --git a/src/main/java/gl/ao/serengeti/storage/StorageScheduler.java b/src/main/java/gl/ao/serengeti/storage/StorageScheduler.java
index c020950..3ea1f57 100644
--- a/src/main/java/gl/ao/serengeti/storage/StorageScheduler.java
+++ b/src/main/java/gl/ao/serengeti/storage/StorageScheduler.java
@@ -47,7 +47,7 @@ public boolean performPersistToDisk() {
DatabaseObject dbo = Storage.databases.get(key);
String dbName = dbo.name;
- List tables = dbo.tables;
+ List tables = dbo.tables;
byte[] data = dbo.returnDBObytes();
Path file = Paths.get(Globals.data_path + dbName + Globals.meta_extention);
diff --git a/src/test/java/gl/ao/serengeti/storage/StorageTest.java b/src/test/java/gl/ao/serengeti/storage/StorageTest.java
new file mode 100644
index 0000000..38a3852
--- /dev/null
+++ b/src/test/java/gl/ao/serengeti/storage/StorageTest.java
@@ -0,0 +1,48 @@
+package gl.ao.serengeti.storage;
+
+import gl.ao.serengeti.Serengeti;
+import org.junit.jupiter.api.*;
+
+import java.net.MalformedURLException;
+import java.net.URISyntaxException;
+import java.util.Random;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+class StorageTest {
+ Serengeti serengeti = new Serengeti();
+ Storage storage;
+
+ @BeforeEach
+ void beforeEach() {
+ storage = new Storage();
+ storage.deleteEverything();
+ }
+
+ @AfterEach
+ void afterEach() {
+ storage.deleteEverything();
+ }
+
+ @Test
+ void test_that_database_created_successfully() {
+ String random_db_name = String.format("test_db-%d", new Random().nextInt());
+
+ storage.createDatabase(random_db_name);
+
+ assertTrue(storage.getDatabases().contains(random_db_name));
+ }
+
+ @Test
+ void test_that_multiple_databases_created_successfully() {
+ storage.deleteEverything();
+
+ for (int i = 0; i < 100; i++) {
+ String random_db_name = String.format("test_db-%d", new Random().nextInt());
+
+ storage.createDatabase(random_db_name);
+
+ assertTrue(storage.getDatabases().contains(random_db_name));
+ }
+ }
+}