add export, fix read issues, decrease read/write queue to sync, fix s…

…tartup
NucleoTeam · Nov 15, 2023 · ca16b6f · ca16b6f
1 parent c851047
commit ca16b6f
Show file tree

Hide file tree

Showing 42 changed files with 878 additions and 609 deletions.
diff --git a/.gitignore b/.gitignore
@@ -3,4 +3,5 @@ out
 build
 .idea
 NucleoCore.iml
-data
+data/*.dat
+export/*.json
diff --git a/buildSrc/src/main/groovy/nucleodb.library.gradle b/buildSrc/src/main/groovy/nucleodb.library.gradle
@@ -3,7 +3,7 @@ plugins {
 }
 
 group = 'com.nucleodb'
-version = '1.9.41'
+version = '1.9.42'
 
 repositories {
     mavenCentral()

diff --git a/cli/restore/src/main/java/com/nucleocore/library/cli/DBRestore.java b/cli/restore/src/main/java/com/nucleocore/library/cli/DBRestore.java
@@ -1,7 +1,7 @@
 package com.nucleocore.library.cli;
 
 import com.nucleocore.library.NucleoDB;
-import com.nucleocore.library.database.tables.DataTable;
+import com.nucleocore.library.database.tables.table.DataTable;
 import com.nucleocore.library.database.utils.Serializer;
 import com.nucleocore.library.database.utils.StartupRun;
 

diff --git a/data/readme.txt b/data/readme.txt
@@ -0,0 +1 @@
+current db state saved here
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -1,42 +1,128 @@
-version: '2'
+version: "3.7"
+name: redpanda-quickstart
+networks:
+  redpanda_network:
+    driver: bridge
+volumes:
+  redpanda-0: null
+  redpanda-1: null
+  redpanda-2: null
 services:
-  zookeeper:
-    image: confluentinc/cp-zookeeper:latest
-    environment:
-      ZOOKEEPER_CLIENT_PORT: 2181
-      ZOOKEEPER_TICK_TIME: 2000
+  redpanda-0:
+    command:
+      - redpanda
+      - start
+      - --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:19092
+      # Address the broker advertises to clients that connect to the Kafka API.
+      # Use the internal addresses to connect to the Redpanda brokers'
+      # from inside the same Docker network.
+      # Use the external addresses to connect to the Redpanda brokers'
+      # from outside the Docker network.
+      - --advertise-kafka-addr internal://redpanda-0:9092,external://localhost:19092
+      - --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:18082
+      # Address the broker advertises to clients that connect to the HTTP Proxy.
+      - --advertise-pandaproxy-addr internal://redpanda-0:8082,external://localhost:18082
+      - --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:18081
+      # Redpanda brokers use the RPC API to communicate with eachother internally.
+      - --rpc-addr redpanda-0:33145
+      - --advertise-rpc-addr redpanda-0:33145
+      # Tells Seastar (the framework Redpanda uses under the hood) to use 1 core on the system.
+      - --smp 1
+      # The amount of memory to make available to Redpanda.
+      - --memory 1G
+      # Mode dev-container uses well-known configuration properties for development in containers.
+      - --mode dev-container
+      # enable logs for debugging.
+      - --default-log-level=debug
+    image: docker.redpanda.com/redpandadata/redpanda:v23.2.15
+    container_name: redpanda-0
     volumes:
-      - zookeeper_db:/var/lib/zookeeper/data
-      - zookeeper_log:/var/lib/zookeeper/log
-
-  kafka:
-    image: confluentinc/cp-kafka:latest
-    depends_on:
-      - zookeeper
+      - redpanda-0:/var/lib/redpanda/data
+    networks:
+      - redpanda_network
     ports:
-      - "29092:29092"
-    environment:
-      KAFKA_BROKER_ID: 1
-      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
-      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
-      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
-      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
-      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      - 18081:18081
+      - 18082:18082
+      - 19092:19092
+      - 19644:9644
+  redpanda-1:
+    command:
+      - redpanda
+      - start
+      - --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:29092
+      - --advertise-kafka-addr internal://redpanda-1:9092,external://localhost:29092
+      - --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:28082
+      - --advertise-pandaproxy-addr internal://redpanda-1:8082,external://localhost:28082
+      - --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:28081
+      - --rpc-addr redpanda-1:33145
+      - --advertise-rpc-addr redpanda-1:33145
+      - --smp 1
+      - --memory 1G
+      - --mode dev-container
+      - --default-log-level=debug
+      - --seeds redpanda-0:33145
+    image: docker.redpanda.com/redpandadata/redpanda:v23.2.15
+    container_name: redpanda-1
     volumes:
-      - kafka_db:/var/lib/kafka/data
-  memcache:
+      - redpanda-1:/var/lib/redpanda/data
+    networks:
+      - redpanda_network
     ports:
-      - "11211:11211"
-    build:
-      dockerfile: ./memcache/Dockerfile
+      - 28081:28081
+      - 28082:28082
+      - 29092:29092
+      - 29644:9644
+    depends_on:
+      - redpanda-0
+  redpanda-2:
+    command:
+      - redpanda
+      - start
+      - --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:39092
+      - --advertise-kafka-addr internal://redpanda-2:9092,external://localhost:39092
+      - --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:38082
+      - --advertise-pandaproxy-addr internal://redpanda-2:8082,external://localhost:38082
+      - --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:38081
+      - --rpc-addr redpanda-2:33145
+      - --advertise-rpc-addr redpanda-2:33145
+      - --smp 1
+      - --memory 1G
+      - --mode dev-container
+      - --default-log-level=debug
+      - --seeds redpanda-0:33145
+    image: docker.redpanda.com/redpandadata/redpanda:v23.2.15
+    container_name: redpanda-2
     volumes:
-      - data_backup:/data/
+      - redpanda-2:/var/lib/redpanda/data
+    networks:
+      - redpanda_network
+    ports:
+      - 38081:38081
+      - 38082:38082
+      - 39092:39092
+      - 39644:9644
+    depends_on:
+      - redpanda-0
+  console:
+    container_name: redpanda-console
+    image: docker.redpanda.com/redpandadata/console:v2.3.5
+    networks:
+      - redpanda_network
+    entrypoint: /bin/sh
+    command: -c 'echo "$$CONSOLE_CONFIG_FILE" > /tmp/config.yml; /app/console'
     environment:
-      PORT: 11211
-      KAFKA_HOSTS: kafka:9092
-      KAFKA_TOPIC: memcache
-volumes:
-  data_backup:
-  kafka_db:
-  zookeeper_db:
-  zookeeper_log:
+      CONFIG_FILEPATH: /tmp/config.yml
+      CONSOLE_CONFIG_FILE: |
+        kafka:
+          brokers: ["redpanda-0:9092"]
+          schemaRegistry:
+            enabled: true
+            urls: ["http://redpanda-0:8081"]
+        redpanda:
+          adminApi:
+            enabled: true
+            urls: ["http://redpanda-0:9644"]
+    ports:
+      - 8080:8080
+    depends_on:
+      - redpanda-0
diff --git a/export/readme.txt b/export/readme.txt
@@ -0,0 +1 @@
+exports saved in this folder
diff --git a/library/src/main/java/com/nucleocore/library/NucleoDB.java b/library/src/main/java/com/nucleocore/library/NucleoDB.java
@@ -2,18 +2,18 @@
 
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.google.common.collect.Queues;
-import com.nucleocore.library.database.tables.ConnectionHandler;
+import com.nucleocore.library.database.tables.connection.ConnectionConfig;
+import com.nucleocore.library.database.tables.connection.ConnectionHandler;
 import com.nucleocore.library.database.tables.annotation.Index;
 import com.nucleocore.library.database.tables.annotation.Relationship;
 import com.nucleocore.library.database.tables.annotation.Relationships;
 import com.nucleocore.library.database.tables.annotation.Table;
-import com.nucleocore.library.database.utils.Serializer;
 import com.nucleocore.library.database.utils.TreeSetExt;
 import com.nucleocore.library.database.utils.index.TreeIndex;
 import com.nucleocore.library.database.utils.sql.SQLHandler;
-import com.nucleocore.library.database.tables.DataTable;
-import com.nucleocore.library.database.tables.DataTableBuilder;
-import com.nucleocore.library.database.utils.DataEntry;
+import com.nucleocore.library.database.tables.table.DataTable;
+import com.nucleocore.library.database.tables.table.DataTableBuilder;
+import com.nucleocore.library.database.tables.table.DataEntry;
 import com.nucleocore.library.database.utils.StartupRun;
 import net.sf.jsqlparser.JSQLParserException;
 import net.sf.jsqlparser.parser.CCJSqlParserUtil;
@@ -27,8 +27,10 @@
 import java.lang.reflect.Field;
 import java.util.*;
 import java.util.concurrent.CountDownLatch;
+import java.util.logging.Logger;
 
 public class NucleoDB{
+  private static Logger logger = Logger.getLogger(DataTable.class.getName());
   private TreeMap<String, DataTable> tables = new TreeMap<>();
   static String latestSave = "";
 
@@ -37,33 +39,68 @@ public class NucleoDB{
   public NucleoDB() {
   }
 
-  public class TableData{
 
+  public enum DBType{
+    NO_LOCAL,
+    EXPORT,
+    ALL;
   }
 
   public NucleoDB(String bootstrap, String packageToScan) {
-    connectionHandler = new ConnectionHandler(this, bootstrap);
+    this(bootstrap, packageToScan, DBType.ALL);
+  }
+
+  public NucleoDB(String bootstrap, String packageToScan, DBType dbType) {
     Set<Class<?>> types = new Reflections(packageToScan).getTypesAnnotatedWith(Table.class);
-    CountDownLatch latch = new CountDownLatch(types.size());
+    CountDownLatch latch = new CountDownLatch(types.size()+1);
+    ConnectionConfig config = new ConnectionConfig();
+    config.setBootstrap(bootstrap);
+    config.setStartupRun(new StartupRun(){
+      public void run(ConnectionHandler connectionHandler) {
+        latch.countDown();
+      }
+    });
+    switch (dbType) {
+      case NO_LOCAL -> {
+        config.setSaveChanges(false);
+        config.setLoadSaved(false);
+      }
+      case EXPORT -> config.setJsonExport(true);
+    }
+    connectionHandler = new ConnectionHandler(this, config);
+
     Set<DataTableBuilder> tables = new TreeSetExt<>();
     Map<String, Set<String>> indexes = new TreeMap<>();
 
-    types.stream().forEach(type->{
+    types.stream().forEach(type -> {
       String tableName = type.getAnnotation(Table.class).value();
       processTableClass(tableName, indexes, type);
-      tables.add(launchTable(bootstrap, tableName, type, new StartupRun(){
-        public void run(DataTable table) {
-          latch.countDown();
-        }
-      }));
+      switch (dbType) {
+        case ALL -> tables.add(launchTable(bootstrap, tableName, type, new StartupRun(){
+          public void run(DataTable table) {
+            latch.countDown();
+          }
+        }));
+        case NO_LOCAL -> tables.add(launchLocalOnlyTable(bootstrap, tableName, type, new StartupRun(){
+          public void run(DataTable table) {
+            latch.countDown();
+          }
+        }));
+        case EXPORT -> tables.add(launchExportOnlyTable(bootstrap, tableName, type, new StartupRun(){
+          public void run(DataTable table) {
+            latch.countDown();
+          }
+        }));
+      }
     });
-    tables.stream().forEach(table->{
+    tables.stream().forEach(table -> {
       table.setIndexes(indexes.get(table.getConfig().getTable()).toArray(new String[0]));
       table.build();
     });
     try {
+      logger.info("NucleoDB Starting");
       latch.await();
-      System.out.println("NucleoDB Started");
+      logger.info("NucleoDB Started");
     } catch (InterruptedException e) {
       throw new RuntimeException(e);
     }
@@ -91,7 +128,7 @@ public Set<DataEntry> getRelated(DataEntry dataEntry, Class clazz) {
         if (relationship.clazz() == clazz) {
           getRelatedByRelationship(dataEntry, set, localTableName, relationship);
         } else {
-          //Serializer.log("Relation ignored");
+          //logger.info("Relation ignored");
         }
       }
     }
@@ -107,7 +144,7 @@ public Set<DataEntry> getRelatedRemote(DataEntry dataEntry, Class clazz, String
         if (relationship.clazz() == clazz && index.equals(relationship.remoteKey())) {
           getRelatedByRelationship(dataEntry, set, localTableName, relationship);
         } else {
-          //Serializer.log("Relation ignored");
+          //logger.info("Relation ignored");
         }
       }
     }
@@ -123,7 +160,7 @@ public Set<DataEntry> getRelatedLocal(DataEntry dataEntry, Class clazz, String i
         if (relationship.clazz() == clazz && index.equals(relationship.localKey())) {
           getRelatedByRelationship(dataEntry, set, localTableName, relationship);
         } else {
-          //Serializer.log("Relation ignored");
+          //logger.info("Relation ignored");
         }
       }
     }
@@ -134,9 +171,9 @@ private void getRelatedByRelationship(DataEntry dataEntry, Set<DataEntry> set, S
     if (relationship.clazz().isAnnotationPresent(Table.class)) {
       Table remoteTable = (Table) relationship.clazz().getAnnotation(Table.class);
       String remoteTableName = remoteTable.value();
-      //Serializer.log("getting for relationship from " + localTableName + " to " + remoteTableName);
+      //logger.info("getting for relationship from " + localTableName + " to " + remoteTableName);
       try {
-        //Serializer.log(relationship.localKey());
+        //logger.info(relationship.localKey());
         List<Object> values = new TreeIndex().getValues(Queues.newLinkedBlockingDeque(Arrays.asList(relationship.localKey().split("\\."))), dataEntry.getData());
         if (values.size() > 0) {
           for (Object value : values) {
@@ -147,7 +184,7 @@ private void getRelatedByRelationship(DataEntry dataEntry, Set<DataEntry> set, S
         e.printStackTrace();
       }
     } else {
-      Serializer.log("Target class not a NucleoDB Table.");
+      logger.info("Target class not a NucleoDB Table.");
     }
 
   }
@@ -295,11 +332,18 @@ public DataTableBuilder launchReadOnlyTable(String bootstrap, String table, Clas
   }
 
   public DataTableBuilder launchLocalOnlyTable(String bootstrap, String table, Class clazz) {
-    return DataTableBuilder.create(bootstrap, table, clazz).setRead(false).setWrite(false).setDb(this);
+    return DataTableBuilder.create(bootstrap, table, clazz).setLoadSave(false).setSaveChanges(false).setDb(this);
   }
 
   public DataTableBuilder launchLocalOnlyTable(String bootstrap, String table, Class clazz, StartupRun startupRun) {
-    return DataTableBuilder.create(bootstrap, table, clazz).setRead(false).setWrite(false).setDb(this).setStartupRun(startupRun);
+    return DataTableBuilder.create(bootstrap, table, clazz).setLoadSave(false).setSaveChanges(false).setDb(this).setStartupRun(startupRun);
+  }
+
+  public DataTableBuilder launchExportOnlyTable(String bootstrap, String table, Class clazz) {
+    return DataTableBuilder.create(bootstrap, table, clazz).setJSONExport(true).setDb(this);
+  }
+  public DataTableBuilder launchExportOnlyTable(String bootstrap, String table, Class clazz, StartupRun startupRun) {
+    return DataTableBuilder.create(bootstrap, table, clazz).setJSONExport(true).setDb(this).setStartupRun(startupRun);
   }
 
   public DataTableBuilder launchWriteOnlyTable(String bootstrap, String table, Class clazz) {

diff --git a/library/src/main/java/com/nucleocore/library/database/modifications/ConnectionCreate.java b/library/src/main/java/com/nucleocore/library/database/modifications/ConnectionCreate.java
@@ -1,9 +1,7 @@
 package com.nucleocore.library.database.modifications;
 
-import com.nucleocore.library.database.tables.Connection;
-import com.nucleocore.library.database.utils.DataEntry;
+import com.nucleocore.library.database.tables.connection.Connection;
 
-import java.io.IOException;
 import java.time.Instant;
 import java.util.UUID;