From e8c0782c8dc5b7229fdc4744410e27709461e2fb Mon Sep 17 00:00:00 2001 From: Yanyang Wang Date: Wed, 30 Sep 2020 14:00:31 +0100 Subject: [PATCH 01/58] basic cleanup --- .../core/components/Spout/SpoutTrait.scala | 159 ++++++++---------- .../com/raphtory/spouts/FirehoseSpout.scala | 20 +-- 2 files changed, 77 insertions(+), 102 deletions(-) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/SpoutTrait.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/SpoutTrait.scala index c3864e7377..4a357cd86d 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/SpoutTrait.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/SpoutTrait.scala @@ -2,21 +2,19 @@ package com.raphtory.core.components.Spout import akka.actor.Actor import akka.actor.ActorLogging -import akka.actor.Cancellable import akka.actor.Timers import akka.cluster.pubsub.DistributedPubSub import akka.cluster.pubsub.DistributedPubSubMediator -import akka.pattern.ask -import akka.util.Timeout +import com.raphtory.core.components.Spout.SpoutTrait.Message.IsSafe +import com.raphtory.core.components.Spout.SpoutTrait.Message.StartSpout +import com.raphtory.core.components.Spout.SpoutTrait.Message.StateCheck import com.raphtory.core.model.communication._ -import com.raphtory.core.utils.SchedulerUtil import kamon.Kamon + +import scala.concurrent.ExecutionContext //import kamon.metric.CounterMetric //import kamon.metric.GaugeMetric -import scala.collection.mutable -import scala.concurrent.Await -import scala.concurrent.duration.Duration import scala.concurrent.duration._ import scala.language.postfixOps @@ -24,116 +22,93 @@ import scala.language.postfixOps // e.g. BlockChainSpout val name = "Blockchain Spout" // Log.debug that read 'Spout' should then read 'Blockchain Spout' trait SpoutTrait extends Actor with ActorLogging with Timers { - case class StartSpout() - private var safe = false - implicit val executionContext = context.system.dispatchers.lookup("spout-dispatcher") - - private val scheduledTaskMap: mutable.HashMap[String, Cancellable] = mutable.HashMap[String, Cancellable]() - val spoutTuples = Kamon.counter("Raphtory_Spout_Tuples").withTag("actor",self.path.name) - var count = 0 - protected def recordUpdate(): Unit = { + // todo: wvv should assign the dispatcher when create the actor + // implicit val executionContext: ExecutionContext = context.system.dispatchers.lookup("spout-dispatcher") + implicit val executionContext: ExecutionContext = context.system.dispatcher + + private val spoutTuples = Kamon.counter("Raphtory_Spout_Tuples").withTag("actor", self.path.name) + private var count = 0 + private def recordUpdate(): Unit = { spoutTuples.increment() count += 1 } - final protected val mediator = DistributedPubSub(context.system).mediator - mediator ! DistributedPubSubMediator.Put(self) + private val mediator = DistributedPubSub(context.system).mediator override def preStart() { log.debug("Spout is being started.") - scheduleTasks() + mediator ! DistributedPubSubMediator.Put(self) + context.system.scheduler.scheduleOnce(7 seconds, self, StateCheck) + context.system.scheduler.scheduleOnce(1 seconds, self, IsSafe) } - override def postStop() { - val allTasksCancelled = scheduledTaskMap.forall { - case (key, task) => - SchedulerUtil.cancelTask(key, task) - } - if (!allTasksCancelled) log.warning("Failed to cancel all scheduled tasks post stop.") - } + final override def receive: Receive = work(false) - final override def receive: Receive = { - case msg: String if msg == "stateCheck" => processStateCheckMessage(msg) - case msg: String if msg == "isSafe" => processIsSafeMessage(msg) - case x => ProcessSpoutTask(x) // TODO How do we know this is a spout task? Add a trait which spout messages extend + private def work(safe: Boolean): Receive = { + case StateCheck => processStateCheckMessage(safe) + case ClusterStatusResponse(clusterUp) => + context.become(work(clusterUp)) + context.system.scheduler.scheduleOnce(1 second, self, StateCheck) + case IsSafe => processIsSafeMessage(safe) + case x => ProcessSpoutTask(x) // TODO How do we know this is a spout task? Add a trait which spout messages extend + case unhandled => log.error(s"Unable to handle message [$unhandled].") } - def start(): Unit = safe = true - - def stop(): Unit = safe = false - - protected def ProcessSpoutTask(receivedMessage: Any) - - private def processStateCheckMessage(msg: String): Unit = { - log.debug(s"Spout received [{}] message.", msg) + // todo: wvv what is actually can be received here? At least we will create a trait + def ProcessSpoutTask(receivedMessage: Any) - if (!safe) - try { - implicit val timeout: Timeout = Timeout(10 seconds) - - val sendMessage = ClusterStatusRequest() - val sendPath = "/user/WatchDog" - - log.debug(s"Sending DPSM message [{}] to path [{}].", sendMessage, sendPath) - - val future = - mediator ? DistributedPubSubMediator.Send(sendPath, sendMessage, localAffinity = false) - - safe = Await.result(future, timeout.duration).asInstanceOf[ClusterStatusResponse].clusterUp - - } catch { case _: java.util.concurrent.TimeoutException => safe = false } - } - - private def processIsSafeMessage(msg: String): Option[Cancellable] = { - log.debug(s"Spout received [{}] message.", msg) - - if (safe) { - val startSpoutCancellable = - SchedulerUtil.scheduleTaskOnce(delay = 1 millisecond, receiver = self, message = StartSpout) - scheduledTaskMap.put("startSpout", startSpoutCancellable) - } else { - val isSafeCancellable = SchedulerUtil.scheduleTaskOnce(delay = 1 second, receiver = self, message = "isSafe") - scheduledTaskMap.put("isSafe", isSafeCancellable) + private def processStateCheckMessage(safe: Boolean): Unit = { + log.debug(s"Spout is handling [StateCheck] message.") + if (!safe) { + val sendMessage = ClusterStatusRequest() + val sendPath = "/user/WatchDog" + log.debug(s"Sending DPSM message [$sendMessage] to path [$sendPath].") + mediator ! DistributedPubSubMediator.Send(sendPath, sendMessage, localAffinity = false) } } - def AllocateSpoutTask(duration: Duration, task: Any): Cancellable = { - val taskCancellable = SchedulerUtil.scheduleTaskOnce(Duration(duration._1, duration._2), self, task) - - // TODO - // scheduledTaskMap.put("passNameAsArgument", taskCancellable) - - taskCancellable + private def processIsSafeMessage(safe: Boolean): Unit = { + log.debug(s"Spout is handling [IsSafe] message.") + if (safe) + self ! StartSpout + else + context.system.scheduler.scheduleOnce(delay = 1 second, receiver = self, message = IsSafe) } + // need more works if we want to make it right. We should never use Any type e.g. AllocateTuple, AllocateTrackedTuple + // I will recommend to have + // case class StringSpout(value: String) extends SpoutGoing + // so all can be handled by the same method and all type safe. Following should be removed protected def sendTuple(command: String): Unit = { - log.debug("The command [{}] received for send.", command) + log.debug(s"The command [$command] received for send.") + val index = count % 10 recordUpdate() - if(count%100==0) - mediator ! DistributedPubSubMediator.Send(s"/user/router/routerWorker_${count % 10}", AllocateTrackedTuple(System.currentTimeMillis(),command), localAffinity = false) - else - mediator ! DistributedPubSubMediator.Send(s"/user/router/routerWorker_${count % 10}", AllocateTuple(command), localAffinity = false) } + val message = + if (count % 100 == 0) + AllocateTrackedTuple(System.currentTimeMillis(), command) + else + AllocateTuple(command) + mediator ! DistributedPubSubMediator.Send(s"/user/router/routerWorker_$index", message, localAffinity = false) + } protected def sendTuple[T <: SpoutGoing](command: T): Unit = { - log.debug("The command [{}] received for send.", command) + log.debug(s"The command [$command] received for send.") + val index = count % 10 recordUpdate() - if(count%100==0) - mediator ! DistributedPubSubMediator.Send(s"/user/router/routerWorker_${count % 10}", AllocateTrackedTuple(System.currentTimeMillis(),command), localAffinity = false) - else - mediator ! DistributedPubSubMediator.Send(s"/user/router/routerWorker_${count % 10}", AllocateTuple(command), localAffinity = false) + val message = + if (count % 100 == 0) + AllocateTrackedTuple(System.currentTimeMillis(), command) + else + AllocateTuple(command) + mediator ! DistributedPubSubMediator.Send(s"/user/router/routerWorker_$index", message, localAffinity = false) } +} - - private def scheduleTasks(): Unit = { - log.debug("Preparing to schedule tasks in Spout.") - - val stateCheckCancellable = - SchedulerUtil.scheduleTask(initialDelay = 7 seconds, interval = 1 second, receiver = self, message = "stateCheck") - scheduledTaskMap.put("stateCheck", stateCheckCancellable) - - val isSafeCancellable = - SchedulerUtil.scheduleTaskOnce(delay = 1 seconds, receiver = self, message = "isSafe") - scheduledTaskMap.put("isSafe", isSafeCancellable) +object SpoutTrait { + object Message { + case object StartSpout + case object StateCheck + case object IsSafe } } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/spouts/FirehoseSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/spouts/FirehoseSpout.scala index 7582312fc9..9fb8ee4b01 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/spouts/FirehoseSpout.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/spouts/FirehoseSpout.scala @@ -4,21 +4,21 @@ import java.io.{BufferedReader, File, FileReader} import java.time.LocalDateTime import com.raphtory.core.components.Spout.SpoutTrait +import com.raphtory.core.components.Spout.SpoutTrait.Message.StartSpout import scala.concurrent.duration.{Duration, MILLISECONDS, NANOSECONDS, SECONDS} import scala.io.Source -class FirehoseSpout extends SpoutTrait { +final case class FirehoseSpout() extends SpoutTrait { + log.info("initialise FirehoseSpout") + private val directory = System.getenv().getOrDefault("FILE_SPOUT_DIRECTORY", "/app").trim + private val fileName = System.getenv().getOrDefault("FILE_SPOUT_FILENAME", "").trim //gabNetwork500.csv + private val dropHeader = System.getenv().getOrDefault("FILE_SPOUT_DROP_HEADER", "false").trim.toBoolean + private var JUMP = System.getenv().getOrDefault("FILE_SPOUT_BLOCK_SIZE", "10").trim.toInt + private var INCREMENT = System.getenv().getOrDefault("FILE_SPOUT_INCREMENT", "1").trim.toInt + private var TIME = System.getenv().getOrDefault("FILE_SPOUT_TIME", "60").trim.toInt - println("Start: " + LocalDateTime.now()) - val directory = System.getenv().getOrDefault("FILE_SPOUT_DIRECTORY", "/app").trim - val fileName = System.getenv().getOrDefault("FILE_SPOUT_FILENAME", "").trim //gabNetwork500.csv - val dropHeader = System.getenv().getOrDefault("FILE_SPOUT_DROP_HEADER", "false").trim.toBoolean - var JUMP = System.getenv().getOrDefault("FILE_SPOUT_BLOCK_SIZE", "10").trim.toInt - var INCREMENT = System.getenv().getOrDefault("FILE_SPOUT_INCREMENT", "1").trim.toInt - var TIME = System.getenv().getOrDefault("FILE_SPOUT_TIME", "60").trim.toInt - - var directoryPosition = 0 + var directoryPosition = 0 val filesToRead = if(fileName.isEmpty) getListOfFiles(directory) From d39e348b654ee2c513f555493333bdbf265c4f9c Mon Sep 17 00:00:00 2001 From: Yanyang Wang Date: Thu, 1 Oct 2020 21:10:24 +0100 Subject: [PATCH 02/58] FireHoseSpout cleanup --- .../core/components/Spout/SpoutTrait.scala | 42 ++-- .../communication/raphtoryMessages.scala | 1 + .../com/raphtory/spouts/EtherFileReader.scala | 95 --------- .../scala/com/raphtory/spouts/FileSpout.scala | 84 -------- .../com/raphtory/spouts/FirehoseSpout.scala | 185 ++++++++++++------ .../com/raphtory/spouts/FirehoseSpoutEx.scala | 92 --------- 6 files changed, 137 insertions(+), 362 deletions(-) delete mode 100644 mainproject/cluster/src/main/scala/com/raphtory/spouts/EtherFileReader.scala delete mode 100644 mainproject/cluster/src/main/scala/com/raphtory/spouts/FileSpout.scala delete mode 100644 mainproject/cluster/src/main/scala/com/raphtory/spouts/FirehoseSpoutEx.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/SpoutTrait.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/SpoutTrait.scala index 4a357cd86d..87d7fec020 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/SpoutTrait.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/SpoutTrait.scala @@ -5,9 +5,10 @@ import akka.actor.ActorLogging import akka.actor.Timers import akka.cluster.pubsub.DistributedPubSub import akka.cluster.pubsub.DistributedPubSubMediator -import com.raphtory.core.components.Spout.SpoutTrait.Message.IsSafe -import com.raphtory.core.components.Spout.SpoutTrait.Message.StartSpout -import com.raphtory.core.components.Spout.SpoutTrait.Message.StateCheck +import com.raphtory.core.components.Spout.SpoutTrait.CommonMessage.IsSafe +import com.raphtory.core.components.Spout.SpoutTrait.CommonMessage.StartSpout +import com.raphtory.core.components.Spout.SpoutTrait.CommonMessage.StateCheck +import com.raphtory.core.components.Spout.SpoutTrait.DomainMessage import com.raphtory.core.model.communication._ import kamon.Kamon @@ -21,7 +22,7 @@ import scala.language.postfixOps // TODO Add val name which sub classes that extend this trait must overwrite // e.g. BlockChainSpout val name = "Blockchain Spout" // Log.debug that read 'Spout' should then read 'Blockchain Spout' -trait SpoutTrait extends Actor with ActorLogging with Timers { +trait SpoutTrait[Domain <: DomainMessage, Out <: SpoutGoing] extends Actor with ActorLogging with Timers { // todo: wvv should assign the dispatcher when create the actor // implicit val executionContext: ExecutionContext = context.system.dispatchers.lookup("spout-dispatcher") implicit val executionContext: ExecutionContext = context.system.dispatcher @@ -50,12 +51,14 @@ trait SpoutTrait extends Actor with ActorLogging with Timers { context.become(work(clusterUp)) context.system.scheduler.scheduleOnce(1 second, self, StateCheck) case IsSafe => processIsSafeMessage(safe) - case x => ProcessSpoutTask(x) // TODO How do we know this is a spout task? Add a trait which spout messages extend + case StartSpout => startSpout() + case x: Domain => handleDomainMessage(x) case unhandled => log.error(s"Unable to handle message [$unhandled].") } - // todo: wvv what is actually can be received here? At least we will create a trait - def ProcessSpoutTask(receivedMessage: Any) + def startSpout(): Unit + + def handleDomainMessage(message: Domain): Unit private def processStateCheckMessage(safe: Boolean): Unit = { log.debug(s"Spout is handling [StateCheck] message.") @@ -75,40 +78,23 @@ trait SpoutTrait extends Actor with ActorLogging with Timers { context.system.scheduler.scheduleOnce(delay = 1 second, receiver = self, message = IsSafe) } - // need more works if we want to make it right. We should never use Any type e.g. AllocateTuple, AllocateTrackedTuple - // I will recommend to have - // case class StringSpout(value: String) extends SpoutGoing - // so all can be handled by the same method and all type safe. Following should be removed - protected def sendTuple(command: String): Unit = { + protected def sendTuple(command: Out): Unit = { log.debug(s"The command [$command] received for send.") - val index = count % 10 - recordUpdate() - val message = - if (count % 100 == 0) - AllocateTrackedTuple(System.currentTimeMillis(), command) - else - AllocateTuple(command) - mediator ! DistributedPubSubMediator.Send(s"/user/router/routerWorker_$index", message, localAffinity = false) - } - - protected def sendTuple[T <: SpoutGoing](command: T): Unit = { - log.debug(s"The command [$command] received for send.") - val index = count % 10 - recordUpdate() val message = if (count % 100 == 0) AllocateTrackedTuple(System.currentTimeMillis(), command) else AllocateTuple(command) - mediator ! DistributedPubSubMediator.Send(s"/user/router/routerWorker_$index", message, localAffinity = false) + mediator ! DistributedPubSubMediator.Send(s"/user/router/routerWorker_${count % 10}", message, localAffinity = false) } } object SpoutTrait { - object Message { + object CommonMessage { case object StartSpout case object StateCheck case object IsSafe } + trait DomainMessage } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/model/communication/raphtoryMessages.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/model/communication/raphtoryMessages.scala index 603a4c2db1..dabc712570 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/model/communication/raphtoryMessages.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/model/communication/raphtoryMessages.scala @@ -20,6 +20,7 @@ sealed trait TrackedGraphUpdate{ } trait SpoutGoing +final case class StringSpoutGoing(value: String) extends SpoutGoing case class RouterUp(id: Int) case class PartitionUp(id: Int) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/spouts/EtherFileReader.scala b/mainproject/cluster/src/main/scala/com/raphtory/spouts/EtherFileReader.scala deleted file mode 100644 index 7d7d12437d..0000000000 --- a/mainproject/cluster/src/main/scala/com/raphtory/spouts/EtherFileReader.scala +++ /dev/null @@ -1,95 +0,0 @@ -package com.raphtory.spouts - -import java.io.{BufferedReader, File, FileReader} -import java.time.LocalDateTime - -import com.raphtory.core.components.Spout.SpoutTrait - -import scala.concurrent.duration.{Duration, MILLISECONDS, NANOSECONDS, SECONDS} -import scala.io.Source - -class EtherFileReader extends SpoutTrait { - - println("Start: " + LocalDateTime.now()) - val directory = System.getenv().getOrDefault("FILE_SPOUT_DIRECTORY", "/app").trim - val fileName = System.getenv().getOrDefault("FILE_SPOUT_FILENAME", "").trim //gabNetwork500.csv - val dropHeader = System.getenv().getOrDefault("FILE_SPOUT_DROP_HEADER", "false").trim.toBoolean - var JUMP = System.getenv().getOrDefault("FILE_SPOUT_BLOCK_SIZE", "100").trim.toInt - var INCREMENT = System.getenv().getOrDefault("FILE_SPOUT_INCREMENT", "1").trim.toInt - var TIME = System.getenv().getOrDefault("FILE_SPOUT_TIME", "60").trim.toInt - - var directoryPosition = 0 - - val filesToRead = if(fileName.isEmpty) - getListOfFiles(directory) - else - Array(directory + "/" + fileName) - - var currentFile = fileToArray(directoryPosition) - - object sortOrdering extends Ordering[String] { - def massage(filename:String) = filename.split("_")(0).replaceAll("/app/transactions","").toInt - def compare(key1: String, key2: String) = massage(key1).compareTo(massage(key2)) - } - - protected def ProcessSpoutTask(message: Any): Unit = message match { - case StartSpout => { - AllocateSpoutTask(Duration(1, NANOSECONDS), "nextLineBLock") - AllocateSpoutTask(Duration(60, SECONDS), "increase") - } - case "increase" => JUMP += INCREMENT ;AllocateSpoutTask(Duration(TIME, SECONDS), "increase") - case "nextLineBLock" => nextLineBlock() - case "nextFile" => nextFile() - case _ => println("message not recognized!") - } - - def nextLineBlock() = { - try { - for (i <- 1 to JUMP) { - val line = currentFile.readLine() - if(line!=null) - sendTuple(line) - else - throw new Exception - } - AllocateSpoutTask(Duration(1, MILLISECONDS), "nextLineBLock") - } - catch { - case e:Exception => AllocateSpoutTask(Duration(1, NANOSECONDS), "nextFile") - } - } - - def nextFile() = { - directoryPosition += 1 - if (filesToRead.length > directoryPosition) { - currentFile = fileToArray(directoryPosition) - AllocateSpoutTask(Duration(1, NANOSECONDS), "nextLineBLock") - } - else { - println("All files read "+ LocalDateTime.now()) - } - } - - - def fileToArray(pos:Int) ={ - println(s"Now reading ${filesToRead(pos)}") - if(dropHeader){ - val br = new BufferedReader(new FileReader(filesToRead(pos))) - br.readLine() - br - } - else - new BufferedReader(new FileReader(filesToRead(pos))) - } - - def getListOfFiles(dir: String):Array[String] = { - val d = new File(dir) - if (d.exists && d.isDirectory) { - d.listFiles.filter(f=> f.isFile && !f.isHidden).map(f=> f.getCanonicalPath).sorted(sortOrdering) - } else { - Array[String]() - } - } - -} - diff --git a/mainproject/cluster/src/main/scala/com/raphtory/spouts/FileSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/spouts/FileSpout.scala deleted file mode 100644 index 3b49e4534d..0000000000 --- a/mainproject/cluster/src/main/scala/com/raphtory/spouts/FileSpout.scala +++ /dev/null @@ -1,84 +0,0 @@ -package com.raphtory.spouts - -import java.io.File -import java.time.LocalDateTime - -import com.raphtory.core.components.Spout.SpoutTrait - -import scala.concurrent.duration.{Duration, NANOSECONDS,MILLISECONDS} -import scala.io.Source - -class FileSpout extends SpoutTrait { - - println("Start: " + LocalDateTime.now()) - println(System.getProperty("user.dir")) - - val directory = System.getenv().getOrDefault("FILE_SPOUT_DIRECTORY", "/Users/imasgo/Downloads/splitcsv-1ce17e2c-d1fb-4dcb-8305-b6d0b1b34fa4-results").trim - val fileName = System.getenv().getOrDefault("FILE_SPOUT_FILENAME", "transactions9000000_9100000-4.csv").trim //gabNetwork500.csv - val dropHeader = System.getenv().getOrDefault("FILE_SPOUT_DROP_HEADER", "false").trim.toBoolean - val JUMP = System.getenv().getOrDefault("FILE_SPOUT_BLOCK_SIZE", "50").trim.toInt - - var filePosition = 0 - var directoryPosition = 0 - - val filesToRead = if(fileName.isEmpty) - getListOfFiles(directory) - else - Array(directory + "/" + fileName) - - var currentFile = fileToArray(directoryPosition) - - - protected def ProcessSpoutTask(message: Any): Unit = message match { - case StartSpout => AllocateSpoutTask(Duration(1, NANOSECONDS), "nextLineBLock") - - case "nextLineBLock" => nextLineBlock() - case "nextFile" => nextFile() - case _ => println("message not recognized!") - } - - def nextLineBlock() = { - try { - for (i <- 1 to JUMP) { - sendTuple(currentFile(filePosition)) - filePosition += 1 - } - AllocateSpoutTask(Duration(1, MILLISECONDS), "nextLineBLock") - } - catch { - case e:Exception => AllocateSpoutTask(Duration(1, NANOSECONDS), "nextFile") - } - } - - def nextFile() = { - directoryPosition += 1 - if (filesToRead.length > directoryPosition) { - currentFile = fileToArray(directoryPosition) - AllocateSpoutTask(Duration(1, NANOSECONDS), "nextLineBLock") - } - else { - println("All files read "+ LocalDateTime.now()) - } - } - - - def fileToArray(pos:Int) ={ - println(s"Now reading ${filesToRead(pos)}") - if(dropHeader) - Source.fromFile(filesToRead(pos)).getLines.drop(1).toArray - else - print(Source.fromFile(filesToRead(pos)).getLines.toArray) - Source.fromFile(filesToRead(pos)).getLines.toArray - } - - def getListOfFiles(dir: String):Array[String] = { - val d = new File(dir) - if (d.exists && d.isDirectory) { - d.listFiles.filter(f=> f.isFile && !f.isHidden).map(f=> f.getCanonicalPath) - } else { - Array[String]() - } - } - -} - diff --git a/mainproject/cluster/src/main/scala/com/raphtory/spouts/FirehoseSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/spouts/FirehoseSpout.scala index 9fb8ee4b01..d23b377623 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/spouts/FirehoseSpout.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/spouts/FirehoseSpout.scala @@ -1,91 +1,150 @@ package com.raphtory.spouts -import java.io.{BufferedReader, File, FileReader} -import java.time.LocalDateTime +import java.io.BufferedReader +import java.io.File +import java.io.FileReader import com.raphtory.core.components.Spout.SpoutTrait -import com.raphtory.core.components.Spout.SpoutTrait.Message.StartSpout +import com.raphtory.core.model.communication.StringSpoutGoing +import com.raphtory.spouts.FirehoseSpout.Message.FireHouseDomain +import com.raphtory.spouts.FirehoseSpout.Message.Increase +import com.raphtory.spouts.FirehoseSpout.Message.NextFile +import com.raphtory.spouts.FirehoseSpout.Message.NextLineBlock +import com.typesafe.scalalogging.LazyLogging -import scala.concurrent.duration.{Duration, MILLISECONDS, NANOSECONDS, SECONDS} -import scala.io.Source +import scala.annotation.tailrec +import scala.concurrent.duration._ -final case class FirehoseSpout() extends SpoutTrait { +final case class FirehoseSpout() extends SpoutTrait[FireHouseDomain, StringSpoutGoing] { log.info("initialise FirehoseSpout") - private val directory = System.getenv().getOrDefault("FILE_SPOUT_DIRECTORY", "/app").trim - private val fileName = System.getenv().getOrDefault("FILE_SPOUT_FILENAME", "").trim //gabNetwork500.csv + private val directory = System.getenv().getOrDefault("FILE_SPOUT_DIRECTORY", "/app").trim + private val fileName = System.getenv().getOrDefault("FILE_SPOUT_FILENAME", "").trim //gabNetwork500.csv private val dropHeader = System.getenv().getOrDefault("FILE_SPOUT_DROP_HEADER", "false").trim.toBoolean - private var JUMP = System.getenv().getOrDefault("FILE_SPOUT_BLOCK_SIZE", "10").trim.toInt - private var INCREMENT = System.getenv().getOrDefault("FILE_SPOUT_INCREMENT", "1").trim.toInt - private var TIME = System.getenv().getOrDefault("FILE_SPOUT_TIME", "60").trim.toInt + private val JUMP = System.getenv().getOrDefault("FILE_SPOUT_BLOCK_SIZE", "10").trim.toInt + private val INCREMENT = System.getenv().getOrDefault("FILE_SPOUT_INCREMENT", "1").trim.toInt + private val TIME = System.getenv().getOrDefault("FILE_SPOUT_TIME", "60").trim.toInt - var directoryPosition = 0 + private var fileManager = FileManager(directory, fileName, dropHeader, JUMP) - val filesToRead = if(fileName.isEmpty) - getListOfFiles(directory) - else - Array(directory + "/" + fileName) + def startSpout(): Unit = { + self ! NextLineBlock + // todo: wvv not sure why we need to keep increasing + context.system.scheduler.scheduleOnce(TIME.seconds, self, Increase) + } - var currentFile = fileToArray(directoryPosition) + def handleDomainMessage(message: FireHouseDomain): Unit = message match { + case Increase => + if (fileManager.allCompleted) + log.info("All files read") + else { + fileManager = fileManager.increaseBlockSize(INCREMENT) + context.system.scheduler.scheduleOnce(TIME.seconds, self, Increase) + } + case NextLineBlock => + if (fileManager.allCompleted) + log.info("All files read") + else { + val (newFileManager, block) = fileManager.nextLineBlock() + fileManager = newFileManager + block.foreach(str => sendTuple(StringSpoutGoing(str))) + self ! NextLineBlock + } + case NextFile => + if (fileManager.allCompleted) + log.info("All files read") + else { + fileManager = fileManager.nextFile() + self ! NextLineBlock + } + } +} - protected def ProcessSpoutTask(message: Any): Unit = message match { - case StartSpout => { - AllocateSpoutTask(Duration(1, NANOSECONDS), "nextLineBLock") - AllocateSpoutTask(Duration(60, SECONDS), "increase") - } - case "increase" => JUMP += INCREMENT ;AllocateSpoutTask(Duration(TIME, SECONDS), "increase") - case "nextLineBLock" => nextLineBlock() - case "nextFile" => nextFile() - case _ => println("message not recognized!") +object FirehoseSpout { + object Message { + sealed trait FireHouseDomain + case object Increase extends FireHouseDomain + case object NextLineBlock extends FireHouseDomain + case object NextFile extends FireHouseDomain } +} - def nextLineBlock() = { - try { - for (i <- 1 to JUMP) { - val line = currentFile.readLine() - if(line!=null) - sendTuple(line) - else - throw new Exception +case class FileManager private ( + currentFileReader: Option[BufferedReader], + restFiles: List[File], + dropHeader: Boolean, + blockSize: Int +) extends LazyLogging { + def nextFile(): FileManager = this.copy(currentFileReader = None) + + lazy val allCompleted: Boolean = currentFileReader.isEmpty && restFiles.isEmpty + + def increaseBlockSize(inc: Int): FileManager = this.copy(blockSize = blockSize + inc) + + def nextLineBlock(): (FileManager, List[String]) = currentFileReader match { + case None => + restFiles match { + case Nil => (this, List.empty) + case head :: tail => + val reader = getFileReader(head) + val (block, endOfFile) = readBlockAndIsEnd(reader) + val currentReader = if (endOfFile) None else Some(reader) + (this.copy(currentFileReader = currentReader, restFiles = tail), block) } - AllocateSpoutTask(Duration(1, MILLISECONDS), "nextLineBLock") - } - catch { - case e:Exception => AllocateSpoutTask(Duration(1, NANOSECONDS), "nextFile") - } - } + case Some(reader) => + val (block, endOfFile) = readBlockAndIsEnd(reader) + if (endOfFile) (this.copy(currentFileReader = None), block) + else (this, block) - def nextFile() = { - directoryPosition += 1 - if (filesToRead.length > directoryPosition) { - currentFile = fileToArray(directoryPosition) - AllocateSpoutTask(Duration(1, NANOSECONDS), "nextLineBLock") - } - else { - println("All files read "+ LocalDateTime.now()) - } } + private def readBlockAndIsEnd(reader: BufferedReader): (List[String], Boolean) = { + @tailrec + def rec(count: Int, result: List[String]): (List[String], Boolean) = + if (count > 0) { + val line = reader.readLine() + if (line != null) + rec(count - 1, result :+ line) + else (result, true) + } else (result, false) + rec(blockSize, List.empty) + } - def fileToArray(pos:Int) ={ - println(s"Now reading ${filesToRead(pos)}") - if(dropHeader){ - val br = new BufferedReader(new FileReader(filesToRead(pos))) + private def getFileReader(file: File): BufferedReader = { + logger.info(s"Reading file ${file.getCanonicalPath}") + if (dropHeader) { + val br = new BufferedReader(new FileReader(file)) br.readLine() br - } - else - new BufferedReader(new FileReader(filesToRead(pos))) + } else + new BufferedReader(new FileReader(file)) } +} - def getListOfFiles(dir: String):Array[String] = { +object FileManager extends LazyLogging { + def apply(dir: String, fileName: String, dropHeader: Boolean, blockSize: Int): FileManager = { + val filesToRead = + if (fileName.isEmpty) + getListOfFiles(dir) + else { + val file = new File(dir + "/" + fileName) + if (file.exists && file.isFile) + List(file) + else { + logger.error(s"File $dir/$fileName does not exist or is not file ") + List.empty + } + } + FileManager(None, filesToRead, dropHeader, blockSize) + } + + private def getListOfFiles(dir: String): List[File] = { val d = new File(dir) - if (d.exists && d.isDirectory) { - d.listFiles.filter(f=> f.isFile && !f.isHidden).map(f=> f.getCanonicalPath).sorted - } else { - Array[String]() + if (d.exists && d.isDirectory) + d.listFiles.toList.filter(f => f.isFile && !f.isHidden) + else { + logger.error(s"Directory $dir does not exist or is not directory") + List.empty } } - } - diff --git a/mainproject/cluster/src/main/scala/com/raphtory/spouts/FirehoseSpoutEx.scala b/mainproject/cluster/src/main/scala/com/raphtory/spouts/FirehoseSpoutEx.scala deleted file mode 100644 index 00a0ff7ffc..0000000000 --- a/mainproject/cluster/src/main/scala/com/raphtory/spouts/FirehoseSpoutEx.scala +++ /dev/null @@ -1,92 +0,0 @@ -package com.raphtory.spouts - -import java.io.{BufferedReader, File, FileReader} -import java.time.LocalDateTime - -import com.raphtory.core.components.Spout.SpoutTrait - -import scala.concurrent.duration.{Duration, MILLISECONDS, NANOSECONDS, SECONDS} -import scala.io.Source - -class FirehoseSpoutEx extends SpoutTrait { - - println("Start: " + LocalDateTime.now()) - val directory = System.getenv().getOrDefault("FILE_SPOUT_DIRECTORY", "/app").trim - val fileName = System.getenv().getOrDefault("FILE_SPOUT_FILENAME", "").trim //gabNetwork500.csv - val dropHeader = System.getenv().getOrDefault("FILE_SPOUT_DROP_HEADER", "false").trim.toBoolean - var JUMP = System.getenv().getOrDefault("FILE_SPOUT_BLOCK_SIZE", "100").trim.toInt - var INCREMENT = System.getenv().getOrDefault("FILE_SPOUT_INCREMENT", "1").trim.toInt - var TIME = System.getenv().getOrDefault("FILE_SPOUT_TIME", "60").trim.toInt - - var directoryPosition = 0 - - val filesToRead = if(fileName.isEmpty) - getListOfFiles(directory) - else - Array(directory + "/" + fileName) - - var currentFile = fileToArray(directoryPosition) - - - protected def ProcessSpoutTask(message: Any): Unit = message match { - case StartSpout => { - AllocateSpoutTask(Duration(1, NANOSECONDS), "nextLineBLock") - AllocateSpoutTask(Duration(60, SECONDS), "increase") - } - case "increase" => JUMP += (JUMP/10) ;AllocateSpoutTask(Duration(TIME, SECONDS), "increase") - case "nextLineBLock" => nextLineBlock() - case "nextFile" => nextFile() - case _ => println("message not recognized!") - } - - def nextLineBlock() = { - try { - for (i <- 1 to JUMP) { - val line = currentFile.readLine() - if(line!=null) - sendTuple(line) - else - throw new Exception - } - AllocateSpoutTask(Duration(1, MILLISECONDS), "nextLineBLock") - } - catch { - case e:Exception => AllocateSpoutTask(Duration(1, NANOSECONDS), "nextFile") - } - } - - def nextFile() = { - directoryPosition += 1 - if (filesToRead.length > directoryPosition) { - currentFile = fileToArray(directoryPosition) - AllocateSpoutTask(Duration(1, NANOSECONDS), "nextLineBLock") - } - else { - println("All files read "+ LocalDateTime.now()) - } - } - - - def fileToArray(pos:Int) ={ - println(s"Now reading ${filesToRead(pos)}") - if(dropHeader){ - val br = new BufferedReader(new FileReader(filesToRead(pos))) - br.readLine() - br - } - else - new BufferedReader(new FileReader(filesToRead(pos))) - } - - def getListOfFiles(dir: String):Array[String] = { - val d = new File(dir) - if (d.exists && d.isDirectory) { - d.listFiles.filter(f=> f.isFile && !f.isHidden).map(f=> f.getCanonicalPath).sorted - } else { - Array[String]() - } - } - -} - - From 48f05c67190b231206f9c8aa928ba53b6faf5f37 Mon Sep 17 00:00:00 2001 From: Yanyang Wang Date: Fri, 2 Oct 2020 11:41:36 +0100 Subject: [PATCH 03/58] clean up KafkaSpout --- .../com/raphtory/spouts/FirehoseSpout.scala | 7 +- .../com/raphtory/spouts/KafkaSpout.scala | 113 +++++++++--------- 2 files changed, 61 insertions(+), 59 deletions(-) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/spouts/FirehoseSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/spouts/FirehoseSpout.scala index d23b377623..bfb85d1988 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/spouts/FirehoseSpout.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/spouts/FirehoseSpout.scala @@ -69,7 +69,7 @@ object FirehoseSpout { } } -case class FileManager private ( +final case class FileManager private ( currentFileReader: Option[BufferedReader], restFiles: List[File], dropHeader: Boolean, @@ -105,7 +105,10 @@ case class FileManager private ( val line = reader.readLine() if (line != null) rec(count - 1, result :+ line) - else (result, true) + else { + reader.close() + (result, true) + } } else (result, false) rec(blockSize, List.empty) } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/spouts/KafkaSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/spouts/KafkaSpout.scala index 5f608d4739..e3b301b3e9 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/spouts/KafkaSpout.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/spouts/KafkaSpout.scala @@ -4,77 +4,76 @@ import java.util import java.util.Properties import com.raphtory.core.components.Spout.SpoutTrait +import com.raphtory.core.components.Spout.SpoutTrait.DomainMessage +import com.raphtory.core.model.communication.StringSpoutGoing +import com.raphtory.spouts.KafkaSpout.Message.Domain +import com.raphtory.spouts.KafkaSpout.Message.NewBlock import org.apache.kafka.clients.consumer.KafkaConsumer +import scala.annotation.tailrec import scala.collection.JavaConverters._ -import scala.concurrent.duration.Duration -import scala.concurrent.duration.MILLISECONDS -import scala.concurrent.duration.SECONDS -import java.util.concurrent.LinkedBlockingQueue - -import akka.actor.Props -import com.raphtory.core.components.Router.RouterManager - -import scala.concurrent.duration.Duration import scala.concurrent.duration._ -import com.raphtory.core.utils.SchedulerUtil - import scala.util.Random -class KafkaSpout extends SpoutTrait { - println("Starting kafka") - var kafkaServer = System.getenv().getOrDefault("KAFKA_ADDRESS", "127.0.0.1").trim - var kafkaIP = System.getenv().getOrDefault("KAFKA_PORT", "9092").trim - var offset = System.getenv().getOrDefault("KAFKA_OFFSET", "earliest").trim - val x = new Random().nextLong() - var groupID = System.getenv().getOrDefault("KAFKA_GROUP", "group" + x).trim - var topic = System.getenv().getOrDefault("KAFKA_TOPIC", "sample_topic").trim - var restart = System.getenv().getOrDefault("RESTART_RATE", "10").trim - val queue = new LinkedBlockingQueue[String] +final case class KafkaSpout() extends SpoutTrait[Domain, StringSpoutGoing] { + log.info("initialising KafkaSpout") + private val kafkaServer = System.getenv().getOrDefault("KAFKA_ADDRESS", "127.0.0.1").trim + private val kafkaIp = System.getenv().getOrDefault("KAFKA_PORT", "9092").trim + private val offset = System.getenv().getOrDefault("KAFKA_OFFSET", "earliest").trim + private val groupId = System.getenv().getOrDefault("KAFKA_GROUP", "group" + Random.nextLong()).trim + private val topic = System.getenv().getOrDefault("KAFKA_TOPIC", "sample_topic").trim + private val restart = System.getenv().getOrDefault("RESTART_RATE", "10").trim.toInt + private val startingSpeed = System.getenv().getOrDefault("STARTING_SPEED", "1000").trim.toInt - val props = new Properties() - props.put("bootstrap.servers", s"$kafkaServer:$kafkaIP") - props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer") - props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer") - props.put("auto.offset.reset", offset) - props.put("group.id", groupID) - val consumer: KafkaConsumer[String, String] = new KafkaConsumer[String, String](props) - consumer.subscribe(util.Arrays.asList(topic)) + private var kafkaManager = KafkaManager(kafkaServer, kafkaIp, groupId, topic, offset) + override def startSpout(): Unit = + self ! NewBlock - val helper = context.system.actorOf(Props(new KafkaSpoutBackPressure(queue)), "Spout_Helper") - - protected def ProcessSpoutTask(message: Any): Unit = message match { - case StartSpout => AllocateSpoutTask(Duration(1, MILLISECONDS), "newLine") - case "newLine" => consumeFromKafka() - case _ => println("message not recognized!") + def handleDomainMessage(message: Domain): Unit = message match { + case NewBlock => + val (newManager, block) = kafkaManager.nextNLine(startingSpeed / 100) + kafkaManager = newManager + block.foreach(str => sendTuple(StringSpoutGoing(str))) + context.system.scheduler.scheduleOnce(restart.millis, self, NewBlock) } +} - def consumeFromKafka() = { - //println("Consuming") - val record = consumer.poll(java.time.Duration.ofMillis(3000)).asScala - for (data <- record.iterator) { - //sendTuple(data.value()) - helper ! KafkaData(data.value()) - } - AllocateSpoutTask(Duration(restart.toInt, MILLISECONDS), "newLine") +object KafkaSpout { + object Message { + sealed trait Domain extends DomainMessage + case object NewBlock extends Domain } } -case class KafkaData(data:String) -class KafkaSpoutBackPressure(queue:LinkedBlockingQueue[String]) extends SpoutTrait { - var startingSpeed = System.getenv().getOrDefault("STARTING_SPEED", "1000").trim.toInt - override protected def ProcessSpoutTask(receivedMessage: Any): Unit = receivedMessage match { - case StartSpout => AllocateSpoutTask(Duration(1, MILLISECONDS), "newLine") - case KafkaData(data) => queue.put(data) - case "newLine" => consumeFromQueue - case _ => println("message not recognized!") + +final case class KafkaManager private (buffer: Stream[String], consumer: KafkaConsumer[String, String]) { + private def poll(): KafkaManager = { + // this is blocking operation which may waster some resource. + // But it should only block when no data which may still make sense. + val incoming = consumer.poll(java.time.Duration.ofMillis(3000)).asScala.toList.map(_.value()) + this.copy(buffer = buffer ++ incoming) } - def consumeFromQueue() = { - for(i<-0 to startingSpeed/100){ - if(!queue.isEmpty) { - sendTuple(queue.take()) - } + + @tailrec + def nextNLine(blockSize: Int): (KafkaManager, List[String]) = + if (buffer.isEmpty) + poll().nextNLine(blockSize) + else { + val (take, rest) = buffer.splitAt(blockSize) + (this.copy(buffer = rest), take.toList) } - AllocateSpoutTask(Duration(10, MILLISECONDS), "newLine") +} + +object KafkaManager { + def apply(server: String, ip: String, groupId: String, topic: String, offset: String): KafkaManager = { + val props = new Properties() + props.put("bootstrap.servers", s"$server:$ip") + props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer") + props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer") + props.put("auto.offset.reset", offset) + props.put("group.id", groupId) + val consumer: KafkaConsumer[String, String] = new KafkaConsumer[String, String](props) + consumer.subscribe(util.Arrays.asList(topic)) + KafkaManager(Stream.empty, consumer) } } From 48e9572b784ba71dabde5dc0dd21bfcb7ab0e306 Mon Sep 17 00:00:00 2001 From: Yanyang Wang Date: Fri, 9 Oct 2020 22:05:17 +0100 Subject: [PATCH 04/58] optimise Router --- .../RaphtoryReplicator.scala | 24 +- .../components/Router/RouterManager.scala | 109 +++----- .../core/components/Router/RouterWorker.scala | 232 ++++++++---------- 3 files changed, 147 insertions(+), 218 deletions(-) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/ClusterManagement/RaphtoryReplicator.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/ClusterManagement/RaphtoryReplicator.scala index 1a17a21890..c669a76d5d 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/ClusterManagement/RaphtoryReplicator.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/ClusterManagement/RaphtoryReplicator.scala @@ -1,29 +1,20 @@ package com.raphtory.core.components.ClusterManagement -import akka.actor.Actor -import akka.actor.ActorLogging -import akka.actor.ActorRef -import akka.actor.Cancellable -import akka.actor.Props -import akka.cluster.pubsub.DistributedPubSub -import akka.cluster.pubsub.DistributedPubSubMediator +import akka.actor.{Actor, ActorLogging, ActorRef, Cancellable, Props} +import akka.cluster.pubsub.{DistributedPubSub, DistributedPubSubMediator} import akka.cluster.pubsub.DistributedPubSubMediator.SubscribeAck import akka.pattern.ask import akka.util.Timeout import com.raphtory.core.components.PartitionManager.Workers.IngestionWorker -import com.raphtory.core.components.PartitionManager.Archivist -import com.raphtory.core.components.PartitionManager.Reader -import com.raphtory.core.components.PartitionManager.Writer +import com.raphtory.core.components.PartitionManager.{Archivist, Reader, Writer} import com.raphtory.core.components.Router.RouterManager import com.raphtory.core.model.communication._ import com.raphtory.core.storage.EntityStorage -import com.raphtory.core.utils.SchedulerUtil -import com.raphtory.core.utils.Utils +import com.raphtory.core.utils.{SchedulerUtil, Utils} import scala.collection.mutable import scala.collection.parallel.mutable.ParTrieMap -import scala.concurrent.Await -import scala.concurrent.Future +import scala.concurrent.{Await, Future} import scala.concurrent.duration._ import scala.language.postfixOps @@ -159,7 +150,10 @@ class RaphtoryReplicator(actorType: String, initialManagerCount: Int, routerName def createNewRouter(assignedId: Int): Unit = { log.info(s"Router $assignedId has come online.") - actorRef = context.system.actorOf(Props(new RouterManager(myId, currentCount, routerName)), "router") + actorRef = context.system.actorOf( + Props(new RouterManager(myId, currentCount, routerName)).withDispatcher("misc-dispatcher"), + "router" + ) } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterManager.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterManager.scala index d0284f1ce5..9af89ad43f 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterManager.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterManager.scala @@ -3,107 +3,76 @@ package com.raphtory.core.components.Router import akka.actor.Actor import akka.actor.ActorLogging import akka.actor.ActorRef -import akka.actor.Cancellable import akka.actor.Props import akka.cluster.pubsub.DistributedPubSub import akka.cluster.pubsub.DistributedPubSubMediator +import com.raphtory.core.components.Router.RouterManager.Message.KeepAlive import com.raphtory.core.model.communication._ -import com.raphtory.core.utils.SchedulerUtil -import com.raphtory.core.utils.Utils.getManager -import kamon.Kamon + +import scala.concurrent.ExecutionContext //import kamon.metric.CounterMetric //import kamon.metric.GaugeMetric -import scala.collection.mutable -import scala.collection.parallel.mutable.ParTrieMap import scala.concurrent.duration._ import scala.language.postfixOps // TODO break object RouterManager { getProps = { routerManager(arg1, arg2...): Props }} -class RouterManager(val routerId: Int, val initialManagerCount: Int, slaveType: String) +// todo slave type should be Props +final case class RouterManager(routerId: Int, initialManagerCount: Int, slaveType: String) extends Actor with ActorLogging { - implicit val executionContext = context.system.dispatchers.lookup("misc-dispatcher") - - private var managerCount: Int = initialManagerCount - private var count = 0 + implicit val executionContext: ExecutionContext = context.system.dispatcher - private val children = 10 - private var childMap: ParTrieMap[Int, ActorRef] = ParTrieMap[Int, ActorRef]() - private val scheduledTaskMap: mutable.HashMap[String, Cancellable] = mutable.HashMap[String, Cancellable]() + private val childrenNumber = 10 + private val children = (0 until childrenNumber).map { i => + context.actorOf( + Props(Class.forName(slaveType), routerId, i, initialManagerCount).withDispatcher("router-dispatcher"), + s"routerWorker_$i" + ) + }.toList - final protected val mediator = DistributedPubSub(context.system).mediator + private val mediator = DistributedPubSub(context.system).mediator mediator ! DistributedPubSubMediator.Put(self) // val kGauge: GaugeMetric = Kamon.gauge("raphtory.benchmarker") // val kCounter: CounterMetric = Kamon.counter("raphtory.counters") override def preStart(): Unit = { - log.debug("RouterManager [{}} is being started.", routerId) - - scheduleTasks() - - for (i <- 0 until children) { - childMap.put(i, context.actorOf(Props(Class.forName(slaveType), routerId, i, initialManagerCount).withDispatcher("router-dispatcher"), s"routerWorker_$i")) - } - } - - override def postStop() { - val allTasksCancelled = scheduledTaskMap.forall { - case (key, task) => - SchedulerUtil.cancelTask(key, task) - } - - if (!allTasksCancelled) log.warning("Failed to cancel all scheduled tasks post stop.") + log.debug(s"RouterManager [$routerId] is being started.") + context.system.scheduler.schedule(0 seconds, 10 seconds, self, KeepAlive) } - override def receive: Receive = { - case msg: String if msg == "keep_alive" => processKeepAliveMessage(msg) - case msg: UpdatedCounter => processUpdatedCounterRequest(msg) + override def receive: Receive = work(initialManagerCount, children) + + private def work(managerCount: Int, children: List[ActorRef]): Receive = { + case KeepAlive => processKeepAliveMessage() + case msg: UpdatedCounter => + log.debug(s"RouterManager [$routerId] received [$msg] request.") + + if (managerCount < msg.newValue) { + log.debug("UpdatedCounter is larger than current managerCount. Bumping managerCount to new value.") + context.become(work(managerCount, children)) + } + children.foreach { actorRef => + log.debug("Propagating UpdatedCounter to child [{}].", actorRef.path) + actorRef ! msg + } + case unhandled => log.warning(s"cannot handle $unhandled") } - private def processKeepAliveMessage(msg: String): Unit = { - log.debug(s"RouterManager [{}] received [{}] message.", routerId, msg) + private def processKeepAliveMessage(): Unit = { + log.debug(s"RouterManager [$routerId] received [KeepAlive] message.") val sendMessage = RouterUp(routerId) val sendPath = "/user/WatchDog" - log.debug(s"Sending DPSM message [{}] to path [{}].", sendMessage, sendPath) - + log.debug(s"Sending DPSM message [$sendMessage] to path [$sendPath].") mediator ! DistributedPubSubMediator.Send(sendPath, sendMessage, localAffinity = false) } +} - def processUpdatedCounterRequest(msg: UpdatedCounter): Unit = { - log.debug(s"RouterManager [{}] received [{}] request.", routerId, msg) - - if (managerCount < msg.newValue) { - log.debug("UpdatedCounter is larger than current managerCount. Bumping managerCount to new value.") - - managerCount = msg.newValue - } - - childMap.values.foreach { actorRef => - log.debug("Propagating UpdatedCounter to child [{}].", actorRef.path) - - actorRef ! UpdatedCounter(msg.newValue) - } - } - - - final protected def getManagerCount: Int = - this.managerCount - - private def scheduleTasks(): Unit = { - log.debug("Preparing to schedule tasks in RouterManager [{}].", routerId) - - val keepAliveCancellable = - SchedulerUtil - .scheduleTask(initialDelay = 0 seconds, interval = 10 seconds, receiver = self, message = "keep_alive") - scheduledTaskMap.put("keep_alive", keepAliveCancellable) +object RouterManager { + object Message { + case object KeepAlive } - - def toPartitionManager[T <: GraphUpdate](message: T): Unit = - mediator ! DistributedPubSubMediator - .Send(getManager(message.srcID, getManagerCount), message, localAffinity = false) - } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala index f1303bb258..79ca540585 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala @@ -1,170 +1,136 @@ package com.raphtory.core.components.Router -import java.util.concurrent.atomic.AtomicInteger - -import akka.actor.{Actor, ActorLogging, Cancellable} +import akka.actor.Actor +import akka.actor.ActorLogging import akka.cluster.pubsub.DistributedPubSub import akka.cluster.pubsub.DistributedPubSubMediator -import com.raphtory.core.model.communication.{AllocateTrackedTuple, AllocateTuple, EdgeAdd, EdgeAddWithProperties, EdgeDelete, GraphUpdate, RouterWorkerTimeSync, TrackedEdgeAdd, TrackedEdgeAddWithProperties, TrackedEdgeDelete, TrackedVertexAdd, TrackedVertexAddWithProperties, TrackedVertexDelete, UpdateArrivalTime, UpdatedCounter, VertexAdd, VertexAddWithProperties, VertexDelete, WatermarkTime} -import com.raphtory.core.model.graphentities.Vertex -import com.raphtory.core.utils.{SchedulerUtil, Utils} +import com.raphtory.core.components.Router.RouterWorker.CommonMessage.TimeBroadcast +import com.raphtory.core.model.communication._ +import com.raphtory.core.utils.Utils import com.raphtory.core.utils.Utils.getManager import kamon.Kamon -import scala.concurrent.duration._ -import scala.collection.mutable import scala.collection.parallel.mutable.ParTrieMap +import scala.concurrent.ExecutionContext +import scala.concurrent.duration._ import scala.util.hashing.MurmurHash3 // TODO Add val name which sub classes that extend this trait must overwrite // e.g. BlockChainRouter val name = "Blockchain Router" // Log.debug that read 'Router' should then read 'Blockchain Router' -trait RouterWorker extends Actor with ActorLogging { - - val routerId: Int - val workerID: Int - var newestTime:Long = 0 - var trackedMessage = false - var trackedTime = 0L - implicit val executionContext = context.system.dispatchers.lookup("router-dispatcher") +abstract class RouterWorker[In <: SpoutGoing](val routerId: Int, val workerID: Int, val initialManagerCount: Int) + extends Actor + with ActorLogging { + implicit val executionContext: ExecutionContext = context.system.dispatcher private val messageIDs = ParTrieMap[String, Int]() - /** Private and protected values */ - private var managerCount: Int = initialManagerCount - val writerArray = Utils.getAllWriterWorkers(managerCount) - val routerWorkerUpdates = Kamon.counter("Raphtory_Router_Output").withTag("Router",routerId).withTag("Worker",workerID) + private val routerWorkerUpdates = + Kamon.counter("Raphtory_Router_Output").withTag("Router", routerId).withTag("Worker", workerID) - protected def initialManagerCount: Int - protected def parseTuple(value: Any) + // todo: wvv let people know parseTuple will create a list of update message + // and this trait will handle logic to send to graph + protected def parseTuple(value: In): List[GraphUpdate] final protected val mediator = DistributedPubSub(context.system).mediator mediator ! DistributedPubSubMediator.Put(self) - private val scheduledTaskMap: mutable.HashMap[String, Cancellable] = mutable.HashMap[String, Cancellable]() override def preStart(): Unit = { - log.debug("RouterWorker [{}] is being started.", routerId) - scheduleTasks() - } - - - - private def scheduleTasks(): Unit = { - log.debug("Preparing to schedule tasks in Spout.") - - val broadcastancellable = - SchedulerUtil.scheduleTask(initialDelay = 5 seconds, interval = 5 second, receiver = self, message = "timeBroadcast") - scheduledTaskMap.put("timeBroadcast", broadcastancellable) - } - override def postStop() { - val allTasksCancelled = scheduledTaskMap.forall { - case (key, task) => - SchedulerUtil.cancelTask(key, task) - } - if (!allTasksCancelled) log.warning("Failed to cancel all scheduled tasks post stop.") - } - - override def receive: Receive = { - case req: UpdatedCounter => processUpdatedCounterRequest(req) - case req: AllocateTuple => processAllocateJobRequest(req) - case req: AllocateTrackedTuple => processAlocateTrackedRequest(req) - case "timeBroadcast" => broadcastToPartitions() - case x => log.warning("RouterWorker received unknown [{}] message.", x) - } - - private def broadcastToPartitions() = { - for (worker <- writerArray ) { - - mediator ! DistributedPubSubMediator.Send(worker,RouterWorkerTimeSync(newestTime,s"${routerId}_${workerID}",getMessageIDForWriter(worker)),false) - } + log.debug(s"RouterWorker [$routerId] is being started.") + context.system.scheduler + .schedule(initialDelay = 5.seconds, interval = 5.second, receiver = self, message = TimeBroadcast) } - final protected def getManagerCount: Int = managerCount - - def assignID(uniqueChars: String): Long = MurmurHash3.stringHash(uniqueChars) - - def processUpdatedCounterRequest(req: UpdatedCounter): Unit = { - log.debug(s"RouterWorker [{}] received [{}] request.", routerId, req) - - if (managerCount < req.newValue) managerCount = req.newValue - } - - def processAlocateTrackedRequest(req:AllocateTrackedTuple):Unit = { - trackedMessage=true - trackedTime = req.wallClock - log.debug(s"RouterWorker [{}] received [{}] request.", routerId, req) - parseTuple(req.record) - } - - def processAllocateJobRequest(req: AllocateTuple): Unit = { - log.debug(s"RouterWorker [{}] received [{}] request.", routerId, req) - parseTuple(req.record) + override def receive: Receive = work(initialManagerCount, 0L, 0L) + + private def work(managerCount: Int, trackedTime: Long, newestTime: Long): Receive = { + case msg: UpdatedCounter => + log.debug(s"RouterWorker [$routerId] received [$msg] request.") + if (managerCount < msg.newValue) context.become(work(msg.newValue, trackedTime, newestTime)) + + case AllocateTuple(record: In) => //todo: wvv AllocateTuple should hold type of record instead of using Any + log.debug(s"RouterWorker [$routerId] received AllocateTuple[$record] request.") + parseTupleAndSendGraph(record, managerCount, false, trackedTime).foreach(newNewestTime => + context.become(work(managerCount, trackedTime, newNewestTime)) + ) + + case msg @ AllocateTrackedTuple( + wallClock, + record: In + ) => //todo: wvv AllocateTrackedTuple should hold type of record instead of using Any + log.debug(s"RouterWorker [$routerId] received [$msg] request.") + val newNewestTime = parseTupleAndSendGraph(record, managerCount, true, wallClock).getOrElse(newestTime) + context.become(work(managerCount, wallClock, newNewestTime)) + + case TimeBroadcast => + Utils.getAllWriterWorkers(managerCount).foreach { workerPath => + mediator ! DistributedPubSubMediator.Send( + workerPath, + RouterWorkerTimeSync(newestTime, s"${routerId}_$workerID", getMessageIDForWriter(workerPath)), + false + ) + } + case unhandled => log.warning(s"RouterWorker received unknown [$unhandled] message.") } - def sendGraphUpdate[T <: GraphUpdate](message: T): Unit = { + protected def assignID(uniqueChars: String): Long = MurmurHash3.stringHash(uniqueChars) + + private def parseTupleAndSendGraph( + record: In, + managerCount: Int, + trackedMessage: Boolean, + trackedTime: Long + ): Option[Long] = + parseTuple(record).map(update => sendGraphUpdate(update, managerCount, trackedMessage, trackedTime)).lastOption + + private def sendGraphUpdate( + message: GraphUpdate, + managerCount: Int, + trackedMessage: Boolean, + trackedTime: Long + ): Long = { routerWorkerUpdates.increment() - val path = getManager(message.srcID, getManagerCount) - val id = getMessageIDForWriter(path) - if(trackedMessage){ - trackedMessage=false - mediator ! DistributedPubSubMediator.Send("/user/WatermarkManager",UpdateArrivalTime(trackedTime,message.msgTime), localAffinity = false) - message match { - case m:VertexAdd => - newestTime = m.msgTime - mediator ! DistributedPubSubMediator.Send(path ,TrackedVertexAdd(s"${routerId}_${workerID}",id,trackedTime,m) , localAffinity = false) - case m:VertexAddWithProperties => - newestTime = m.msgTime - mediator ! DistributedPubSubMediator.Send(path ,TrackedVertexAddWithProperties(s"${routerId}_${workerID}",id,trackedTime,m) , localAffinity = false) - case m:EdgeAdd => - newestTime = m.msgTime - mediator ! DistributedPubSubMediator.Send(path ,TrackedEdgeAdd(s"${routerId}_${workerID}",id,trackedTime,m) , localAffinity = false) - case m:EdgeAddWithProperties => - newestTime = m.msgTime - mediator ! DistributedPubSubMediator.Send(path ,TrackedEdgeAddWithProperties(s"${routerId}_${workerID}",id,trackedTime,m) , localAffinity = false) - case m:VertexDelete => - newestTime = m.msgTime - mediator ! DistributedPubSubMediator.Send(path ,TrackedVertexDelete(s"${routerId}_${workerID}",id,trackedTime,m) , localAffinity = false) - case m:EdgeDelete => - newestTime = m.msgTime - mediator ! DistributedPubSubMediator.Send(path ,TrackedEdgeDelete(s"${routerId}_${workerID}",id,trackedTime,m) , localAffinity = false) - } - } - else { - message match { - case m: VertexAdd => - newestTime = m.msgTime - mediator ! DistributedPubSubMediator.Send(path, TrackedVertexAdd(s"${routerId}_${workerID}", id, -1, m), localAffinity = false) - case m: VertexAddWithProperties => - newestTime = m.msgTime - mediator ! DistributedPubSubMediator.Send(path, TrackedVertexAddWithProperties(s"${routerId}_${workerID}", id, -1, m), localAffinity = false) - case m: EdgeAdd => - newestTime = m.msgTime - mediator ! DistributedPubSubMediator.Send(path, TrackedEdgeAdd(s"${routerId}_${workerID}", id, -1, m), localAffinity = false) - case m: EdgeAddWithProperties => - newestTime = m.msgTime - mediator ! DistributedPubSubMediator.Send(path, TrackedEdgeAddWithProperties(s"${routerId}_${workerID}", id, -1, m), localAffinity = false) - case m: VertexDelete => - newestTime = m.msgTime - mediator ! DistributedPubSubMediator.Send(path, TrackedVertexDelete(s"${routerId}_${workerID}", id, -1, m), localAffinity = false) - case m: EdgeDelete => - newestTime = m.msgTime - mediator ! DistributedPubSubMediator.Send(path, TrackedEdgeDelete(s"${routerId}_${workerID}", id, -1, m), localAffinity = false) - } + val path = getManager(message.srcID, managerCount) + val id = getMessageIDForWriter(path) + val trackedTimeToUse = if (trackedMessage) trackedTime else -1L + + val sentMessage = message match { + case m: VertexAdd => + TrackedVertexAdd(s"${routerId}_$workerID", id, trackedTimeToUse, m) + case m: VertexAddWithProperties => + TrackedVertexAddWithProperties(s"${routerId}_$workerID", id, trackedTimeToUse, m) + case m: EdgeAdd => + TrackedEdgeAdd(s"${routerId}_$workerID", id, trackedTimeToUse, m) + case m: EdgeAddWithProperties => + TrackedEdgeAddWithProperties(s"${routerId}_$workerID", id, trackedTimeToUse, m) + case m: VertexDelete => + TrackedVertexDelete(s"${routerId}_$workerID", id, trackedTimeToUse, m) + case m: EdgeDelete => + TrackedEdgeDelete(s"${routerId}_$workerID", id, trackedTimeToUse, m) } + log.debug(s"RouterWorker sending message [$sentMessage] to PubSub") + if (trackedMessage) + mediator ! DistributedPubSubMediator + .Send("/user/WatermarkManager", UpdateArrivalTime(trackedTime, message.msgTime), localAffinity = false) - - log.debug("RouterWorker sending message [{}] to PubSub", message) + mediator ! DistributedPubSubMediator.Send(path, sentMessage, localAffinity = false) + message.msgTime } - private def getMessageIDForWriter(path:String) ={ + + private def getMessageIDForWriter(path: String) = messageIDs.get(path) match { - case Some(messageid) => - messageIDs put (path,messageid+1) - messageid + case Some(messageId) => + messageIDs put (path, messageId + 1) + messageId case None => - messageIDs put (path,1) + messageIDs put (path, 1) 0 } - } +} +object RouterWorker { + object CommonMessage { + case object TimeBroadcast + } } From 3d75f7a0b2deb1bc916c18119afda775b537ec39 Mon Sep 17 00:00:00 2001 From: imanehaf Date: Thu, 22 Oct 2020 19:32:44 +0100 Subject: [PATCH 05/58] extended to weighted LPA + filtered cooccurence ingestion --- mainproject/build.sbt | 10 +- .../core/analysis/Algorithms/FlowGraph.scala | 52 ---------- .../core/analysis/Algorithms/LPA.scala | 24 ++--- .../routers/CooccurrenceMatrixRouter.scala | 6 +- .../CooccurrenceMatrixRouterFiltered.scala | 38 ++++++++ .../spouts/CooccurrenceMatrixSpout.scala | 96 ++++++++++++------- .../CooccurrenceMatrixSpoutFiltered.scala | 52 ++++++++++ 7 files changed, 174 insertions(+), 104 deletions(-) delete mode 100644 mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/FlowGraph.scala create mode 100644 mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixRouterFiltered.scala create mode 100644 mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/spouts/CooccurrenceMatrixSpoutFiltered.scala diff --git a/mainproject/build.sbt b/mainproject/build.sbt index ec9bd8db7e..c5b9a90307 100644 --- a/mainproject/build.sbt +++ b/mainproject/build.sbt @@ -89,10 +89,10 @@ lazy val basicSettings = Seq( ) lazy val dockerStuff = Seq( - maintainer := "Ben Steer ", + maintainer := "Imane Hafnaoui ", dockerBaseImage := "miratepuffin/raphtory-redis:latest", - dockerRepository := Some("narnolddd"), - dockerExposedPorts := Seq(2551, 8080, 2552, 1600, 11600,8081,46339,9100), + dockerRepository := Some("tsukitsune"), + dockerExposedPorts := Seq(2551, 8080, 2552, 1600, 11600,8081) ) lazy val root = Project(id = "raphtory", base = file(".")) aggregate (cluster) @@ -154,6 +154,6 @@ lazy val cluster = project ) .settings( javaAgents += "org.aspectj" % "aspectjweaver" % "1.8.13", - javaAgents += "io.kamon" % "kanela-agent" % "1.0.6", - javaOptions in Universal += "-Dorg.aspectj.tracing.factory=default -XX:+UnlockExperimentalVMOptions -XX:+UseShenandoahGC -XX:+UseStringDeduplication" + javaAgents += "io.kamon" % "kanela-agent" % "1.0.5", + javaOptions in Universal += "-Dorg.aspectj.tracing.factory=default" ) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/FlowGraph.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/FlowGraph.scala deleted file mode 100644 index b815d6b94a..0000000000 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/FlowGraph.scala +++ /dev/null @@ -1,52 +0,0 @@ -package com.raphtory.core.analysis.Algorithms - -import com.raphtory.core.analysis.API.Analyser -import com.raphtory.core.utils.Utils - -import scala.collection.mutable.ArrayBuffer - -class FlowGraph(args:Array[String]) extends Analyser(args){ - - override def setup(): Unit = {} - - override def analyse(): Unit = {} - - override def returnResults(): Any = { - val locV = view.getVertices().filter{ vertex => vertex.Type() == "Location" } - if(locV.size > 1){ - var flow = List[(Long, Long, Int)]() - //TODO Revisit with Imane - locV.foreach { vertU => -// val neighU = vertU.getIncEdges.keySet -// val excU = locV - vertU.ID() -// excU.foreach { v => -// val neighV = view.getVertex(v._2).getIncEdges.keySet -// val com = (neighU & neighV).size -// flow = (u._2.getId, v._2.getId, com) :: flow -// } -// } - flow - } - } - } - - override def defineMaxSteps(): Int = 100 - - override def processResults(results: ArrayBuffer[Any], timeStamp: Long, viewCompleteTime: Long): Unit = { - val endResults = results.filter(_ != (())).asInstanceOf[ArrayBuffer[List[(Long, Long, Int)]]].filter(!_.isEmpty).flatten - val output_file = System.getenv().getOrDefault("PROJECT_OUTPUT", "/home/tsunade/qmul/results/flow-output.csv").trim - val startTime = System.currentTimeMillis() - try { - val totalV = endResults.map(x=>x._1).toSet.size - val totalE = endResults.map(x => x._1).size - val maxFlow = if (endResults.map(_._3).nonEmpty) endResults.map(_._3).max else 0 - val busyEdgeArray = endResults.sortBy(_._3)(Ordering[Int].reverse).map(x=>(x._1,x._2)).take(5) - val text = - s"""{"time":$timeStamp, "vertices":$totalV,"edges":$totalE,"max flow":$maxFlow,"busy roads":$busyEdgeArray,"viewTime":$viewCompleteTime,"concatTime":${System.currentTimeMillis() - startTime}}""" - Utils.writeLines(output_file, text, "") - println(text) - } catch { - case e: UnsupportedOperationException => println(s"No activity for view at $timeStamp") - } - } -} diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/LPA.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/LPA.scala index fa4dd7dfc8..40b8469f34 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/LPA.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/LPA.scala @@ -13,27 +13,27 @@ class LPA(args:Array[String]) extends Analyser(args){ override def setup(): Unit = { view.getVertices().foreach { vertex => - val lab = scala.util.Random.nextLong() - vertex.setState("lpalabel", lab)//vertex.ID()) - vertex.messageAllNeighbours(lab)//vertex.ID()) -// vertex.setState("lpalabel", vertex.ID()) -// vertex messageAllNeighbours vertex.ID() + val lab = vertex.ID()//scala.util.Random.nextLong() + vertex.setState("lpalabel", lab) + vertex.messageAllNeighbours((vertex.ID(),lab)) } } override def analyse(): Unit = { view.getMessagedVertices().foreach { vertex => val vlabel = vertex.getState[Long]("lpalabel") - val gp = vertex.messageQueue[Long] - gp.append(vlabel) - val newLabel = labelProbability(gp.groupBy(identity)) + val vfreq = vertex.getPropertyValue("Frequency").getOrElse(1L).asInstanceOf[Long] + val neigh = (vertex.getIncEdges ++ vertex.getOutEdges).map{e=> e.ID()->e.getPropertyValue("Frequency").getOrElse(1L).asInstanceOf[Long]}.toMap + val gp = vertex.messageQueue[(Long, Long)].map{v => (v._2, neigh(v._1))} + gp.append((vlabel, vfreq)) + val newLabel = labelProbability(gp.groupBy(_._1).mapValues(_.map(_._2))) if (newLabel == vlabel) { vertex.voteToHalt() }else { vertex.setState("lpalabel", newLabel) } - vertex messageAllNeighbours newLabel - doSomething(vertex, gp.toArray) + vertex.messageAllNeighbours((vertex.ID(),newLabel)) + doSomething(vertex, gp.map(_._1).toArray) } } @@ -41,7 +41,7 @@ class LPA(args:Array[String]) extends Analyser(args){ // label probability function to get rid of oscillation phenomena of synchronous LPA from [1] // [1] https://doi.org/10.1016/j.neucom.2014.04.084 - val f = gp.map{lab => Math.pow(lab._2.size, 2)} + val f = gp.map{lab => Math.pow(lab._2.sum, 2)}//TODO: implement full method from paper val p = f.toArray.map(i => i/f.toArray.sum) val gpp = gp.keys zip p gpp.filter(x=>x._2 == gpp.maxBy(_._2)._2).maxBy(_._1)._1 @@ -78,7 +78,7 @@ class LPA(args:Array[String]) extends Analyser(args){ val grouped = endResults.flatten.groupBy(f => f._1).mapValues(x => x.flatMap(_._2)) val groupedNonIslands = grouped.filter(x => x._2.size > 1) val biggest = grouped.maxBy(_._2.size)._2.size - val sorted = groupedNonIslands.toArray.sortBy(_._2.size)(sortOrdering)// + val sorted = grouped.toArray.sortBy(_._2.size)(sortOrdering)// val top5 = sorted.map(x=>x._2.size).take(5) val total = grouped.size val totalWithoutIslands = groupedNonIslands.size diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixRouter.scala index bfbdd88f09..4b0fbcc4a9 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixRouter.scala @@ -11,10 +11,10 @@ class CooccurrenceMatrixRouter(override val routerId: Int, override val workerID val occurenceTime = dp.head.toLong//DateFormatting(dp.head) //.slice(4, dp.head.length) try { dp = dp.last.split("\t") - val srcClusterId = assignID(dp.head) + val srcClusterId = dp.head.toLong//assignID(dp.head) val len = dp.length for (i <- 1 until len by 2) { - val dstClusterId = assignID(dp(i)) + val dstClusterId = dp(i).toLong//assignID(dp(i)) val coocWeight = dp(i + 1).toLong sendGraphUpdate( @@ -22,7 +22,7 @@ class CooccurrenceMatrixRouter(override val routerId: Int, override val workerID msgTime = occurenceTime, srcID = srcClusterId, dstID = dstClusterId, - Properties(DoubleProperty("Frequency", coocWeight)) + Properties(LongProperty("Frequency", coocWeight)) ) ) } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixRouterFiltered.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixRouterFiltered.scala new file mode 100644 index 0000000000..aa67abf310 --- /dev/null +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixRouterFiltered.scala @@ -0,0 +1,38 @@ +package com.raphtory.examples.wordSemantic.routers + +import com.raphtory.core.components.Router.RouterWorker +import com.raphtory.core.model.communication._ + +class CooccurrenceMatrixRouterFiltered(override val routerId: Int, override val workerID:Int, override val initialManagerCount: Int) extends RouterWorker { + val THR = System.getenv().getOrDefault("COOC_FREQ_THRESHOLD ", "0.05").trim.toDouble + + def parseTuple(record: Any): Unit = { + //println(record) + var dp = record.asInstanceOf[String].split(" ").map(_.trim) + val occurenceTime = dp.head.toLong//DateFormatting(dp.head) //.slice(4, dp.head.length) + val scale = dp(1).toDouble + try { + dp = dp.last.split("\t") + val srcClusterId = assignID(dp.head) + val len = dp.length + for (i <- 1 until len by 2) { + if ((dp(i+1).toLong/scale) >= THR) { + val dstClusterId = assignID(dp(i)) + val coocWeight = dp(i + 1).toLong + + sendGraphUpdate( + EdgeAddWithProperties( + msgTime = occurenceTime, + srcID = srcClusterId, + dstID = dstClusterId, + Properties(LongProperty("Frequency", coocWeight), + DoubleProperty("ScaledFreq", coocWeight/scale)) + ) + ) + } + } + }catch { + case e: Exception => println(e, dp.length, record.asInstanceOf[String]) + } + } +} diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/spouts/CooccurrenceMatrixSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/spouts/CooccurrenceMatrixSpout.scala index 4c352f7d4a..2e62cf301f 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/spouts/CooccurrenceMatrixSpout.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/spouts/CooccurrenceMatrixSpout.scala @@ -1,23 +1,44 @@ package com.raphtory.examples.wordSemantic.spouts -import com.raphtory.spouts.{EtherFileReader, FileSpout} +import java.io.{BufferedReader, File, FileReader} +import java.time.LocalDateTime + +import com.raphtory.core.components.Spout.SpoutTrait +import com.raphtory.spouts.EtherFileReader import scala.concurrent.duration._ import scala.language.postfixOps -class CooccurrenceMatrixSpout extends EtherFileReader { - var cnt = 1 +class CooccurrenceMatrixSpout extends SpoutTrait { + + println("Start: " + LocalDateTime.now()) + val directory = System.getenv().getOrDefault("FILE_SPOUT_DIRECTORY", "/app").trim + val fileName = System.getenv().getOrDefault("FILE_SPOUT_FILENAME", "").trim //gabNetwork500.csv + val dropHeader = System.getenv().getOrDefault("FILE_SPOUT_DROP_HEADER", "false").trim.toBoolean + var JUMP = System.getenv().getOrDefault("FILE_SPOUT_BLOCK_SIZE", "100").trim.toInt + var INCREMENT = System.getenv().getOrDefault("FILE_SPOUT_INCREMENT", "1").trim.toInt + var TIME = System.getenv().getOrDefault("FILE_SPOUT_TIME", "60").trim.toInt + + var directoryPosition = 0 + + val filesToRead = if(fileName.isEmpty) + getListOfFiles(directory) + else + Array(directory + "/" + fileName) + + var currentFile = fileToArray(directoryPosition) val JUMP2 = 20 var posSlice = 1 var cline = currentFile.readLine() -// if (cline != null) { var currentLine = cline.split("\t") + var filename = filesToRead(directoryPosition) //D-200001_merge_occ + var time = filename.split('/').last.stripPrefix("D-").stripSuffix("_merge_occ").toLong * 1000000000L + var cnt = time + 1 - // var currentLine = Array("")} -override protected def ProcessSpoutTask(message: Any): Unit = message match { + override protected def ProcessSpoutTask(message: Any): Unit = message match { case StartSpout => AllocateSpoutTask(Duration(1, NANOSECONDS), "nextLineSlice") case "nextLineSlice" => nextLineSlice() @@ -29,8 +50,8 @@ override protected def ProcessSpoutTask(message: Any): Unit = message match { try { if (posSlice <= currentLine.length-1) { val head = currentLine(0) - for (i<- 1 to Set(JUMP, currentLine.length/JUMP2).min+1) { - val currentSlice = currentLine.slice(posSlice, Set(posSlice + JUMP2, currentLine.length).min) + for (i<- 1 to Set(JUMP, currentLine.length-posSlice/JUMP2).min) { + val currentSlice = currentLine.slice(posSlice, posSlice + JUMP2) sendTuple(cnt.toString + ' ' + head + "\t" + currentSlice.mkString("\t")) posSlice += JUMP2 } @@ -45,11 +66,8 @@ override protected def ProcessSpoutTask(message: Any): Unit = message match { } } - override def nextLineBlock() = { - //val filename = filesToRead(directoryPosition) //D-200001_merge_occ - // var time = filename.split('/').last.stripPrefix("D-").stripSuffix("_merge_occ") + def nextLineBlock() = { try { - //filePosition += 1 cnt += 1 cline = currentFile.readLine() currentLine = cline.split("\t") @@ -59,24 +77,38 @@ override protected def ProcessSpoutTask(message: Any): Unit = message match { case e:Exception => AllocateSpoutTask(Duration(1, NANOSECONDS), "nextFile") } } -// override def nextLineBlock() = { -// try { -// var del = 1000 -// val cline = currentFile.readLine() -// if(cline!=null) { -// val line = cline.split("\t") -// val head = line(0) -// for (i <- 1 until line.length - 1 by JUMP2) { -// sendTuple(cnt.toString + ' ' + head + "\t" + line.slice(i, i + JUMP2).mkString("\t")) -// } -// cnt += 1 -// //del = 15 * line.length -// } -// -// AllocateSpoutTask(Duration(del, NANOSECONDS), "nextLineBLock") -// } -// catch { -// case e:Exception => AllocateSpoutTask(Duration(1, NANOSECONDS), "nextFile") -// } -// } + + def nextFile() = { + directoryPosition += 1 + if (filesToRead.length > directoryPosition) { + currentFile = fileToArray(directoryPosition) + filename = filesToRead(directoryPosition) //D-200001_merge_occ + time = filename.split('/').last.stripPrefix("D-").stripSuffix("_merge_occ").toLong * 1000000000L + cnt = time + 1 + AllocateSpoutTask(Duration(1, NANOSECONDS), "nextLineBLock") + } + else { + println("All files read "+ LocalDateTime.now()) + } + } + + def fileToArray(pos:Int) ={ + println(s"Now reading ${filesToRead(pos)}") + if(dropHeader){ + val br = new BufferedReader(new FileReader(filesToRead(pos))) + br.readLine() + br + } + else + new BufferedReader(new FileReader(filesToRead(pos))) + } + + def getListOfFiles(dir: String):Array[String] = { + val d = new File(dir) + if (d.exists && d.isDirectory) { + d.listFiles.filter(f=> f.isFile && !f.isHidden).map(f=> f.getCanonicalPath).sorted + } else { + Array[String]() + } + } } \ No newline at end of file diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/spouts/CooccurrenceMatrixSpoutFiltered.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/spouts/CooccurrenceMatrixSpoutFiltered.scala new file mode 100644 index 0000000000..4133008853 --- /dev/null +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/spouts/CooccurrenceMatrixSpoutFiltered.scala @@ -0,0 +1,52 @@ +package com.raphtory.examples.wordSemantic.spouts + +import java.io.{BufferedReader, File, FileReader} +import java.time.LocalDateTime + +import com.raphtory.core.components.Spout.SpoutTrait + +import scala.concurrent.duration._ +import scala.language.postfixOps + +class CooccurrenceMatrixSpoutFiltered extends CooccurrenceMatrixSpout { + var freq = currentLine.drop(2).grouped(2).map(_.head.toInt).toArray + var scale = scalling(freq) + + override def nextLineSlice() = { + try { + if (posSlice <= currentLine.length-1) { + val head = currentLine(0) + for (i<- 1 to Set(JUMP, currentLine.length-posSlice/JUMP2).min) { + val currentSlice = currentLine.slice(posSlice, posSlice + JUMP2) + sendTuple(cnt.toString + ' ' + scale.toString + ' ' + head + "\t" + currentSlice.mkString("\t")) + posSlice += JUMP2 + } + AllocateSpoutTask(Duration(1, MILLISECONDS), "nextLineSlice") + } + else { + posSlice = 1 + AllocateSpoutTask(Duration(1, NANOSECONDS), "nextLineBLock") + } + }catch { + case e: Exception => println(e, posSlice) + } + } + + override def nextLineBlock() = { + try { + cnt += 1 + cline = currentFile.readLine() + currentLine = cline.split("\t") + freq = currentLine.drop(2).grouped(2).map(_.head.toInt).toArray + scale = scalling(freq) + AllocateSpoutTask(Duration(1, NANOSECONDS), "nextLineSlice") + } + catch { + case e:Exception => AllocateSpoutTask(Duration(1, NANOSECONDS), "nextFile") + } + } + + def scalling(freq: Array[Int]): Double = { + math.sqrt(freq.map(math.pow(_, 2)).sum) + } +} \ No newline at end of file From 4702b752d1c8d0cce1fe4692c984892538823c03 Mon Sep 17 00:00:00 2001 From: Miratepuffin Date: Thu, 29 Oct 2020 00:55:16 +0000 Subject: [PATCH 06/58] Fixed all example Spouts Removed Duplicate/Old Spouts --- .../core/components/Spout/SpoutTrait.scala | 10 +- .../blockchain/BlockchainCommunication.scala | 17 --- .../spouts/BitcoinExampleSpout.scala | 50 ------- .../spouts/ChainalysisABSpout.scala | 38 ----- .../blockchain/spouts/DashcoinSpout.scala | 59 -------- .../blockchain/spouts/LitecoinSpout.scala | 54 ------- .../spouts/ReadFromCsvFileSpout.scala | 31 ---- .../citationNetwork/old/CitationRouter.scala | 45 ------ .../citationNetwork/old/CitationSpout.scala | 29 ---- .../examples/gab/actors/GabExampleSpout.scala | 42 ------ .../examples/gab/actors/GabKafkaSpout.scala | 37 ----- .../GabPostGraphRouter.scala | 2 +- .../{actors => routers}/GabRawRouter.scala | 2 +- .../GabUserGraphRouter.scala | 2 +- .../gab/{actors => spouts}/GabRawSpout.scala | 23 +-- .../{actors => spouts}/GabSampledSpout.scala | 20 ++- .../examples/ldbc/spouts/LDBCSpout.scala | 42 ------ .../raphtory/examples/lotr/LOTRSpout.scala | 15 +- .../examples/test/actors/RandomSpout.scala | 42 +++--- .../test/actors/TriangleTestSpout.scala | 23 +-- .../spouts/TrackAndTraceSpout.scala | 31 ---- .../examples/tsvnet/SamplerSpout.scala | 106 +++++++------- .../raphtory/examples/tsvnet/TSVSpout.scala | 80 +++++----- .../examples/twitterRumour/rumourSpout.scala | 138 +++++++++--------- .../spouts/CooccurrenceMatrixSpout.scala | 78 +++++----- .../{FirehoseSpout.scala => FileSpout.scala} | 28 ++-- .../com/raphtory/spouts/KafkaSpout.scala | 21 +-- .../blockchain}/BitcoinSpout.scala | 25 ++-- .../blockchain}/EthereumGethSpout.scala | 32 ++-- .../blockchain}/EthereumPostgresSpout.scala | 29 ++-- .../tests/TemporalTriangleCountExample.scala | 8 +- 31 files changed, 351 insertions(+), 808 deletions(-) delete mode 100644 mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/BlockchainCommunication.scala delete mode 100644 mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/spouts/BitcoinExampleSpout.scala delete mode 100644 mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/spouts/ChainalysisABSpout.scala delete mode 100644 mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/spouts/DashcoinSpout.scala delete mode 100644 mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/spouts/LitecoinSpout.scala delete mode 100644 mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/spouts/ReadFromCsvFileSpout.scala delete mode 100644 mainproject/cluster/src/main/scala/com/raphtory/examples/citationNetwork/old/CitationRouter.scala delete mode 100644 mainproject/cluster/src/main/scala/com/raphtory/examples/citationNetwork/old/CitationSpout.scala delete mode 100644 mainproject/cluster/src/main/scala/com/raphtory/examples/gab/actors/GabExampleSpout.scala delete mode 100644 mainproject/cluster/src/main/scala/com/raphtory/examples/gab/actors/GabKafkaSpout.scala rename mainproject/cluster/src/main/scala/com/raphtory/examples/gab/{actors => routers}/GabPostGraphRouter.scala (97%) rename mainproject/cluster/src/main/scala/com/raphtory/examples/gab/{actors => routers}/GabRawRouter.scala (99%) rename mainproject/cluster/src/main/scala/com/raphtory/examples/gab/{actors => routers}/GabUserGraphRouter.scala (98%) rename mainproject/cluster/src/main/scala/com/raphtory/examples/gab/{actors => spouts}/GabRawSpout.scala (73%) rename mainproject/cluster/src/main/scala/com/raphtory/examples/gab/{actors => spouts}/GabSampledSpout.scala (69%) delete mode 100644 mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/spouts/LDBCSpout.scala delete mode 100644 mainproject/cluster/src/main/scala/com/raphtory/examples/trackAndTrace/spouts/TrackAndTraceSpout.scala rename mainproject/cluster/src/main/scala/com/raphtory/spouts/{FirehoseSpout.scala => FileSpout.scala} (85%) rename mainproject/cluster/src/main/scala/com/raphtory/{examples/blockchain/spouts => spouts/blockchain}/BitcoinSpout.scala (76%) rename mainproject/cluster/src/main/scala/com/raphtory/{examples/blockchain/spouts => spouts/blockchain}/EthereumGethSpout.scala (86%) rename mainproject/cluster/src/main/scala/com/raphtory/{examples/blockchain/spouts => spouts/blockchain}/EthereumPostgresSpout.scala (72%) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/SpoutTrait.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/SpoutTrait.scala index 87d7fec020..bb00ca2f87 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/SpoutTrait.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/SpoutTrait.scala @@ -1,8 +1,6 @@ package com.raphtory.core.components.Spout -import akka.actor.Actor -import akka.actor.ActorLogging -import akka.actor.Timers +import akka.actor.{Actor, ActorLogging, Cancellable, Timers} import akka.cluster.pubsub.DistributedPubSub import akka.cluster.pubsub.DistributedPubSubMediator import com.raphtory.core.components.Spout.SpoutTrait.CommonMessage.IsSafe @@ -88,6 +86,10 @@ trait SpoutTrait[Domain <: DomainMessage, Out <: SpoutGoing] extends Actor with AllocateTuple(command) mediator ! DistributedPubSubMediator.Send(s"/user/router/routerWorker_${count % 10}", message, localAffinity = false) } + def AllocateSpoutTask(duration: FiniteDuration, task: Any): Cancellable = { + val taskCancellable = context.system.scheduler.scheduleOnce(duration, self, task) + taskCancellable + } } object SpoutTrait { @@ -95,6 +97,8 @@ object SpoutTrait { case object StartSpout case object StateCheck case object IsSafe + case object Next extends BasicDomain } trait DomainMessage + sealed trait BasicDomain } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/BlockchainCommunication.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/BlockchainCommunication.scala deleted file mode 100644 index bcd53b3363..0000000000 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/BlockchainCommunication.scala +++ /dev/null @@ -1,17 +0,0 @@ -package com.raphtory.examples.blockchain - -import com.raphtory.core.analysis.API.WorkerID -import com.raphtory.core.model.communication.SpoutGoing -import spray.json.JsValue - -import scala.collection.mutable.ArrayBuffer - -case class BitcoinTransaction(time: JsValue, block: Int, blockID: JsValue, transaction: JsValue) extends SpoutGoing -case class LitecoinTransaction(time: JsValue, block: Int, blockID: JsValue, transaction: JsValue) extends SpoutGoing - -case class CoinsAquiredPayload( - workerID: WorkerID, - wallets: ArrayBuffer[(String, Double)], - highestBlock: Int, - blockhash: String -) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/spouts/BitcoinExampleSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/spouts/BitcoinExampleSpout.scala deleted file mode 100644 index 33bb61266d..0000000000 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/spouts/BitcoinExampleSpout.scala +++ /dev/null @@ -1,50 +0,0 @@ -package com.raphtory.examples.blockchain.spouts - -import com.raphtory.core.components.Spout.SpoutTrait -import com.raphtory.examples.blockchain.BitcoinTransaction -import spray.json._ - -import scala.concurrent.duration._ -import scala.io.Source -import scala.language.postfixOps - -class BitcoinExampleSpout extends SpoutTrait { - - var blockcount = 1 - var folder = System.getenv().getOrDefault("BITCOIN_DIRECTORY", "/app/blocks/").trim - - //************* MESSAGE HANDLING BLOCK - override def ProcessSpoutTask(message: Any): Unit = message match { - case StartSpout => AllocateSpoutTask(Duration(1, MILLISECONDS), "parseBlock") - case "parseBlock" => - getTransactions() - blockcount += 1 - case _ => println("message not recognized!") - } - - def getTransactions(): Unit = { - if (blockcount > 20000) return - val result = readNextBlock().parseJson.asJsObject - val time = result.fields("time") - val blockID = result.fields("hash") - for (transaction <- result.asJsObject().fields("tx").asInstanceOf[JsArray].elements) - sendTuple(BitcoinTransaction(time, blockcount, blockID, transaction)) - //val time = transaction.asJsObject.fields("time") - - AllocateSpoutTask(Duration(1, MILLISECONDS), "parseBlock") - } - - def readNextBlock(): String = { - //val bufferedSource = Source.fromFile(s"/Users/lagordamotoneta/Documents/QMUL/QMUL/project/Datasets/blocks/$blockcount.txt") - - val bufferedSource = Source.fromFile(s"$folder/$blockcount.txt") - - var block = "" - for (line <- bufferedSource.getLines) - block += line - bufferedSource.close - blockcount += 1 - block - } - -} diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/spouts/ChainalysisABSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/spouts/ChainalysisABSpout.scala deleted file mode 100644 index dc74358471..0000000000 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/spouts/ChainalysisABSpout.scala +++ /dev/null @@ -1,38 +0,0 @@ -package com.raphtory.examples.blockchain.spouts - -import java.time.LocalDateTime - -import com.raphtory.core.components.Spout.SpoutTrait -import scala.concurrent.duration._ -import scala.io.Source -import scala.language.postfixOps - -class ChainalysisABSpout extends SpoutTrait { - - val file_name = System.getenv().getOrDefault("DATA_FILENAME", "/home/tsunade/qmul/datasets/chainalysis/abmsample-merged.csv").trim - val fl = Source.fromFile(file_name) - var linesNumber = Source.fromFile(file_name).getLines().size - 1 - val fileLines = fl.getLines.drop(1)//.toArray - // upstream/master - var position = 0 - val JUMP = 100 - println("Starting File ingestion: " + LocalDateTime.now()) - println("Lines :" + linesNumber) - - override def ProcessSpoutTask(message: Any): Unit = message match { - case StartSpout => AllocateSpoutTask(Duration(1, NANOSECONDS), "newLine") - case "newLine" => - - if (position < linesNumber) { - try { - val l = fileLines.take(JUMP).mkString("\n") - sendTuple(l) - AllocateSpoutTask(Duration(1, NANOSECONDS), "newLine") - }catch{ - case e: NoSuchElementException => println("End of file!") - } - position += JUMP - } - case _ => println("message not recognized!") - } -} diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/spouts/DashcoinSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/spouts/DashcoinSpout.scala deleted file mode 100644 index 599744825e..0000000000 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/spouts/DashcoinSpout.scala +++ /dev/null @@ -1,59 +0,0 @@ -package com.raphtory.examples.blockchain.spouts - -import com.raphtory.core.components.Spout.SpoutTrait -import com.raphtory.examples.blockchain.LitecoinTransaction -import org.joda.time.DateTime -import org.joda.time.DateTimeZone -import scalaj.http.Http -import scalaj.http.HttpRequest -import spray.json._ - -import scala.concurrent.duration._ -import scala.language.postfixOps - -class DashcoinSpout extends SpoutTrait { - - var blockcount = 1 - val rpcuser = System.getenv().getOrDefault("DASHCOIN_USERNAME", "").trim - val rpcpassword = System.getenv().getOrDefault("DASHCOIN_PASSWORD", "").trim - val serverAddress = System.getenv().getOrDefault("DASHCOIN_NODE", "").trim - val id = "scala-jsonrpc" - val baseRequest = Http(serverAddress).auth(rpcuser, rpcpassword).header("content-type", "text/plain") - - //************* MESSAGE HANDLING BLOCK - override def ProcessSpoutTask(message: Any): Unit = message match { - case StartSpout => AllocateSpoutTask(Duration(1, MILLISECONDS), "parseBlock") - case "parseBlock" => running() - case _ => println("message not recognized!") - } - - def running(): Unit = - try { - for (i <- 1 to 10) { - getTransactions() - blockcount += 1 - if (blockcount % 1000 == 0) println(s"Parsed block $blockcount at ${DateTime.now(DateTimeZone.UTC).getMillis}") - } - AllocateSpoutTask(Duration(1, NANOSECONDS), "parseBlock") - - } catch { - case e: java.net.SocketTimeoutException => - case e: spray.json.DeserializationException => AllocateSpoutTask(Duration(10, SECONDS), "parseBlock") - } - - def getTransactions(): Unit = { - val re = request("getblockhash", blockcount.toString).execute().body.toString.parseJson.asJsObject - val blockID = re.fields("result") - val blockData = request("getblock", s"$blockID,2").execute().body.toString.parseJson.asJsObject - val result = blockData.fields("result") - val time = result.asJsObject.fields("time") - for (transaction <- result.asJsObject().fields("tx").asInstanceOf[JsArray].elements) - sendTuple(LitecoinTransaction(time, blockcount, blockID, transaction)) - //val time = transaction.asJsObject.fields("time") - } - - def request(command: String, params: String = ""): HttpRequest = - baseRequest.postData(s"""{"jsonrpc": "1.0", "id":"$id", "method": "$command", "params": [$params] }""") - -} -//def request(command: String, params: String = ""): HttpRequest = baseRequest.postData(s"""{"jsonrpc": "1.0", "id":"$id", "method": "$command", "params": [$params] }""") diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/spouts/LitecoinSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/spouts/LitecoinSpout.scala deleted file mode 100644 index 6bb94d4d6d..0000000000 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/spouts/LitecoinSpout.scala +++ /dev/null @@ -1,54 +0,0 @@ -package com.raphtory.examples.blockchain.spouts - -import com.raphtory.core.components.Spout.SpoutTrait -import com.raphtory.examples.blockchain.LitecoinTransaction -import scalaj.http.Http -import scalaj.http.HttpRequest -import spray.json._ - -import scala.concurrent.duration._ -import scala.language.postfixOps - -class LitecoinSpout extends SpoutTrait { - - var blockcount = 1 - val rpcuser = System.getenv().getOrDefault("LITECOIN_USERNAME", "").trim - val rpcpassword = System.getenv().getOrDefault("LITECOIN_PASSWORD", "").trim - val serverAddress = System.getenv().getOrDefault("LITECOIN_NODE", "").trim - val id = "scala-jsonrpc" - val baseRequest = Http(serverAddress).auth(rpcuser, rpcpassword).header("content-type", "text/plain") - - //************* MESSAGE HANDLING BLOCK - override def ProcessSpoutTask(message: Any): Unit = message match { - case StartSpout => AllocateSpoutTask(Duration(1, MILLISECONDS), "parseBlock") - case "parseBlock" => running() - case _ => println("message not recognized!") - } - - def running(): Unit = - try { - getTransactions() - blockcount += 1 - AllocateSpoutTask(Duration(1, NANOSECONDS), "parseBlock") - if (blockcount % 100 == 0) println("Currently Calling for block $blockcount") - } catch { - case e: java.net.SocketTimeoutException => - case e: spray.json.DeserializationException => AllocateSpoutTask(Duration(10, SECONDS), "parseBlock") - } - - def getTransactions(): Unit = { - val re = request("getblockhash", blockcount.toString).execute().body.toString.parseJson.asJsObject - val blockID = re.fields("result") - val blockData = request("getblock", s"$blockID,2").execute().body.toString.parseJson.asJsObject - val result = blockData.fields("result") - val time = result.asJsObject.fields("time") - for (transaction <- result.asJsObject().fields("tx").asInstanceOf[JsArray].elements) - sendTuple(LitecoinTransaction(time, blockcount, blockID, transaction)) - //val time = transaction.asJsObject.fields("time") - } - - def request(command: String, params: String = ""): HttpRequest = - baseRequest.postData(s"""{"jsonrpc": "1.0", "id":"$id", "method": "$command", "params": [$params] }""") - -} -//def request(command: String, params: String = ""): HttpRequest = baseRequest.postData(s"""{"jsonrpc": "1.0", "id":"$id", "method": "$command", "params": [$params] }""") diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/spouts/ReadFromCsvFileSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/spouts/ReadFromCsvFileSpout.scala deleted file mode 100644 index 6b6e1eca22..0000000000 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/spouts/ReadFromCsvFileSpout.scala +++ /dev/null @@ -1,31 +0,0 @@ -//package com.raphtory.examples.blockchain.spouts -// -//import com.raphtory.core.components.Spout.SpoutTrait -// -//import scala.io.Source -// -//class ReadFromCsvFileSpout extends SpoutTrait { -// -// var currentBlock = System.getenv().getOrDefault("SPOUT_ETHEREUM_START_BLOCK_INDEX", "9014194").trim.toInt -// var highestBlock = System.getenv().getOrDefault("SPOUT_ETHEREUM_MAXIMUM_BLOCK_INDEX", "999999999").trim.toInt -// val nodeIP = System.getenv().getOrDefault("SPOUT_ETHEREUM_IP_ADDRESS", "127.0.0.1").trim -// val nodePort = System.getenv().getOrDefault("SPOUT_ETHEREUM_PORT", "8545").trim -// -// val filename = "Macintosh HD/Users/imasgo/downloads/transactions9000000_9100000.csv" -// for (line <- Source.fromFile(filename).getLines) { -// println(line) -// } -// -// override protected def ProcessSpoutTask(message: Any): Unit = message match { -// case StartSpout => pullNextBlock() -// case "nextBlock" => pullNextBlock() -// } -// -// -// -//// read csv -//// for each line in the csv DO SOMETHING (what???) -// -// -// -//} diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/citationNetwork/old/CitationRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/citationNetwork/old/CitationRouter.scala deleted file mode 100644 index 7c2cdeb637..0000000000 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/citationNetwork/old/CitationRouter.scala +++ /dev/null @@ -1,45 +0,0 @@ -package com.raphtory.examples.citationNetwork.old - -import java.text.SimpleDateFormat - -import com.raphtory.core.components.Router.RouterWorker -import com.raphtory.core.model.communication._ - -class CitationRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int) extends RouterWorker { - - def parseTuple(record: Any): Unit = { - - val fileLine = record.asInstanceOf[String].split(",").map(_.trim) - //extract the values from the data source in the form of: - // 0-sourceNode,1-targetNode,2-sourceCitedTargetOn,3-targetCreationDate,4-targetLastCitedOn - val sourceNode = fileLine(0).toInt - val targetNode = fileLine(1).toInt - val sourceCitedTargetOn = dateToUnixTime(timestamp = fileLine(2)) - val targetCreationDate = dateToUnixTime(timestamp = fileLine(3)) - val targetLastCitedOn = dateToUnixTime(timestamp = fileLine(4)) - - //create sourceNode - sendGraphUpdate(VertexAdd(sourceCitedTargetOn, sourceNode)) - //create destinationNode - sendGraphUpdate(VertexAdd(targetCreationDate, targetNode)) - //create edge - sendGraphUpdate(EdgeAdd(sourceCitedTargetOn, sourceNode, targetNode)) - - if (sourceCitedTargetOn == targetLastCitedOn) - sendGraphUpdate(EdgeDelete(targetLastCitedOn, sourceNode, targetNode)) - - } - - def dateToUnixTime(timestamp: => String): Long = { - //if(timestamp == null) return null; - println(timestamp) - val sdf = new SimpleDateFormat("dd/MM/yyyy") - println(sdf) - val dt = sdf.parse(timestamp) - println(dt) - val epoch = dt.getTime() - println(epoch) - epoch / 1000 - - } -} diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/citationNetwork/old/CitationSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/citationNetwork/old/CitationSpout.scala deleted file mode 100644 index 5b55498c3b..0000000000 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/citationNetwork/old/CitationSpout.scala +++ /dev/null @@ -1,29 +0,0 @@ -package com.raphtory.examples.citationNetwork.old - -import com.raphtory.core.components.Spout.SpoutTrait - -import scala.concurrent.duration._ -import scala.io.Source -import scala.language.postfixOps - -class CitationSpout extends SpoutTrait { - - //file is read. Please note that the first line is dropped, this in case the file has headers - val fileLines = Source.fromFile("/Users/lagordamotoneta/Scala/small.csv").getLines.drop(1).toArray - var position = 0 - var linesNumber = fileLines.length - println(linesNumber) - - protected def ProcessSpoutTask(message: Any): Unit = message match { - case StartSpout => AllocateSpoutTask(Duration(1, MILLISECONDS), "newLine") - case "newLine" => - if (position < linesNumber) { - var line = fileLines(position) - sendTuple(line) - position += 1 - AllocateSpoutTask(Duration(1, MILLISECONDS), "newLine") - } - case _ => println("message not recognized!") - } - -} diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/actors/GabExampleSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/actors/GabExampleSpout.scala deleted file mode 100644 index c009f5c40e..0000000000 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/actors/GabExampleSpout.scala +++ /dev/null @@ -1,42 +0,0 @@ -package com.raphtory.examples.gab.actors - -import java.time.LocalDateTime - -import com.raphtory.core.components.Spout.SpoutTrait - -import scala.concurrent.duration._ -import scala.io -import scala.io.Source -import scala.language.postfixOps - -class GabExampleSpout extends SpoutTrait { - - val directory = System.getenv().getOrDefault("GAB_DIRECTORY", "/app").trim - val file_name = System.getenv().getOrDefault("GAB_FILE_NAME", "gab.csv").trim - val fileLines = Source.fromFile(directory + "/" + file_name).getLines.drop(1).toArray - // upstream/master - var position = 0 - var linesNumber = fileLines.length - println("Start: " + LocalDateTime.now()) - - - protected def ProcessSpoutTask(message: Any): Unit = message match { - case StartSpout => AllocateSpoutTask(Duration(1, NANOSECONDS), "newLine") - case "newLine" => - try { - if (position < linesNumber) { - for (i <- 1 to 100) { - var line = fileLines(position) - sendTuple(line) - position += 1 - } - AllocateSpoutTask(Duration(1, NANOSECONDS), "newLine") - } - else { - println("Finished ingestion") - System.gc() - } - }catch {case e:Exception => println("Finished ingestion")} - case _ => println("message not recognized!") - } -} diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/actors/GabKafkaSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/actors/GabKafkaSpout.scala deleted file mode 100644 index 148952b360..0000000000 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/actors/GabKafkaSpout.scala +++ /dev/null @@ -1,37 +0,0 @@ -package com.raphtory.examples.gab.actors - -import java.util -import java.util.Properties - -import com.raphtory.core.components.Spout.SpoutTrait -import org.apache.kafka.clients.consumer.KafkaConsumer - -import scala.collection.JavaConverters._ -import scala.concurrent.duration.Duration -import scala.concurrent.duration.MILLISECONDS -import scala.concurrent.duration.SECONDS -import scala.util.Random -class GabKafkaSpout extends SpoutTrait { - val x = new Random().nextLong() - val props = new Properties() - props.put("bootstrap.servers", "moe.eecs.qmul.ac.uk:9092") - props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer") - props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer") - props.put("auto.offset.reset", "earliest") - props.put("group.id", "group" + x) - val consumer: KafkaConsumer[String, String] = new KafkaConsumer[String, String](props) - consumer.subscribe(util.Arrays.asList("gabResortedGraph")) - - protected def ProcessSpoutTask(message: Any): Unit = message match { - case StartSpout => AllocateSpoutTask(Duration(1, MILLISECONDS), "newLine") - case "newLine" => consumeFromKafka() - case _ => println("message not recognized!") - } - - def consumeFromKafka() = { - val record = consumer.poll(1000).asScala - for (data <- record.iterator) - sendTuple(data.value()) - AllocateSpoutTask(Duration(1, MILLISECONDS), "newLine") - } -} diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/actors/GabPostGraphRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabPostGraphRouter.scala similarity index 97% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/gab/actors/GabPostGraphRouter.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabPostGraphRouter.scala index 5493677f78..eaa40174ce 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/actors/GabPostGraphRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabPostGraphRouter.scala @@ -1,4 +1,4 @@ -package com.raphtory.examples.gab.actors +package com.raphtory.examples.gab.routers import java.text.SimpleDateFormat diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/actors/GabRawRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabRawRouter.scala similarity index 99% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/gab/actors/GabRawRouter.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabRawRouter.scala index 4e637cb022..2517b274ba 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/actors/GabRawRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabRawRouter.scala @@ -1,4 +1,4 @@ -package com.raphtory.examples.gab.actors +package com.raphtory.examples.gab.routers import java.time.OffsetDateTime diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/actors/GabUserGraphRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabUserGraphRouter.scala similarity index 98% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/gab/actors/GabUserGraphRouter.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabUserGraphRouter.scala index 0b17e0b167..313ed9b995 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/actors/GabUserGraphRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabUserGraphRouter.scala @@ -1,4 +1,4 @@ -package com.raphtory.examples.gab.actors +package com.raphtory.examples.gab.routers import java.text.SimpleDateFormat diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/actors/GabRawSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/spouts/GabRawSpout.scala similarity index 73% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/gab/actors/GabRawSpout.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/gab/spouts/GabRawSpout.scala index 1754885428..2956b5c886 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/actors/GabRawSpout.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/spouts/GabRawSpout.scala @@ -1,16 +1,17 @@ -package com.raphtory.examples.gab.actors +package com.raphtory.examples.gab.spouts import akka.actor.Cancellable import ch.qos.logback.classic.Level -import com.mongodb.casbah.Imports.MongoConnection -import com.mongodb.casbah.Imports._ +import com.mongodb.casbah.Imports.{MongoConnection, _} import com.raphtory.core.components.Spout.SpoutTrait +import com.raphtory.core.components.Spout.SpoutTrait.BasicDomain +import com.raphtory.core.components.Spout.SpoutTrait.CommonMessage.Next +import com.raphtory.core.model.communication.StringSpoutGoing import org.slf4j.LoggerFactory -import scala.concurrent.duration._ import scala.language.postfixOps -final class GabRawSpout extends SpoutTrait { +final class GabRawSpout extends SpoutTrait[BasicDomain,StringSpoutGoing] { //private val redis = new RedisClient("moe", 6379) //private val redisKey = "gab-posts" @@ -29,9 +30,8 @@ final class GabRawSpout extends SpoutTrait { // private val mongoLogger = Logger.getLogger("org.mongodb.driver.cluster") // mongoLogger.setLevel(Level.OFF) - override protected def ProcessSpoutTask(message: Any): Unit = message match { - case StartSpout => AllocateSpoutTask(Duration(1, MILLISECONDS), "parsePost") - case "parsePost" => running() + override def handleDomainMessage(message: BasicDomain): Unit = message match { + case Next => running() } def running(): Unit = { @@ -39,7 +39,7 @@ final class GabRawSpout extends SpoutTrait { postMin += window postMax += window println(s"Current min post is $postMin, max post is $postMax, last call retrieved $count posts") - AllocateSpoutTask(Duration(10, MILLISECONDS), "parsePost") + self ! Next } @@ -50,7 +50,7 @@ final class GabRawSpout extends SpoutTrait { val data = x.get("data").toString.drop(2).dropRight(1).replaceAll("""\\"""", "").replaceAll("""\\""", "") count += 1 //println(data) - sendTuple(data) + sendTuple(StringSpoutGoing(data)) } catch { case e: Throwable => println("Cannot parse record") @@ -58,5 +58,8 @@ final class GabRawSpout extends SpoutTrait { return count } + override def startSpout(): Unit = self ! Next + + } //redis-server --dir /home/moe/ben/gab --dbfilename gab.rdb --daemonize yes diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/actors/GabSampledSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/spouts/GabSampledSpout.scala similarity index 69% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/gab/actors/GabSampledSpout.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/gab/spouts/GabSampledSpout.scala index ae0184d960..17e97d28dd 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/actors/GabSampledSpout.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/spouts/GabSampledSpout.scala @@ -1,15 +1,17 @@ -package com.raphtory.examples.gab.actors +package com.raphtory.examples.gab.spouts import java.time.LocalDateTime import com.raphtory.core.components.Spout.SpoutTrait +import com.raphtory.core.components.Spout.SpoutTrait.BasicDomain +import com.raphtory.core.components.Spout.SpoutTrait.CommonMessage.Next +import com.raphtory.core.model.communication.StringSpoutGoing -import scala.concurrent.duration.{Duration, NANOSECONDS} import scala.io.Source import scala.util.Random import scala.util.control.Breaks.{break, breakable} -class GabSampledSpout extends SpoutTrait { +class GabSampledSpout extends SpoutTrait[BasicDomain,StringSpoutGoing] { val prop = 1.0/10 val r = new Random() @@ -22,9 +24,8 @@ class GabSampledSpout extends SpoutTrait { var linesNumber = fileLines.length println("Start: " + LocalDateTime.now()) - protected def ProcessSpoutTask(message: Any): Unit = message match { - case StartSpout => AllocateSpoutTask(Duration(1, NANOSECONDS), "newLine") - case "newLine" => + override def handleDomainMessage(message: BasicDomain): Unit = message match { + case Next => try { if (position < linesNumber) { for (i <- 1 to 100) { @@ -35,12 +36,12 @@ class GabSampledSpout extends SpoutTrait { break } else { val line = fileLines(position) - sendTuple(line) + sendTuple(StringSpoutGoing(line)) position += 1 } } } - AllocateSpoutTask(Duration(1, NANOSECONDS), "newLine") + self ! Next } else { @@ -50,4 +51,7 @@ class GabSampledSpout extends SpoutTrait { catch {case e:Exception => println("Finished ingestion")} case _ => println("message not recognized!") } + + override def startSpout(): Unit = self ! Next + } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/spouts/LDBCSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/spouts/LDBCSpout.scala deleted file mode 100644 index 2e82d01767..0000000000 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/spouts/LDBCSpout.scala +++ /dev/null @@ -1,42 +0,0 @@ -package com.raphtory.examples.ldbc.spouts - -import java.time.LocalDateTime - -import com.raphtory.core.components.Spout.SpoutTrait - -import scala.concurrent.duration._ -import scala.io -import scala.language.postfixOps -import scala.concurrent.duration.Duration -import scala.concurrent.duration.NANOSECONDS - -class LDBCSpout extends SpoutTrait { - - val directory = System - .getenv() - .getOrDefault("LDBC_DIRECTORY", "/Users/mirate/Documents/phd/ldbc_snb_datagen/social_network/dynamic") - .trim - - val peopleFile = io.Source.fromFile(directory + "/" + "person_0_0.csv").getLines.drop(1).toArray - val friendFile = io.Source.fromFile(directory + "/" + "person_knows_person_0_0.csv").getLines.drop(1).toArray - // upstream/master - var position = 1 - var people = peopleFile.length - var friends = friendFile.length - println("Start: " + LocalDateTime.now()) - - protected def ProcessSpoutTask(message: Any): Unit = message match { - case StartSpout => AllocateSpoutTask(Duration(1, NANOSECONDS), "newLine") - case "newLine" => - for (i <- 1 to 100) { - if (position < people) sendTuple("person|" + peopleFile(position)) - if (position < friends) sendTuple("person_knows_person|" + friendFile(position)) - position += 1 - } - if (position > friends) - println("ingestion Finished") - else - AllocateSpoutTask(Duration(1, NANOSECONDS), "newLine") - case _ => println("message not recognized!") - } -} diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRSpout.scala index 1b8660fc35..a126e34207 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRSpout.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRSpout.scala @@ -3,11 +3,14 @@ package com.raphtory.examples.lotr import java.time.LocalDateTime import com.raphtory.core.components.Spout.SpoutTrait +import com.raphtory.core.components.Spout.SpoutTrait.BasicDomain +import com.raphtory.core.components.Spout.SpoutTrait.CommonMessage.{Next} +import com.raphtory.core.model.communication.StringSpoutGoing import scala.concurrent.duration.{Duration, MILLISECONDS, NANOSECONDS} import scala.io -class LOTRSpout extends SpoutTrait { +class LOTRSpout extends SpoutTrait[BasicDomain,StringSpoutGoing] { // Relating to where the file is val directory = System.getenv().getOrDefault("LOTR_DIRECTORY", "/Users/naomiarnold/CODE/Raphtory/LOTR").trim @@ -19,15 +22,15 @@ class LOTRSpout extends SpoutTrait { var linesNumber = fileLines.length println("Start: " + LocalDateTime.now()) - protected def ProcessSpoutTask(message: Any): Unit = message match { - case StartSpout => AllocateSpoutTask(Duration(1, MILLISECONDS), "newLine") - case "newLine" => + override def handleDomainMessage(message: BasicDomain): Unit = message match { + case Next => if (position < linesNumber) { var line = fileLines(position) - sendTuple(line) + sendTuple(StringSpoutGoing(line)) position += 1 - AllocateSpoutTask(Duration(1, NANOSECONDS), "newLine") + self ! Next } case _ => println("message not recognized!") } + override def startSpout(): Unit = self ! Next } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomSpout.scala index 72c2227089..e9d15bf389 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomSpout.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomSpout.scala @@ -1,13 +1,16 @@ package com.raphtory.examples.test.actors import com.raphtory.core.components.Spout.SpoutTrait +import com.raphtory.core.components.Spout.SpoutTrait.DomainMessage +import com.raphtory.core.model.communication.StringSpoutGoing import com.raphtory.core.utils.Utils +import com.raphtory.examples.test.actors.RandomSpout.Message.{Increase, RandomDomain, Required,Next} import scala.concurrent.duration._ import scala.language.postfixOps import scala.util.Random -class RandomSpout extends SpoutTrait { +class RandomSpout extends SpoutTrait[RandomDomain,StringSpoutGoing] { var totalCount = 100 var freq = 1000 @@ -15,32 +18,20 @@ class RandomSpout extends SpoutTrait { var pool = System.getenv().getOrDefault("ENTITY_POOL", "1000000").toInt var msgID = 0 - override def preStart() { //set up partition to report how many messages it has processed in the last X seconds - super.preStart() - println(s"Prestarting ($freq Hz) Entity pool = $pool Ramp flag = $increase") - //context.system.scheduler.schedule(Duration(40, SECONDS), Duration(5, MINUTES), self, "stop") - println("Only sending adds") - } - - protected def ProcessSpoutTask(message: Any): Unit = message match { - case StartSpout => AllocateSpoutTask(Duration(1, MILLISECONDS), "random") - case "required" => + override def handleDomainMessage(message: RandomDomain): Unit = message match { + case Required => freq = System.getenv().getOrDefault("UPDATES_FREQ", "10000").toInt; println(s"Full start ($freq Hz) Entity pool = $pool Ramp flag = $increase") // (Updates/s) - Hz - case "increase" => + case Increase => if (increase) { freq += 1000 println(s"Frequency increased, new frequency: $freq at ${Utils.nowTimeStamp()}") } - case "random" => genRandomCommands(freq / 1000) - case "stop" => stop() + case Next => genRandomCommands(freq / 1000) case _ => println("message not recognized!") } - //context.system.scheduler.scheduleOnce(Duration(1, MINUTES), self, "required") - //context.system.scheduler.schedule(Duration(90, SECONDS), Duration(30, SECONDS), self, "increase") - //context.system.scheduler.schedule(Duration(10, SECONDS), Duration(1, MILLISECONDS), self, "random") def distribution(): String = { val random = Random.nextFloat() @@ -52,9 +43,9 @@ class RandomSpout extends SpoutTrait { def genRandomCommands(number: Int): Unit = { (1 to number) foreach (_ => { - sendTuple(distribution()) + sendTuple(StringSpoutGoing(distribution())) }) - AllocateSpoutTask(Duration(1, NANOSECONDS), "random") + self ! Next } def genVertexAdd(): String = @@ -113,8 +104,15 @@ class RandomSpout extends SpoutTrait { properties } - def running(): Unit = - genRandomCommands(totalCount) - //totalCount+=1000 + override def startSpout(): Unit = ??? } + +object RandomSpout { + object Message { + sealed trait RandomDomain extends DomainMessage + case object Increase extends RandomDomain + case object Required extends RandomDomain + case object Next extends RandomDomain + } +} \ No newline at end of file diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleTestSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleTestSpout.scala index a02f166466..3d4ebd5ca9 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleTestSpout.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleTestSpout.scala @@ -1,16 +1,21 @@ package com.raphtory.examples.test.actors import com.raphtory.core.components.Spout.SpoutTrait +import com.raphtory.core.components.Spout.SpoutTrait.BasicDomain +import com.raphtory.core.components.Spout.SpoutTrait.CommonMessage.Next +import com.raphtory.core.model.communication.StringSpoutGoing -class TriangleTestSpout extends SpoutTrait { - override protected def ProcessSpoutTask(receivedMessage: Any) = receivedMessage match { - case StartSpout => - sendTuple("1,1,2") - sendTuple("2,2,3") - sendTuple("3,3,1") - sendTuple("4,3,4") - sendTuple("5,4,1") - sendTuple("6,1,3") +class TriangleTestSpout extends SpoutTrait[BasicDomain,StringSpoutGoing] { + override def handleDomainMessage(message: BasicDomain): Unit = message match { + case Next => + sendTuple(StringSpoutGoing("1,1,2")) + sendTuple(StringSpoutGoing("2,2,3")) + sendTuple(StringSpoutGoing("3,3,1")) + sendTuple(StringSpoutGoing("4,3,4")) + sendTuple(StringSpoutGoing("5,4,1")) + sendTuple(StringSpoutGoing("6,1,3")) } + + override def startSpout(): Unit = self ! Next } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/trackAndTrace/spouts/TrackAndTraceSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/trackAndTrace/spouts/TrackAndTraceSpout.scala deleted file mode 100644 index d4d61dbf52..0000000000 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/trackAndTrace/spouts/TrackAndTraceSpout.scala +++ /dev/null @@ -1,31 +0,0 @@ -package com.raphtory.examples.trackAndTrace.spouts - -import java.time.LocalDateTime - -import com.raphtory.core.components.Spout.SpoutTrait - -import scala.concurrent.duration._ -import scala.io -import scala.language.postfixOps - -class TrackAndTraceSpout extends SpoutTrait { - - val file_name = System.getenv().getOrDefault("TRACK_AND_TRACE_FILENAME", "/Users/mirate/Documents/phd/locationdataexample.csv").trim - val fileLines = io.Source.fromFile(file_name).getLines.drop(3).toArray - // upstream/master - var position = 0 - var linesNumber = fileLines.length - println("Starting File ingestion: " + LocalDateTime.now()) - println("Lines :" + linesNumber) - - protected def ProcessSpoutTask(message: Any): Unit = message match { - case StartSpout => AllocateSpoutTask(Duration(1, NANOSECONDS), "newLine") - case "newLine" => - if (position < linesNumber) { - sendTuple(fileLines(position)) - position += 1 - AllocateSpoutTask(Duration(1, NANOSECONDS), "newLine") - } - case _ => println("message not recognized!") - } -} \ No newline at end of file diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/SamplerSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/SamplerSpout.scala index 6f1c2f25ff..1d056df8af 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/SamplerSpout.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/SamplerSpout.scala @@ -1,53 +1,53 @@ -package com.raphtory.examples.tsvnet - -import java.time.LocalDateTime - -import com.raphtory.core.components.Spout.SpoutTrait - -import scala.concurrent.duration.{Duration, NANOSECONDS} -import scala.io -import scala.util.Random -import scala.util.control.Breaks._ - -class SamplerSpout extends SpoutTrait { - - val prop = 1.0/10 - val r = new Random() - val directory = System.getenv().getOrDefault("TSV_DIRECTORY", "/app").trim - val file_name = System.getenv().getOrDefault("TSV_FILE_NAME", "sx_reordered.txt").trim - val fileLines = io.Source.fromFile(directory + "/" + file_name).getLines.drop(1).toArray - // upstream/master - var position = 0 - var linesNumber = fileLines.length - println("Start: " + LocalDateTime.now()) - - protected def ProcessSpoutTask(message: Any): Unit = message match { - case StartSpout => AllocateSpoutTask(Duration(1, NANOSECONDS), "newLine") - case "newLine" => - try { - if (position < linesNumber) { - for (i <- 1 to 100) { - val randFloat = r.nextFloat() - breakable { - if (randFloat > prop) { - position +=1 - break - } else { - val line = fileLines(position) - sendTuple(line) - position += 1 - } - } - } - AllocateSpoutTask(Duration(1, NANOSECONDS), "newLine") - } - else - { - println("Finished ingestion") - } - } - catch {case e:Exception => println("Finished ingestion")} - case _ => println("message not recognized!") - } - -} +//package com.raphtory.examples.tsvnet +// +//import java.time.LocalDateTime +// +//import com.raphtory.core.components.Spout.SpoutTrait +// +//import scala.concurrent.duration.{Duration, NANOSECONDS} +//import scala.io +//import scala.util.Random +//import scala.util.control.Breaks._ +// +//class SamplerSpout extends SpoutTrait { +// +// val prop = 1.0/10 +// val r = new Random() +// val directory = System.getenv().getOrDefault("TSV_DIRECTORY", "/app").trim +// val file_name = System.getenv().getOrDefault("TSV_FILE_NAME", "sx_reordered.txt").trim +// val fileLines = io.Source.fromFile(directory + "/" + file_name).getLines.drop(1).toArray +// // upstream/master +// var position = 0 +// var linesNumber = fileLines.length +// println("Start: " + LocalDateTime.now()) +// +// protected def ProcessSpoutTask(message: Any): Unit = message match { +// case StartSpout => AllocateSpoutTask(Duration(1, NANOSECONDS), "newLine") +// case "newLine" => +// try { +// if (position < linesNumber) { +// for (i <- 1 to 100) { +// val randFloat = r.nextFloat() +// breakable { +// if (randFloat > prop) { +// position +=1 +// break +// } else { +// val line = fileLines(position) +// sendTuple(line) +// position += 1 +// } +// } +// } +// AllocateSpoutTask(Duration(1, NANOSECONDS), "newLine") +// } +// else +// { +// println("Finished ingestion") +// } +// } +// catch {case e:Exception => println("Finished ingestion")} +// case _ => println("message not recognized!") +// } +// +//} diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/TSVSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/TSVSpout.scala index 16719ded27..2373386c3d 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/TSVSpout.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/TSVSpout.scala @@ -1,40 +1,40 @@ -package com.raphtory.examples.tsvnet - -import java.time.LocalDateTime - -import com.raphtory.core.components.Spout.SpoutTrait - -import scala.concurrent.duration._ -import scala.io -import scala.language.postfixOps - -/** Spout for network datasets of the form SRC_NODE_ID DEST_NODE_ID TIMESTAMP */ -class TSVSpout extends SpoutTrait { - - val directory = System.getenv().getOrDefault("TSV_DIRECTORY", "/app").trim - val file_name = System.getenv().getOrDefault("TSV_FILE_NAME", "sx_reordered.txt").trim - val fileLines = io.Source.fromFile(directory + "/" + file_name).getLines.drop(1).toArray - // upstream/master - var position = 0 - var linesNumber = fileLines.length - println("Start: " + LocalDateTime.now()) - - protected def ProcessSpoutTask(message: Any): Unit = message match { - case StartSpout => AllocateSpoutTask(Duration(1, NANOSECONDS), "newLine") - case "newLine" => - try { - if (position < linesNumber) { - for (i <- 0 to 100) { - var line = fileLines(position) - sendTuple(line) - position += 1 - } - AllocateSpoutTask(Duration(1, NANOSECONDS), "newLine") - } - else { - println("Finished ingestion") - } - }catch {case e:Exception => println("Finished ingestion")} - case _ => println("message not recognized!") - } -} +//package com.raphtory.examples.tsvnet +// +//import java.time.LocalDateTime +// +//import com.raphtory.core.components.Spout.SpoutTrait +// +//import scala.concurrent.duration._ +//import scala.io +//import scala.language.postfixOps +// +///** Spout for network datasets of the form SRC_NODE_ID DEST_NODE_ID TIMESTAMP */ +//class TSVSpout extends SpoutTrait { +// +// val directory = System.getenv().getOrDefault("TSV_DIRECTORY", "/app").trim +// val file_name = System.getenv().getOrDefault("TSV_FILE_NAME", "sx_reordered.txt").trim +// val fileLines = io.Source.fromFile(directory + "/" + file_name).getLines.drop(1).toArray +// // upstream/master +// var position = 0 +// var linesNumber = fileLines.length +// println("Start: " + LocalDateTime.now()) +// +// protected def ProcessSpoutTask(message: Any): Unit = message match { +// case StartSpout => AllocateSpoutTask(Duration(1, NANOSECONDS), "newLine") +// case "newLine" => +// try { +// if (position < linesNumber) { +// for (i <- 0 to 100) { +// var line = fileLines(position) +// sendTuple(line) +// position += 1 +// } +// AllocateSpoutTask(Duration(1, NANOSECONDS), "newLine") +// } +// else { +// println("Finished ingestion") +// } +// }catch {case e:Exception => println("Finished ingestion")} +// case _ => println("message not recognized!") +// } +//} diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/twitterRumour/rumourSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/twitterRumour/rumourSpout.scala index 6c5855e71e..6093a7e4f2 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/twitterRumour/rumourSpout.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/twitterRumour/rumourSpout.scala @@ -1,69 +1,69 @@ -package com.raphtory.examples.twitterRumour - -import akka.actor.ActorContext -import com.raphtory.core.components.Spout.SpoutTrait - -import scala.concurrent.duration.Duration -import scala.concurrent.duration.MILLISECONDS -import scala.concurrent.duration.SECONDS -import java.io.File - -class rumourSpout extends SpoutTrait { - var directory = System.getenv().getOrDefault("PHEME_DIRECTORY", "/home/tsunade/qmul/datasets/pheme-rnr-dataset").trim - var r_case = System.getenv().getOrDefault("RUMOUR_CASE", "/sydneysiege").trim - val rpath = directory + r_case + "/rumours" - val nrpath = directory + r_case + "/non-rumours" - val max_r = 500 //Option(new File(rpath).list.size).getOrElse(0) - val max_nr = 500 //Option(new File(nrpath).list.size).getOrElse(0) - - var rtweetCounter = 0 - var nrtweetCounter = 0 - - override protected def ProcessSpoutTask(message: Any): Unit = - // println(message) - if ((rtweetCounter < max_r) || (nrtweetCounter < max_nr)) - //println("inside if stm") - message match { - case StartSpout => - AllocateSpoutTask(Duration(1, SECONDS), "rumourTweets") - AllocateSpoutTask(Duration(1, SECONDS), "nonRumourTweets") - case "rumourTweets" => - if (rtweetCounter < max_r) { - // println("sending ingest cmd r...") - running(rpath, position = rtweetCounter, "R__") - rtweetCounter += 1 - context.system.scheduler.scheduleOnce(Duration(1, SECONDS), self, "rumourTweets") - } - case "nonRumourTweets" => - if (nrtweetCounter < max_nr) { - // println("sending ingest cmd nr...") - running(nrpath, position = nrtweetCounter, "nR__") - nrtweetCounter += 1 - context.system.scheduler.scheduleOnce(Duration(1, SECONDS), self, "nonRumourTweets") - } - case _ => println("message not recognized!") - } - - def ingestTweet(path: String, rnr: String): Unit = { - // println("inside ingest....") - val tweet_l = new File(path) - if (tweet_l.exists && tweet_l.isDirectory) - tweet_l.listFiles.filter(_.isFile).toList.map(_.toString).foreach { tw_path: String => - sendTuple(rnr + tw_path) - // println("running sendTuple...") - } - } - - def running(rpath: String, position: Int, rnr: String): Unit = { - val rumours = new File(rpath) - // println("inside runnign...") - - if (rumours.exists && rumours.isDirectory) { - - //println("inside runnign if...") - val r_tweet = rumours.listFiles.toList.map(_.toString) - ingestTweet(r_tweet(position) + "/source-tweet", rnr) - ingestTweet(r_tweet(position) + "/reactions", rnr) - } - } -} +//package com.raphtory.examples.twitterRumour +// +//import akka.actor.ActorContext +//import com.raphtory.core.components.Spout.SpoutTrait +// +//import scala.concurrent.duration.Duration +//import scala.concurrent.duration.MILLISECONDS +//import scala.concurrent.duration.SECONDS +//import java.io.File +// +//class rumourSpout extends SpoutTrait { +// var directory = System.getenv().getOrDefault("PHEME_DIRECTORY", "/home/tsunade/qmul/datasets/pheme-rnr-dataset").trim +// var r_case = System.getenv().getOrDefault("RUMOUR_CASE", "/sydneysiege").trim +// val rpath = directory + r_case + "/rumours" +// val nrpath = directory + r_case + "/non-rumours" +// val max_r = 500 //Option(new File(rpath).list.size).getOrElse(0) +// val max_nr = 500 //Option(new File(nrpath).list.size).getOrElse(0) +// +// var rtweetCounter = 0 +// var nrtweetCounter = 0 +// +// override protected def ProcessSpoutTask(message: Any): Unit = +// // println(message) +// if ((rtweetCounter < max_r) || (nrtweetCounter < max_nr)) +// //println("inside if stm") +// message match { +// case StartSpout => +// AllocateSpoutTask(Duration(1, SECONDS), "rumourTweets") +// AllocateSpoutTask(Duration(1, SECONDS), "nonRumourTweets") +// case "rumourTweets" => +// if (rtweetCounter < max_r) { +// // println("sending ingest cmd r...") +// running(rpath, position = rtweetCounter, "R__") +// rtweetCounter += 1 +// context.system.scheduler.scheduleOnce(Duration(1, SECONDS), self, "rumourTweets") +// } +// case "nonRumourTweets" => +// if (nrtweetCounter < max_nr) { +// // println("sending ingest cmd nr...") +// running(nrpath, position = nrtweetCounter, "nR__") +// nrtweetCounter += 1 +// context.system.scheduler.scheduleOnce(Duration(1, SECONDS), self, "nonRumourTweets") +// } +// case _ => println("message not recognized!") +// } +// +// def ingestTweet(path: String, rnr: String): Unit = { +// // println("inside ingest....") +// val tweet_l = new File(path) +// if (tweet_l.exists && tweet_l.isDirectory) +// tweet_l.listFiles.filter(_.isFile).toList.map(_.toString).foreach { tw_path: String => +// sendTuple(rnr + tw_path) +// // println("running sendTuple...") +// } +// } +// +// def running(rpath: String, position: Int, rnr: String): Unit = { +// val rumours = new File(rpath) +// // println("inside runnign...") +// +// if (rumours.exists && rumours.isDirectory) { +// +// //println("inside runnign if...") +// val r_tweet = rumours.listFiles.toList.map(_.toString) +// ingestTweet(r_tweet(position) + "/source-tweet", rnr) +// ingestTweet(r_tweet(position) + "/reactions", rnr) +// } +// } +//} diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/spouts/CooccurrenceMatrixSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/spouts/CooccurrenceMatrixSpout.scala index 2fc30ea650..66a731b27d 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/spouts/CooccurrenceMatrixSpout.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/spouts/CooccurrenceMatrixSpout.scala @@ -1,39 +1,39 @@ -package com.raphtory.examples.wordSemantic.spouts - -import java.time.LocalDateTime - -import com.raphtory.core.components.Spout.SpoutTrait - -import scala.concurrent.duration._ -import scala.io.Source -import scala.language.postfixOps - -class CooccurrenceMatrixSpout extends SpoutTrait { - - val file_name = System.getenv().getOrDefault("DATA_FILENAME", "/home/tsunade/qmul/datasets/word_semantics/test_file_0.txt").trim - val fl = Source.fromFile(file_name) - val fileLines = fl.getLines//.toArray - var time = file_name.replace(".txt", "").split('_').last - // upstream/master -// var position = 0 -// var linesNumber = fl.length - println("Starting File ingestion: " + LocalDateTime.now()) -// println("Lines :" + linesNumber) - - override def ProcessSpoutTask(message: Any): Unit = message match { - case StartSpout => AllocateSpoutTask(Duration(1, NANOSECONDS), "newLine") - case "newLine" => - - // if (position < linesNumber) { - //val line = if(fileLines.hasNext) fileLines.next() else "" - try { - sendTuple(time+' '+fileLines.next()) - AllocateSpoutTask(Duration(1, NANOSECONDS), "newLine") - }catch{ - case e: NoSuchElementException => println("End of file!") - } - // position += 1 - // } - case _ => println("message not recognized!") - } -} \ No newline at end of file +//package com.raphtory.examples.wordSemantic.spouts +// +//import java.time.LocalDateTime +// +//import com.raphtory.core.components.Spout.SpoutTrait +// +//import scala.concurrent.duration._ +//import scala.io.Source +//import scala.language.postfixOps +// +//class CooccurrenceMatrixSpout extends SpoutTrait { +// +// val file_name = System.getenv().getOrDefault("DATA_FILENAME", "/home/tsunade/qmul/datasets/word_semantics/test_file_0.txt").trim +// val fl = Source.fromFile(file_name) +// val fileLines = fl.getLines//.toArray +// var time = file_name.replace(".txt", "").split('_').last +// // upstream/master +//// var position = 0 +//// var linesNumber = fl.length +// println("Starting File ingestion: " + LocalDateTime.now()) +//// println("Lines :" + linesNumber) +// +// override def ProcessSpoutTask(message: Any): Unit = message match { +// case StartSpout => AllocateSpoutTask(Duration(1, NANOSECONDS), "newLine") +// case "newLine" => +// +// // if (position < linesNumber) { +// //val line = if(fileLines.hasNext) fileLines.next() else "" +// try { +// sendTuple(time+' '+fileLines.next()) +// AllocateSpoutTask(Duration(1, NANOSECONDS), "newLine") +// }catch{ +// case e: NoSuchElementException => println("End of file!") +// } +// // position += 1 +// // } +// case _ => println("message not recognized!") +// } +//} \ No newline at end of file diff --git a/mainproject/cluster/src/main/scala/com/raphtory/spouts/FirehoseSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/spouts/FileSpout.scala similarity index 85% rename from mainproject/cluster/src/main/scala/com/raphtory/spouts/FirehoseSpout.scala rename to mainproject/cluster/src/main/scala/com/raphtory/spouts/FileSpout.scala index bfb85d1988..f11694265c 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/spouts/FirehoseSpout.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/spouts/FileSpout.scala @@ -5,34 +5,34 @@ import java.io.File import java.io.FileReader import com.raphtory.core.components.Spout.SpoutTrait +import com.raphtory.core.components.Spout.SpoutTrait.DomainMessage import com.raphtory.core.model.communication.StringSpoutGoing -import com.raphtory.spouts.FirehoseSpout.Message.FireHouseDomain -import com.raphtory.spouts.FirehoseSpout.Message.Increase -import com.raphtory.spouts.FirehoseSpout.Message.NextFile -import com.raphtory.spouts.FirehoseSpout.Message.NextLineBlock +import com.raphtory.spouts.FileSpout.Message.FileDomain +import com.raphtory.spouts.FileSpout.Message.Increase +import com.raphtory.spouts.FileSpout.Message.NextFile +import com.raphtory.spouts.FileSpout.Message.NextLineBlock import com.typesafe.scalalogging.LazyLogging import scala.annotation.tailrec import scala.concurrent.duration._ -final case class FirehoseSpout() extends SpoutTrait[FireHouseDomain, StringSpoutGoing] { - log.info("initialise FirehoseSpout") +final case class FileSpout() extends SpoutTrait[FileDomain, StringSpoutGoing] { + log.info("initialise FileSpout") private val directory = System.getenv().getOrDefault("FILE_SPOUT_DIRECTORY", "/app").trim private val fileName = System.getenv().getOrDefault("FILE_SPOUT_FILENAME", "").trim //gabNetwork500.csv private val dropHeader = System.getenv().getOrDefault("FILE_SPOUT_DROP_HEADER", "false").trim.toBoolean private val JUMP = System.getenv().getOrDefault("FILE_SPOUT_BLOCK_SIZE", "10").trim.toInt - private val INCREMENT = System.getenv().getOrDefault("FILE_SPOUT_INCREMENT", "1").trim.toInt + private val INCREMENT = System.getenv().getOrDefault("FILE_SPOUT_INCREMENT", "0").trim.toInt private val TIME = System.getenv().getOrDefault("FILE_SPOUT_TIME", "60").trim.toInt private var fileManager = FileManager(directory, fileName, dropHeader, JUMP) def startSpout(): Unit = { self ! NextLineBlock - // todo: wvv not sure why we need to keep increasing context.system.scheduler.scheduleOnce(TIME.seconds, self, Increase) } - def handleDomainMessage(message: FireHouseDomain): Unit = message match { + def handleDomainMessage(message: FileDomain): Unit = message match { case Increase => if (fileManager.allCompleted) log.info("All files read") @@ -60,12 +60,12 @@ final case class FirehoseSpout() extends SpoutTrait[FireHouseDomain, StringSpout } } -object FirehoseSpout { +object FileSpout { object Message { - sealed trait FireHouseDomain - case object Increase extends FireHouseDomain - case object NextLineBlock extends FireHouseDomain - case object NextFile extends FireHouseDomain + sealed trait FileDomain extends DomainMessage + case object Increase extends FileDomain + case object NextLineBlock extends FileDomain + case object NextFile extends FileDomain } } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/spouts/KafkaSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/spouts/KafkaSpout.scala index e3b301b3e9..9a2b5bfd17 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/spouts/KafkaSpout.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/spouts/KafkaSpout.scala @@ -4,10 +4,9 @@ import java.util import java.util.Properties import com.raphtory.core.components.Spout.SpoutTrait -import com.raphtory.core.components.Spout.SpoutTrait.DomainMessage +import com.raphtory.core.components.Spout.SpoutTrait.CommonMessage.Next +import com.raphtory.core.components.Spout.SpoutTrait.{BasicDomain, DomainMessage} import com.raphtory.core.model.communication.StringSpoutGoing -import com.raphtory.spouts.KafkaSpout.Message.Domain -import com.raphtory.spouts.KafkaSpout.Message.NewBlock import org.apache.kafka.clients.consumer.KafkaConsumer import scala.annotation.tailrec @@ -15,7 +14,7 @@ import scala.collection.JavaConverters._ import scala.concurrent.duration._ import scala.util.Random -final case class KafkaSpout() extends SpoutTrait[Domain, StringSpoutGoing] { +final case class KafkaSpout() extends SpoutTrait[BasicDomain, StringSpoutGoing] { log.info("initialising KafkaSpout") private val kafkaServer = System.getenv().getOrDefault("KAFKA_ADDRESS", "127.0.0.1").trim private val kafkaIp = System.getenv().getOrDefault("KAFKA_PORT", "9092").trim @@ -28,23 +27,17 @@ final case class KafkaSpout() extends SpoutTrait[Domain, StringSpoutGoing] { private var kafkaManager = KafkaManager(kafkaServer, kafkaIp, groupId, topic, offset) override def startSpout(): Unit = - self ! NewBlock + self ! Next - def handleDomainMessage(message: Domain): Unit = message match { - case NewBlock => + def handleDomainMessage(message: BasicDomain): Unit = message match { + case Next => val (newManager, block) = kafkaManager.nextNLine(startingSpeed / 100) kafkaManager = newManager block.foreach(str => sendTuple(StringSpoutGoing(str))) - context.system.scheduler.scheduleOnce(restart.millis, self, NewBlock) + context.system.scheduler.scheduleOnce(restart.millis, self, Next) } } -object KafkaSpout { - object Message { - sealed trait Domain extends DomainMessage - case object NewBlock extends Domain - } -} final case class KafkaManager private (buffer: Stream[String], consumer: KafkaConsumer[String, String]) { private def poll(): KafkaManager = { diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/spouts/BitcoinSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/spouts/blockchain/BitcoinSpout.scala similarity index 76% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/spouts/BitcoinSpout.scala rename to mainproject/cluster/src/main/scala/com/raphtory/spouts/blockchain/BitcoinSpout.scala index b3caa78a2d..5558bd5cdd 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/spouts/BitcoinSpout.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/spouts/blockchain/BitcoinSpout.scala @@ -1,19 +1,21 @@ -package com.raphtory.examples.blockchain.spouts +package com.raphtory.spouts.blockchain import java.io.File import java.io.PrintWriter import com.raphtory.core.components.Spout.SpoutTrait -import com.raphtory.examples.blockchain.BitcoinTransaction +import com.raphtory.core.components.Spout.SpoutTrait.BasicDomain +import com.raphtory.core.components.Spout.SpoutTrait.CommonMessage.Next +import com.raphtory.core.model.communication.SpoutGoing +import com.raphtory.spouts.blockchain.BitcoinSpout.Message.{BitcoinDomain, ParseBlock} import scalaj.http.Http import scalaj.http.HttpRequest import spray.json._ -import scala.concurrent.duration._ import scala.language.postfixOps import scala.sys.process._ -class BitcoinSpout extends SpoutTrait { +class BitcoinSpout extends SpoutTrait[BasicDomain, BitcoinTransaction] { var blockcount = 1 val rpcuser = System.getenv().getOrDefault("BITCOIN_USERNAME", "").trim @@ -22,16 +24,16 @@ class BitcoinSpout extends SpoutTrait { val id = "scala-jsonrpc" val baseRequest = Http(serverAddress).auth(rpcuser, rpcpassword).header("content-type", "text/plain") + //************* MESSAGE HANDLING BLOCK - override def ProcessSpoutTask(message: Any): Unit = message match { - case StartSpout => AllocateSpoutTask(Duration(1, MILLISECONDS), "parseBlock") - case "parseBlock" => + def handleDomainMessage(message: BasicDomain): Unit = message match { + case Next => try { getTransactions() blockcount += 1 - AllocateSpoutTask(Duration(1, MILLISECONDS), "parseBlock") + self ! Next } catch { - case e: java.net.SocketTimeoutException => AllocateSpoutTask(Duration(1, MILLISECONDS), "parseBlock") + case e: java.net.SocketTimeoutException => self ! Next } case _ => println("message not recognized!") } @@ -64,5 +66,10 @@ class BitcoinSpout extends SpoutTrait { } + override def startSpout(): Unit = self ! Next + } + +case class BitcoinTransaction(time: JsValue, block: Int, blockID: JsValue, transaction: JsValue) extends SpoutGoing + //def request(command: String, params: String = ""): HttpRequest = baseRequest.postData(s"""{"jsonrpc": "1.0", "id":"$id", "method": "$command", "params": [$params] }""") diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/spouts/EthereumGethSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/spouts/blockchain/EthereumGethSpout.scala similarity index 86% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/spouts/EthereumGethSpout.scala rename to mainproject/cluster/src/main/scala/com/raphtory/spouts/blockchain/EthereumGethSpout.scala index 998d54ea05..32d0165de3 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/spouts/EthereumGethSpout.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/spouts/blockchain/EthereumGethSpout.scala @@ -1,4 +1,4 @@ -package com.raphtory.examples.blockchain.spouts +package com.raphtory.spouts.blockchain import java.net.InetAddress import java.util.NoSuchElementException @@ -18,17 +18,16 @@ import scala.sys.process._ import scalaj.http.Http import scalaj.http.HttpRequest import spray.json._ -import akka.http.scaladsl.unmarshalling.Unmarshal -import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._ import akka.stream.ActorMaterializer +import com.raphtory.core.components.Spout.SpoutTrait.CommonMessage.Next +import com.raphtory.core.components.Spout.SpoutTrait.{BasicDomain, DomainMessage} +import com.raphtory.core.model.communication.StringSpoutGoing +import com.raphtory.spouts.FirehoseSpout.Message.FireHouseDomain import spray.json.DefaultJsonProtocol._ -import scala.concurrent.duration._ -import scala.concurrent.Await -case class EthResult(blockHash:Option[String],blockNumber:Option[String],from:Option[String],gas:Option[String],gasPrice:Option[String],hash:Option[String],input:Option[String],nonce:Option[String],r:Option[String],s:Option[String],to:Option[String],transactionIndex:Option[String],v:Option[String],value:Option[String]) -case class EthTransaction(id:Option[String],jsonrpc:Option[String],result:EthResult) -class EthereumGethSpout extends SpoutTrait { + +class EthereumGethSpout extends SpoutTrait[BasicDomain, StringSpoutGoing] { var currentBlock = System.getenv().getOrDefault("SPOUT_ETHEREUM_START_BLOCK_INDEX", "9014194").trim.toInt var highestBlock = System.getenv().getOrDefault("SPOUT_ETHEREUM_MAXIMUM_BLOCK_INDEX", "10026447").trim.toInt @@ -49,9 +48,8 @@ class EthereumGethSpout extends SpoutTrait { else println(s"Connecting to Ethereum RPC \n Address:${hostname2Ip(nodeIP)} \n Port:$nodePort") - override protected def ProcessSpoutTask(message: Any): Unit = message match { - case StartSpout => pullNextBlock() - case "nextBlock" => pullNextBlock() + override def handleDomainMessage(message: BasicDomain): Unit = message match { + case Next => pullNextBlock() } def pullNextBlock(): Unit = { @@ -68,7 +66,7 @@ class EthereumGethSpout extends SpoutTrait { val trasnactionBlock = executeBatchRequest(transactions.dropRight(1)+"]") val transList = trasnactionBlock.parseJson.convertTo[List[EthTransaction]] transList.foreach(t => { //try needed to ignore contracts //todo include them - try{sendTuple(s"${t.result.blockNumber.get},${t.result.from.get},${t.result.to.get},${t.result.value.get}")} + try{sendTuple(StringSpoutGoing(s"${t.result.blockNumber.get},${t.result.from.get},${t.result.to.get},${t.result.value.get}"))} catch {case e:NoSuchElementException =>} }) @@ -95,13 +93,15 @@ class EthereumGethSpout extends SpoutTrait { baseRequest.postData(s"""{"jsonrpc": "2.0", "id":"100", "method": "$command", "params": [$params]}""") def executeRequest(command: String, params: String = "") = { - print(params) - print(request(command, params).execute().body.toString) request(command, params).execute().body.toString.parseJson.asJsObject - - } def hostname2Ip(hostname: String): String = InetAddress.getByName(hostname).getHostAddress() + override def startSpout(): Unit = self ! Next + } + +case class EthResult(blockHash:Option[String],blockNumber:Option[String],from:Option[String],gas:Option[String],gasPrice:Option[String],hash:Option[String],input:Option[String],nonce:Option[String],r:Option[String],s:Option[String],to:Option[String],transactionIndex:Option[String],v:Option[String],value:Option[String]) +case class EthTransaction(id:Option[String],jsonrpc:Option[String],result:EthResult) + diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/spouts/EthereumPostgresSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/spouts/blockchain/EthereumPostgresSpout.scala similarity index 72% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/spouts/EthereumPostgresSpout.scala rename to mainproject/cluster/src/main/scala/com/raphtory/spouts/blockchain/EthereumPostgresSpout.scala index 598a468b81..c302200677 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/spouts/EthereumPostgresSpout.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/spouts/blockchain/EthereumPostgresSpout.scala @@ -1,17 +1,15 @@ -package com.raphtory.examples.blockchain.spouts +package com.raphtory.spouts.blockchain -import cats.effect.Blocker -import cats.effect.IO +import cats.effect.{Blocker, IO} import com.raphtory.core.components.Spout.SpoutTrait +import com.raphtory.core.components.Spout.SpoutTrait.BasicDomain +import com.raphtory.core.components.Spout.SpoutTrait.CommonMessage.Next +import com.raphtory.core.model.communication.StringSpoutGoing import doobie.implicits._ import doobie.util.ExecutionContexts import doobie.util.transactor.Transactor -import scala.concurrent.duration.Duration -import scala.concurrent.duration.MILLISECONDS -import scala.concurrent.duration.SECONDS - -class EthereumPostgresSpout extends SpoutTrait { +class EthereumPostgresSpout extends SpoutTrait[BasicDomain, StringSpoutGoing] { var startBlock = System.getenv().getOrDefault("STARTING_BLOCK", "46147").trim.toInt //first block to have a transaction by default val batchSize = System.getenv().getOrDefault("BLOCK_BATCH_SIZE", "100").trim.toInt //number of blocks to pull each query val maxblock = System.getenv().getOrDefault("MAX_BLOCK", "8828337").trim.toInt //Maximum block in database to stop querying once this is reached @@ -30,12 +28,12 @@ class EthereumPostgresSpout extends SpoutTrait { Blocker.liftExecutionContext(ExecutionContexts.synchronous) ) - override def ProcessSpoutTask(message: Any): Unit = message match { - case StartSpout => AllocateSpoutTask(Duration(1, MILLISECONDS), "nextBatch") - case "nextBatch" => running() + override def handleDomainMessage(message: BasicDomain): Unit = message match { + case Next => running() case _ => println("message not recognized!") } + protected def running(): Unit = { sql"select from_address, to_address, value,block_timestamp from transactions where block_number >= $startBlock AND block_number < ${startBlock + batchSize} " .query[ @@ -44,10 +42,13 @@ class EthereumPostgresSpout extends SpoutTrait { .to[List] // ConnectionIO[List[String]] .transact(dbconnector) // IO[List[String]] .unsafeRunSync // List[String] - .foreach(x => sendTuple(x.toString())) //send each transaction to the routers + .foreach(x => sendTuple(StringSpoutGoing(x.toString()))) //send each transaction to the routers startBlock += batchSize //increment batch for the next query - if (startBlock > maxblock) stop() //if we have reached the max block we stop querying the database - AllocateSpoutTask(Duration(1, MILLISECONDS), "nextBatch") // line up the next batch + if (startBlock <= maxblock) //if we have reached the max block we stop querying the database + self ! Next // line up the next batch } + + override def startSpout(): Unit = self ! Next + } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/tests/TemporalTriangleCountExample.scala b/mainproject/cluster/src/main/scala/com/raphtory/tests/TemporalTriangleCountExample.scala index c506419f4f..33e8b856e3 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/tests/TemporalTriangleCountExample.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/tests/TemporalTriangleCountExample.scala @@ -25,13 +25,13 @@ object TemporalTriangleCountExample extends App{ system.actorOf(Props(new WatchDog(partitionNumber, minimumRouters)), "WatchDog") - //var SpoutName ="com.raphtory.examples.test.actors.TriangleTestSpout" + var SpoutName ="com.raphtory.examples.test.actors.TriangleTestSpout" //var SpoutName = "com.raphtory.examples.gab.actors.GabExampleSpout" - var SpoutName = "com.raphtory.spouts.FirehoseSpout" + //var SpoutName = "com.raphtory.spouts.FirehoseSpout" system.actorOf(Props(Class.forName(SpoutName)), "Spout") - //var routerClassName = "com.raphtory.examples.test.actors.TriangleTestRouter" - var routerClassName = "com.raphtory.examples.blockchain.routers.FirehoseKafkaRouter" + var routerClassName = "com.raphtory.examples.test.actors.TriangleTestRouter" + //var routerClassName = "com.raphtory.examples.blockchain.routers.FirehoseKafkaRouter" system.actorOf(Props(RaphtoryReplicator("Router", 1, routerClassName)), s"Routers") system.actorOf(Props(RaphtoryReplicator("Partition Manager", 1)), s"PartitionManager") From c8d90c2f48c9fd8a8914b7f5cbf42c6ab2e35b56 Mon Sep 17 00:00:00 2001 From: Miratepuffin Date: Thu, 29 Oct 2020 01:30:07 +0000 Subject: [PATCH 07/58] Initial Router example fix --- .../blockchain/routers/BitcoinRouter.scala | 59 ++++++++----------- .../examples/test/actors/RandomRouter.scala | 43 ++++++++------ 2 files changed, 48 insertions(+), 54 deletions(-) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/BitcoinRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/BitcoinRouter.scala index 6fbcc044be..5c82c2bf81 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/BitcoinRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/BitcoinRouter.scala @@ -1,18 +1,15 @@ package com.raphtory.examples.blockchain.routers import com.raphtory.core.components.Router.RouterWorker -import com.raphtory.core.model.communication.EdgeAdd -import com.raphtory.core.model.communication.EdgeAddWithProperties -import com.raphtory.core.model.communication.Properties -import com.raphtory.core.model.communication.StringProperty -import com.raphtory.core.model.communication.VertexAddWithProperties -import com.raphtory.examples.blockchain.BitcoinTransaction +import com.raphtory.core.model.communication._ +import com.raphtory.spouts.blockchain.BitcoinTransaction import spray.json.JsArray +import scala.collection.mutable.ListBuffer -class BitcoinRouter(override val routerId: Int,override val workerID:Int,val initialManagerCount: Int) extends RouterWorker { +class BitcoinRouter(override val routerId: Int,override val workerID:Int,override val initialManagerCount: Int) + extends RouterWorker[BitcoinTransaction](routerId,workerID,initialManagerCount) { - def parseTuple(record: Any): Unit = { - val value = record.asInstanceOf[BitcoinTransaction] + override protected def parseTuple(value:BitcoinTransaction): List[GraphUpdate] = { val transaction = value.transaction val time = value.time @@ -26,6 +23,7 @@ class BitcoinRouter(override val routerId: Int,override val workerID:Int,val ini val vouts = transaction.asJsObject.fields("vout") var total: Double = 0 + var commands = new ListBuffer[GraphUpdate]() for (vout <- vouts.asInstanceOf[JsArray].elements) { val voutOBJ = vout.asJsObject() var value = voutOBJ.fields("value").toString @@ -40,26 +38,21 @@ class BitcoinRouter(override val routerId: Int,override val workerID:Int,val ini //println(s"Edge $timeAsLong, ${txid.hashCode}, ${address.hashCode}, $n, $value") //creates vertex for the receiving wallet - sendGraphUpdate( - VertexAddWithProperties( + + commands += VertexAddWithProperties( msgTime = timeAsLong, srcID = address.hashCode, - properties = Properties(StringProperty("type", "address"), StringProperty("address", address)) - ) - ) + properties = Properties(StringProperty("type", "address"), + StringProperty("address", address))) //creates edge between the transaction and the wallet - sendGraphUpdate( - EdgeAddWithProperties( + commands += EdgeAddWithProperties( msgTime = timeAsLong, srcID = txid.hashCode, dstID = address.hashCode, - properties = Properties(StringProperty("n", n), StringProperty("value", value)) - ) - ) + properties = Properties(StringProperty("n", n), StringProperty("value", value))) } - sendGraphUpdate( - VertexAddWithProperties( + commands += VertexAddWithProperties( msgTime = timeAsLong, srcID = txid.hashCode, properties = Properties( @@ -69,22 +62,19 @@ class BitcoinRouter(override val routerId: Int,override val workerID:Int,val ini StringProperty("total", total.toString), StringProperty("blockhash", blockID.toString), StringProperty("block", block.toString) - ) - ) - ) + )) if (vins.toString().contains("coinbase")) { //creates the coingen node //TODO change so only added once - sendGraphUpdate( - VertexAddWithProperties( + commands += VertexAddWithProperties( msgTime = timeAsLong, srcID = "coingen".hashCode, - properties = Properties(StringProperty("type", "coingen")) - ) - ) + properties = Properties(StringProperty("type", "coingen"))) + //creates edge between coingen and the transaction - sendGraphUpdate(EdgeAdd(msgTime = timeAsLong, srcID = "coingen".hashCode, dstID = txid.hashCode)) + commands += EdgeAdd(msgTime = timeAsLong, srcID = "coingen".hashCode, dstID = txid.hashCode) + commands.toList } else for (vin <- vins.asInstanceOf[JsArray].elements) { val vinOBJ = vin.asJsObject() @@ -92,15 +82,14 @@ class BitcoinRouter(override val routerId: Int,override val workerID:Int,val ini val prevtxid = vinOBJ.fields("txid").toString //no need to create node for prevtxid as should already exist //creates edge between the prev transaction and current transaction - sendGraphUpdate( - EdgeAddWithProperties( + commands += EdgeAddWithProperties( msgTime = timeAsLong, srcID = prevtxid.hashCode, dstID = txid.hashCode, - properties = Properties(StringProperty("vout", prevVout)) - ) - ) + properties = Properties(StringProperty("vout", prevVout))) + } + commands.toList } } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomRouter.scala index 911fe35459..67d46d66a3 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomRouter.scala @@ -1,7 +1,7 @@ package com.raphtory.examples.test.actors import com.raphtory.core.components.Router.RouterWorker -import com.raphtory.core.model.communication._ +import com.raphtory.core.model.communication.{GraphUpdate, _} import spray.json._ /** @@ -14,25 +14,29 @@ import spray.json._ * the correct case Class which can then be passed to the graph manager * which will then pass it to the graph partition dealing with the associated vertex */ -class RandomRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int) extends RouterWorker { +class RandomRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int) + extends RouterWorker[StringSpoutGoing](routerId,workerID,initialManagerCount) { //************* MESSAGE HANDLING BLOCK - def parseTuple(record: Any): Unit = { - val command = record.asInstanceOf[String] + override protected def parseTuple(value:StringSpoutGoing): List[GraphUpdate] = { + val command = value.asInstanceOf[String] val parsedOBJ = command.parseJson.asJsObject //get the json object val commandKey = parsedOBJ.fields //get the command type - if (commandKey.contains("VertexAdd")) vertexAdd(parsedOBJ.getFields("VertexAdd").head.asJsObject) + if (commandKey.contains("VertexAdd")) + List(vertexAdd(parsedOBJ.getFields("VertexAdd").head.asJsObject)) //else if(commandKey.contains("VertexUpdateProperties")) vertexUpdateProperties(parsedOBJ.getFields("VertexUpdateProperties").head.asJsObject) - else if (commandKey.contains("VertexRemoval")) vertexRemoval(parsedOBJ.getFields("VertexRemoval").head.asJsObject) + else if (commandKey.contains("VertexRemoval")) + List(vertexRemoval(parsedOBJ.getFields("VertexRemoval").head.asJsObject)) else if (commandKey.contains("EdgeAdd")) - edgeAdd(parsedOBJ.getFields("EdgeAdd").head.asJsObject) //if addVertex, parse to handling function + List(edgeAdd(parsedOBJ.getFields("EdgeAdd").head.asJsObject)) //if addVertex, parse to handling function // else if(commandKey.contains("EdgeUpdateProperties")) edgeUpdateProperties(parsedOBJ.getFields("EdgeUpdateProperties").head.asJsObject) - else if (commandKey.contains("EdgeRemoval")) edgeRemoval(parsedOBJ.getFields("EdgeRemoval").head.asJsObject) - else println(command) + else if (commandKey.contains("EdgeRemoval")) + List(edgeRemoval(parsedOBJ.getFields("EdgeRemoval").head.asJsObject)) + else List() } - def vertexAdd(command: JsObject): Unit = { + def vertexAdd(command: JsObject) = { val msgTime = command.fields("messageID").toString().toLong val srcId = command.fields("srcID").toString().toInt //extract the srcID if (command.fields.contains("properties")) { //if there are properties within the command @@ -41,9 +45,9 @@ class RandomRouter(override val routerId: Int,override val workerID:Int, overrid // properties = properties updated (pair._1, pair._2.toString()) // }) //send the srcID and properties to the graph manager - sendGraphUpdate(VertexAddWithProperties(msgTime, srcId, properties)) + VertexAddWithProperties(msgTime, srcId, properties) } else - sendGraphUpdate(VertexAdd(msgTime, srcId)) + VertexAdd(msgTime, srcId) // if there are not any properties, just send the srcID } @@ -55,13 +59,13 @@ class RandomRouter(override val routerId: Int,override val workerID:Int, overrid // sendGraphUpdate(VertexUpdateProperties(msgTime,srcId,properties)) //send the srcID and properties to the graph parition // } - def vertexRemoval(command: JsObject): Unit = { + def vertexRemoval(command: JsObject):GraphUpdate = { val msgTime = command.fields("messageID").toString().toLong val srcId = command.fields("srcID").toString().toInt //extract the srcID - sendGraphUpdate(VertexDelete(msgTime, srcId)) + VertexDelete(msgTime, srcId) } - def edgeAdd(command: JsObject): Unit = { + def edgeAdd(command: JsObject):GraphUpdate = { val msgTime = command.fields("messageID").toString().toLong val srcId = command.fields("srcID").toString().toInt //extract the srcID val dstId = command.fields("dstID").toString().toInt //extract the dstID @@ -70,8 +74,8 @@ class RandomRouter(override val routerId: Int,override val workerID:Int, overrid // command.fields("properties").asJsObject.fields.foreach( pair => { //add all of the pairs to the map // properties = properties updated (pair._1,pair._2.toString()) // }) - sendGraphUpdate(EdgeAddWithProperties(msgTime, srcId, dstId, properties)) - } else sendGraphUpdate(EdgeAdd(msgTime, srcId, dstId)) + EdgeAddWithProperties(msgTime, srcId, dstId, properties) + } else EdgeAdd(msgTime, srcId, dstId) } // def edgeUpdateProperties(command:JsObject):Unit={ @@ -83,10 +87,11 @@ class RandomRouter(override val routerId: Int,override val workerID:Int, overrid // sendGraphUpdate(EdgeUpdateProperties(msgTime,srcId,dstId,properties))//send the srcID, dstID and properties to the graph manager // } - def edgeRemoval(command: JsObject): Unit = { + def edgeRemoval(command: JsObject):GraphUpdate = { val msgTime = command.fields("messageID").toString().toLong val srcId = command.fields("srcID").toString().toInt //extract the srcID val dstId = command.fields("dstID").toString().toInt //extract the dstID - sendGraphUpdate(EdgeDelete(msgTime, srcId, dstId)) + EdgeDelete(msgTime, srcId, dstId) } + } From 9ddfbd4d2f80912620864b786e9105214126fc42 Mon Sep 17 00:00:00 2001 From: miratepuffin Date: Thu, 29 Oct 2020 16:20:46 +0000 Subject: [PATCH 08/58] Deleting more unneeded examples --- .../routers/ChABClus2ClusRouter.scala | 2 +- .../examples/tsvnet/SamplerSpout.scala | 108 +++++++++--------- .../raphtory/examples/tsvnet/TSVSpout.scala | 40 ------- .../twitterRumour/rumourInteractRouter.scala | 64 ----------- .../examples/twitterRumour/rumourSpout.scala | 69 ----------- .../spouts/CooccurrenceMatrixSpout.scala | 78 ++++++------- .../spouts/blockchain/BitcoinSpout.scala | 1 - 7 files changed, 95 insertions(+), 267 deletions(-) delete mode 100644 mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/TSVSpout.scala delete mode 100644 mainproject/cluster/src/main/scala/com/raphtory/examples/twitterRumour/rumourInteractRouter.scala delete mode 100644 mainproject/cluster/src/main/scala/com/raphtory/examples/twitterRumour/rumourSpout.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusRouter.scala index 9247ce86e8..efb37cf19b 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusRouter.scala @@ -9,7 +9,7 @@ import com.raphtory.core.components.Router.RouterWorker import com.raphtory.core.model.communication.Type import com.raphtory.core.model.communication._ -class ChABClus2ClusRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int) extends RouterWorker { +class ChABClus2ClusRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int) extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount) { def parseTuple(record: Any): Unit = { val dp = formatLine(record.asInstanceOf[String].split(",").map(_.trim)) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/SamplerSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/SamplerSpout.scala index 1d056df8af..9ec2f671d0 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/SamplerSpout.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/SamplerSpout.scala @@ -1,53 +1,55 @@ -//package com.raphtory.examples.tsvnet -// -//import java.time.LocalDateTime -// -//import com.raphtory.core.components.Spout.SpoutTrait -// -//import scala.concurrent.duration.{Duration, NANOSECONDS} -//import scala.io -//import scala.util.Random -//import scala.util.control.Breaks._ -// -//class SamplerSpout extends SpoutTrait { -// -// val prop = 1.0/10 -// val r = new Random() -// val directory = System.getenv().getOrDefault("TSV_DIRECTORY", "/app").trim -// val file_name = System.getenv().getOrDefault("TSV_FILE_NAME", "sx_reordered.txt").trim -// val fileLines = io.Source.fromFile(directory + "/" + file_name).getLines.drop(1).toArray -// // upstream/master -// var position = 0 -// var linesNumber = fileLines.length -// println("Start: " + LocalDateTime.now()) -// -// protected def ProcessSpoutTask(message: Any): Unit = message match { -// case StartSpout => AllocateSpoutTask(Duration(1, NANOSECONDS), "newLine") -// case "newLine" => -// try { -// if (position < linesNumber) { -// for (i <- 1 to 100) { -// val randFloat = r.nextFloat() -// breakable { -// if (randFloat > prop) { -// position +=1 -// break -// } else { -// val line = fileLines(position) -// sendTuple(line) -// position += 1 -// } -// } -// } -// AllocateSpoutTask(Duration(1, NANOSECONDS), "newLine") -// } -// else -// { -// println("Finished ingestion") -// } -// } -// catch {case e:Exception => println("Finished ingestion")} -// case _ => println("message not recognized!") -// } -// -//} +package com.raphtory.examples.tsvnet + +import java.time.LocalDateTime + +import com.raphtory.core.components.Spout.SpoutTrait +import com.raphtory.core.components.Spout.SpoutTrait.BasicDomain +import com.raphtory.core.model.communication.StringSpoutGoing +import monix.tail.Iterant.Next + +import scala.concurrent.duration.{Duration, NANOSECONDS} +import scala.io +import scala.util.Random +import scala.util.control.Breaks._ + +class SamplerSpout extends SpoutTrait[BasicDomain,StringSpoutGoing] { + + val prop = 1.0/10 + val r = new Random() + val directory = System.getenv().getOrDefault("TSV_DIRECTORY", "/app").trim + val file_name = System.getenv().getOrDefault("TSV_FILE_NAME", "sx_reordered.txt").trim + val fileLines = io.Source.fromFile(directory + "/" + file_name).getLines.drop(1).toArray + // upstream/master + var position = 0 + var linesNumber = fileLines.length + println("Start: " + LocalDateTime.now()) + + protected def ProcessSpoutTask(message: Any): Unit = message match { + case Next => + try { + if (position < linesNumber) { + for (i <- 1 to 100) { + val randFloat = r.nextFloat() + breakable { + if (randFloat > prop) { + position +=1 + break + } else { + val line = fileLines(position) + sendTuple(line) + position += 1 + } + } + } + AllocateSpoutTask(Duration(1, NANOSECONDS), "newLine") + } + else + { + println("Finished ingestion") + } + } + catch {case e:Exception => println("Finished ingestion")} + case _ => println("message not recognized!") + } + +} diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/TSVSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/TSVSpout.scala deleted file mode 100644 index 2373386c3d..0000000000 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/TSVSpout.scala +++ /dev/null @@ -1,40 +0,0 @@ -//package com.raphtory.examples.tsvnet -// -//import java.time.LocalDateTime -// -//import com.raphtory.core.components.Spout.SpoutTrait -// -//import scala.concurrent.duration._ -//import scala.io -//import scala.language.postfixOps -// -///** Spout for network datasets of the form SRC_NODE_ID DEST_NODE_ID TIMESTAMP */ -//class TSVSpout extends SpoutTrait { -// -// val directory = System.getenv().getOrDefault("TSV_DIRECTORY", "/app").trim -// val file_name = System.getenv().getOrDefault("TSV_FILE_NAME", "sx_reordered.txt").trim -// val fileLines = io.Source.fromFile(directory + "/" + file_name).getLines.drop(1).toArray -// // upstream/master -// var position = 0 -// var linesNumber = fileLines.length -// println("Start: " + LocalDateTime.now()) -// -// protected def ProcessSpoutTask(message: Any): Unit = message match { -// case StartSpout => AllocateSpoutTask(Duration(1, NANOSECONDS), "newLine") -// case "newLine" => -// try { -// if (position < linesNumber) { -// for (i <- 0 to 100) { -// var line = fileLines(position) -// sendTuple(line) -// position += 1 -// } -// AllocateSpoutTask(Duration(1, NANOSECONDS), "newLine") -// } -// else { -// println("Finished ingestion") -// } -// }catch {case e:Exception => println("Finished ingestion")} -// case _ => println("message not recognized!") -// } -//} diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/twitterRumour/rumourInteractRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/twitterRumour/rumourInteractRouter.scala deleted file mode 100644 index de9b1a3b74..0000000000 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/twitterRumour/rumourInteractRouter.scala +++ /dev/null @@ -1,64 +0,0 @@ -package com.raphtory.examples.twitterRumour - -import java.text.SimpleDateFormat - -import com.raphtory.core.components.Router.RouterWorker -import com.raphtory.core.model.communication._ -import spray.json._ - -import scala.io.Source - -class rumourInteractRouter(override val routerId: Int,override val workerID:Int, val initialManagerCount: Int) extends RouterWorker { - - override protected def parseTuple(cmd: Any): Unit = { - //println("im at router top...") - val List(r_status, tweet) = cmd.asInstanceOf[String].split("__").toList - val json = Source.fromFile(tweet) - for (line <- json.getLines) { - // println("reading json"+cmd) - var user = line.parseJson.asJsObject.fields("user").asJsObject - val post = line.parseJson.asJsObject - - val replyTime = post.fields("created_at").toString.toString.split("\"")(1) - val source = user.fields("id").toString - // if (source.toLong <0){println("this is converting worng.."+source) - // sys.exit()} - val dist = post.fields("in_reply_to_user_id").toString - - if (dist != "null") - sendGraphUpdate( - EdgeAddWithProperties( - getTwitterDate(replyTime), - source.toLong, - dist.toLong, - properties = Properties(ImmutableProperty("rumourStatus", r_status)) - ) - ) - else - sendGraphUpdate( - VertexAddWithProperties( - getTwitterDate(replyTime), - source.toLong, - properties = Properties(ImmutableProperty("rumourStatus", r_status)) - ) - ) - } - } - - def getTwitterDate(date: String): Long = { - // println(">>> converting time...") - val twitter = "EEE MMM dd HH:mm:ss ZZZZZ yyyy" - val sf = new SimpleDateFormat(twitter) - //println(date) - try - //println("converted time ///"+t) - return sf.parse(date).getTime() - catch { - case e: Throwable => - println("-----time not properly converting" + date) - sys.exit() - return 0 - } - - } -} diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/twitterRumour/rumourSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/twitterRumour/rumourSpout.scala deleted file mode 100644 index 6093a7e4f2..0000000000 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/twitterRumour/rumourSpout.scala +++ /dev/null @@ -1,69 +0,0 @@ -//package com.raphtory.examples.twitterRumour -// -//import akka.actor.ActorContext -//import com.raphtory.core.components.Spout.SpoutTrait -// -//import scala.concurrent.duration.Duration -//import scala.concurrent.duration.MILLISECONDS -//import scala.concurrent.duration.SECONDS -//import java.io.File -// -//class rumourSpout extends SpoutTrait { -// var directory = System.getenv().getOrDefault("PHEME_DIRECTORY", "/home/tsunade/qmul/datasets/pheme-rnr-dataset").trim -// var r_case = System.getenv().getOrDefault("RUMOUR_CASE", "/sydneysiege").trim -// val rpath = directory + r_case + "/rumours" -// val nrpath = directory + r_case + "/non-rumours" -// val max_r = 500 //Option(new File(rpath).list.size).getOrElse(0) -// val max_nr = 500 //Option(new File(nrpath).list.size).getOrElse(0) -// -// var rtweetCounter = 0 -// var nrtweetCounter = 0 -// -// override protected def ProcessSpoutTask(message: Any): Unit = -// // println(message) -// if ((rtweetCounter < max_r) || (nrtweetCounter < max_nr)) -// //println("inside if stm") -// message match { -// case StartSpout => -// AllocateSpoutTask(Duration(1, SECONDS), "rumourTweets") -// AllocateSpoutTask(Duration(1, SECONDS), "nonRumourTweets") -// case "rumourTweets" => -// if (rtweetCounter < max_r) { -// // println("sending ingest cmd r...") -// running(rpath, position = rtweetCounter, "R__") -// rtweetCounter += 1 -// context.system.scheduler.scheduleOnce(Duration(1, SECONDS), self, "rumourTweets") -// } -// case "nonRumourTweets" => -// if (nrtweetCounter < max_nr) { -// // println("sending ingest cmd nr...") -// running(nrpath, position = nrtweetCounter, "nR__") -// nrtweetCounter += 1 -// context.system.scheduler.scheduleOnce(Duration(1, SECONDS), self, "nonRumourTweets") -// } -// case _ => println("message not recognized!") -// } -// -// def ingestTweet(path: String, rnr: String): Unit = { -// // println("inside ingest....") -// val tweet_l = new File(path) -// if (tweet_l.exists && tweet_l.isDirectory) -// tweet_l.listFiles.filter(_.isFile).toList.map(_.toString).foreach { tw_path: String => -// sendTuple(rnr + tw_path) -// // println("running sendTuple...") -// } -// } -// -// def running(rpath: String, position: Int, rnr: String): Unit = { -// val rumours = new File(rpath) -// // println("inside runnign...") -// -// if (rumours.exists && rumours.isDirectory) { -// -// //println("inside runnign if...") -// val r_tweet = rumours.listFiles.toList.map(_.toString) -// ingestTweet(r_tweet(position) + "/source-tweet", rnr) -// ingestTweet(r_tweet(position) + "/reactions", rnr) -// } -// } -//} diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/spouts/CooccurrenceMatrixSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/spouts/CooccurrenceMatrixSpout.scala index 66a731b27d..2fc30ea650 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/spouts/CooccurrenceMatrixSpout.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/spouts/CooccurrenceMatrixSpout.scala @@ -1,39 +1,39 @@ -//package com.raphtory.examples.wordSemantic.spouts -// -//import java.time.LocalDateTime -// -//import com.raphtory.core.components.Spout.SpoutTrait -// -//import scala.concurrent.duration._ -//import scala.io.Source -//import scala.language.postfixOps -// -//class CooccurrenceMatrixSpout extends SpoutTrait { -// -// val file_name = System.getenv().getOrDefault("DATA_FILENAME", "/home/tsunade/qmul/datasets/word_semantics/test_file_0.txt").trim -// val fl = Source.fromFile(file_name) -// val fileLines = fl.getLines//.toArray -// var time = file_name.replace(".txt", "").split('_').last -// // upstream/master -//// var position = 0 -//// var linesNumber = fl.length -// println("Starting File ingestion: " + LocalDateTime.now()) -//// println("Lines :" + linesNumber) -// -// override def ProcessSpoutTask(message: Any): Unit = message match { -// case StartSpout => AllocateSpoutTask(Duration(1, NANOSECONDS), "newLine") -// case "newLine" => -// -// // if (position < linesNumber) { -// //val line = if(fileLines.hasNext) fileLines.next() else "" -// try { -// sendTuple(time+' '+fileLines.next()) -// AllocateSpoutTask(Duration(1, NANOSECONDS), "newLine") -// }catch{ -// case e: NoSuchElementException => println("End of file!") -// } -// // position += 1 -// // } -// case _ => println("message not recognized!") -// } -//} \ No newline at end of file +package com.raphtory.examples.wordSemantic.spouts + +import java.time.LocalDateTime + +import com.raphtory.core.components.Spout.SpoutTrait + +import scala.concurrent.duration._ +import scala.io.Source +import scala.language.postfixOps + +class CooccurrenceMatrixSpout extends SpoutTrait { + + val file_name = System.getenv().getOrDefault("DATA_FILENAME", "/home/tsunade/qmul/datasets/word_semantics/test_file_0.txt").trim + val fl = Source.fromFile(file_name) + val fileLines = fl.getLines//.toArray + var time = file_name.replace(".txt", "").split('_').last + // upstream/master +// var position = 0 +// var linesNumber = fl.length + println("Starting File ingestion: " + LocalDateTime.now()) +// println("Lines :" + linesNumber) + + override def ProcessSpoutTask(message: Any): Unit = message match { + case StartSpout => AllocateSpoutTask(Duration(1, NANOSECONDS), "newLine") + case "newLine" => + + // if (position < linesNumber) { + //val line = if(fileLines.hasNext) fileLines.next() else "" + try { + sendTuple(time+' '+fileLines.next()) + AllocateSpoutTask(Duration(1, NANOSECONDS), "newLine") + }catch{ + case e: NoSuchElementException => println("End of file!") + } + // position += 1 + // } + case _ => println("message not recognized!") + } +} \ No newline at end of file diff --git a/mainproject/cluster/src/main/scala/com/raphtory/spouts/blockchain/BitcoinSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/spouts/blockchain/BitcoinSpout.scala index 5558bd5cdd..518dfec7e3 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/spouts/blockchain/BitcoinSpout.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/spouts/blockchain/BitcoinSpout.scala @@ -7,7 +7,6 @@ import com.raphtory.core.components.Spout.SpoutTrait import com.raphtory.core.components.Spout.SpoutTrait.BasicDomain import com.raphtory.core.components.Spout.SpoutTrait.CommonMessage.Next import com.raphtory.core.model.communication.SpoutGoing -import com.raphtory.spouts.blockchain.BitcoinSpout.Message.{BitcoinDomain, ParseBlock} import scalaj.http.Http import scalaj.http.HttpRequest import spray.json._ From a2abaf21196787cf2335e42cba8bcbda68e28209 Mon Sep 17 00:00:00 2001 From: miratepuffin Date: Thu, 29 Oct 2020 17:21:14 +0000 Subject: [PATCH 09/58] Fixed spouts post merging in all branches --- .../examples/tsvnet/SamplerSpout.scala | 3 +- .../spouts/CooccurrenceMatrixSpout.scala | 28 +++++++++++++------ .../CooccurrenceMatrixSpoutFiltered.scala | 3 +- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/SamplerSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/SamplerSpout.scala index 9ec2f671d0..24ef35966f 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/SamplerSpout.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/SamplerSpout.scala @@ -8,7 +8,6 @@ import com.raphtory.core.model.communication.StringSpoutGoing import monix.tail.Iterant.Next import scala.concurrent.duration.{Duration, NANOSECONDS} -import scala.io import scala.util.Random import scala.util.control.Breaks._ @@ -18,7 +17,7 @@ class SamplerSpout extends SpoutTrait[BasicDomain,StringSpoutGoing] { val r = new Random() val directory = System.getenv().getOrDefault("TSV_DIRECTORY", "/app").trim val file_name = System.getenv().getOrDefault("TSV_FILE_NAME", "sx_reordered.txt").trim - val fileLines = io.Source.fromFile(directory + "/" + file_name).getLines.drop(1).toArray + val fileLines = scala.io.Source.fromFile(directory + "/" + file_name).getLines.drop(1).toArray // upstream/master var position = 0 var linesNumber = fileLines.length diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/spouts/CooccurrenceMatrixSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/spouts/CooccurrenceMatrixSpout.scala index 2e62cf301f..21729334cc 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/spouts/CooccurrenceMatrixSpout.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/spouts/CooccurrenceMatrixSpout.scala @@ -5,12 +5,14 @@ import java.io.{BufferedReader, File, FileReader} import java.time.LocalDateTime import com.raphtory.core.components.Spout.SpoutTrait -import com.raphtory.spouts.EtherFileReader +import com.raphtory.core.components.Spout.SpoutTrait.{BasicDomain, DomainMessage} +import com.raphtory.core.model.communication.StringSpoutGoing +import com.raphtory.examples.wordSemantic.spouts.CooccurrenceMatrixSpout.Message.{CooccuranceDomain, NextFile, NextLineBlock, NextLineSlice} import scala.concurrent.duration._ import scala.language.postfixOps -class CooccurrenceMatrixSpout extends SpoutTrait { +class CooccurrenceMatrixSpout extends SpoutTrait[CooccuranceDomain,StringSpoutGoing] { println("Start: " + LocalDateTime.now()) val directory = System.getenv().getOrDefault("FILE_SPOUT_DIRECTORY", "/app").trim @@ -38,12 +40,11 @@ class CooccurrenceMatrixSpout extends SpoutTrait { - override protected def ProcessSpoutTask(message: Any): Unit = message match { - case StartSpout => AllocateSpoutTask(Duration(1, NANOSECONDS), "nextLineSlice") + def handleDomainMessage(message: CooccuranceDomain): Unit = message match { - case "nextLineSlice" => nextLineSlice() - case "nextLineBLock" => nextLineBlock() - case "nextFile" => nextFile() + case NextLineSlice => nextLineSlice() + case NextLineBlock => nextLineBlock() + case NextFile => nextFile() case _ => println("message not recognized!") } def nextLineSlice() = { @@ -52,7 +53,7 @@ class CooccurrenceMatrixSpout extends SpoutTrait { val head = currentLine(0) for (i<- 1 to Set(JUMP, currentLine.length-posSlice/JUMP2).min) { val currentSlice = currentLine.slice(posSlice, posSlice + JUMP2) - sendTuple(cnt.toString + ' ' + head + "\t" + currentSlice.mkString("\t")) + sendTuple(StringSpoutGoing(cnt.toString + ' ' + head + "\t" + currentSlice.mkString("\t"))) posSlice += JUMP2 } AllocateSpoutTask(Duration(1, MILLISECONDS), "nextLineSlice") @@ -111,4 +112,15 @@ class CooccurrenceMatrixSpout extends SpoutTrait { Array[String]() } } + + override def startSpout(): Unit = self ! NextLineSlice +} + +object CooccurrenceMatrixSpout { + object Message { + sealed trait CooccuranceDomain extends DomainMessage + case object NextLineSlice extends CooccuranceDomain + case object NextLineBlock extends CooccuranceDomain + case object NextFile extends CooccuranceDomain + } } \ No newline at end of file diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/spouts/CooccurrenceMatrixSpoutFiltered.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/spouts/CooccurrenceMatrixSpoutFiltered.scala index 4133008853..783a2fdab6 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/spouts/CooccurrenceMatrixSpoutFiltered.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/spouts/CooccurrenceMatrixSpoutFiltered.scala @@ -4,6 +4,7 @@ import java.io.{BufferedReader, File, FileReader} import java.time.LocalDateTime import com.raphtory.core.components.Spout.SpoutTrait +import com.raphtory.core.model.communication.StringSpoutGoing import scala.concurrent.duration._ import scala.language.postfixOps @@ -18,7 +19,7 @@ class CooccurrenceMatrixSpoutFiltered extends CooccurrenceMatrixSpout { val head = currentLine(0) for (i<- 1 to Set(JUMP, currentLine.length-posSlice/JUMP2).min) { val currentSlice = currentLine.slice(posSlice, posSlice + JUMP2) - sendTuple(cnt.toString + ' ' + scale.toString + ' ' + head + "\t" + currentSlice.mkString("\t")) + sendTuple(StringSpoutGoing(cnt.toString + ' ' + scale.toString + ' ' + head + "\t" + currentSlice.mkString("\t"))) posSlice += JUMP2 } AllocateSpoutTask(Duration(1, MILLISECONDS), "nextLineSlice") From 03ba1c32330175edf67efce7ea6b6b70d867e1d2 Mon Sep 17 00:00:00 2001 From: miratepuffin Date: Thu, 29 Oct 2020 23:17:45 +0000 Subject: [PATCH 10/58] Finalised Merge of all PRs and branches --- .../src/main/resources/application.conf | 2 +- .../core/components/Router/RouterWorker.scala | 2 +- .../core/components/Spout/SpoutTrait.scala | 2 +- .../blockchain/routers/BitcoinRouter.scala | 10 +- .../routers/ChABClus2ClusRouter.scala | 21 +- .../routers/ChABClus2ClusRouterNoprop.scala | 19 +- .../routers/ChainalysisABRouter.scala | 22 +- .../blockchain/routers/DashcoinRouter.scala | 40 ++-- .../routers/EthereumGethRouter.scala | 25 ++- .../routers/EthereumKafkaRouter.scala | 27 ++- .../routers/EthereumTransactionRouter.scala | 26 +-- .../routers/FirehoseKafkaRouter.scala | 22 +- .../blockchain/routers/LitecoinRouter.scala | 42 ++-- .../citationNetwork/CitationRouter.scala | 18 +- .../gab/routers/GabPostGraphRouter.scala | 19 +- .../examples/gab/routers/GabRawRouter.scala | 195 +++++++++--------- .../gab/routers/GabUserGraphRouter.scala | 18 +- .../examples/ldbc/routers/LDBCOldRouter.scala | 21 +- .../examples/ldbc/routers/LDBCRouter.scala | 24 +-- .../raphtory/examples/lotr/LOTRRouter.scala | 16 +- .../examples/test/actors/RandomRouter.scala | 4 +- .../test/actors/TriangleTestRouter.scala | 13 +- .../routers/TrackAndTraceRouter.scala | 19 +- .../examples/tsvnet/SamplerSpout.scala | 8 +- .../raphtory/examples/tsvnet/TSVRouter.scala | 21 +- .../routers/CooccurrenceMatrixRouter.scala | 17 +- .../CooccurrenceMatrixRouterFiltered.scala | 15 +- .../com/raphtory/routers/RouterCSV.scala | 26 --- .../spouts/blockchain/EthereumGethSpout.scala | 1 - .../com/raphtory/tests/SingleNodeTest.scala | 2 - 30 files changed, 361 insertions(+), 336 deletions(-) delete mode 100644 mainproject/cluster/src/main/scala/com/raphtory/routers/RouterCSV.scala diff --git a/mainproject/cluster/src/main/resources/application.conf b/mainproject/cluster/src/main/resources/application.conf index 7d094844da..8ab8f88d55 100644 --- a/mainproject/cluster/src/main/resources/application.conf +++ b/mainproject/cluster/src/main/resources/application.conf @@ -163,7 +163,7 @@ akka { # Note that it might take up to 1 tick to stop the Timer, so setting the # tick-duration to a high value will make shutting down the actor system # take longer. - tick-duration = 1ms + tick-duration = 10ms # The timer uses a circular wheel of buckets to store the timer tasks. # This should be set such that the majority of scheduled timeouts (for high diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala index 79ca540585..3237ee4450 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala @@ -30,7 +30,7 @@ abstract class RouterWorker[In <: SpoutGoing](val routerId: Int, val workerID: I // todo: wvv let people know parseTuple will create a list of update message // and this trait will handle logic to send to graph - protected def parseTuple(value: In): List[GraphUpdate] + protected def parseTuple(tuple: In): List[GraphUpdate] final protected val mediator = DistributedPubSub(context.system).mediator mediator ! DistributedPubSubMediator.Put(self) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/SpoutTrait.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/SpoutTrait.scala index bb00ca2f87..a40dbaf0cc 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/SpoutTrait.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/SpoutTrait.scala @@ -100,5 +100,5 @@ object SpoutTrait { case object Next extends BasicDomain } trait DomainMessage - sealed trait BasicDomain + sealed trait BasicDomain extends DomainMessage } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/BitcoinRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/BitcoinRouter.scala index 5c82c2bf81..a4bdb2bd6a 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/BitcoinRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/BitcoinRouter.scala @@ -9,12 +9,12 @@ import scala.collection.mutable.ListBuffer class BitcoinRouter(override val routerId: Int,override val workerID:Int,override val initialManagerCount: Int) extends RouterWorker[BitcoinTransaction](routerId,workerID,initialManagerCount) { - override protected def parseTuple(value:BitcoinTransaction): List[GraphUpdate] = { + override protected def parseTuple(tuple:BitcoinTransaction): List[GraphUpdate] = { - val transaction = value.transaction - val time = value.time - val blockID = value.blockID - val block = value.block + val transaction = tuple.transaction + val time = tuple.time + val blockID = tuple.blockID + val block = tuple.block val timeAsString = time.toString val timeAsLong = timeAsString.toLong diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusRouter.scala index efb37cf19b..94860b9693 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusRouter.scala @@ -1,18 +1,19 @@ package com.raphtory.examples.blockchain.routers import java.text.SimpleDateFormat - import java.util.Calendar - import com.raphtory.core.components.Router.RouterWorker import com.raphtory.core.model.communication.Type import com.raphtory.core.model.communication._ -class ChABClus2ClusRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int) extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount) { +import scala.collection.mutable.ListBuffer + +class ChABClus2ClusRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int) + extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount) { - def parseTuple(record: Any): Unit = { - val dp = formatLine(record.asInstanceOf[String].split(",").map(_.trim)) + override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { + val dp = formatLine(tuple.value.split(",").map(_.trim)) val transactionTime = dp.time val srcClusterId = dp.srcCluster val dstClusterId = dp.dstCluster @@ -20,10 +21,12 @@ class ChABClus2ClusRouter(override val routerId: Int,override val workerID:Int, val btcAmount = dp.amount val usdAmount = dp.usd - sendGraphUpdate(VertexAdd(msgTime = transactionTime, srcID = srcClusterId, Type("Cluster"))) - sendGraphUpdate(VertexAdd(msgTime = transactionTime, srcID = dstClusterId, Type("Cluster"))) + val commands = new ListBuffer[GraphUpdate]() - sendGraphUpdate( + commands+= VertexAdd(msgTime = transactionTime, srcID = srcClusterId, Type("Cluster")) + commands+=(VertexAdd(msgTime = transactionTime, srcID = dstClusterId, Type("Cluster"))) + + commands+=( EdgeAddWithProperties(msgTime = transactionTime, srcID = srcClusterId, dstID = dstClusterId, @@ -33,6 +36,7 @@ class ChABClus2ClusRouter(override val routerId: Int,override val workerID:Int, Type("Transfer") ) ) + commands.toList } //converts the line into a case class which has all of the data via the correct name and type @@ -58,4 +62,5 @@ class ChABClus2ClusRouter(override val routerId: Int,override val workerID:Int, txid: Long, //ID of transaction, can be similar for many records usd: Double //Amount of transaction in USD ) + } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusRouterNoprop.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusRouterNoprop.scala index de6fd001fa..615341a944 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusRouterNoprop.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusRouterNoprop.scala @@ -7,18 +7,21 @@ import com.raphtory.core.components.Router.RouterWorker import com.raphtory.core.model.communication.Type import com.raphtory.core.model.communication._ -class ChABClus2ClusRouterNoprop(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int) extends RouterWorker { +import scala.collection.mutable.ListBuffer - def parseTuple(record: Any): Unit = { - val dp = formatLine(record.asInstanceOf[String].split(",").map(_.trim)) +class ChABClus2ClusRouterNoprop(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int) + extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount) { + + override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { + val dp = formatLine(tuple.value.split(",").map(_.trim)) val transactionTime = dp.time val srcClusterId = dp.srcCluster val dstClusterId = dp.dstCluster - - sendGraphUpdate(VertexAdd(msgTime = transactionTime, srcID = srcClusterId, Type("Cluster"))) - sendGraphUpdate(VertexAdd(msgTime = transactionTime, srcID = dstClusterId, Type("Cluster"))) - sendGraphUpdate(EdgeAdd(msgTime = transactionTime, srcID = srcClusterId, dstID = dstClusterId, Type("Transfer"))) - + val commands = new ListBuffer[GraphUpdate]() + commands+=(VertexAdd(msgTime = transactionTime, srcID = srcClusterId, Type("Cluster"))) + commands+=(VertexAdd(msgTime = transactionTime, srcID = dstClusterId, Type("Cluster"))) + commands+=(EdgeAdd(msgTime = transactionTime, srcID = srcClusterId, dstID = dstClusterId, Type("Transfer"))) + commands.toList } //converts the line into a case class which has all of the data via the correct name and type diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChainalysisABRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChainalysisABRouter.scala index ee948f51e0..b848b25212 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChainalysisABRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChainalysisABRouter.scala @@ -6,22 +6,26 @@ import com.raphtory.core.components.Router.RouterWorker import com.raphtory.core.model.communication.Type import com.raphtory.core.model.communication._ -class ChainalysisABRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int) extends RouterWorker { +import scala.collection.mutable.ListBuffer - def parseTuple(record: Any): Unit = { - val dp = formatLine(record.asInstanceOf[String].split(",").map(_.trim)) +class ChainalysisABRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int) + extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount) { + + override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { + val dp = formatLine(tuple.value.split(",").map(_.trim)) val transactionTime = dp.time val srcClusterId = dp.srcCluster val dstClusterId = dp.dstCluster val transactionId = dp.txid val btcAmount = dp.amount val usdAmount = dp.usd + val commands = new ListBuffer[GraphUpdate]() - sendGraphUpdate(VertexAdd(msgTime = transactionTime, srcID = srcClusterId, Type("Cluster"))) - sendGraphUpdate(VertexAdd(msgTime = transactionTime, srcID = dstClusterId, Type("Cluster"))) - sendGraphUpdate(VertexAdd(msgTime = transactionTime, srcID = transactionId, Type("Transaction"))) + commands+=(VertexAdd(msgTime = transactionTime, srcID = srcClusterId, Type("Cluster"))) + commands+=(VertexAdd(msgTime = transactionTime, srcID = dstClusterId, Type("Cluster"))) + commands+=(VertexAdd(msgTime = transactionTime, srcID = transactionId, Type("Transaction"))) - sendGraphUpdate( + commands+=( EdgeAddWithProperties(msgTime = transactionTime, srcID = srcClusterId, dstID = transactionId, @@ -29,7 +33,7 @@ class ChainalysisABRouter(override val routerId: Int,override val workerID:Int, Type("Incoming Payment") ) ) - sendGraphUpdate( + commands+=( EdgeAddWithProperties(msgTime = transactionTime, srcID = transactionId, dstID = dstClusterId, @@ -37,7 +41,7 @@ class ChainalysisABRouter(override val routerId: Int,override val workerID:Int, Type("Outgoing Payment") ) ) - + commands.toList } //converts the line into a case class which has all of the data via the correct name and type diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/DashcoinRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/DashcoinRouter.scala index 1cc97b47b5..174abce111 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/DashcoinRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/DashcoinRouter.scala @@ -1,25 +1,21 @@ package com.raphtory.examples.blockchain.routers import com.raphtory.core.components.Router.RouterWorker -import com.raphtory.core.model.communication.EdgeAdd -import com.raphtory.core.model.communication.EdgeAddWithProperties -import com.raphtory.core.model.communication.Properties -import com.raphtory.core.model.communication.StringProperty -import com.raphtory.core.model.communication.VertexAdd -import com.raphtory.core.model.communication.VertexAddWithProperties -import com.raphtory.examples.blockchain.LitecoinTransaction +import com.raphtory.core.model.communication.{EdgeAdd, EdgeAddWithProperties, GraphUpdate, Properties, StringProperty, StringSpoutGoing, VertexAdd, VertexAddWithProperties} +import com.raphtory.spouts.blockchain.BitcoinTransaction import spray.json.JsArray +import scala.collection.mutable.ListBuffer import scala.util.hashing.MurmurHash3 -class DashcoinRouter(override val routerId: Int,override val workerID:Int, val initialManagerCount: Int) extends RouterWorker { +class DashcoinRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int) + extends RouterWorker[BitcoinTransaction](routerId,workerID, initialManagerCount) { - def parseTuple(record: Any): Unit = { - val value = record.asInstanceOf[LitecoinTransaction] - val transaction = value.transaction - val time = value.time - val blockID = value.blockID - val block = value.block + override protected def parseTuple(tuple: BitcoinTransaction): List[GraphUpdate] = { + val transaction = tuple.transaction + val time = tuple.time + val blockID = tuple.blockID + val block = tuple.block val timeAsString = time.toString val timeAsLong = (timeAsString.toLong) * 1000 @@ -29,7 +25,7 @@ class DashcoinRouter(override val routerId: Int,override val workerID:Int, val i val locktime = transaction.asJsObject.fields("locktime") val version = transaction.asJsObject.fields("version") var total: Double = 0 - + val commands = new ListBuffer[GraphUpdate]() for (vout <- vouts.asInstanceOf[JsArray].elements) { val voutOBJ = vout.asJsObject() var value = voutOBJ.fields("value").toString @@ -45,7 +41,7 @@ class DashcoinRouter(override val routerId: Int,override val workerID:Int, val i else value = "0" //TODO deal with people burning money //creates vertex for the receiving wallet - sendGraphUpdate( + commands+=( VertexAddWithProperties( msgTime = timeAsLong, srcID = MurmurHash3.stringHash(address), @@ -57,7 +53,7 @@ class DashcoinRouter(override val routerId: Int,override val workerID:Int, val i ) ) //creates edge between the transaction and the wallet - sendGraphUpdate( + commands+=( EdgeAddWithProperties( msgTime = timeAsLong, srcID = MurmurHash3.stringHash(txid), @@ -67,7 +63,7 @@ class DashcoinRouter(override val routerId: Int,override val workerID:Int, val i ) } - sendGraphUpdate( + commands+=( VertexAddWithProperties( msgTime = timeAsLong, srcID = MurmurHash3.stringHash(txid), @@ -86,10 +82,10 @@ class DashcoinRouter(override val routerId: Int,override val workerID:Int, val i if (vins.toString().contains("coinbase")) { //creates the coingen node //toPartitionManager(VertexAddWithProperties(msgTime = timeAsLong, srcID = MurmurHash3.stringHash("coingen"), properties = Map[String,String](("type","coingen")))) - sendGraphUpdate(VertexAdd(msgTime = timeAsLong, srcID = MurmurHash3.stringHash("coingen"))) + commands+=(VertexAdd(msgTime = timeAsLong, srcID = MurmurHash3.stringHash("coingen"))) //creates edge between coingen and the transaction - sendGraphUpdate( + commands+=( EdgeAdd( msgTime = timeAsLong, srcID = MurmurHash3.stringHash("coingen"), @@ -104,7 +100,7 @@ class DashcoinRouter(override val routerId: Int,override val workerID:Int, val i val sequence = vinOBJ.fields("sequence").toString //no need to create node for prevtxid as should already exist //creates edge between the prev transaction and current transaction - sendGraphUpdate( + commands+=( EdgeAddWithProperties( msgTime = timeAsLong, srcID = MurmurHash3.stringHash(prevtxid), @@ -112,7 +108,9 @@ class DashcoinRouter(override val routerId: Int,override val workerID:Int, val i properties = Properties(StringProperty("vout", prevVout), StringProperty("sequence", sequence)) ) ) + } + commands.toList } } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumGethRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumGethRouter.scala index 7a8b8d6637..f3892ac7e3 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumGethRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumGethRouter.scala @@ -2,24 +2,22 @@ package com.raphtory.examples.blockchain.routers import akka.http.scaladsl.unmarshalling.Unmarshal import com.raphtory.core.components.Router.RouterWorker -import com.raphtory.core.model.communication.EdgeAddWithProperties -import com.raphtory.core.model.communication.ImmutableProperty -import com.raphtory.core.model.communication.Properties -import com.raphtory.core.model.communication.StringProperty -import com.raphtory.core.model.communication.VertexAddWithProperties +import com.raphtory.core.model.communication.{EdgeAddWithProperties, GraphUpdate, ImmutableProperty, Properties, StringProperty, StringSpoutGoing, VertexAddWithProperties} import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._ import akka.stream.ActorMaterializer import spray.json._ +import scala.collection.mutable.ListBuffer import scala.util.hashing.MurmurHash3 -class EthereumGethRouter(override val routerId: Int,override val workerID:Int, val initialManagerCount: Int) extends RouterWorker { +class EthereumGethRouter(override val routerId: Int,override val workerID:Int,override val initialManagerCount: Int) + extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount) { def hexToInt(hex: String) = Integer.parseInt(hex.drop(2), 16) print(routerId) - override protected def parseTuple(value: Any): Unit = { - print(value) - val transaction = value.toString.split(",") + override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { + print(tuple) + val transaction = tuple.value.split(",") val blockNumber = hexToInt(transaction(0)) val from = transaction(1).replaceAll("\"", "").toLowerCase @@ -30,14 +28,15 @@ class EthereumGethRouter(override val routerId: Int,override val workerID:Int, v print(from) print(to) + val commands = new ListBuffer[GraphUpdate]() - sendGraphUpdate( + commands+=( VertexAddWithProperties(blockNumber, sourceNode, properties = Properties(ImmutableProperty("id", from))) ) - sendGraphUpdate( + commands+=( VertexAddWithProperties(blockNumber, destinationNode, properties = Properties(ImmutableProperty("id", to))) ) - sendGraphUpdate( + commands+=( EdgeAddWithProperties( blockNumber, sourceNode, @@ -45,7 +44,7 @@ class EthereumGethRouter(override val routerId: Int,override val workerID:Int, v properties = Properties(StringProperty("value", sent)) ) ) - + commands.toList } } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumKafkaRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumKafkaRouter.scala index 888d71f9a0..6f5e81d1fb 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumKafkaRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumKafkaRouter.scala @@ -2,23 +2,21 @@ package com.raphtory.examples.blockchain.routers import akka.http.scaladsl.unmarshalling.Unmarshal import com.raphtory.core.components.Router.RouterWorker -import com.raphtory.core.model.communication.EdgeAddWithProperties -import com.raphtory.core.model.communication.ImmutableProperty -import com.raphtory.core.model.communication.Properties -import com.raphtory.core.model.communication.StringProperty -import com.raphtory.core.model.communication.VertexAddWithProperties +import com.raphtory.core.model.communication.{EdgeAddWithProperties, GraphUpdate, ImmutableProperty, Properties, StringProperty, StringSpoutGoing, VertexAddWithProperties} import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._ import akka.stream.ActorMaterializer import spray.json._ +import scala.collection.mutable.ListBuffer import scala.util.hashing.MurmurHash3 -class EthereumKafkaRouter(override val routerId: Int,override val workerID:Int, val initialManagerCount: Int) extends RouterWorker { +class EthereumKafkaRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int) + extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount) { def hexToInt(hex: String) = Integer.parseInt(hex.drop(2), 16) - override protected def parseTuple(value: Any): Unit = { + override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { - val transaction = value.toString.split(",") + val transaction = tuple.value.split(",") - if(transaction(1).equals("block_number")) return + if(transaction(1).equals("block_number")) return List() print(transaction) @@ -29,7 +27,7 @@ class EthereumKafkaRouter(override val routerId: Int,override val workerID:Int, // val to = transaction(2).replaceAll("\"", "").toLowerCase // val sent = transaction(5).replaceAll("\"", "") - if(transaction(2).equals("block_number")) return + if(transaction(2).equals("block_number")) return List() val blockNumber = transaction(2).toInt val from = transaction(4).replaceAll("\"", "").toLowerCase @@ -38,14 +36,15 @@ class EthereumKafkaRouter(override val routerId: Int,override val workerID:Int, val sourceNode = assignID(from) //hash the id to get a vertex ID val destinationNode = assignID(to) //hash the id to get a vertex ID + val commands = new ListBuffer[GraphUpdate]() - sendGraphUpdate( + commands+=( VertexAddWithProperties(blockNumber, sourceNode, properties = Properties(ImmutableProperty("id", from))) ) - sendGraphUpdate( + commands+=( VertexAddWithProperties(blockNumber, destinationNode, properties = Properties(ImmutableProperty("id", to))) ) - sendGraphUpdate( + commands+=( EdgeAddWithProperties( blockNumber, sourceNode, @@ -53,7 +52,7 @@ class EthereumKafkaRouter(override val routerId: Int,override val workerID:Int, properties = Properties(StringProperty("value", sent)) ) ) - + commands.toList } } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumTransactionRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumTransactionRouter.scala index 24f9b33eb9..116bf24fd8 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumTransactionRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumTransactionRouter.scala @@ -1,28 +1,28 @@ package com.raphtory.examples.blockchain.routers import com.raphtory.core.components.Router.RouterWorker -import com.raphtory.core.model.communication.EdgeAddWithProperties -import com.raphtory.core.model.communication.Properties -import com.raphtory.core.model.communication.StringProperty -import com.raphtory.core.model.communication.VertexAddWithProperties +import com.raphtory.core.model.communication.{EdgeAddWithProperties, GraphUpdate, Properties, StringProperty, StringSpoutGoing, VertexAddWithProperties} +import scala.collection.mutable.ListBuffer import scala.util.hashing.MurmurHash3 -class EthereumTransactionRouter(override val routerId: Int,override val workerID:Int, val initialManagerCount: Int) extends RouterWorker { +class EthereumTransactionRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int) + extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount) { - override protected def parseTuple(value: Any): Unit = { - val components = value.toString.drop(1).dropRight(1).split(",") + override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { + val components = tuple.value.drop(1).dropRight(1).split(",") val creationDate = components(3).toLong * 1000 //seconds to miliseconds val sourceNode = MurmurHash3.stringHash(components(0)) //hash the id to get a vertex ID - sendGraphUpdate( + val commands = new ListBuffer[GraphUpdate]() + commands+=( VertexAddWithProperties(creationDate, sourceNode, Properties(StringProperty("id", components(0)))) ) //create the source node and add the wallet ID as a property if (components(1).nonEmpty) { //money being sent to an actual user val targetNode = MurmurHash3.stringHash(components(1)) //hash the id of the to wallet to get a vertex ID - sendGraphUpdate( + commands+=( VertexAddWithProperties(creationDate, targetNode, Properties(StringProperty("id", components(1)))) ) //create the destination vertex - sendGraphUpdate( + commands+=( EdgeAddWithProperties( creationDate, sourceNode, @@ -32,8 +32,8 @@ class EthereumTransactionRouter(override val routerId: Int,override val workerID ) //create the edge between them adding the value as a property } else { //burnt cash val targetNode = MurmurHash3.stringHash("null") - sendGraphUpdate(VertexAddWithProperties(creationDate, targetNode, Properties(StringProperty("id", "null")))) - sendGraphUpdate( + commands+=(VertexAddWithProperties(creationDate, targetNode, Properties(StringProperty("id", "null")))) + commands+=( EdgeAddWithProperties( creationDate, sourceNode, @@ -42,6 +42,6 @@ class EthereumTransactionRouter(override val routerId: Int,override val workerID ) ) } - + commands.toList } } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/FirehoseKafkaRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/FirehoseKafkaRouter.scala index 93023528a2..31f03c1739 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/FirehoseKafkaRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/FirehoseKafkaRouter.scala @@ -2,24 +2,28 @@ package com.raphtory.examples.blockchain.routers import akka.http.scaladsl.unmarshalling.Unmarshal import com.raphtory.core.components.Router.RouterWorker -import com.raphtory.core.model.communication.{DoubleProperty, EdgeAdd, EdgeAddWithProperties, EdgeDelete, ImmutableProperty, LongProperty, Properties, StringProperty, VertexAdd, VertexAddWithProperties, VertexDelete} +import com.raphtory.core.model.communication.{DoubleProperty, EdgeAdd, EdgeAddWithProperties, EdgeDelete, GraphUpdate, ImmutableProperty, LongProperty, Properties, StringProperty, StringSpoutGoing, VertexAdd, VertexAddWithProperties, VertexDelete} import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._ import akka.stream.ActorMaterializer import spray.json._ +import scala.collection.mutable.ListBuffer import scala.util.Random import scala.util.hashing.MurmurHash3 import scala.math.BigInt -class FirehoseKafkaRouter(override val routerId: Int,override val workerID:Int, val initialManagerCount: Int) extends RouterWorker { +class FirehoseKafkaRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int) + extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount) { var DELETEPERCENT = System.getenv().getOrDefault("ETHER_DELETE_PERCENT", "0").trim.toDouble/100 var DELETESEED = System.getenv().getOrDefault("ETHER_DELETE_SEED", "123").trim.toInt val random = new Random(DELETESEED) def hexToInt(hex: String) = Integer.parseInt(hex.drop(2), 16) - override protected def parseTuple(value: Any): Unit = { + + override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { //if(value.toString.contains("0xa09871aeadf4994ca12f5c0b6056bbd1d343c029")) println(value.toString) - val transaction = value.toString.split(",") - if(transaction(1).equals("block_number")) return + val transaction = tuple.value.split(",") + if(transaction(1).equals("block_number")) return List() val blockNumber = transaction(2).toInt + val commands = new ListBuffer[GraphUpdate]() val from = transaction(4).replaceAll("\"", "").toLowerCase val to = transaction(5).replaceAll("\"", "").toLowerCase @@ -28,18 +32,18 @@ class FirehoseKafkaRouter(override val routerId: Int,override val workerID:Int, val destinationNode = assignID(to) //hash the id to get a vertex ID //if(from.contains("0xa09871aeadf4994ca12f5c0b6056bbd1d343c029".toLowerCase())) println(from) //if(to.contains("0xa09871aeadf4994ca12f5c0b6056bbd1d343c029".toLowerCase())) println(to) - sendGraphUpdate(VertexAddWithProperties(blockNumber, sourceNode, properties = Properties(ImmutableProperty("id", from)))) + commands+=(VertexAddWithProperties(blockNumber, sourceNode, properties = Properties(ImmutableProperty("id", from)))) // if(random.nextDouble()<=DELETEPERCENT) // sendGraphUpdate(VertexDelete(blockNumber+1,sourceNode)) - sendGraphUpdate(VertexAddWithProperties(blockNumber, destinationNode, properties = Properties(ImmutableProperty("id", to)))) + commands+=(VertexAddWithProperties(blockNumber, destinationNode, properties = Properties(ImmutableProperty("id", to)))) // if(random.nextDouble()<=DELETEPERCENT) // sendGraphUpdate(VertexDelete(blockNumber+1,destinationNode)) - sendGraphUpdate(EdgeAddWithProperties(blockNumber, sourceNode, destinationNode,properties = Properties(DoubleProperty("value", sent)))) + commands+=(EdgeAddWithProperties(blockNumber, sourceNode, destinationNode,properties = Properties(DoubleProperty("value", sent)))) // if(random.nextDouble()<=DELETEPERCENT) // sendGraphUpdate(EdgeDelete(blockNumber+1,sourceNode,destinationNode)) - + commands.toList } } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/LitecoinRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/LitecoinRouter.scala index 092c8442fb..435fa2423e 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/LitecoinRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/LitecoinRouter.scala @@ -1,24 +1,21 @@ package com.raphtory.examples.blockchain.routers import com.raphtory.core.components.Router.RouterWorker -import com.raphtory.core.model.communication.EdgeAdd -import com.raphtory.core.model.communication.EdgeAddWithProperties -import com.raphtory.core.model.communication.Properties -import com.raphtory.core.model.communication.StringProperty -import com.raphtory.core.model.communication.VertexAddWithProperties -import com.raphtory.examples.blockchain.LitecoinTransaction +import com.raphtory.core.model.communication.{EdgeAdd, EdgeAddWithProperties, GraphUpdate, Properties, StringProperty, VertexAddWithProperties} +import com.raphtory.spouts.blockchain.BitcoinTransaction import spray.json.JsArray +import scala.collection.mutable.ListBuffer import scala.util.hashing.MurmurHash3 -class LitecoinRouter(override val routerId: Int,override val workerID:Int, val initialManagerCount: Int) extends RouterWorker { +class LitecoinRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int) + extends RouterWorker[BitcoinTransaction](routerId,workerID,initialManagerCount) { - def parseTuple(record: Any): Unit = { - val value = record.asInstanceOf[LitecoinTransaction] - val transaction = value.transaction - val time = value.time - val blockID = value.blockID - val block = value.block + override protected def parseTuple(tuple:BitcoinTransaction): List[GraphUpdate] = { + val transaction = tuple.transaction + val time = tuple.time + val blockID = tuple.blockID + val block = tuple.block val timeAsString = time.toString val timeAsLong = (timeAsString.toLong) * 1000 @@ -28,7 +25,7 @@ class LitecoinRouter(override val routerId: Int,override val workerID:Int, val i val locktime = transaction.asJsObject.fields("locktime") val version = transaction.asJsObject.fields("version") var total: Double = 0 - + val commands = new ListBuffer[GraphUpdate]() for (vout <- vouts.asInstanceOf[JsArray].elements) { val voutOBJ = vout.asJsObject() var value = voutOBJ.fields("value").toString @@ -44,7 +41,7 @@ class LitecoinRouter(override val routerId: Int,override val workerID:Int, val i else value = "0" //TODO deal with people burning money //creates vertex for the receiving wallet - sendGraphUpdate( + commands+=( VertexAddWithProperties( msgTime = timeAsLong, srcID = MurmurHash3.stringHash(address), @@ -56,7 +53,7 @@ class LitecoinRouter(override val routerId: Int,override val workerID:Int, val i ) ) //creates edge between the transaction and the wallet - sendGraphUpdate( + commands+=( EdgeAddWithProperties( msgTime = timeAsLong, srcID = MurmurHash3.stringHash(txid), @@ -66,7 +63,7 @@ class LitecoinRouter(override val routerId: Int,override val workerID:Int, val i ) } - sendGraphUpdate( + commands+=( VertexAddWithProperties( msgTime = timeAsLong, srcID = MurmurHash3.stringHash(txid), @@ -85,7 +82,7 @@ class LitecoinRouter(override val routerId: Int,override val workerID:Int, val i if (vins.toString().contains("coinbase")) { //creates the coingen node - sendGraphUpdate( + commands+=( VertexAddWithProperties( msgTime = timeAsLong, srcID = MurmurHash3.stringHash("coingen"), @@ -94,14 +91,14 @@ class LitecoinRouter(override val routerId: Int,override val workerID:Int, val i ) //creates edge between coingen and the transaction - sendGraphUpdate( + commands+=( EdgeAdd( msgTime = timeAsLong, srcID = MurmurHash3.stringHash("coingen"), dstID = MurmurHash3.stringHash(txid) ) ) - } else + } else { for (vin <- vins.asInstanceOf[JsArray].elements) { val vinOBJ = vin.asJsObject() val prevVout = vinOBJ.fields("vout").toString @@ -109,7 +106,7 @@ class LitecoinRouter(override val routerId: Int,override val workerID:Int, val i val sequence = vinOBJ.fields("sequence").toString //no need to create node for prevtxid as should already exist //creates edge between the prev transaction and current transaction - sendGraphUpdate( + commands+=( EdgeAddWithProperties( msgTime = timeAsLong, srcID = MurmurHash3.stringHash(prevtxid), @@ -118,6 +115,9 @@ class LitecoinRouter(override val routerId: Int,override val workerID:Int, val i ) ) } + + } + commands.toList } } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/citationNetwork/CitationRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/citationNetwork/CitationRouter.scala index f6cc07518a..392939ce35 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/citationNetwork/CitationRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/citationNetwork/CitationRouter.scala @@ -5,10 +5,12 @@ import java.text.SimpleDateFormat import com.raphtory.core.components.Router.RouterWorker import com.raphtory.core.model.communication._ -class CitationRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int) extends RouterWorker { +import scala.collection.mutable.ListBuffer - def parseTuple(record: Any): Unit = { - val fileLine = record.asInstanceOf[String].split(",").map(_.trim) //take the tuple and split on , as we are only interested in the first 4 fields +class CitationRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int) + extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount) { + override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { + val fileLine = tuple.value.split(",").map(_.trim) //take the tuple and split on , as we are only interested in the first 4 fields // title_paper,year,volume,title,pages,number,journal,author,ENTRYTYPE,ID val sourceTitle = fileLine(0) val destinationTitle = fileLine(3) @@ -17,16 +19,16 @@ class CitationRouter(override val routerId: Int,override val workerID:Int, overr val sourceID = assignID(sourceTitle) val destinationID = assignID(destinationTitle) - + val commands = new ListBuffer[GraphUpdate]() //create sourceNode - sendGraphUpdate(VertexAddWithProperties( + commands+=(VertexAddWithProperties( sourceYear, //when it happened ?? sourceID, // the id of the node Properties(ImmutableProperty("title", sourceTitle)), //properties for the node Type("Publication")) //node type ) //create destinationNode - sendGraphUpdate(VertexAddWithProperties( + commands+=(VertexAddWithProperties( destinationYear, destinationID, Properties(ImmutableProperty("title", destinationTitle)), @@ -34,13 +36,13 @@ class CitationRouter(override val routerId: Int,override val workerID:Int, overr ) //create edge - sendGraphUpdate(EdgeAdd( + commands+=(EdgeAdd( sourceYear, //time of edge ?? sourceID, //source of edge destinationID, //destination of edge Type("Cited") // edge type )) - + commands.toList } } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabPostGraphRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabPostGraphRouter.scala index eaa40174ce..6071f7c42a 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabPostGraphRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabPostGraphRouter.scala @@ -5,6 +5,8 @@ import java.text.SimpleDateFormat import com.raphtory.core.components.Router.RouterWorker import com.raphtory.core.model.communication._ +import scala.collection.mutable.ListBuffer + // The lines sent by the Gab mining spout are read and processed accordingly. //In this router we needed to transform the data that was sent by the spout by turning it into a epoch value (long value) // in order to be accepted by Raphtory to create the corresponding entity. @@ -12,25 +14,26 @@ import com.raphtory.core.model.communication._ // is equal to -1. Columns 1 and 4 correspond to the postId and parentPostid in the file. // Then either the vertex or the edge are created accordingly. -class GabPostGraphRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int) extends RouterWorker { +class GabPostGraphRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int) + extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount) { - def parseTuple(record: Any): Unit = { - val fileLine = record.asInstanceOf[String].split(";").map(_.trim) + override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { + val fileLine = tuple.value.split(";").map(_.trim) //user wise // val sourceNode=fileLine(2).toInt // val targetNode=fileLine(5).toInt - + val commands = new ListBuffer[GraphUpdate]() //comment wise val sourceNode = fileLine(1).toInt val targetNode = fileLine(4).toInt if (targetNode > 0) { val creationDate = dateToUnixTime(timestamp = fileLine(0).slice(0, 19)) - sendGraphUpdate(VertexAdd(creationDate, sourceNode)) - sendGraphUpdate(VertexAdd(creationDate, targetNode)) - sendGraphUpdate(EdgeAdd(creationDate, sourceNode, targetNode)) + commands+=(VertexAdd(creationDate, sourceNode)) + commands+=(VertexAdd(creationDate, targetNode)) + commands+=(EdgeAdd(creationDate, sourceNode, targetNode)) } - + commands.toList } def dateToUnixTime(timestamp: => String): Long = { diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabRawRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabRawRouter.scala index 2517b274ba..fe98bcf272 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabRawRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabRawRouter.scala @@ -3,13 +3,12 @@ package com.raphtory.examples.gab.routers import java.time.OffsetDateTime import com.raphtory.core.components.Router.RouterWorker -import com.raphtory.core.model.communication.EdgeAddWithProperties -import com.raphtory.core.model.communication.Properties -import com.raphtory.core.model.communication.StringProperty -import com.raphtory.core.model.communication.VertexAddWithProperties +import com.raphtory.core.model.communication.{EdgeAddWithProperties, GraphUpdate, Properties, StringProperty, StringSpoutGoing, VertexAddWithProperties} import com.raphtory.examples.gab.rawgraphmodel.GabPost import spray.json._ +import scala.collection.mutable.ListBuffer + /** * The Graph Manager is the top level actor in this system (under the stream) * which tracks all the graph partitions - passing commands processed by the 'command processor' actors @@ -20,113 +19,121 @@ import spray.json._ * the correct case Class which can then be passed to the graph manager * which will then pass it to the graph partition dealing with the associated vertex */ -final class GabRawRouter(override val routerId: Int,override val workerID:Int, val initialManagerCount: Int) extends RouterWorker { +final class GabRawRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int) + extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount) { + import com.raphtory.examples.gab.rawgraphmodel.GabJsonProtocol._ + private val nullStr = "null" - override def parseTuple(record: Any): Unit = + + override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { + val commands = new ListBuffer[GraphUpdate]() try { - val command = record.asInstanceOf[String] - val post = command.parseJson.convertTo[GabPost] + val command = tuple.value + val post = command.parseJson.convertTo[GabPost] sendPostToPartitions(post) } catch { case e: Exception => println("Could not parse post") } - //val parsedOBJ: Command = command.parseJson.convertTo[Command] - //val manager = getManager(parsedOBJ.value.srcId, getManagerCount) - //mediator ! DistributedPubSubMediator.Send(manager, parsedOBJ.value, false) + //val parsedOBJ: Command = command.parseJson.convertTo[Command] + //val manager = getManager(parsedOBJ.value.srcId, getManagerCount) + //mediator ! DistributedPubSubMediator.Send(manager, parsedOBJ.value, false) + + def sendPostToPartitions(post: GabPost, recursiveCall: Boolean = false, parent: Int = 0): Unit = { + val postUUID = post.id.get.toInt + val timestamp = OffsetDateTime.parse(post.created_at.get).toEpochSecond + commands += ( + VertexAddWithProperties( + timestamp, + postUUID, + Properties( + StringProperty("user", post.user match { + case Some(u) => u.id.toString + case None => nullStr + }), + StringProperty("likeCount", post.like_count match { + case Some(likeCount) => likeCount.toString + case None => nullStr + }), + StringProperty("score", post.score match { + case Some(score) => score.toString + case None => nullStr + }), + StringProperty("topic", post.topic match { + case Some(topic) => topic.id + case None => nullStr + }), + StringProperty("type", "post") + ) + ) + ) - def sendPostToPartitions(post: GabPost, recursiveCall: Boolean = false, parent: Int = 0): Unit = { - val postUUID = post.id.get.toInt - val timestamp = OffsetDateTime.parse(post.created_at.get).toEpochSecond - sendGraphUpdate( + post.user match { + case Some(user) => + val userUUID: Int = "user".hashCode() + user.id //TODO improve in case of clashes + commands += ( VertexAddWithProperties( - timestamp, - postUUID, - Properties( - StringProperty("user", post.user match { - case Some(u) => u.id.toString - case None => nullStr - }), - StringProperty("likeCount", post.like_count match { - case Some(likeCount) => likeCount.toString - case None => nullStr - }), - StringProperty("score", post.score match { - case Some(score) => score.toString - case None => nullStr - }), - StringProperty("topic", post.topic match { - case Some(topic) => topic.id - case None => nullStr - }), - StringProperty("type", "post") - ) + timestamp, + userUUID, + Properties( + StringProperty("type", "user"), + StringProperty("id", user.id.toString), + StringProperty("name", user.name), + StringProperty("username", user.username), + StringProperty("verified", user.verified.toString) + ) + ) ) - ) - - post.user match { - case Some(user) => - val userUUID: Int = "user".hashCode() + user.id //TODO improve in case of clashes - sendGraphUpdate( - VertexAddWithProperties( - timestamp, - userUUID, - Properties( - StringProperty("type", "user"), - StringProperty("id", user.id.toString), - StringProperty("name", user.name), - StringProperty("username", user.username), - StringProperty("verified", user.verified.toString) - ) - ) - ) - sendGraphUpdate( - EdgeAddWithProperties(timestamp, userUUID, postUUID, Properties((StringProperty("type", "userToPost")))) - ) - sendGraphUpdate( - EdgeAddWithProperties(timestamp, postUUID, userUUID, Properties(StringProperty("type", "postToUser"))) - ) - case None => - } + commands += ( + EdgeAddWithProperties(timestamp, userUUID, postUUID, Properties((StringProperty("type", "userToPost")))) + ) + commands += ( + EdgeAddWithProperties(timestamp, postUUID, userUUID, Properties(StringProperty("type", "postToUser"))) + ) + case None => + } - post.topic match { - case Some(topic) => - val topicUUID: Int = Math.pow(2, 24).toInt + (topic.id.hashCode()) - sendGraphUpdate( - VertexAddWithProperties( - timestamp, - topicUUID, - Properties( - StringProperty("created_at", topic.created_at), - StringProperty("category", topic.category.toString), - StringProperty("title", topic.title.getOrElse("null")), - StringProperty("type", "topic"), - StringProperty("id", topic.id) - ) - ) - ) + post.topic match { + case Some(topic) => + val topicUUID: Int = Math.pow(2, 24).toInt + (topic.id.hashCode()) + commands += ( + VertexAddWithProperties( + timestamp, + topicUUID, + Properties( + StringProperty("created_at", topic.created_at), + StringProperty("category", topic.category.toString), + StringProperty("title", topic.title.getOrElse("null")), + StringProperty("type", "topic"), + StringProperty("id", topic.id) + ) + ) + ) - sendGraphUpdate( - EdgeAddWithProperties(timestamp, postUUID, topicUUID, Properties(StringProperty("type", "postToTopic"))) - ) - case None => - } + commands += ( + EdgeAddWithProperties(timestamp, postUUID, topicUUID, Properties(StringProperty("type", "postToTopic"))) + ) + case None => + } - // Edge from child to parent post - if (recursiveCall && parent != 0) - sendGraphUpdate( - EdgeAddWithProperties(timestamp, postUUID, parent, Properties(StringProperty("type", "childToParent"))) - ) - post.parent match { - case Some(p) => - if (!recursiveCall) // Allow only one recursion per post + // Edge from child to parent post + if (recursiveCall && parent != 0) + commands += ( + EdgeAddWithProperties(timestamp, postUUID, parent, Properties(StringProperty("type", "childToParent"))) + ) + post.parent match { + case Some(p) => + if (!recursiveCall) // Allow only one recursion per post //println("Found parent post: Recursion!") - sendPostToPartitions(p, true, postUUID) - case None => + sendPostToPartitions(p, true, postUUID) + case None => + } + } + commands.toList } - } + diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabUserGraphRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabUserGraphRouter.scala index 313ed9b995..629fc2f3cd 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabUserGraphRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabUserGraphRouter.scala @@ -6,6 +6,7 @@ import com.raphtory.core.components.Router.RouterWorker import com.raphtory.core.model.communication.Type import com.raphtory.core.model.communication._ +import scala.collection.mutable.ListBuffer import scala.util.Random // The lines sent by the Gab mining spout are read and processed accordingly. @@ -15,27 +16,28 @@ import scala.util.Random // is equal to -1. Columns 2 and 5 correspond to the userid and parentUserid in the file. // Then either the vertex or the edge are created accordingly. -class GabUserGraphRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int) extends RouterWorker { +class GabUserGraphRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int) + extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount) { - def parseTuple(record: Any): Unit = { - val fileLine = record.asInstanceOf[String].split(";").map(_.trim) + override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { + val fileLine = tuple.value.split(";").map(_.trim) //user wise val sourceNode = fileLine(2).toInt val targetNode = fileLine(5).toInt - + val commands = new ListBuffer[GraphUpdate]() //comment wise // val sourceNode=fileLine(1).toInt //val targetNode=fileLine(4).toInt if (targetNode > 0 && targetNode != sourceNode) { val creationDate = dateToUnixTime(timestamp = fileLine(0).slice(0, 19)) - sendGraphUpdate(VertexAdd(creationDate, sourceNode, Type("User"))) - sendGraphUpdate(VertexAdd(creationDate, targetNode, Type("User"))) - sendGraphUpdate(EdgeAdd(creationDate, sourceNode, targetNode, Type("User to User"))) + commands+=(VertexAdd(creationDate, sourceNode, Type("User"))) + commands+=(VertexAdd(creationDate, targetNode, Type("User"))) + commands+=(EdgeAdd(creationDate, sourceNode, targetNode, Type("User to User"))) // sendGraphUpdate(VertexAddWithProperties(creationDate, sourceNode, Properties(StringProperty("test1","value1"),StringProperty("test2","Value2")),Type("User"))) // sendGraphUpdate(VertexAddWithProperties(creationDate, targetNode, Properties(StringProperty("test1","value1"),StringProperty("test2","Value2")),Type("User"))) // sendGraphUpdate(EdgeAddWithProperties(creationDate, sourceNode, targetNode, Properties(StringProperty("test1","value1"),StringProperty("test2","Value2")),Type("User To User"))) } - + commands.toList } def dateToUnixTime(timestamp: => String): Long = { diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCOldRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCOldRouter.scala index 8e8930a0a2..92b847c270 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCOldRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCOldRouter.scala @@ -4,24 +4,24 @@ import java.text.SimpleDateFormat import java.util.Date import com.raphtory.core.components.Router.RouterWorker -import com.raphtory.core.model.communication.EdgeAdd -import com.raphtory.core.model.communication.EdgeDelete -import com.raphtory.core.model.communication.Type -import com.raphtory.core.model.communication.VertexAdd -import com.raphtory.core.model.communication.VertexDelete +import com.raphtory.core.model.communication.{EdgeAdd, EdgeDelete, GraphUpdate, StringSpoutGoing, Type, VertexAdd, VertexDelete} -class LDBCOldRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int) extends RouterWorker { - override protected def parseTuple(value: Any): Unit = { +import scala.collection.mutable.ListBuffer - val fileLine = value.asInstanceOf[String].split("\\|") +class LDBCOldRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int) + extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount) { + override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { + val fileLine = tuple.asInstanceOf[String].split("\\|") + val commands = new ListBuffer[GraphUpdate]() //val deletionDate:Long = new SimpleDateFormat("yyyy-MM-ddHH:mm:ss.SSS").parse(date2).getTime() fileLine(0) match { case "person" => val date = fileLine(6).substring(0, 10) + fileLine(5).substring(11, 23); //extract the day of the event //val date2 = fileLine(2).substring(0, 10) + fileLine(1).substring(11, 23); //extract the day of the event val creationDate: Long = new SimpleDateFormat("yyyy-MM-ddHH:mm:ss.SSS").parse(date).getTime() - sendGraphUpdate(VertexAdd(creationDate, assignID("person" + fileLine(1)), Type("person"))) + commands+=(VertexAdd(creationDate, assignID("person" + fileLine(1)), Type("person"))) + commands.toList //sendGraphUpdate(VertexAdd(creationDate, fileLine(3).toLong,Type("person"))) // sendGraphUpdate(VertexDelete(deletionDate, assignID("person"+fileLine(3)))) case "person_knows_person" => @@ -29,7 +29,7 @@ class LDBCOldRouter(override val routerId: Int,override val workerID:Int, overri //val date2 = fileLine(2).substring(0, 10) + fileLine(1).substring(11, 23); //extract the day of the event val creationDate: Long = new SimpleDateFormat("yyyy-MM-ddHH:mm:ss.SSS").parse(date).getTime() //sendGraphUpdate(EdgeAdd(creationDate, fileLine(3).toLong,fileLine(4).toLong,Type("person_knows_person"))) - sendGraphUpdate( + commands+=( EdgeAdd( creationDate, assignID("person" + fileLine(1)), @@ -37,6 +37,7 @@ class LDBCOldRouter(override val routerId: Int,override val workerID:Int, overri Type("person_knows_person") ) ) + commands.toList //sendGraphUpdate(EdgeDelete(deletionDate, assignID("person"+fileLine(3)),assignID("person"+fileLine(4)))) } } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCRouter.scala index 1c223241f1..131a99c3ca 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCRouter.scala @@ -4,17 +4,16 @@ import java.text.SimpleDateFormat import java.util.Date import com.raphtory.core.components.Router.RouterWorker -import com.raphtory.core.model.communication.EdgeAdd -import com.raphtory.core.model.communication.EdgeDelete -import com.raphtory.core.model.communication.Type -import com.raphtory.core.model.communication.VertexAdd -import com.raphtory.core.model.communication.VertexDelete +import com.raphtory.core.model.communication.{EdgeAdd, EdgeDelete, GraphUpdate, StringSpoutGoing, Type, VertexAdd, VertexDelete} import com.raphtory.examples.test.actors.RandomSpout -class LDBCRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int) extends RouterWorker { - override protected def parseTuple(value: Any): Unit = { +import scala.collection.mutable.ListBuffer - val fileLine = value.asInstanceOf[String].split("\\|") +class LDBCRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int) + extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount) { + override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { + val commands = new ListBuffer[GraphUpdate]() + val fileLine = tuple.asInstanceOf[String].split("\\|") val date = fileLine(1).substring(0, 10) + fileLine(1).substring(11, 23); //extract the day of the event val date2 = fileLine(2).substring(0, 10) + fileLine(1).substring(11, 23); //extract the day of the event val creationDate: Long = new SimpleDateFormat("yyyy-MM-ddHH:mm:ss.SSS").parse(date).getTime() @@ -23,13 +22,13 @@ class LDBCRouter(override val routerId: Int,override val workerID:Int, override val edgeDeletion = sys.env.getOrElse("LDBC_EDGE_DELETION", "false").trim.toBoolean fileLine(0) match { case "person" => - sendGraphUpdate(VertexAdd(creationDate, assignID("person" + fileLine(3)), Type("person"))) + commands+=(VertexAdd(creationDate, assignID("person" + fileLine(3)), Type("person"))) //sendGraphUpdate(VertexAdd(creationDate, fileLine(3).toLong,Type("person"))) if(vertexDeletion) - sendGraphUpdate(VertexDelete(deletionDate, assignID("person" + fileLine(3)))) + commands+=(VertexDelete(deletionDate, assignID("person" + fileLine(3)))) case "person_knows_person" => //sendGraphUpdate(EdgeAdd(creationDate, fileLine(3).toLong,fileLine(4).toLong,Type("person_knows_person"))) - sendGraphUpdate( + commands+=( EdgeAdd( creationDate, assignID("person" + fileLine(3)), @@ -38,8 +37,9 @@ class LDBCRouter(override val routerId: Int,override val workerID:Int, override ) ) if(edgeDeletion) - sendGraphUpdate(EdgeDelete(deletionDate, assignID("person"+fileLine(3)),assignID("person"+fileLine(4)))) + commands+=(EdgeDelete(deletionDate, assignID("person"+fileLine(3)),assignID("person"+fileLine(4)))) } + commands.toList } } //2012-11-01T09:28:01.185+00:00|2019-07-22T11:24:24.362+00:00|35184372093644|Jose|Garcia|female|1988-05-20|111.68.47.44|Firefox diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRRouter.scala index c3940a0784..e19dc3dcaf 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRRouter.scala @@ -3,12 +3,15 @@ package com.raphtory.examples.lotr import com.raphtory.core.components.Router.RouterWorker import com.raphtory.core.model.communication.{EdgeAdd, Type, VertexAdd, _} -class LOTRRouter(override val routerId: Int, override val workerID:Int, override val initialManagerCount: Int) extends RouterWorker { +import scala.collection.mutable.ListBuffer - def parseTuple(record: Any): Unit = { +class LOTRRouter(override val routerId: Int, override val workerID:Int, override val initialManagerCount: Int) + extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount) { - val fileLine = record.asInstanceOf[String].split(",").map(_.trim) + override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { + val fileLine = tuple.value.split(",").map(_.trim) + val commands = new ListBuffer[GraphUpdate]() val sourceNode = fileLine(0) val srcID = assignID(sourceNode) @@ -17,8 +20,9 @@ class LOTRRouter(override val routerId: Int, override val workerID:Int, override val timeStamp = fileLine(2).toLong - sendGraphUpdate(VertexAddWithProperties(timeStamp, srcID, Properties(ImmutableProperty("name",sourceNode)),Type("Character"))) - sendGraphUpdate(VertexAddWithProperties(timeStamp, tarID, Properties(ImmutableProperty("name",targetNode)),Type("Character"))) - sendGraphUpdate(EdgeAdd(timeStamp,srcID,tarID, Type("Character Co-occurence"))) + commands+=(VertexAddWithProperties(timeStamp, srcID, Properties(ImmutableProperty("name",sourceNode)),Type("Character"))) + commands+=(VertexAddWithProperties(timeStamp, tarID, Properties(ImmutableProperty("name",targetNode)),Type("Character"))) + commands+=(EdgeAdd(timeStamp,srcID,tarID, Type("Character Co-occurence"))) + commands.toList } } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomRouter.scala index 67d46d66a3..2b5661991d 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomRouter.scala @@ -19,8 +19,8 @@ class RandomRouter(override val routerId: Int,override val workerID:Int, overrid //************* MESSAGE HANDLING BLOCK - override protected def parseTuple(value:StringSpoutGoing): List[GraphUpdate] = { - val command = value.asInstanceOf[String] + override protected def parseTuple(tuple:StringSpoutGoing): List[GraphUpdate] = { + val command = tuple.asInstanceOf[String] val parsedOBJ = command.parseJson.asJsObject //get the json object val commandKey = parsedOBJ.fields //get the command type if (commandKey.contains("VertexAdd")) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleTestRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleTestRouter.scala index 2d7bc834b0..54e34d8668 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleTestRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleTestRouter.scala @@ -2,14 +2,17 @@ package com.raphtory.examples.test.actors import com.raphtory.core.components.Router.RouterWorker import com.raphtory.core.components.Spout.SpoutTrait -import com.raphtory.core.model.communication.{EdgeAdd, VertexAddWithProperties} +import com.raphtory.core.model.communication.{EdgeAdd, GraphUpdate, StringSpoutGoing, VertexAddWithProperties} -class TriangleTestRouter (override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int) extends RouterWorker { - override protected def parseTuple(tuple: Any): Unit = { - val command = tuple.asInstanceOf[String].split(",") +import scala.collection.mutable.ListBuffer + +class TriangleTestRouter (override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int) + extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount) { + override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { + val command = tuple.value.split(",") val msgTime = command(0).toLong val sourceID = command(1).toLong val destID = command(2).toLong - sendGraphUpdate(EdgeAdd(msgTime, sourceID, destID)) + List((EdgeAdd(msgTime, sourceID, destID))) } } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/trackAndTrace/routers/TrackAndTraceRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/trackAndTrace/routers/TrackAndTraceRouter.scala index e167588e48..ae52cc25b5 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/trackAndTrace/routers/TrackAndTraceRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/trackAndTrace/routers/TrackAndTraceRouter.scala @@ -5,24 +5,27 @@ import java.text.SimpleDateFormat import com.raphtory.core.components.Router.RouterWorker import com.raphtory.core.model.communication.Type import com.raphtory.core.model.communication._ + +import scala.collection.mutable.ListBuffer import scala.util.control.Breaks._ -class TrackAndTraceRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int) extends RouterWorker { +class TrackAndTraceRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int) + extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount) { val EARTH_EQU = 6378137.0 //m val EARTH_POL = 6356752.3142 //m val STEPSIZE = System.getenv().getOrDefault("MAP_GRID_SIZE", "100").trim.toDouble //m - def parseTuple(record: Any): Unit = { - val datapoint = lineToDatapoint(record.asInstanceOf[String].split(",").map(_.trim)) + override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { + val datapoint = lineToDatapoint(tuple.value.split(",").map(_.trim)) val eventTime = datapoint.time val userID = datapoint.userId val latitude = datapoint.latitude val longitude = datapoint.longitude val locationID = locationIDGenerator(latitude, longitude) + val commands = new ListBuffer[GraphUpdate]() + commands+=(VertexAdd(eventTime, userID, Type("User"))) - sendGraphUpdate(VertexAdd(eventTime, userID, Type("User"))) - - sendGraphUpdate( + commands+=( VertexAddWithProperties( eventTime, locationID, @@ -31,8 +34,8 @@ class TrackAndTraceRouter(override val routerId: Int,override val workerID:Int, ) ) - sendGraphUpdate(EdgeAdd(eventTime, userID, locationID, Type("User Visted Location"))) - + commands+=(EdgeAdd(eventTime, userID, locationID, Type("User Visted Location"))) + commands.toList } //converts the line into a case class which has all of the data via the correct name and type diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/SamplerSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/SamplerSpout.scala index 24ef35966f..14577bde87 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/SamplerSpout.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/SamplerSpout.scala @@ -4,8 +4,8 @@ import java.time.LocalDateTime import com.raphtory.core.components.Spout.SpoutTrait import com.raphtory.core.components.Spout.SpoutTrait.BasicDomain +import com.raphtory.core.components.Spout.SpoutTrait.CommonMessage.Next import com.raphtory.core.model.communication.StringSpoutGoing -import monix.tail.Iterant.Next import scala.concurrent.duration.{Duration, NANOSECONDS} import scala.util.Random @@ -23,7 +23,7 @@ class SamplerSpout extends SpoutTrait[BasicDomain,StringSpoutGoing] { var linesNumber = fileLines.length println("Start: " + LocalDateTime.now()) - protected def ProcessSpoutTask(message: Any): Unit = message match { + override def handleDomainMessage(message: BasicDomain): Unit = message match { case Next => try { if (position < linesNumber) { @@ -35,7 +35,7 @@ class SamplerSpout extends SpoutTrait[BasicDomain,StringSpoutGoing] { break } else { val line = fileLines(position) - sendTuple(line) + sendTuple(StringSpoutGoing(line)) position += 1 } } @@ -51,4 +51,6 @@ class SamplerSpout extends SpoutTrait[BasicDomain,StringSpoutGoing] { case _ => println("message not recognized!") } + override def startSpout(): Unit = self ! Next + } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/TSVRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/TSVRouter.scala index 3e5260c753..efd784bba6 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/TSVRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/TSVRouter.scala @@ -1,13 +1,16 @@ package com.raphtory.examples.tsvnet import com.raphtory.core.components.Router.RouterWorker -import com.raphtory.core.model.communication.{EdgeAdd, Type, VertexAdd} +import com.raphtory.core.model.communication.{EdgeAdd, GraphUpdate, StringSpoutGoing, Type, VertexAdd} -/** Spout for network datasets of the form SRC_NODE_ID DEST_NODE_ID TIMESTAMP */ -class TSVRouter(override val routerId: Int, override val workerID:Int, override val initialManagerCount: Int) extends RouterWorker { +import scala.collection.mutable.ListBuffer - def parseTuple(record: Any): Unit = { - val fileLine = record.asInstanceOf[String].split(" ").map(_.trim) +/** Spout for network datasets of the form SRC_NODE_ID DEST_NODE_ID TIMESTAMP */ +class TSVRouter(override val routerId: Int, override val workerID:Int, override val initialManagerCount: Int) + extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount) { + val commands = new ListBuffer[GraphUpdate]() + override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { + val fileLine = tuple.value.split(" ").map(_.trim) //user wise val sourceNode = fileLine(0).toInt val targetNode = fileLine(1).toInt @@ -20,11 +23,11 @@ class TSVRouter(override val routerId: Int, override val workerID:Int, override //val targetNode=fileLine(4).toInt if (targetNode > 0) { val creationDate = fileLine(2).toLong - sendGraphUpdate(VertexAdd(creationDate, sourceNode, Type("User"))) - sendGraphUpdate(VertexAdd(creationDate, targetNode, Type("User"))) - sendGraphUpdate(EdgeAdd(creationDate, sourceNode, targetNode, Type("User to User"))) + commands+=(VertexAdd(creationDate, sourceNode, Type("User"))) + commands+=(VertexAdd(creationDate, targetNode, Type("User"))) + commands+=(EdgeAdd(creationDate, sourceNode, targetNode, Type("User to User"))) } - + commands.toList } } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixRouter.scala index 4b0fbcc4a9..419e5d5203 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixRouter.scala @@ -3,21 +3,26 @@ package com.raphtory.examples.wordSemantic.routers import com.raphtory.core.components.Router.RouterWorker import com.raphtory.core.model.communication.{Type, _} -class CooccurrenceMatrixRouter(override val routerId: Int, override val workerID:Int, override val initialManagerCount: Int) extends RouterWorker { +import scala.collection.mutable.ListBuffer - def parseTuple(record: Any): Unit = { +class CooccurrenceMatrixRouter(override val routerId: Int, override val workerID:Int, override val initialManagerCount: Int) + extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount) { + + override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { //println(record) - var dp = record.asInstanceOf[String].split(" ").map(_.trim) + var dp = tuple.value.split(" ").map(_.trim) + val commands = new ListBuffer[GraphUpdate]() val occurenceTime = dp.head.toLong//DateFormatting(dp.head) //.slice(4, dp.head.length) try { dp = dp.last.split("\t") val srcClusterId = dp.head.toLong//assignID(dp.head) val len = dp.length + for (i <- 1 until len by 2) { val dstClusterId = dp(i).toLong//assignID(dp(i)) val coocWeight = dp(i + 1).toLong - sendGraphUpdate( + commands+=( EdgeAddWithProperties( msgTime = occurenceTime, srcID = srcClusterId, @@ -26,9 +31,11 @@ class CooccurrenceMatrixRouter(override val routerId: Int, override val workerID ) ) } + }catch { - case e: Exception => println(e, dp.length, record.asInstanceOf[String]) + case e: Exception => println(e, dp.length, tuple.value) } + commands.toList } def DateFormatting(date: String): Long = { diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixRouterFiltered.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixRouterFiltered.scala index aa67abf310..e5dc01c490 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixRouterFiltered.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixRouterFiltered.scala @@ -3,14 +3,18 @@ package com.raphtory.examples.wordSemantic.routers import com.raphtory.core.components.Router.RouterWorker import com.raphtory.core.model.communication._ -class CooccurrenceMatrixRouterFiltered(override val routerId: Int, override val workerID:Int, override val initialManagerCount: Int) extends RouterWorker { +import scala.collection.mutable.ListBuffer + +class CooccurrenceMatrixRouterFiltered(override val routerId: Int, override val workerID:Int, override val initialManagerCount: Int) + extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount) { val THR = System.getenv().getOrDefault("COOC_FREQ_THRESHOLD ", "0.05").trim.toDouble - def parseTuple(record: Any): Unit = { + override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { //println(record) - var dp = record.asInstanceOf[String].split(" ").map(_.trim) + var dp =tuple.value.split(" ").map(_.trim) val occurenceTime = dp.head.toLong//DateFormatting(dp.head) //.slice(4, dp.head.length) val scale = dp(1).toDouble + val commands = new ListBuffer[GraphUpdate]() try { dp = dp.last.split("\t") val srcClusterId = assignID(dp.head) @@ -20,7 +24,7 @@ class CooccurrenceMatrixRouterFiltered(override val routerId: Int, override val val dstClusterId = assignID(dp(i)) val coocWeight = dp(i + 1).toLong - sendGraphUpdate( + commands+= ( EdgeAddWithProperties( msgTime = occurenceTime, srcID = srcClusterId, @@ -32,7 +36,8 @@ class CooccurrenceMatrixRouterFiltered(override val routerId: Int, override val } } }catch { - case e: Exception => println(e, dp.length, record.asInstanceOf[String]) + case e: Exception => println(e, dp.length, tuple.asInstanceOf[String]) } + commands.toList } } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/routers/RouterCSV.scala b/mainproject/cluster/src/main/scala/com/raphtory/routers/RouterCSV.scala deleted file mode 100644 index 42096454b0..0000000000 --- a/mainproject/cluster/src/main/scala/com/raphtory/routers/RouterCSV.scala +++ /dev/null @@ -1,26 +0,0 @@ -package com.raphtory.routers - -import com.raphtory.core.components.Router.RouterWorker -import com.raphtory.core.model.communication._ - -class RouterCSV(override val routerId: Int, override val workerID:Int, override val initialManagerCount: Int) extends RouterWorker { - - def parseTuple(record: Any): Unit = { - try { - val dp = record.asInstanceOf[String].split(",").map(_.trim) - val time = dp.head.toLong - val srcId = assignID(dp(1)) - val dstId = assignID(dp(2)) - - sendGraphUpdate( - EdgeAdd( - msgTime = time, - srcID = srcId, - dstID = dstId - ) - ) - }catch { - case e: Exception => println(e, record.asInstanceOf[String]) - } - } -} \ No newline at end of file diff --git a/mainproject/cluster/src/main/scala/com/raphtory/spouts/blockchain/EthereumGethSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/spouts/blockchain/EthereumGethSpout.scala index 32d0165de3..3f72c817b1 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/spouts/blockchain/EthereumGethSpout.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/spouts/blockchain/EthereumGethSpout.scala @@ -22,7 +22,6 @@ import akka.stream.ActorMaterializer import com.raphtory.core.components.Spout.SpoutTrait.CommonMessage.Next import com.raphtory.core.components.Spout.SpoutTrait.{BasicDomain, DomainMessage} import com.raphtory.core.model.communication.StringSpoutGoing -import com.raphtory.spouts.FirehoseSpout.Message.FireHouseDomain import spray.json.DefaultJsonProtocol._ diff --git a/mainproject/cluster/src/main/scala/com/raphtory/tests/SingleNodeTest.scala b/mainproject/cluster/src/main/scala/com/raphtory/tests/SingleNodeTest.scala index c7c11d71c9..658d1503b8 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/tests/SingleNodeTest.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/tests/SingleNodeTest.scala @@ -11,8 +11,6 @@ import com.raphtory.examples.blockchain.analysers.TaintTrackExchangeStop import kamon.Kamon import org.slf4j.LoggerFactory -import com.raphtory.examples.blockchain.routers.EthereumGethRouter -import com.raphtory.examples.blockchain.spouts.EthereumGethSpout import scala.language.postfixOps From 4de6b5d7949e7d6b318e92671f25a68438b776fa Mon Sep 17 00:00:00 2001 From: miratepuffin Date: Tue, 3 Nov 2020 23:19:31 +0000 Subject: [PATCH 11/58] Changed Analysis Task to report earliest seen time --- .../raphtory/core/analysis/Tasks/AnalysisTask.scala | 12 ++++++++++-- .../PartitionManager/Workers/ReaderWorker.scala | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/AnalysisTask.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/AnalysisTask.scala index f43684c627..04b54f691e 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/AnalysisTask.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/AnalysisTask.scala @@ -45,6 +45,7 @@ abstract class AnalysisTask(jobID: String, args:Array[String], analyser: Analyse protected var workersFinishedSuperStep = 0 //Acks from workers to say they have finished the superstep protected var workerResultsReceived = 0 //Acks from workers when sending results back to analysis manager protected var messageLogACKS = 0 //Acks from Workers with the amount of messages they have sent and received + protected var recentlySeenTime = Long.MaxValue //tracking the earliest time seen private var totalReceivedMessages = 0 //Total number of messages received by the workers private var totalSentMessages = 0 //Total number of messages sent by the workers @@ -203,6 +204,7 @@ abstract class AnalysisTask(jobID: String, args:Array[String], analyser: Analyse if (!ok) TimeOKFlag = false TimeOKACKS += 1 + if(timev._2.computationValues = ParTrieMap[String, Any]()) + //storage.vertices.foreach(v=>v._2.computationValues = ParTrieMap[String, Any]()) } } From cca5195b64501e559a7064b304b686bac058f311 Mon Sep 17 00:00:00 2001 From: miratepuffin Date: Tue, 3 Nov 2020 23:25:51 +0000 Subject: [PATCH 12/58] Stopped Analysis Task retrying on error --- .../core/analysis/Tasks/AnalysisTask.scala | 31 ++++++++++++------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/AnalysisTask.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/AnalysisTask.scala index 04b54f691e..f6b02d06bb 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/AnalysisTask.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/AnalysisTask.scala @@ -106,19 +106,26 @@ abstract class AnalysisTask(jobID: String, args:Array[String], analyser: Analyse } protected def processResultsWrapper(timeStamp: Long) = { - if(saveData) { - val mongo = MongoClient(MongoClientURI(s"mongodb://${InetAddress.getByName(mongoIP).getHostAddress()}:$mongoPort")) - val buffer = new java.util.ArrayList[DBObject]() - processResults(timeStamp) - analyser.getPublishedData().foreach(data => { - buffer.add(JSON.parse(data).asInstanceOf[DBObject]) - }) - analyser.clearPublishedData() - mongo.getDB(dbname).getCollection(jobID).insert(buffer) - buffer.clear() + try{ + if(saveData) { + val mongo = MongoClient(MongoClientURI(s"mongodb://${InetAddress.getByName(mongoIP).getHostAddress()}:$mongoPort")) + val buffer = new java.util.ArrayList[DBObject]() + processResults(timeStamp) + analyser.getPublishedData().foreach(data => { + buffer.add(JSON.parse(data).asInstanceOf[DBObject]) + }) + analyser.clearPublishedData() + mongo.getDB(dbname).getCollection(jobID).insert(buffer) + buffer.clear() + } + else + processResults(timeStamp) + } + catch { + case e:Exception => e.printStackTrace() } - else - processResults(timeStamp) + + } From a4bf783086ea43024f61fde9c77eff4e7e67fa74 Mon Sep 17 00:00:00 2001 From: miratepuffin Date: Wed, 4 Nov 2020 00:10:04 +0000 Subject: [PATCH 13/58] Removed old safe time conception from watermarking --- .../Workers/IngestionWorker.scala | 42 ++++++++----------- .../raphtory/core/storage/EntityStorage.scala | 3 -- 2 files changed, 17 insertions(+), 28 deletions(-) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/IngestionWorker.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/IngestionWorker.scala index 6b132aec73..1ad8a3fe6d 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/IngestionWorker.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/IngestionWorker.scala @@ -14,7 +14,7 @@ import scala.concurrent.duration._ import scala.collection.mutable import scala.collection.parallel.mutable.ParTrieMap -case class queueItem(routerEpoch:Int,timestamp:Long,safe:Boolean)extends Ordered[queueItem] { +case class queueItem(routerEpoch:Int,timestamp:Long)extends Ordered[queueItem] { def compare(that: queueItem): Int = that.routerEpoch-this.routerEpoch } @@ -92,7 +92,7 @@ class IngestionWorker(workerId: Int,partitionID:Int, storage: EntityStorage) ext private def vertexAddTrack(srcID: Long, msgTime: Long,routerID:String,routerTime:Int,spoutTime:Long): Unit = { //Vertex Adds the message time straight into queue as no sync storage.timings(msgTime) - addToWatermarkQueue(routerID,routerTime,msgTime,safe=false,spoutTime = spoutTime) + addToWatermarkQueue(routerID,routerTime,msgTime,spoutTime = spoutTime) } @@ -115,7 +115,7 @@ class IngestionWorker(workerId: Int,partitionID:Int, storage: EntityStorage) ext private def edgeAddTrack(srcID: Long, dstID: Long, msgTime: Long,local:Boolean,routerID:String,routerTime:Int,spoutTime:Long): Unit = { storage.timings(msgTime) if(local) {//if the edge is totally handled by this worker then we are safe to add to watermark queue - addToWatermarkQueue(routerID,routerTime,msgTime,safe=false,spoutTime = spoutTime) + addToWatermarkQueue(routerID,routerTime,msgTime,spoutTime = spoutTime) } } @@ -140,12 +140,12 @@ class IngestionWorker(workerId: Int,partitionID:Int, storage: EntityStorage) ext def processRemoteReturnDeathsRequest(req: RemoteReturnDeaths): Unit = { //when the new edge add is responded to we can say it is synced log.debug("IngestionWorker [{}] received [{}] request.", workerId, req) storage.remoteReturnDeaths(req.msgTime, req.srcID, req.dstID, req.kills) - addToWatermarkQueue(req.routerID,req.routerTime,req.msgTime,safe=false,spoutTime = req.spoutTime) + addToWatermarkQueue(req.routerID,req.routerTime,req.msgTime,spoutTime = req.spoutTime) } def processEdgeSyncAck(req: EdgeSyncAck) = { //when the edge isn't new we will get this response instead log.debug("IngestionWorker [{}] received [{}] request.", workerId, req) - addToWatermarkQueue(req.routerID,req.routerTime,req.msgTime,safe=false,spoutTime = req.spoutTime) + addToWatermarkQueue(req.routerID,req.routerTime,req.msgTime,spoutTime = req.spoutTime) } def processDstAddForOtherWorkerRequest(req: DstAddForOtherWorker): Unit = { //local worker asking this one to deal with an incoming edge @@ -158,7 +158,7 @@ class IngestionWorker(workerId: Int,partitionID:Int, storage: EntityStorage) ext def processDstResponseFromOtherWorkerRequest(req: DstResponseFromOtherWorker): Unit = { //local worker responded for a new edge so can watermark, if existing edge will just be an ack log.debug("IngestionWorker [{}] received [{}] request.", workerId, req) storage.vertexWorkerRequestEdgeHandler(req.msgTime, req.srcForEdge, req.dstID, req.removeList) - addToWatermarkQueue(req.routerID,req.routerTime,req.msgTime,safe=false,spoutTime = req.spoutTime) + addToWatermarkQueue(req.routerID,req.routerTime,req.msgTime,spoutTime = req.spoutTime) } @@ -173,7 +173,7 @@ class IngestionWorker(workerId: Int,partitionID:Int, storage: EntityStorage) ext private def edgeDeleteTrack(msgTime: Long,local:Boolean,routerID:String,routerTime:Int,spoutTime:Long): Unit = { storage.timings(msgTime) if(local) { //if the edge is totally handled by this worker then we are safe to add to watermark queue - addToWatermarkQueue(routerID,routerTime,msgTime,safe=false,spoutTime = spoutTime) + addToWatermarkQueue(routerID,routerTime,msgTime,spoutTime = spoutTime) } } @@ -208,7 +208,7 @@ class IngestionWorker(workerId: Int,partitionID:Int, storage: EntityStorage) ext } private def vertexDeleteTrack(srcID: Long, msgTime: Long,routerID:String,routerTime:Int,totalCount:Int,spoutTime:Long): Unit = { if(totalCount==0) //if there are no outgoing edges it is safe to watermark - addToWatermarkQueue(routerID,routerTime,msgTime,safe=false,spoutTime = spoutTime) + addToWatermarkQueue(routerID,routerTime,msgTime,spoutTime = spoutTime) else{ vDeleteCountdownMap put ((routerID,routerTime),new AtomicInteger(totalCount)) } @@ -218,7 +218,7 @@ class IngestionWorker(workerId: Int,partitionID:Int, storage: EntityStorage) ext def processVertexRemoveSyncAck(req:VertexRemoveSyncAck)= { vDeleteCountdownMap.get((req.routerID,req.routerTime)) match { case Some(integer) => if(integer.decrementAndGet()==0){ - addToWatermarkQueue(req.routerID,req.routerTime,req.msgTime,safe=false,spoutTime = req.spoutTime) + addToWatermarkQueue(req.routerID,req.routerTime,req.msgTime,spoutTime = req.spoutTime) vDeleteCountdownMap.remove((req.routerID,req.routerTime)) //todo improve this datastructure } case None => println("Should never Happen") @@ -245,15 +245,15 @@ class IngestionWorker(workerId: Int,partitionID:Int, storage: EntityStorage) ext interWorkerUpdates.increment() } - private def addToWatermarkQueue(routerID:String,routerTime:Int,msgTime:Long,safe:Boolean,spoutTime:Long) = { + private def addToWatermarkQueue(routerID:String,routerTime:Int,msgTime:Long,spoutTime:Long) = { queuedMessageMap.get(routerID) match { - case Some(queue) => queue += queueItem(routerTime,msgTime,safe) + case Some(queue) => queue += queueItem(routerTime,msgTime) case None => val queue = new mutable.PriorityQueue[queueItem]() - queue += queueItem(routerTime,msgTime,safe) + queue += queueItem(routerTime,msgTime) queuedMessageMap put(routerID,queue) } - if(!safe) synchronisedUpdates.increment() + synchronisedUpdates.increment() processSynchTime(spoutTime) } @@ -269,23 +269,15 @@ class IngestionWorker(workerId: Int,partitionID:Int, storage: EntityStorage) ext }) val timestamps = queueState.map(q => q.timestamp) storage.windowTime = timestamps.min - storage.safeWindowTime = timestamps.max - storage.windowSafe = queueState.map(q=>q.safe).fold(true)(_&&_) //if all safe then safe latestTime.update(storage.newestTime) earliestTime.update(storage.oldestTime) - if(storage.windowSafe) { - mediator ! DistributedPubSubMediator.Send("/user/WatermarkManager",WatermarkTime(storage.safeWindowTime), localAffinity = false) - safeTime.update(storage.safeWindowTime) - } else { - mediator ! DistributedPubSubMediator.Send("/user/WatermarkManager",WatermarkTime(storage.windowTime), localAffinity = false) - safeTime.update(storage.windowTime) - } - + mediator ! DistributedPubSubMediator.Send("/user/WatermarkManager",WatermarkTime(storage.windowTime), localAffinity = false) + safeTime.update(storage.windowTime) } } private def setSafePoint(routerName:String,messageQueue:mutable.PriorityQueue[queueItem]) = { - val default = queueItem(-1,0,false) + val default = queueItem(-1,0) var currentSafePoint = safeMessageMap.get(routerName) match { case Some(value) => value case None => default @@ -300,7 +292,7 @@ class IngestionWorker(workerId: Int,partitionID:Int, storage: EntityStorage) ext private def processRouterTimeSync(req:RouterWorkerTimeSync) ={ storage.timings(req.msgTime) - addToWatermarkQueue(req.routerID,req.routerTime,req.msgTime,true,-1) + addToWatermarkQueue(req.routerID,req.routerTime,req.msgTime,0) } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/storage/EntityStorage.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/storage/EntityStorage.scala index a6278a65e2..5bcd62a9d7 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/storage/EntityStorage.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/storage/EntityStorage.scala @@ -41,9 +41,6 @@ class EntityStorage(partitionID:Int,workerID: Int) { var oldestTime: Long = Long.MaxValue var newestTime: Long = 0 var windowTime: Long = 0 - var safeWindowTime:Long = 0 - var windowSafe:Boolean = false - var lastCompressedAt: Long = 0 val vertexCount = Kamon.counter("Raphtory_Vertex_Count").withTag("actor",s"PartitionWriter_$partitionID").withTag("ID",workerID) val localEdgeCount = Kamon.counter("Raphtory_Local_Edge_Count").withTag("actor",s"PartitionWriter_$partitionID").withTag("ID",workerID) From 12551727b880db09168190e3e3dbcf5f0040100b Mon Sep 17 00:00:00 2001 From: miratepuffin Date: Wed, 4 Nov 2020 15:16:25 +0000 Subject: [PATCH 14/58] Changed watermarking to batch end approach --- .../src/main/scala/com/raphtory/Server.scala | 4 +-- .../analysis/serialisers/Serialiser.scala | 5 +-- .../core/clustersetup/ManagerNode.scala | 4 +-- .../core/clustersetup/RouterNode.scala | 4 +-- .../singlenode/SingleNodeSetup.scala | 4 +-- .../RaphtoryReplicator.scala | 12 +++---- .../ClusterManagement/WatchDog.scala | 2 +- .../Workers/IngestionWorker.scala | 7 ++-- .../Workers/ReaderWorker.scala | 8 ++--- .../components/Router/RouterManager.scala | 6 ++-- .../core/components/Router/RouterWorker.scala | 35 +++++++++++++++++-- .../core/components/Spout/SpoutTrait.scala | 31 +++++++++++----- .../communication/raphtoryMessages.scala | 4 ++- .../scala/com/raphtory/core/utils/Utils.scala | 8 +++++ .../blockchain/routers/BitcoinRouter.scala | 4 +-- .../routers/ChABClus2ClusRouter.scala | 4 +-- .../routers/ChABClus2ClusRouterNoprop.scala | 4 +-- .../routers/ChainalysisABRouter.scala | 4 +-- .../blockchain/routers/DashcoinRouter.scala | 4 +-- .../routers/EthereumGethRouter.scala | 4 +-- .../routers/EthereumKafkaRouter.scala | 4 +-- .../routers/EthereumTransactionRouter.scala | 4 +-- .../routers/FirehoseKafkaRouter.scala | 4 +-- .../blockchain/routers/LitecoinRouter.scala | 4 +-- .../citationNetwork/CitationRouter.scala | 4 +-- .../gab/routers/GabPostGraphRouter.scala | 4 +-- .../examples/gab/routers/GabRawRouter.scala | 4 +-- .../gab/routers/GabUserGraphRouter.scala | 4 +-- .../examples/ldbc/routers/LDBCOldRouter.scala | 4 +-- .../examples/ldbc/routers/LDBCRouter.scala | 4 +-- .../raphtory/examples/lotr/LOTRRouter.scala | 4 +-- .../examples/test/actors/RandomRouter.scala | 4 +-- .../test/actors/TriangleTestRouter.scala | 4 +-- .../test/actors/TriangleTestSpout.scala | 2 ++ .../routers/TrackAndTraceRouter.scala | 4 +-- .../raphtory/examples/tsvnet/TSVRouter.scala | 4 +-- .../routers/CooccurrenceMatrixRouter.scala | 4 +-- .../CooccurrenceMatrixRouterFiltered.scala | 4 +-- .../com/raphtory/tests/CitationExample.scala | 4 +-- .../scala/com/raphtory/tests/NaomiTest.scala | 4 +-- .../com/raphtory/tests/SingleNodeTest.scala | 4 +-- .../tests/TemporalTriangleCountExample.scala | 4 +-- 42 files changed, 151 insertions(+), 93 deletions(-) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/Server.scala b/mainproject/cluster/src/main/scala/com/raphtory/Server.scala index 611a636ed4..e9be690baf 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/Server.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/Server.scala @@ -44,10 +44,10 @@ object Go extends App { SeedNode(seedLoc) case "router" => println("Creating Router") - RouterNode(getConf(), sys.env("PARTITION_MIN").toInt, routerName) + RouterNode(getConf(), sys.env("PARTITION_MIN").toInt, sys.env("ROUTER_MIN").toInt, routerName) case "partitionManager" => println(s"Creating Partition Manager...") - ManagerNode(getConf(), sys.env("PARTITION_MIN").toInt) + ManagerNode(getConf(), sys.env("PARTITION_MIN").toInt, sys.env("ROUTER_MIN").toInt) case "updater" => println("Creating Update Generator") diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/serialisers/Serialiser.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/serialisers/Serialiser.scala index 49298ae88c..58134cae9e 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/serialisers/Serialiser.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/serialisers/Serialiser.scala @@ -33,9 +33,10 @@ abstract class Serialiser extends Analyser(null){ } override def processWindowResults(results: ArrayBuffer[Any], timestamp: Long, windowSize: Long, viewCompleteTime: Long): Unit = { - val serialisedResults = results.asInstanceOf[(Array[String],Array[String])] + val serialisedResults = results.asInstanceOf[ArrayBuffer[(Array[String],Array[String])]] val file = new File(s"$path/Raphtory_Snapshot_${timestamp}_$windowSize.txt") - write(serialisedResults,file) + write((serialisedResults.flatMap(x=>x._1).toArray,serialisedResults.flatMap(x=>x._2).toArray),file) + } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/ManagerNode.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/ManagerNode.scala index 3cfffbc2d2..0cff34b686 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/ManagerNode.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/ManagerNode.scala @@ -7,7 +7,7 @@ import com.raphtory.core.utils.Utils import scala.language.postfixOps -case class ManagerNode(seedLoc: String, partitionCount: Int) extends DocSvr { +case class ManagerNode(seedLoc: String, partitionCount: Int,routerCount:Int) extends DocSvr { implicit val system: ActorSystem = initialiseActorSystem(seeds = List(seedLoc)) @@ -15,7 +15,7 @@ case class ManagerNode(seedLoc: String, partitionCount: Int) extends DocSvr { final val actorName: String = "PartitionManager" system.actorOf( - Props(RaphtoryReplicator(actorType = "Partition Manager", initialManagerCount = partitionCount)), + Props(RaphtoryReplicator(actorType = "Partition Manager", initialManagerCount = partitionCount,initialRouterCount = routerCount)), actorName ) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/RouterNode.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/RouterNode.scala index fe15b88449..3f44b76934 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/RouterNode.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/RouterNode.scala @@ -7,12 +7,12 @@ import akka.actor.ActorSystem import akka.actor.Props import com.raphtory.core.components.ClusterManagement.RaphtoryReplicator -case class RouterNode(seedLoc: String, partitionCount: Int, className: String) extends DocSvr { +case class RouterNode(seedLoc: String, partitionCount: Int, routerCount:Int, className: String) extends DocSvr { implicit val system: ActorSystem = initialiseActorSystem(seeds = List(seedLoc)) final val actorName: String = "Routers" system.actorOf( - Props(RaphtoryReplicator(actorType = "Router", initialManagerCount = partitionCount, routerName = className)), + Props(RaphtoryReplicator(actorType = "Router", initialManagerCount = partitionCount, initialRouterCount = routerCount, routerName = className)), actorName ) } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/singlenode/SingleNodeSetup.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/singlenode/SingleNodeSetup.scala index 9cbb19116c..9d556174f5 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/singlenode/SingleNodeSetup.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/singlenode/SingleNodeSetup.scala @@ -31,8 +31,8 @@ case class SingleNodeSetup( system.actorOf(Props(new SeedActor(this)), "cluster") system.actorOf(Props(new WatchDog(1, 1)), "WatchDog") system.actorOf(Props(new WatermarkManager(managerCount = 1)),"WatermarkManager") - system.actorOf(Props(RaphtoryReplicator("Router", 1, routerClassName)), s"Routers") - system.actorOf(Props(RaphtoryReplicator("Partition Manager", 1)), s"PartitionManager") + system.actorOf(Props(RaphtoryReplicator("Router", 1, 1,routerClassName)), s"Routers") + system.actorOf(Props(RaphtoryReplicator("Partition Manager", 1,1)), s"PartitionManager") system.actorOf(Props[AnalysisManager], s"AnalysisManager") system.actorOf(Props(Class.forName(UpdaterName)), "Spout") diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/ClusterManagement/RaphtoryReplicator.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/ClusterManagement/RaphtoryReplicator.scala index c669a76d5d..60cab5024e 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/ClusterManagement/RaphtoryReplicator.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/ClusterManagement/RaphtoryReplicator.scala @@ -19,14 +19,14 @@ import scala.concurrent.duration._ import scala.language.postfixOps object RaphtoryReplicator { - def apply(actorType: String, initialManagerCount: Int, routerName: String): RaphtoryReplicator = - new RaphtoryReplicator(actorType, initialManagerCount, routerName) + def apply(actorType: String, initialManagerCount: Int,initialRouterCount:Int, routerName: String): RaphtoryReplicator = + new RaphtoryReplicator(actorType, initialManagerCount,initialRouterCount, routerName) - def apply(actorType: String, initialManagerCount: Int): RaphtoryReplicator = - new RaphtoryReplicator(actorType, initialManagerCount, null) + def apply(actorType: String, initialManagerCount: Int,initialRouterCount:Int): RaphtoryReplicator = + new RaphtoryReplicator(actorType, initialManagerCount,initialRouterCount, null) } -class RaphtoryReplicator(actorType: String, initialManagerCount: Int, routerName: String) +class RaphtoryReplicator(actorType: String, initialManagerCount: Int, initialRouterCount:Int, routerName: String) extends Actor with ActorLogging { @@ -151,7 +151,7 @@ class RaphtoryReplicator(actorType: String, initialManagerCount: Int, routerName log.info(s"Router $assignedId has come online.") actorRef = context.system.actorOf( - Props(new RouterManager(myId, currentCount, routerName)).withDispatcher("misc-dispatcher"), + Props(new RouterManager(myId, currentCount, initialRouterCount, routerName)).withDispatcher("misc-dispatcher"), "router" ) } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/ClusterManagement/WatchDog.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/ClusterManagement/WatchDog.scala index b2c52a4631..9beadd0f84 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/ClusterManagement/WatchDog.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/ClusterManagement/WatchDog.scala @@ -91,7 +91,7 @@ class WatchDog(managerCount: Int, minimumRouters: Int) extends Actor with ActorL private def processClusterStatusRequest(req: ClusterStatusRequest): Unit = { log.debug(s"WatchDog received [{}] request.", req) - sender ! ClusterStatusResponse(clusterUp) + sender ! ClusterStatusResponse(clusterUp,pmCounter,roCounter) } private def processRequestPartitionCountRequest(req: RequestPartitionCount): Unit = { diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/IngestionWorker.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/IngestionWorker.scala index 1ad8a3fe6d..80facd6c35 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/IngestionWorker.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/IngestionWorker.scala @@ -262,16 +262,17 @@ class IngestionWorker(workerId: Int,partitionID:Int, storage: EntityStorage) ext } private def processWatermarkRequest() ={ - if(queuedMessageMap nonEmpty) { val queueState = queuedMessageMap.map(queue => { setSafePoint(queue._1, queue._2) }) val timestamps = queueState.map(q => q.timestamp) - storage.windowTime = timestamps.min + val min = timestamps.min + if(storage.windowTime processNextStepRequest(req) case req: NextStepNewAnalyser => processNextStepNewAnalyserRequest(req) case req: Finish => processFinishRequest(req) - case req: FinishNewAnalyser => processFinishNewAnalyserRequest(req) + case req: FinishNewAnalyser => processFinishNewAnalyserRequest(req) case req: VertexMessage => handleVertexMessage(req) case x => log.warning("ReaderWorker [{}] belonging to Reader [{}] received unknown [{}] message.", x) @@ -225,9 +225,7 @@ class ReaderWorker(managerCountVal: Int, managerID: Int, workerId: Int, storage: def processTimeCheckRequest(req: TimeCheck): Unit = { log.debug(s"Reader [{}] received [{}] request.", workerId, req) val timestamp = req.timestamp - - //val newest = if(storage.windowSafe) storage.safeWindowTime else storage.windowTime - val newest = storage.newestTime + val newest = storage.windowTime if (timestamp <= newest) { log.debug("Received timestamp is smaller or equal to newest entityStorage timestamp.") @@ -256,6 +254,8 @@ class ReaderWorker(managerCountVal: Int, managerID: Int, workerId: Int, storage: } } + + private def setProxy(jobID: String, superStep: Int, timestamp: Long, analysisType: AnalysisType.Value, window: Long, windowSet: Array[Long]): Unit = analysisType match { case AnalysisType.live => handleLiveAnalysis(jobID, superStep, timestamp, analysisType, window, windowSet) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterManager.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterManager.scala index 9af89ad43f..eb48e31c15 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterManager.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterManager.scala @@ -18,7 +18,7 @@ import scala.language.postfixOps // TODO break object RouterManager { getProps = { routerManager(arg1, arg2...): Props }} // todo slave type should be Props -final case class RouterManager(routerId: Int, initialManagerCount: Int, slaveType: String) +final case class RouterManager(routerId: Int, initialManagerCount: Int, initialRouterCount:Int, slaveType: String) extends Actor with ActorLogging { implicit val executionContext: ExecutionContext = context.system.dispatcher @@ -26,8 +26,8 @@ final case class RouterManager(routerId: Int, initialManagerCount: Int, slaveTyp private val childrenNumber = 10 private val children = (0 until childrenNumber).map { i => context.actorOf( - Props(Class.forName(slaveType), routerId, i, initialManagerCount).withDispatcher("router-dispatcher"), - s"routerWorker_$i" + Props(Class.forName(slaveType), routerId, i, initialManagerCount,initialRouterCount).withDispatcher("router-dispatcher"), + s"router_${routerId}_Worker_$i" ) }.toList diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala index 3237ee4450..edd0bdf49e 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala @@ -18,7 +18,7 @@ import scala.util.hashing.MurmurHash3 // TODO Add val name which sub classes that extend this trait must overwrite // e.g. BlockChainRouter val name = "Blockchain Router" // Log.debug that read 'Router' should then read 'Blockchain Router' -abstract class RouterWorker[In <: SpoutGoing](val routerId: Int, val workerID: Int, val initialManagerCount: Int) +abstract class RouterWorker[In <: SpoutGoing](val routerId: Int, val workerID: Int, val initialManagerCount: Int,val initialRouterCount:Int) extends Actor with ActorLogging { implicit val executionContext: ExecutionContext = context.system.dispatcher @@ -43,6 +43,8 @@ abstract class RouterWorker[In <: SpoutGoing](val routerId: Int, val workerID: I override def receive: Receive = work(initialManagerCount, 0L, 0L) + + private def work(managerCount: Int, trackedTime: Long, newestTime: Long): Receive = { case msg: UpdatedCounter => log.debug(s"RouterWorker [$routerId] received [$msg] request.") @@ -51,7 +53,8 @@ abstract class RouterWorker[In <: SpoutGoing](val routerId: Int, val workerID: I case AllocateTuple(record: In) => //todo: wvv AllocateTuple should hold type of record instead of using Any log.debug(s"RouterWorker [$routerId] received AllocateTuple[$record] request.") parseTupleAndSendGraph(record, managerCount, false, trackedTime).foreach(newNewestTime => - context.become(work(managerCount, trackedTime, newNewestTime)) + if(newNewestTime>newestTime) + context.become(work(managerCount, trackedTime, newNewestTime)) ) case msg @ AllocateTrackedTuple( @@ -70,9 +73,37 @@ abstract class RouterWorker[In <: SpoutGoing](val routerId: Int, val workerID: I false ) } + case DataFinished => + Utils.getAllWriterWorkers(managerCount).foreach { workerPath => + mediator ! DistributedPubSubMediator.Send( + workerPath, + RouterWorkerTimeSync(newestTime+1, s"${routerId}_$workerID", getMessageIDForWriter(workerPath)), + false + ) + } + Utils.getAllRouterWorkers(initialRouterCount).foreach { workerPath => + println(workerPath) + mediator ! DistributedPubSubMediator.Send( + workerPath, + DataFinishedSync(newestTime+1), + false + ) + } + context.become(work(managerCount, trackedTime, newestTime+1)) + case DataFinishedSync(time) => + println("syncing") + Utils.getAllWriterWorkers(managerCount).foreach { workerPath => + mediator ! DistributedPubSubMediator.Send( + workerPath, + RouterWorkerTimeSync(time, s"${routerId}_$workerID", getMessageIDForWriter(workerPath)), + false + ) + } + context.become(work(managerCount, trackedTime, time)) case unhandled => log.warning(s"RouterWorker received unknown [$unhandled] message.") } + protected def assignID(uniqueChars: String): Long = MurmurHash3.stringHash(uniqueChars) private def parseTupleAndSendGraph( diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/SpoutTrait.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/SpoutTrait.scala index a40dbaf0cc..c487d061d1 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/SpoutTrait.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/SpoutTrait.scala @@ -27,6 +27,9 @@ trait SpoutTrait[Domain <: DomainMessage, Out <: SpoutGoing] extends Actor with private val spoutTuples = Kamon.counter("Raphtory_Spout_Tuples").withTag("actor", self.path.name) private var count = 0 + private var lastRouter = "" + private var partitionManagers = 0 + private var routers = 0 private def recordUpdate(): Unit = { spoutTuples.increment() count += 1 @@ -41,14 +44,14 @@ trait SpoutTrait[Domain <: DomainMessage, Out <: SpoutGoing] extends Actor with context.system.scheduler.scheduleOnce(1 seconds, self, IsSafe) } - final override def receive: Receive = work(false) + final override def receive: Receive = work(false,1,1) - private def work(safe: Boolean): Receive = { + private def work(safe: Boolean,pmCounter:Int,rmCounter:Int): Receive = { case StateCheck => processStateCheckMessage(safe) - case ClusterStatusResponse(clusterUp) => - context.become(work(clusterUp)) + case ClusterStatusResponse(clusterUp,pmCounter,rmCounter) => + context.become(work(clusterUp,pmCounter,rmCounter)) context.system.scheduler.scheduleOnce(1 second, self, StateCheck) - case IsSafe => processIsSafeMessage(safe) + case IsSafe => processIsSafeMessage(safe,pmCounter,rmCounter) case StartSpout => startSpout() case x: Domain => handleDomainMessage(x) case unhandled => log.error(s"Unable to handle message [$unhandled].") @@ -68,14 +71,21 @@ trait SpoutTrait[Domain <: DomainMessage, Out <: SpoutGoing] extends Actor with } } - private def processIsSafeMessage(safe: Boolean): Unit = { + private def processIsSafeMessage(safe: Boolean,pmCount:Int,roCount:Int): Unit = { log.debug(s"Spout is handling [IsSafe] message.") - if (safe) + if (safe) { self ! StartSpout - else + partitionManagers=pmCount + routers=roCount + + } else context.system.scheduler.scheduleOnce(delay = 1 second, receiver = self, message = IsSafe) } + protected def dataFinished():Unit = { + mediator ! DistributedPubSubMediator.Send(lastRouter, DataFinished, localAffinity = false) + } + protected def sendTuple(command: Out): Unit = { log.debug(s"The command [$command] received for send.") recordUpdate() @@ -84,8 +94,11 @@ trait SpoutTrait[Domain <: DomainMessage, Out <: SpoutGoing] extends Actor with AllocateTrackedTuple(System.currentTimeMillis(), command) else AllocateTuple(command) - mediator ! DistributedPubSubMediator.Send(s"/user/router/routerWorker_${count % 10}", message, localAffinity = false) + val mod = count % (routers * 10) + lastRouter=s"/user/router/router_${mod/10}_Worker_${mod%10}" + mediator ! DistributedPubSubMediator.Send(lastRouter, message, localAffinity = false) } + def AllocateSpoutTask(duration: FiniteDuration, task: Any): Cancellable = { val taskCancellable = context.system.scheduler.scheduleOnce(duration, self, task) taskCancellable diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/model/communication/raphtoryMessages.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/model/communication/raphtoryMessages.scala index dabc712570..355b2f520e 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/model/communication/raphtoryMessages.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/model/communication/raphtoryMessages.scala @@ -21,11 +21,13 @@ sealed trait TrackedGraphUpdate{ trait SpoutGoing final case class StringSpoutGoing(value: String) extends SpoutGoing +case class DataFinished() +case class DataFinishedSync(time:Long) case class RouterUp(id: Int) case class PartitionUp(id: Int) case class ClusterStatusRequest() -case class ClusterStatusResponse(clusterUp: Boolean) +case class ClusterStatusResponse(clusterUp: Boolean,pmCounter:Int,roCounter:Int) sealed trait Property { def key: String diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/utils/Utils.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/utils/Utils.scala index 95114cce76..ba1b634675 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/utils/Utils.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/utils/Utils.scala @@ -85,6 +85,14 @@ object Utils { workers.toArray } + def getAllRouterWorkers(managerCount: Int): Array[String] = { + val workers = mutable.ArrayBuffer[String]() + for (i <- 0 until managerCount) + for (j <- 0 until totalWorkers) + workers += s"/user/router/router_${i}_Worker_$j" + workers.toArray + } + def createHistory(history: mutable.TreeMap[Long, Boolean]): String = { if (history.size == 0) return "" diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/BitcoinRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/BitcoinRouter.scala index a4bdb2bd6a..a0e99166bd 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/BitcoinRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/BitcoinRouter.scala @@ -6,8 +6,8 @@ import com.raphtory.spouts.blockchain.BitcoinTransaction import spray.json.JsArray import scala.collection.mutable.ListBuffer -class BitcoinRouter(override val routerId: Int,override val workerID:Int,override val initialManagerCount: Int) - extends RouterWorker[BitcoinTransaction](routerId,workerID,initialManagerCount) { +class BitcoinRouter(override val routerId: Int, override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) + extends RouterWorker[BitcoinTransaction](routerId,workerID,initialManagerCount, initialRouterCount) { override protected def parseTuple(tuple:BitcoinTransaction): List[GraphUpdate] = { diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusRouter.scala index 94860b9693..04e4cad552 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusRouter.scala @@ -9,8 +9,8 @@ import com.raphtory.core.model.communication._ import scala.collection.mutable.ListBuffer -class ChABClus2ClusRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int) - extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount) { +class ChABClus2ClusRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) + extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount,initialRouterCount) { override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { val dp = formatLine(tuple.value.split(",").map(_.trim)) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusRouterNoprop.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusRouterNoprop.scala index 615341a944..786e06c8d3 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusRouterNoprop.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusRouterNoprop.scala @@ -9,8 +9,8 @@ import com.raphtory.core.model.communication._ import scala.collection.mutable.ListBuffer -class ChABClus2ClusRouterNoprop(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int) - extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount) { +class ChABClus2ClusRouterNoprop(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) + extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount,initialRouterCount) { override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { val dp = formatLine(tuple.value.split(",").map(_.trim)) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChainalysisABRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChainalysisABRouter.scala index b848b25212..482b442ab5 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChainalysisABRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChainalysisABRouter.scala @@ -8,8 +8,8 @@ import com.raphtory.core.model.communication._ import scala.collection.mutable.ListBuffer -class ChainalysisABRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int) - extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount) { +class ChainalysisABRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) + extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount,initialRouterCount) { override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { val dp = formatLine(tuple.value.split(",").map(_.trim)) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/DashcoinRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/DashcoinRouter.scala index 174abce111..8eee81a2b2 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/DashcoinRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/DashcoinRouter.scala @@ -8,8 +8,8 @@ import spray.json.JsArray import scala.collection.mutable.ListBuffer import scala.util.hashing.MurmurHash3 -class DashcoinRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int) - extends RouterWorker[BitcoinTransaction](routerId,workerID, initialManagerCount) { +class DashcoinRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) + extends RouterWorker[BitcoinTransaction](routerId,workerID, initialManagerCount,initialRouterCount) { override protected def parseTuple(tuple: BitcoinTransaction): List[GraphUpdate] = { val transaction = tuple.transaction diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumGethRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumGethRouter.scala index f3892ac7e3..e760f673fc 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumGethRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumGethRouter.scala @@ -9,8 +9,8 @@ import spray.json._ import scala.collection.mutable.ListBuffer import scala.util.hashing.MurmurHash3 -class EthereumGethRouter(override val routerId: Int,override val workerID:Int,override val initialManagerCount: Int) - extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount) { +class EthereumGethRouter(override val routerId: Int,override val workerID:Int,override val initialManagerCount: Int, override val initialRouterCount: Int) + extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount, initialRouterCount) { def hexToInt(hex: String) = Integer.parseInt(hex.drop(2), 16) print(routerId) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumKafkaRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumKafkaRouter.scala index 6f5e81d1fb..e9f2eb2664 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumKafkaRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumKafkaRouter.scala @@ -9,8 +9,8 @@ import spray.json._ import scala.collection.mutable.ListBuffer import scala.util.hashing.MurmurHash3 -class EthereumKafkaRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int) - extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount) { +class EthereumKafkaRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) + extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount, initialRouterCount) { def hexToInt(hex: String) = Integer.parseInt(hex.drop(2), 16) override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumTransactionRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumTransactionRouter.scala index 116bf24fd8..f19d4b74db 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumTransactionRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumTransactionRouter.scala @@ -6,8 +6,8 @@ import com.raphtory.core.model.communication.{EdgeAddWithProperties, GraphUpdate import scala.collection.mutable.ListBuffer import scala.util.hashing.MurmurHash3 -class EthereumTransactionRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int) - extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount) { +class EthereumTransactionRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) + extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount, initialRouterCount) { override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { val components = tuple.value.drop(1).dropRight(1).split(",") diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/FirehoseKafkaRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/FirehoseKafkaRouter.scala index 31f03c1739..9f9309248d 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/FirehoseKafkaRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/FirehoseKafkaRouter.scala @@ -11,8 +11,8 @@ import scala.collection.mutable.ListBuffer import scala.util.Random import scala.util.hashing.MurmurHash3 import scala.math.BigInt -class FirehoseKafkaRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int) - extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount) { +class FirehoseKafkaRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) + extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount, initialRouterCount) { var DELETEPERCENT = System.getenv().getOrDefault("ETHER_DELETE_PERCENT", "0").trim.toDouble/100 var DELETESEED = System.getenv().getOrDefault("ETHER_DELETE_SEED", "123").trim.toInt val random = new Random(DELETESEED) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/LitecoinRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/LitecoinRouter.scala index 435fa2423e..5eb7f15c40 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/LitecoinRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/LitecoinRouter.scala @@ -8,8 +8,8 @@ import spray.json.JsArray import scala.collection.mutable.ListBuffer import scala.util.hashing.MurmurHash3 -class LitecoinRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int) - extends RouterWorker[BitcoinTransaction](routerId,workerID,initialManagerCount) { +class LitecoinRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) + extends RouterWorker[BitcoinTransaction](routerId,workerID,initialManagerCount, initialRouterCount) { override protected def parseTuple(tuple:BitcoinTransaction): List[GraphUpdate] = { val transaction = tuple.transaction diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/citationNetwork/CitationRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/citationNetwork/CitationRouter.scala index 392939ce35..02eb85a361 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/citationNetwork/CitationRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/citationNetwork/CitationRouter.scala @@ -7,8 +7,8 @@ import com.raphtory.core.model.communication._ import scala.collection.mutable.ListBuffer -class CitationRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int) - extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount) { +class CitationRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) + extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount, initialRouterCount) { override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { val fileLine = tuple.value.split(",").map(_.trim) //take the tuple and split on , as we are only interested in the first 4 fields // title_paper,year,volume,title,pages,number,journal,author,ENTRYTYPE,ID diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabPostGraphRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabPostGraphRouter.scala index 6071f7c42a..215ffbdcc9 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabPostGraphRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabPostGraphRouter.scala @@ -14,8 +14,8 @@ import scala.collection.mutable.ListBuffer // is equal to -1. Columns 1 and 4 correspond to the postId and parentPostid in the file. // Then either the vertex or the edge are created accordingly. -class GabPostGraphRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int) - extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount) { +class GabPostGraphRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) + extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount, initialRouterCount) { override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { val fileLine = tuple.value.split(";").map(_.trim) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabRawRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabRawRouter.scala index fe98bcf272..8d13d43c89 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabRawRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabRawRouter.scala @@ -19,8 +19,8 @@ import scala.collection.mutable.ListBuffer * the correct case Class which can then be passed to the graph manager * which will then pass it to the graph partition dealing with the associated vertex */ -final class GabRawRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int) - extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount) { +final class GabRawRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) + extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount, initialRouterCount) { import com.raphtory.examples.gab.rawgraphmodel.GabJsonProtocol._ diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabUserGraphRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabUserGraphRouter.scala index 629fc2f3cd..11f34e98e6 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabUserGraphRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabUserGraphRouter.scala @@ -16,8 +16,8 @@ import scala.util.Random // is equal to -1. Columns 2 and 5 correspond to the userid and parentUserid in the file. // Then either the vertex or the edge are created accordingly. -class GabUserGraphRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int) - extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount) { +class GabUserGraphRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) + extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount, initialRouterCount) { override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { val fileLine = tuple.value.split(";").map(_.trim) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCOldRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCOldRouter.scala index 92b847c270..6af88f2931 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCOldRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCOldRouter.scala @@ -8,8 +8,8 @@ import com.raphtory.core.model.communication.{EdgeAdd, EdgeDelete, GraphUpdate, import scala.collection.mutable.ListBuffer -class LDBCOldRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int) - extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount) { +class LDBCOldRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) + extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount, initialRouterCount) { override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { val fileLine = tuple.asInstanceOf[String].split("\\|") diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCRouter.scala index 131a99c3ca..0744940433 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCRouter.scala @@ -9,8 +9,8 @@ import com.raphtory.examples.test.actors.RandomSpout import scala.collection.mutable.ListBuffer -class LDBCRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int) - extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount) { +class LDBCRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) + extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount, initialRouterCount) { override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { val commands = new ListBuffer[GraphUpdate]() val fileLine = tuple.asInstanceOf[String].split("\\|") diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRRouter.scala index e19dc3dcaf..4234430aa5 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRRouter.scala @@ -5,8 +5,8 @@ import com.raphtory.core.model.communication.{EdgeAdd, Type, VertexAdd, _} import scala.collection.mutable.ListBuffer -class LOTRRouter(override val routerId: Int, override val workerID:Int, override val initialManagerCount: Int) - extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount) { +class LOTRRouter(override val routerId: Int, override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) + extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount, initialRouterCount) { override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomRouter.scala index 2b5661991d..201968c93e 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomRouter.scala @@ -14,8 +14,8 @@ import spray.json._ * the correct case Class which can then be passed to the graph manager * which will then pass it to the graph partition dealing with the associated vertex */ -class RandomRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int) - extends RouterWorker[StringSpoutGoing](routerId,workerID,initialManagerCount) { +class RandomRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) + extends RouterWorker[StringSpoutGoing](routerId,workerID,initialManagerCount, initialRouterCount) { //************* MESSAGE HANDLING BLOCK diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleTestRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleTestRouter.scala index 54e34d8668..5ff7a38ed8 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleTestRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleTestRouter.scala @@ -6,8 +6,8 @@ import com.raphtory.core.model.communication.{EdgeAdd, GraphUpdate, StringSpoutG import scala.collection.mutable.ListBuffer -class TriangleTestRouter (override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int) - extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount) { +class TriangleTestRouter (override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) + extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount, initialRouterCount) { override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { val command = tuple.value.split(",") val msgTime = command(0).toLong diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleTestSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleTestSpout.scala index 3d4ebd5ca9..ad12448b48 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleTestSpout.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleTestSpout.scala @@ -15,6 +15,8 @@ class TriangleTestSpout extends SpoutTrait[BasicDomain,StringSpoutGoing] { sendTuple(StringSpoutGoing("5,4,1")) sendTuple(StringSpoutGoing("6,1,3")) + dataFinished() + } override def startSpout(): Unit = self ! Next diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/trackAndTrace/routers/TrackAndTraceRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/trackAndTrace/routers/TrackAndTraceRouter.scala index ae52cc25b5..886d2a6ea7 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/trackAndTrace/routers/TrackAndTraceRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/trackAndTrace/routers/TrackAndTraceRouter.scala @@ -9,8 +9,8 @@ import com.raphtory.core.model.communication._ import scala.collection.mutable.ListBuffer import scala.util.control.Breaks._ -class TrackAndTraceRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int) - extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount) { +class TrackAndTraceRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) + extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount, initialRouterCount) { val EARTH_EQU = 6378137.0 //m val EARTH_POL = 6356752.3142 //m val STEPSIZE = System.getenv().getOrDefault("MAP_GRID_SIZE", "100").trim.toDouble //m diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/TSVRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/TSVRouter.scala index efd784bba6..c68d633ced 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/TSVRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/TSVRouter.scala @@ -6,8 +6,8 @@ import com.raphtory.core.model.communication.{EdgeAdd, GraphUpdate, StringSpoutG import scala.collection.mutable.ListBuffer /** Spout for network datasets of the form SRC_NODE_ID DEST_NODE_ID TIMESTAMP */ -class TSVRouter(override val routerId: Int, override val workerID:Int, override val initialManagerCount: Int) - extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount) { +class TSVRouter(override val routerId: Int, override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) + extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount, initialRouterCount) { val commands = new ListBuffer[GraphUpdate]() override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { val fileLine = tuple.value.split(" ").map(_.trim) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixRouter.scala index 419e5d5203..af38f931a7 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixRouter.scala @@ -5,8 +5,8 @@ import com.raphtory.core.model.communication.{Type, _} import scala.collection.mutable.ListBuffer -class CooccurrenceMatrixRouter(override val routerId: Int, override val workerID:Int, override val initialManagerCount: Int) - extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount) { +class CooccurrenceMatrixRouter(override val routerId: Int, override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) + extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount, initialRouterCount) { override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { //println(record) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixRouterFiltered.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixRouterFiltered.scala index e5dc01c490..32db76b97b 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixRouterFiltered.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixRouterFiltered.scala @@ -5,8 +5,8 @@ import com.raphtory.core.model.communication._ import scala.collection.mutable.ListBuffer -class CooccurrenceMatrixRouterFiltered(override val routerId: Int, override val workerID:Int, override val initialManagerCount: Int) - extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount) { +class CooccurrenceMatrixRouterFiltered(override val routerId: Int, override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) + extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount, initialRouterCount) { val THR = System.getenv().getOrDefault("COOC_FREQ_THRESHOLD ", "0.05").trim.toDouble override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { diff --git a/mainproject/cluster/src/main/scala/com/raphtory/tests/CitationExample.scala b/mainproject/cluster/src/main/scala/com/raphtory/tests/CitationExample.scala index 767b72cd35..d9f163484e 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/tests/CitationExample.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/tests/CitationExample.scala @@ -26,9 +26,9 @@ object CitationExample extends App{ var routerClassName = "com.raphtory.examples.gab.actors.GabUserGraphRouter" //var routerClassName = "com.raphtory.examples.citationNetwork.CitationRouter" - system.actorOf(Props(RaphtoryReplicator("Router", 1, routerClassName)), s"Routers") + system.actorOf(Props(RaphtoryReplicator("Router", 1, 1,routerClassName)), s"Routers") - system.actorOf(Props(RaphtoryReplicator("Partition Manager", 1)), s"PartitionManager") + system.actorOf(Props(RaphtoryReplicator("Partition Manager", 1,1)), s"PartitionManager") system.actorOf(Props[AnalysisManager], s"AnalysisManager") AnalysisRestApi(system) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/tests/NaomiTest.scala b/mainproject/cluster/src/main/scala/com/raphtory/tests/NaomiTest.scala index e31712c9a3..4926daac98 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/tests/NaomiTest.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/tests/NaomiTest.scala @@ -31,8 +31,8 @@ object NaomiTest extends App{ system.actorOf(Props(new WatermarkManager(managerCount = 1)),"WatermarkManager") system.actorOf(Props(new WatchDog(partitionNumber, minimumRouters)), "WatchDog") - system.actorOf(Props(RaphtoryReplicator("Router", 1, routerClassName)), s"Routers") - system.actorOf(Props(RaphtoryReplicator("Partition Manager", 1)), s"PartitionManager") + system.actorOf(Props(RaphtoryReplicator("Router", 1, 1,routerClassName)), s"Routers") + system.actorOf(Props(RaphtoryReplicator("Partition Manager", 1,1)), s"PartitionManager") system.actorOf(Props(Class.forName(SpoutName)), "Spout") val analysisManager = system.actorOf(Props[AnalysisManager], s"AnalysisManager") AnalysisRestApi(system) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/tests/SingleNodeTest.scala b/mainproject/cluster/src/main/scala/com/raphtory/tests/SingleNodeTest.scala index 658d1503b8..deefd8149a 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/tests/SingleNodeTest.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/tests/SingleNodeTest.scala @@ -73,8 +73,8 @@ object SingleNodeTest extends App { system.actorOf(Props(new WatermarkManager(managerCount = 1)),"WatermarkManager") system.actorOf(Props(new WatchDog(partitionNumber, minimumRouters)), "WatchDog") - system.actorOf(Props(RaphtoryReplicator("Router", 1, routerClassName)), s"Routers") - system.actorOf(Props(RaphtoryReplicator("Partition Manager", 1)), s"PartitionManager") + system.actorOf(Props(RaphtoryReplicator("Router", 1,1, routerClassName)), s"Routers") + system.actorOf(Props(RaphtoryReplicator("Partition Manager", 1,1)), s"PartitionManager") system.actorOf(Props(Class.forName(SpoutName)), "Spout") val analysisManager = system.actorOf(Props[AnalysisManager], s"AnalysisManager") //AnalysisRestApi(system) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/tests/TemporalTriangleCountExample.scala b/mainproject/cluster/src/main/scala/com/raphtory/tests/TemporalTriangleCountExample.scala index 33e8b856e3..ca50381a37 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/tests/TemporalTriangleCountExample.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/tests/TemporalTriangleCountExample.scala @@ -32,9 +32,9 @@ object TemporalTriangleCountExample extends App{ var routerClassName = "com.raphtory.examples.test.actors.TriangleTestRouter" //var routerClassName = "com.raphtory.examples.blockchain.routers.FirehoseKafkaRouter" - system.actorOf(Props(RaphtoryReplicator("Router", 1, routerClassName)), s"Routers") + system.actorOf(Props(RaphtoryReplicator("Router", 1, 1,routerClassName)), s"Routers") - system.actorOf(Props(RaphtoryReplicator("Partition Manager", 1)), s"PartitionManager") + system.actorOf(Props(RaphtoryReplicator("Partition Manager", 1,1)), s"PartitionManager") system.actorOf(Props[AnalysisManager], s"AnalysisManager") AnalysisRestApi(system) From 61279c137d5adfccf81ae08969a149a9e361c23a Mon Sep 17 00:00:00 2001 From: miratepuffin Date: Wed, 4 Nov 2020 17:13:30 +0000 Subject: [PATCH 15/58] Fixed watermarking for out of order data --- .../core/components/Router/RouterWorker.scala | 45 ++++++++++--------- .../core/components/Spout/SpoutTrait.scala | 6 ++- .../test/actors/TriangleTestSpout.scala | 21 +++++---- 3 files changed, 43 insertions(+), 29 deletions(-) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala index edd0bdf49e..fc1e31da75 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala @@ -73,33 +73,38 @@ abstract class RouterWorker[In <: SpoutGoing](val routerId: Int, val workerID: I false ) } - case DataFinished => - Utils.getAllWriterWorkers(managerCount).foreach { workerPath => - mediator ! DistributedPubSubMediator.Send( - workerPath, - RouterWorkerTimeSync(newestTime+1, s"${routerId}_$workerID", getMessageIDForWriter(workerPath)), - false - ) - } + case DataFinished => { Utils.getAllRouterWorkers(initialRouterCount).foreach { workerPath => - println(workerPath) mediator ! DistributedPubSubMediator.Send( workerPath, - DataFinishedSync(newestTime+1), + DataFinishedSync(newestTime), false ) } - context.become(work(managerCount, trackedTime, newestTime+1)) - case DataFinishedSync(time) => - println("syncing") - Utils.getAllWriterWorkers(managerCount).foreach { workerPath => - mediator ! DistributedPubSubMediator.Send( - workerPath, - RouterWorkerTimeSync(time, s"${routerId}_$workerID", getMessageIDForWriter(workerPath)), - false - ) + } + + case DataFinishedSync(time) => { + if (time >= newestTime) { + Utils.getAllWriterWorkers(managerCount).foreach { workerPath => + mediator ! DistributedPubSubMediator.Send( + workerPath, + RouterWorkerTimeSync(time, s"${routerId}_$workerID", getMessageIDForWriter(workerPath)), + false + ) + } + context.become(work(managerCount, trackedTime, time)) } - context.become(work(managerCount, trackedTime, time)) + else { + Utils.getAllRouterWorkers(initialRouterCount).foreach { workerPath => + mediator ! DistributedPubSubMediator.Send( + workerPath, + DataFinishedSync(newestTime), + false + ) + } + + } + } case unhandled => log.warning(s"RouterWorker received unknown [$unhandled] message.") } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/SpoutTrait.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/SpoutTrait.scala index c487d061d1..cc7e387ddc 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/SpoutTrait.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/SpoutTrait.scala @@ -30,6 +30,7 @@ trait SpoutTrait[Domain <: DomainMessage, Out <: SpoutGoing] extends Actor with private var lastRouter = "" private var partitionManagers = 0 private var routers = 0 + private var finish = false private def recordUpdate(): Unit = { spoutTuples.increment() count += 1 @@ -83,7 +84,10 @@ trait SpoutTrait[Domain <: DomainMessage, Out <: SpoutGoing] extends Actor with } protected def dataFinished():Unit = { - mediator ! DistributedPubSubMediator.Send(lastRouter, DataFinished, localAffinity = false) + if(!finish) { + mediator ! DistributedPubSubMediator.Send(lastRouter, DataFinished, localAffinity = false) + finish=true + } } protected def sendTuple(command: Out): Unit = { diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleTestSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleTestSpout.scala index ad12448b48..b3bca14961 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleTestSpout.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleTestSpout.scala @@ -6,18 +6,23 @@ import com.raphtory.core.components.Spout.SpoutTrait.CommonMessage.Next import com.raphtory.core.model.communication.StringSpoutGoing class TriangleTestSpout extends SpoutTrait[BasicDomain,StringSpoutGoing] { + override def handleDomainMessage(message: BasicDomain): Unit = message match { case Next => - sendTuple(StringSpoutGoing("1,1,2")) - sendTuple(StringSpoutGoing("2,2,3")) - sendTuple(StringSpoutGoing("3,3,1")) - sendTuple(StringSpoutGoing("4,3,4")) - sendTuple(StringSpoutGoing("5,4,1")) - sendTuple(StringSpoutGoing("6,1,3")) - dataFinished() + + + } + + override def startSpout(): Unit = { + sendTuple(StringSpoutGoing("3,3,1")) + sendTuple(StringSpoutGoing("4,3,4")) + sendTuple(StringSpoutGoing("2,2,3")) + sendTuple(StringSpoutGoing("5,4,1")) + sendTuple(StringSpoutGoing("6,1,3")) + sendTuple(StringSpoutGoing("1,1,2")) + dataFinished() } - override def startSpout(): Unit = self ! Next } From 37797fbec74c1df666b4bb262df45324c92ab519 Mon Sep 17 00:00:00 2001 From: miratepuffin Date: Wed, 4 Nov 2020 17:26:42 +0000 Subject: [PATCH 16/58] Changed property keys to correctly use jobID --- .../analysis/API/entityVisitors/VertexVisitor.scala | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/VertexVisitor.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/VertexVisitor.scala index d9f44c30c2..1bce2c831a 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/VertexVisitor.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/VertexVisitor.scala @@ -113,24 +113,24 @@ class VertexVisitor(v: Vertex, viewJob:ViewJob, superStep: Int, view: GraphLens) def setState(key: String, value: Any): Unit = { - val realkey = key + timestamp + window + val realkey = key + timestamp + window + jobID v.addCompValue(realkey, value) } def getState[T: ClassTag](key: String) = { - val realkey = key + timestamp + window + val realkey = key + timestamp + window + jobID v.getCompValue(realkey).asInstanceOf[T] } def containsState(key: String): Boolean = { - val realkey = key + timestamp + window + val realkey = key + timestamp + window + jobID v.containsCompvalue(realkey) } def getOrSetState[T: ClassTag](key: String, value: Any) = { - val realkey = key + timestamp + window + val realkey = key + timestamp + window + jobID v.getOrSet(realkey, value).asInstanceOf[T] } def appendToState[T: ClassTag](key: String, value: Any) = { //write function later - val realkey = key + timestamp + window + val realkey = key + timestamp + window + jobID if(v.containsCompvalue(realkey)) v.addCompValue(realkey,v.getCompValue(realkey).asInstanceOf[Array[Any]] ++ Array(value)) else From 04d8ccb16f327de530e0d23a0fa7c7b1060812e9 Mon Sep 17 00:00:00 2001 From: miratepuffin Date: Thu, 5 Nov 2020 14:48:02 +0000 Subject: [PATCH 17/58] initial refactor of properties --- .../core/model/graphentities/Entity.scala | 4 +- .../graphentities/ImmutableProperty.scala | 6 +- .../model/graphentities/MutableProperty.scala | 84 ++----------------- .../core/model/graphentities/Property.scala | 4 +- 4 files changed, 14 insertions(+), 84 deletions(-) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/Entity.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/Entity.scala index 8678cbd232..440af72f6d 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/Entity.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/Entity.scala @@ -149,9 +149,9 @@ abstract class Entity(val creationTime: Long, isInitialValue: Boolean, storage: p update (msgTime, value) } case None => - if (immutable) properties.put(key, new ImmutableProperty(msgTime, key, value, storage)) + if (immutable) properties.put(key, new ImmutableProperty(msgTime, value)) else { - properties.put(key, new MutableProperty(msgTime, key, value, storage)) + properties.put(key, new MutableProperty(msgTime, value)) } } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/ImmutableProperty.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/ImmutableProperty.scala index 0bbb068f47..8175117e27 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/ImmutableProperty.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/ImmutableProperty.scala @@ -1,13 +1,11 @@ package com.raphtory.core.model.graphentities -import com.raphtory.core.storage.EntityStorage - -class ImmutableProperty(creationTime: Long, key: String, value: Any, storage: EntityStorage) extends Property(creationTime) { +class ImmutableProperty(creationTime: Long, value: Any) extends Property { var earliestTime: Long = creationTime - def earlist(): Long = creationTime override def valueAt(time: Long): Any = if (time >= earliestTime) value else "" override def valuesAfter(time: Long): Array[Any] = if (time >= earliestTime) Array[Any](value) else Array[Any]() override def update(msgTime: Long, newValue: Any): Unit = if (msgTime <= earliestTime) earliestTime = msgTime override def currentValue(): Any = value + override def creation(): Long = earliestTime } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/MutableProperty.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/MutableProperty.scala index 1f6cfc31c1..5290291b8f 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/MutableProperty.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/MutableProperty.scala @@ -9,76 +9,23 @@ import scala.collection.mutable * Node or Vertice Property. Created by Mirate on 10/03/2017. * * @param creationTime - * @param key Property name * @param value Property value */ -class MutableProperty(creationTime: Long, key: String, value: Any, storage: EntityStorage) extends Property(creationTime) { - private var saved = false - - // Initialize the TreeMap +class MutableProperty(creationTime: Long, value: Any) extends Property { var previousState: mutable.TreeMap[Long, Any] = mutable.TreeMap()(HistoryOrdering) - var compressedState: mutable.TreeMap[Long, Any] = mutable.TreeMap()(HistoryOrdering) - // add in the initial information update(creationTime, value) - def name = key - - def update(msgTime: Long, newValue: Any): Unit = - previousState.put(msgTime, newValue) - def compressHistory(cutoff: Long, entityType: Boolean, workerID: Int): mutable.TreeMap[Long, Any] = { - if (previousState.isEmpty) - return mutable.TreeMap()(HistoryOrdering) //if we have no need to compress, return an empty list - if (previousState.takeRight(1).head._1 >= cutoff) - return mutable.TreeMap()(HistoryOrdering) //if the oldest point is younger than the cut off no need to do anything - var toWrite: mutable.TreeMap[Long, Any] = mutable.TreeMap()(HistoryOrdering) //data which needs to be saved to cassandra - var newPreviousState: mutable.TreeMap[Long, Any] = mutable.TreeMap()(HistoryOrdering) //map which will replace the current history - - var PriorPoint: (Long, Any) = previousState.head - var swapped = false //specify pivot point when crossing over the cutoff as you don't want to compare to historic points which are still changing - previousState.foreach { - case (k, v) => - if (k >= cutoff) //still in read write space so - newPreviousState.put(k, v) //add to new uncompressed history - else { //reached the history which should be compressed - if (swapped && (v != PriorPoint._2)) { //if we have skipped the pivot and the types are different - toWrite.put(PriorPoint._1, PriorPoint._2) //add to toWrite so this can be saved to cassandra - compressedState.put(PriorPoint._1, PriorPoint._2) //add to compressedState in-mem - } - swapped = true - } - PriorPoint = (k, v) - } - if ((PriorPoint._1 < cutoff)) { // - if (compressedState.isEmpty || (compressedState.head._2 != PriorPoint._2)) { //if the last point is before the cut off --if compressedState is empty or the head of the compressed state is different to the final point of the uncompressed state then write - toWrite.put(PriorPoint._1, PriorPoint._2) //add to toWrite so this can be saved to cassandra - compressedState.put(PriorPoint._1, PriorPoint._2) //add to compressedState in-mem - } - } - else - newPreviousState.put( - PriorPoint._1, - PriorPoint._2 - ) //if the last point in the uncompressed history wasn't past the cutoff it needs to go back into the new uncompressed history - previousState = newPreviousState - toWrite - } + var earliest = creationTime + override def creation(): Long = earliest - def getPreviousStateSize(): Int = - previousState.size - - override def equals(obj: scala.Any): Boolean = { - if (obj.isInstanceOf[MutableProperty]) { - val prop2 = obj.asInstanceOf[MutableProperty] - if (name.equals(prop2.name) && (previousState.equals(prop2.previousState))) - return true - return false - } - false + def update(msgTime: Long, newValue: Any): Unit = { + if(msgTime x._1>=time).map(x=>x._2).toArray} - override def toString: String = - s"History: $previousState" - // var toReturn = System.lineSeparator() - // previousState.foreach(p => - // toReturn = s"$toReturn MessageID ${p._1}: ${p._2} -- ${p._2} " + System - // .lineSeparator()) - // s"Property: ${key} ----- Previous State: $toReturn" - - def toStringCurrent: String = { - val toReturn = System.lineSeparator() + - s" MessageID ${previousState.head._1}: ${previousState.head._2} -- ${previousState.head._2} " + - System.lineSeparator() - s"Property: $key ----- Current State: $toReturn" - } - def currentValue: Any = previousState.head._2 def currentTime: Long = previousState.head._1 - def beenSaved(): Boolean = saved + } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/Property.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/Property.scala index 30ac4a744d..5305f71e85 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/Property.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/Property.scala @@ -1,9 +1,9 @@ package com.raphtory.core.model.graphentities -abstract class Property(creationTime:Long) { +abstract class Property { def update(msgTime: Long, newValue: Any): Unit def valueAt(time: Long): Any def valuesAfter(time:Long):Array[Any] def currentValue(): Any - def creation():Long = creationTime + def creation():Long } From eb5b730537093ba704c151ef52bc004198cbdecc Mon Sep 17 00:00:00 2001 From: miratepuffin Date: Thu, 5 Nov 2020 15:50:14 +0000 Subject: [PATCH 18/58] Refactor of entity Classes --- .../API/entityVisitors/EdgeVisitor.scala | 6 - .../API/entityVisitors/EntityVisitor.scala | 12 +- .../API/entityVisitors/VertexVisitor.scala | 6 +- .../core/model/graphentities/Edge.scala | 54 +-------- .../core/model/graphentities/Entity.scala | 106 +----------------- .../core/model/graphentities/SplitEdge.scala | 13 +-- .../core/model/graphentities/Vertex.scala | 10 +- .../raphtory/core/storage/EntityStorage.scala | 18 +-- 8 files changed, 34 insertions(+), 191 deletions(-) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/EdgeVisitor.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/EdgeVisitor.scala index 452ce67021..4593e61f6d 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/EdgeVisitor.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/EdgeVisitor.scala @@ -40,10 +40,4 @@ class EdgeVisitor(edge:Edge,id:Long,viewJob:ViewJob,superStep:Int,view:GraphLens case None => None } - // def getOutgoingEdgePropertyValues(key: String) = - // v.outgoingEdges.map(e => getEdgePropertyValuesAfterTime(e._2, key, timestamp, window).get.values) - // def getIncomingEdgePropertyValues(key: String) = - // v.incomingEdges.map(e => getEdgePropertyValuesAfterTime(e._2, key, timestamp, window).get.values) - - } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/EntityVisitor.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/EntityVisitor.scala index 860ac07490..e108d96a04 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/EntityVisitor.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/EntityVisitor.scala @@ -14,9 +14,15 @@ abstract class EntityVisitor(entity:Entity,viewJob:ViewJob) { def latestActivity() = getHistory.head def earliestActivity() = getHistory.minBy(k=> k._1) - def getPropertySet(): ParTrieMap[String,Any] = entity.properties.filter(p =>{ - p._2.creation()<=viewJob.timestamp - }).map(f => (f._1,f._2.valueAt(viewJob.timestamp))) + def getPropertySet(): ParTrieMap[String,Any] = { + val x =entity.properties.filter(p =>{ + p._2.creation()<=viewJob.timestamp + }).map(f => (f._1,f._2.valueAt(viewJob.timestamp))) + if(x.isEmpty){ + println("empty") + } + x + } def getPropertyValue(key: String): Option[Any] = entity.properties.get(key) match { diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/VertexVisitor.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/VertexVisitor.scala index 1bce2c831a..93fa85e9d5 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/VertexVisitor.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/VertexVisitor.scala @@ -108,10 +108,6 @@ class VertexVisitor(v: Vertex, viewJob:ViewJob, superStep: Int, view: GraphLens) } - - //TODO work on properties - - def setState(key: String, value: Any): Unit = { val realkey = key + timestamp + window + jobID v.addCompValue(realkey, value) @@ -155,7 +151,7 @@ class VertexVisitor(v: Vertex, viewJob:ViewJob, superStep: Int, view: GraphLens) - def visitify(edge:Edge,id:Long) = new EdgeVisitor(edge,id,viewJob,superStep,view,mediator) + private def visitify(edge:Edge,id:Long) = new EdgeVisitor(edge,id,viewJob,superStep,view,mediator) } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/Edge.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/Edge.scala index f5626fda94..55211070bd 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/Edge.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/Edge.scala @@ -19,7 +19,7 @@ object Edge { storage: EntityStorage ) = { - val e = new Edge(workerID, creationTime, srcID, dstID, initialValue = true, storage) + val e = new Edge(workerID, creationTime, srcID, dstID, initialValue = true) e.history = previousState e.properties = properties e @@ -30,45 +30,16 @@ object Edge { /** * Created by Mirate on 01/03/2017. */ -class Edge(workerID: Int, msgTime: Long, srcId: Long, dstId: Long, initialValue: Boolean, storage: EntityStorage) - extends Entity(msgTime, initialValue, storage) { +class Edge(workerID: Int, msgTime: Long, srcId: Long, dstId: Long, initialValue: Boolean) + extends Entity(msgTime, initialValue) { - def killList(vKills: mutable.TreeMap[Long, Boolean]): Unit = - try { - removeList ++= vKills - history ++= vKills - } catch { - case e: java.lang.NullPointerException => - e.printStackTrace() - killList(vKills) - } + def killList(vKills: mutable.TreeMap[Long, Boolean]): Unit = history ++= vKills - def getId: Long = dstId //todo remove def getSrcId: Long = srcId def getDstId: Long = dstId def getWorkerID: Int = workerID -// def getPropertyValuesAfterTime(key : String,time:Long,window:Long) : Option[mutable.TreeMap[Long,Any]] = { -// if (window == -1L) -// properties.get(key) match { -// case Some(p) => Some(p.previousState.filter(x => x._1 <= time)) -// case None => None -// } -// else -// properties.get(key) match { -// case Some(p) => Some(p.previousState.filter(x => x._1 <= time && time - x._1 <= window)) -// case None => None -// } -// } - def viewAt(time: Long): Edge = { - -// if(time > EntityStorage.lastCompressedAt){ -// throw StillWithinLiveGraphException(time) -// } - //if(time < EntityStorage.oldestTime){ - // throw PushedOutOfGraphException(time) - //} var closestTime: Long = 0 var value = false for ((k, v) <- history) @@ -77,9 +48,7 @@ class Edge(workerID: Int, msgTime: Long, srcId: Long, dstId: Long, initialValue: closestTime = k value = v } - //if(value==false) - // throw EntityRemovedAtTimeException(getId) - val edge = new Edge(-1, closestTime, srcId, dstId, value, storage) + val edge = new Edge(-1, closestTime, srcId, dstId, value) for ((k, p) <- properties) { val value = p.valueAt(time) if (!(value equals (""))) @@ -88,18 +57,5 @@ class Edge(workerID: Int, msgTime: Long, srcId: Long, dstId: Long, initialValue: edge } - override def equals(obj: scala.Any): Boolean = { - if (obj.isInstanceOf[Edge]) { - val v2 = obj.asInstanceOf[Edge] //add associated edges - if ((getSrcId == v2.getSrcId) && (getDstId == v2.getDstId) && (history - .equals(v2.history)) && (oldestPoint == v2.oldestPoint) && (newestPoint == newestPoint) && (properties - .equals(v2.properties))) - return true - } - false - } - - override def toString: String = - s"Edge srcID $srcId dstID $dstId \n History $history \n Properties:\n $properties" } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/Entity.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/Entity.scala index 440af72f6d..250b7401bd 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/Entity.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/Entity.scala @@ -14,7 +14,7 @@ import scala.collection.parallel.mutable.ParTrieMap * @param creationTime ID of the message that created the entity * @param isInitialValue Is the first moment this entity is referenced */ -abstract class Entity(val creationTime: Long, isInitialValue: Boolean, storage: EntityStorage) { +abstract class Entity(val creationTime: Long, isInitialValue: Boolean) { // Properties from that entity private var entityType: String = null @@ -22,17 +22,12 @@ abstract class Entity(val creationTime: Long, isInitialValue: Boolean, storage: // History of that entity var history: mutable.TreeMap[Long, Boolean] = mutable.TreeMap(creationTime -> isInitialValue)(HistoryOrdering) - var compressedState: mutable.TreeMap[Long, Boolean] = mutable.TreeMap()(HistoryOrdering) - private var saved = false - var shouldBeWiped = false var oldestPoint: Long = creationTime var newestPoint: Long = creationTime // History of that entity - var removeList: mutable.TreeMap[Long, Boolean] = mutable.TreeMap()(HistoryOrdering) - if (!isInitialValue) - removeList = mutable.TreeMap(creationTime -> isInitialValue)(HistoryOrdering) + def removeList: mutable.TreeMap[Long, Boolean] = history.filter(f=> !f._2) def setType(newType: String): Unit = if (entityType == (null)) entityType = newType def getType: String = if (entityType == null) "" else entityType @@ -44,7 +39,6 @@ abstract class Entity(val creationTime: Long, isInitialValue: Boolean, storage: def kill(msgTime: Long): Unit = { checkOldestNewest(msgTime) - removeList.put(msgTime, false) history.put(msgTime, false) } @@ -60,88 +54,10 @@ abstract class Entity(val creationTime: Long, isInitialValue: Boolean, storage: */ def apply(property: String): Property = properties(property) - def compressHistory(cutoff: Long): mutable.TreeMap[Long, Boolean] = { - if (history.isEmpty) - return mutable.TreeMap()(HistoryOrdering) //if we have no need to compress, return an empty list - if (history.takeRight(1).head._1 >= cutoff) - return mutable.TreeMap()(HistoryOrdering) //if the oldest point is younger than the cut off no need to do anything - var toWrite: mutable.TreeMap[Long, Boolean] = mutable.TreeMap()(HistoryOrdering) //data which needs to be saved to cassandra - var newPreviousState: mutable.TreeMap[Long, Boolean] = mutable.TreeMap()(HistoryOrdering) //map which will replace the current history - - var PriorPoint: (Long, Boolean) = history.head - var swapped = false //specify pivot point when crossing over the cutoff as you don't want to compare to historic points which are still changing - history.foreach { - case (k, v) => - if (k >= cutoff) //still in read write space so - newPreviousState.put(k, v) //add to new uncompressed history - else { //reached the history which should be compressed - if (swapped && (v != PriorPoint._2)) { //if we have skipped the pivot and the types are different - toWrite.put(PriorPoint._1, PriorPoint._2) //add to toWrite so this can be saved to cassandra - compressedState.put(PriorPoint._1, PriorPoint._2) //add to compressedState in-mem - } - swapped = true - } - PriorPoint = (k, v) - } - if ((PriorPoint._1 < cutoff)) { // - if (compressedState.isEmpty || (compressedState.head._2 != PriorPoint._2)) { //if the last point is before the cut off --if compressedState is empty or the head of the compressed state is different to the final point of the uncompressed state then write - toWrite.put(PriorPoint._1, PriorPoint._2) //add to toWrite so this can be saved to cassandra - compressedState.put(PriorPoint._1, PriorPoint._2) //add to compressedState in-mem - } - } else - newPreviousState.put( - PriorPoint._1, - PriorPoint._2 - ) //if the last point in the uncompressed history wasn't past the cutoff it needs to go back into the new uncompressed history - history = newPreviousState - toWrite - } - //val removeFrom = if(compressing) compressedState else previousState - def archive(cutoff: Long, compressing: Boolean, entityType: Boolean, workerID: Int): Boolean = { // - if (history.isEmpty && compressedState.isEmpty) return false //blank node, decide what to do later - if (compressedState.nonEmpty) { - if (compressedState.takeRight(1).head._1 >= cutoff) - return false //if the oldest point is younger than the cut off no need to do anything - var head = compressedState.head._2 //get the head of later - val newCompressedState: mutable.TreeMap[Long, Boolean] = mutable.TreeMap()(HistoryOrdering) - compressedState.foreach { - case (k, v) => - if (k >= cutoff) - newCompressedState put (k, v) - //else recordRemoval(entityType, workerID) - } //for each point if it is safe then keep it - compressedState = newCompressedState //overwrite the compressed state - //properties.foreach{case ((propkey, property)) =>{property.archive(cutoff,compressing,entityType)}}//do the same for all properties //currently properties should exist until entity removed - newestPoint < cutoff && !head //return all points older than cutoff and latest update is deletion - } - false - } - - def archiveOnly(cutoff: Long, entityType: Boolean, workerID: Int): Boolean = { // - if (history.isEmpty) return false //blank node, decide what to do later - if (history.takeRight(1).head._1 >= cutoff) - return false //if the oldest point is younger than the cut off no need to do anything - - var head = history.head._2 //get the head of later - val newPreviousState: mutable.TreeMap[Long, Boolean] = mutable.TreeMap()(HistoryOrdering) - history.foreach { - case (k, v) => - if (k >= cutoff) - newPreviousState put (k, v) - //else recordRemoval(entityType, workerID) - } //for each point if it is safe then keep it - history = newPreviousState //overwrite the compressed state - newestPoint < cutoff && !head //return all points older than cutoff and latest update is deletion - false - } /** * * Add or update the property from an edge or a vertex based, using the operator vertex + (k,v) to add new properties - * - * @param msgTime Message ID where the update came from - * @param key property key - * @param value property value */ def +(msgTime: Long, immutable: Boolean, key: String, value: Any): Unit = properties.get(key) match { @@ -155,23 +71,8 @@ abstract class Entity(val creationTime: Long, isInitialValue: Boolean, storage: } } - def beenSaved(): Boolean = saved - def firstSave(): Unit = saved = true - def wipe() = history = mutable.TreeMap()(HistoryOrdering) - def getHistorySize(): Int = history.size + compressedState.size - - def getUncompressedSize(): Int = history.size - - def getId: Long - def getPropertyCurrentValue(key: String): Option[Any] = - properties.get(key) match { - case Some(p) => Some(p.currentValue) - case None => None - } - - protected val stateCache = ParTrieMap[Long, Long]() protected def closestTime(time: Long): (Long, Boolean) = { var closestTime: Long = -1 @@ -209,5 +110,4 @@ abstract class Entity(val creationTime: Long, isInitialValue: Boolean, storage: } -//def latestRouterCheck(newRouter:Int):Boolean = newRouter==latestRouter -//def updateLatestRouter(newRouter:Int):Unit = latestRouter = newRouter + diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/SplitEdge.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/SplitEdge.scala index cee30416f0..975f8580e8 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/SplitEdge.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/SplitEdge.scala @@ -15,11 +15,9 @@ object SplitEdge { srcID: Long, dstID: Long, previousState: mutable.TreeMap[Long, Boolean], - properties: ParTrieMap[String, Property], - remotePartitionId: Int, - storage: EntityStorage + properties: ParTrieMap[String, Property] ) = { - val e = new SplitEdge(workerID: Int, creationTime, srcID, dstID, initialValue = true, remotePartitionId, storage) + val e = new SplitEdge(workerID: Int, creationTime, srcID, dstID, initialValue = true) e.history = previousState e.properties = properties e @@ -38,9 +36,6 @@ class SplitEdge( msgTime: Long, srcID: Long, dstID: Long, - initialValue: Boolean, - remotePartitionId: Int, - storage: EntityStorage -) extends Edge(workerID, msgTime, srcID, dstID, initialValue, storage) { - def remotePartitionID = remotePartitionId + initialValue: Boolean) + extends Edge(workerID, msgTime, srcID, dstID, initialValue) { } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/Vertex.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/Vertex.scala index c89c07e62f..d09fc655ae 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/Vertex.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/Vertex.scala @@ -15,7 +15,7 @@ object Vertex { properties: ParTrieMap[String, Property], storage: EntityStorage ) = { - val v = new Vertex(creationTime, vertexId, initialValue = true, storage) + val v = new Vertex(creationTime, vertexId, initialValue = true) v.history = previousState //v.associatedEdges = associatedEdges v.properties = properties @@ -24,8 +24,8 @@ object Vertex { } -class Vertex(msgTime: Long, val vertexId: Long, initialValue: Boolean, storage: EntityStorage) - extends Entity(msgTime, initialValue, storage) { +class Vertex(msgTime: Long, val vertexId: Long, initialValue: Boolean) + extends Entity(msgTime, initialValue) { var incomingEdges = ParTrieMap[Long, Edge]() //Map of all edges associated with the vertex @@ -37,8 +37,6 @@ class Vertex(msgTime: Long, val vertexId: Long, initialValue: Boolean, storage: var multiQueue = new VertexMutliQueue() //Map of queues for all ongoing processing var computationValues = ParTrieMap[String, Any]() //Partial results kept between supersteps in calculation - override def getId = vertexId //get the vertexID - //Functions for adding associated edges to this vertex def incrementEdgesRequiringSync() =edgesRequiringSync+=1 def getEdgesRequringSync() =edgesRequiringSync @@ -87,6 +85,4 @@ class Vertex(msgTime: Long, val vertexId: Long, initialValue: Boolean, storage: else true } else false - override def toString: String = s"Vertex ID $vertexId \n History $history \n //Properties:\n $properties \n" - } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/storage/EntityStorage.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/storage/EntityStorage.scala index 5bcd62a9d7..fd0adbfa4c 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/storage/EntityStorage.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/storage/EntityStorage.scala @@ -80,7 +80,7 @@ class EntityStorage(partitionID:Int,workerID: Int) { v revive msgTime //add the history point v case None => //if it does not exist - val v = new Vertex(msgTime, srcId, initialValue = true, storage = this) //create a new vertex + val v = new Vertex(msgTime, srcId, initialValue = true) //create a new vertex vertexCount.increment() if (!(vertexType == null)) v.setType(vertexType.name) vertices put (srcId, v) //put it in the map @@ -96,7 +96,7 @@ class EntityStorage(partitionID:Int,workerID: Int) { case Some(vertex) => vertex case None => vertexCount.increment() - val vertex = new Vertex(msgTime, id, initialValue = true, this) + val vertex = new Vertex(msgTime, id, initialValue = true) vertices put (id, vertex) vertex wipe () vertex @@ -158,7 +158,7 @@ class EntityStorage(partitionID:Int,workerID: Int) { v case None => //if the removal has arrived before the creation vertexCount.increment() - val v = new Vertex(msgTime, srcId, initialValue = false, this) //create a placeholder + val v = new Vertex(msgTime, srcId, initialValue = false) //create a placeholder vertices put (srcId, v) //add it to the map v } @@ -254,10 +254,10 @@ class EntityStorage(partitionID:Int,workerID: Int) { present = true case None => //if it does not if (local) { - edge = new Edge(workerID, msgTime, srcId, dstId, initialValue = true, this) //create the new edge, local or remote + edge = new Edge(workerID, msgTime, srcId, dstId, initialValue = true) //create the new edge, local or remote localEdgeCount.increment() } else { - edge = new SplitEdge(workerID, msgTime, srcId, dstId, initialValue = true, getPartition(dstId, managerCount), this) + edge = new SplitEdge(workerID, msgTime, srcId, dstId, initialValue = true) masterSplitEdgeCount.increment() } if (!(edgeType == null)) edge.setType(edgeType.name) @@ -310,7 +310,7 @@ class EntityStorage(partitionID:Int,workerID: Int) { spoutTime:Long ): Unit = { val dstVertex = vertexAdd(msgTime, dstId, vertexType = null) //create or revive the destination node - val edge = new SplitEdge(workerID, msgTime, srcId, dstId, initialValue = true, getPartition(srcId, managerCount), this) + val edge = new SplitEdge(workerID, msgTime, srcId, dstId, initialValue = true) copySplitEdgeCount.increment() dstVertex addIncomingEdge (edge) //add the edge to the associated edges of the destination node val deaths = dstVertex.removeList //get the destination node deaths @@ -349,10 +349,10 @@ class EntityStorage(partitionID:Int,workerID: Int) { case None => if (local) { localEdgeCount.increment() - edge = new Edge(workerID, msgTime, srcId, dstId, initialValue = false, this) + edge = new Edge(workerID, msgTime, srcId, dstId, initialValue = false) } else { masterSplitEdgeCount.increment() - edge = new SplitEdge(workerID, msgTime, srcId, dstId, initialValue = false, getPartition(dstId, managerCount), this) + edge = new SplitEdge(workerID, msgTime, srcId, dstId, initialValue = false) } srcVertex addOutgoingEdge (edge) // add the edge to the associated edges of the source node } @@ -440,7 +440,7 @@ class EntityStorage(partitionID:Int,workerID: Int) { val dstVertex = getVertexOrPlaceholder(msgTime, dstId) dstVertex.incrementEdgesRequiringSync() copySplitEdgeCount.increment() - val edge = new SplitEdge(workerID, msgTime, srcId, dstId, initialValue = false, getPartition(srcId, managerCount), this) + val edge = new SplitEdge(workerID, msgTime, srcId, dstId, initialValue = false) dstVertex addIncomingEdge (edge) //add the edge to the destination nodes associated list val deaths = dstVertex.removeList //get the destination node deaths edge killList srcDeaths //pass source node death lists to the edge From 72da35c19070116f5929ba3946ae24376787a5ec Mon Sep 17 00:00:00 2001 From: miratepuffin Date: Thu, 5 Nov 2020 15:50:45 +0000 Subject: [PATCH 19/58] adding lotr data for unit testing --- .../scala/com/raphtory/examples/lotr/lotr.csv | 2649 +++++++++++++++++ 1 file changed, 2649 insertions(+) create mode 100644 mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/lotr.csv diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/lotr.csv b/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/lotr.csv new file mode 100644 index 0000000000..897532fcf5 --- /dev/null +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/lotr.csv @@ -0,0 +1,2649 @@ +Gandalf,Elrond,33 +Frodo,Bilbo,114 +Blanco,Marcho,146 +Frodo,Bilbo,205 +Thorin,Gandalf,270 +Thorin,Bilbo,270 +Gandalf,Bilbo,270 +Gollum,Bilbo,286 +Gollum,Bilbo,306 +Gollum,Bilbo,308 +Bilbo,Elrond,317 +Frodo,Samwise,319 +Gandalf,Bilbo,320 +Gollum,Bilbo,324 +Frodo,Gandalf,329 +Peregrin,Elessar,356 +Arwen,Aragorn,358 +Barahir,Faramir,359 +Bilbo,Findegil,360 +Meriadoc,Peregrin,363 +Peregrin,Elendil,368 +Galadriel,Celeborn,374 +Frodo,Bilbo,387 +Frodo,Bilbo,388 +Frodo,Bilbo,389 +Frodo,Bilbo,390 +Frodo,Bilbo,393 +Frodo,Bilbo,399 +Hamfast,Bilbo,402 +Gandalf,Bilbo,483 +Gandalf,Bilbo,543 +Frodo,Gandalf,555 +Frodo,Bilbo,555 +Gandalf,Bilbo,555 +Frodo,Bilbo,562 +Gandalf,Bilbo,730 +Frodo,Bilbo,808 +Frodo,Bilbo,815 +Gandalf,Bilbo,843 +Frodo,Gandalf,861 +Frodo,Bilbo,898 +Frodo,Merry,929 +Frodo,Merry,930 +Frodo,Bilbo,942 +Frodo,Bilbo,944 +Odo,Frodo,959 +Frodo,Bilbo,1038 +Meriadoc,Peregrin,1042 +Meriadoc,Merry,1042 +Meriadoc,Pippin,1042 +Meriadoc,Fredegar,1042 +Peregrin,Merry,1042 +Peregrin,Pippin,1042 +Peregrin,Fredegar,1042 +Merry,Pippin,1042 +Merry,Fredegar,1042 +Pippin,Fredegar,1042 +Pippin,Merry,1044 +Pippin,Bilbo,1044 +Merry,Bilbo,1044 +Frodo,Gandalf,1056 +Sam,Halfast,1089 +Frodo,Bilbo,1106 +Frodo,Gandalf,1130 +Gandalf,Bilbo,1135 +Gandalf,Bilbo,1156 +Frodo,Gandalf,1160 +Frodo,Saruman,1185 +Gandalf,Bilbo,1229 +Frodo,Gandalf,1234 +Frodo,Gandalf,1241 +Isildur,Elendil,1309 +Isildur,Gil-galad,1309 +Isildur,Sauron,1309 +Elendil,Gil-galad,1309 +Elendil,Sauron,1309 +Gil-galad,Sauron,1309 +Sméagol,Déagol,1324 +Sméagol,Déagol,1330 +Sméagol,Déagol,1331 +Sméagol,Déagol,1336 +Frodo,Gollum,1356 +Frodo,Gollum,1359 +Frodo,Gandalf,1390 +Frodo,Gollum,1400 +Frodo,Bilbo,1401 +Isildur,Gollum,1407 +Isildur,Déagol,1407 +Gollum,Déagol,1407 +Frodo,Gandalf,1417 +Isildur,Elendil,1420 +Frodo,Gollum,1429 +Gollum,Déagol,1441 +Gollum,Bilbo,1459 +Gandalf,Bilbo,1469 +Frodo,Bilbo,1484 +Frodo,Gollum,1484 +Bilbo,Gollum,1484 +Frodo,Bilbo,1567 +Frodo,Gollum,1567 +Bilbo,Gollum,1567 +Gandalf,Bilbo,1654 +Frodo,Gandalf,1656 +Sam,Gandalf,1677 +Sam,Frodo,1687 +Sam,Frodo,1705 +Sam,Gandalf,1710 +Sam,Gandalf,1711 +Sam,Frodo,1714 +Gandalf,Bilbo,1721 +Frodo,Gandalf,1741 +Gandalf,Bilbo,1748 +Sam,Elrond,1756 +Frodo,Gandalf,1783 +Frodo,Gandalf,1785 +Pippin,Frodo,1789 +Pippin,Merry,1789 +Pippin,Fredegar,1789 +Frodo,Merry,1789 +Frodo,Fredegar,1789 +Merry,Fredegar,1789 +Frodo,Gandalf,1792 +Frodo,Bilbo,1802 +Frodo,Gandalf,1804 +Merry,Fredegar,1806 +Frodo,Gandalf,1809 +Lobelia,Frodo,1816 +Lobelia,Lotho,1816 +Frodo,Lotho,1816 +Pippin,Sam,1825 +Lobelia,Sam,1826 +Lobelia,Frodo,1826 +Sam,Frodo,1826 +Pippin,Sam,1829 +Sam,Frodo,1869 +Sam,Frodo,1871 +Peregrin,Bilbo,1879 +Pippin,Sam,1888 +Pippin,Frodo,1951 +Pippin,Sam,1955 +Pippin,Bilbo,1981 +Frodo,Gandalf,1999 +Pippin,Frodo,2024 +Pippin,Sam,2024 +Frodo,Sam,2024 +Frodo,Gandalf,2056 +Frodo,Bilbo,2083 +Pippin,Frodo,2110 +Pippin,Sam,2110 +Frodo,Sam,2110 +Frodo,Bilbo,2175 +Sam,Frodo,2204 +Pippin,Frodo,2242 +Frodo,Bilbo,2246 +Frodo,Bilbo,2250 +Sam,Frodo,2254 +Sam,Gandalf,2254 +Frodo,Gandalf,2254 +Sam,Frodo,2318 +Sam,Frodo,2358 +Sam,Frodo,2393 +Pippin,Sam,2446 +Pippin,Frodo,2488 +Maggot,Pippin,2490 +Pippin,Frodo,2511 +Pippin,Sam,2511 +Frodo,Sam,2511 +Pippin,Sam,2513 +Maggot,Pippin,2519 +Pippin,Peregrin,2521 +Sam,Frodo,2541 +Maggot,Frodo,2607 +Frodo,Bilbo,2613 +Frodo,Peregrin,2628 +Pippin,Frodo,2633 +Pippin,Sam,2633 +Frodo,Sam,2633 +Pippin,Sam,2659 +Sam,Frodo,2668 +Frodo,Merry,2694 +Pippin,Frodo,2742 +Pippin,Merry,2742 +Pippin,Sam,2742 +Frodo,Merry,2742 +Frodo,Sam,2742 +Merry,Sam,2742 +Maggot,Frodo,2756 +Meriadoc,Frodo,2786 +Pippin,Bilbo,2794 +Maggot,Frodo,2821 +Pippin,Frodo,2829 +Maggot,Merry,2834 +Pippin,Frodo,2835 +Maggot,Frodo,2837 +Maggot,Bilbo,2837 +Frodo,Bilbo,2837 +Maggot,Merry,2838 +Pippin,Merry,2845 +Frodo,Merry,2850 +Pippin,Frodo,2857 +Sam,Frodo,2882 +Sam,Merry,2919 +Sam,Frodo,2922 +Frodo,Gandalf,2925 +Sam,Frodo,2938 +Pippin,Merry,2949 +Gandalf,Merry,2972 +Frodo,Gandalf,2999 +Gandalf,Fredegar,3060 +Pippin,Merry,3239 +Sam,Frodo,3265 +Pippin,Frodo,3271 +Pippin,Sam,3271 +Frodo,Sam,3271 +Sam,Frodo,3278 +Pippin,Sam,3282 +Pippin,Merry,3291 +Sam,Frodo,3316 +Sam,Frodo,3337 +Sam,Frodo,3345 +Sam,Merry,3356 +Frodo,Goldberry,3461 +Tom,Goldberry,3482 +Tom,Goldberry,3499 +Tom,Goldberry,3514 +Tom,Frodo,3526 +Pippin,Merry,3544 +Tom,Goldberry,3616 +Tom,Goldberry,3686 +Tom,Frodo,3702 +Tom,Frodo,3703 +Tom,Gandalf,3703 +Tom,Bilbo,3703 +Frodo,Gandalf,3703 +Frodo,Bilbo,3703 +Gandalf,Bilbo,3703 +Tom,Frodo,3705 +Tom,Frodo,3714 +Tom,Gandalf,3718 +Frodo,Merry,3721 +Tom,Frodo,3733 +Pippin,Sam,3837 +Pippin,Merry,3837 +Sam,Merry,3837 +Frodo,Gandalf,3914 +Frodo,Bilbo,3914 +Gandalf,Bilbo,3914 +Pippin,Sam,3919 +Pippin,Merry,3919 +Sam,Merry,3919 +Pippin,Sam,3938 +Pippin,Merry,3938 +Sam,Merry,3938 +Frodo,Merry,3946 +Tom,Frodo,3960 +Pippin,Sam,3976 +Pippin,Merry,3976 +Sam,Merry,3976 +Tom,Frodo,3988 +Tom,Merry,4037 +Pippin,Sam,4046 +Pippin,Merry,4046 +Sam,Merry,4046 +Tom,Goldberry,4060 +Butterbur,Barliman,4126 +Tom,Goldberry,4131 +Butterbur,Barliman,4261 +Sam,Frodo,4269 +Nob,Butterbur,4332 +Pippin,Frodo,4342 +Pippin,Sam,4342 +Frodo,Sam,4342 +Butterbur,Barliman,4353 +Pippin,Sam,4380 +Frodo,Butterbur,4389 +Frodo,Bilbo,4418 +Butterbur,Bilbo,4421 +Pippin,Sam,4482 +Pippin,Frodo,4565 +Pippin,Sam,4565 +Frodo,Sam,4565 +Sam,Frodo,4620 +Pippin,Frodo,4629 +Pippin,Sam,4629 +Frodo,Sam,4629 +Sam,Frodo,4692 +Sam,Frodo,4697 +Nob,Butterbur,4714 +Pippin,Sam,4734 +Pippin,Butterbur,4736 +Frodo,Butterbur,4749 +Pippin,Frodo,4902 +Pippin,Sam,4902 +Frodo,Sam,4902 +Frodo,Gandalf,4910 +Pippin,Gandalf,4926 +Pippin,Sam,4930 +Nob,Merry,5012 +Frodo,Merry,5089 +Nob,Merry,5090 +Nob,Gandalf,5090 +Merry,Gandalf,5090 +Bob,Bill,5228 +Bob,Bill,5229 +Butterbur,Merry,5242 +Tom,Butterbur,5249 +Nob,Butterbur,5268 +Nob,Bob,5268 +Butterbur,Bob,5268 +Pippin,Frodo,5274 +Pippin,Merry,5274 +Pippin,Sam,5274 +Frodo,Merry,5274 +Frodo,Sam,5274 +Merry,Sam,5274 +Nob,Bob,5276 +Pippin,Frodo,5410 +Elendil,Gil-galad,5434 +Pippin,Sam,5468 +Frodo,Merry,5470 +Merry,Gandalf,5500 +Frodo,Merry,5534 +Sam,Peregrin,5538 +Pippin,Gandalf,5544 +Merry,Gil-galad,5615 +Pippin,Frodo,5619 +Pippin,Merry,5619 +Frodo,Merry,5619 +Frodo,Gandalf,5620 +Lúthien,Beren,5653 +Lúthien,Barahir,5653 +Beren,Barahir,5653 +Beren,Lúthien,5654 +Beren,Thingol,5654 +Lúthien,Thingol,5654 +Beren,Barahir,5657 +Beren,Thingol,5657 +Barahir,Thingol,5657 +Beren,Lúthien,5660 +Beren,Sauron,5660 +Beren,Thingol,5660 +Lúthien,Sauron,5660 +Lúthien,Thingol,5660 +Sauron,Thingol,5660 +Beren,Thingol,5667 +Beren,Elwing,5667 +Beren,Eärendil,5667 +Beren,Lúthien,5667 +Beren,Dior,5667 +Thingol,Elwing,5667 +Thingol,Eärendil,5667 +Thingol,Lúthien,5667 +Thingol,Dior,5667 +Elwing,Eärendil,5667 +Elwing,Lúthien,5667 +Elwing,Dior,5667 +Eärendil,Lúthien,5667 +Eärendil,Dior,5667 +Lúthien,Dior,5667 +Sam,Merry,5682 +Pippin,Frodo,5683 +Pippin,Merry,5713 +Sam,Frodo,5714 +Sam,Frodo,5752 +Pippin,Merry,5758 +Frodo,Gandalf,5775 +Sam,Merry,5999 +Pippin,Merry,6010 +Sam,Frodo,6011 +Pippin,Merry,6012 +Sam,Frodo,6020 +Frodo,Bilbo,6037 +Frodo,Bilbo,6044 +Gandalf,Bilbo,6118 +Frodo,Bilbo,6126 +Frodo,Bilbo,6127 +Glorfindel,Elrond,6161 +Glorfindel,Frodo,6195 +Elrond,Aragorn,6200 +Glorfindel,Frodo,6220 +Pippin,Sam,6225 +Pippin,Merry,6225 +Sam,Merry,6225 +Glorfindel,Frodo,6267 +Frodo,Lúthien,6325 +Frodo,Gandalf,6381 +Butterbur,Bill,6429 +Gandalf,Barliman,6431 +Frodo,Gandalf,6531 +Glorfindel,Aragorn,6549 +Frodo,Gandalf,6578 +Glorfindel,Elrond,6583 +Sam,Frodo,6628 +Glorfindel,Gandalf,6660 +Glorfindel,Elrond,6660 +Gandalf,Elrond,6660 +Glorfindel,Frodo,6661 +Glorfindel,Gandalf,6661 +Glorfindel,Elrond,6661 +Frodo,Gandalf,6661 +Frodo,Elrond,6661 +Gandalf,Elrond,6661 +Frodo,Elrond,6669 +Arwen,Frodo,6673 +Arwen,Lúthien,6673 +Arwen,Elrond,6673 +Frodo,Lúthien,6673 +Frodo,Elrond,6673 +Lúthien,Elrond,6673 +Frodo,Elrond,6676 +Pippin,Frodo,6682 +Pippin,Merry,6682 +Frodo,Merry,6682 +Thorin,Glóin,6693 +Gandalf,Elrond,6701 +Frodo,Glóin,6709 +Frodo,Beorn,6710 +Frodo,Grimbeorn,6710 +Beorn,Grimbeorn,6710 +Bard,Bain,6717 +Bard,Brand,6717 +Bain,Brand,6717 +Dwalin,Bombur,6728 +Dwalin,Glóin,6728 +Dwalin,Bofur,6728 +Bombur,Glóin,6728 +Bombur,Bofur,6728 +Glóin,Bofur,6728 +Ori,Balin,6730 +Ori,Óin,6730 +Balin,Óin,6730 +Frodo,Glóin,6736 +Smaug,Bilbo,6747 +Frodo,Glóin,6748 +Arwen,Elrond,6754 +Frodo,Gandalf,6757 +Sam,Frodo,6810 +Sam,Bilbo,6810 +Frodo,Bilbo,6810 +Gandalf,Elrond,6833 +Gandalf,Bilbo,6835 +Frodo,Bilbo,6855 +Frodo,Bilbo,6935 +Eärendil,Elrond,6969 +Frodo,Bilbo,6987 +Frodo,Aragorn,6994 +Arwen,Frodo,6995 +Sam,Bilbo,7006 +Sam,Bilbo,7009 +Frodo,Elrond,7018 +Gandalf,Bilbo,7024 +Gandalf,Elrond,7037 +Sam,Frodo,7040 +Sam,Bilbo,7040 +Frodo,Bilbo,7040 +Frodo,Gandalf,7041 +Glorfindel,Glóin,7047 +Glorfindel,Frodo,7047 +Glóin,Frodo,7047 +Frodo,Elrond,7048 +Gimli,Glóin,7051 +Elrond,Glorfindel,7052 +Elrond,Galdor,7052 +Elrond,Erestor,7052 +Elrond,Círdan,7052 +Glorfindel,Galdor,7052 +Glorfindel,Erestor,7052 +Glorfindel,Círdan,7052 +Galdor,Erestor,7052 +Galdor,Círdan,7052 +Erestor,Círdan,7052 +Legolas,Thranduil,7053 +Frodo,Bilbo,7058 +Gandalf,Elrond,7059 +Gandalf,Boromir,7059 +Elrond,Boromir,7059 +Frodo,Glóin,7064 +Balin,Óin,7077 +Balin,Ori,7077 +Óin,Ori,7077 +Elrond,Sauron,7127 +Elrond,Sauron,7128 +Isildur,Elendil,7134 +Isildur,Anárion,7134 +Elendil,Anárion,7134 +Elendil,Gil-galad,7135 +Elendil,Sauron,7135 +Gil-galad,Sauron,7135 +Gil-galad,Elrond,7142 +Elwing,Eärendil,7145 +Elwing,Lúthien,7145 +Elwing,Dior,7145 +Eärendil,Lúthien,7145 +Eärendil,Dior,7145 +Lúthien,Dior,7145 +Elendil,Gil-galad,7148 +Isildur,Elendil,7149 +Isildur,Gil-galad,7149 +Isildur,Sauron,7149 +Elendil,Gil-galad,7149 +Elendil,Sauron,7149 +Gil-galad,Sauron,7149 +Gil-galad,Círdan,7162 +Isildur,Valandil,7170 +Isildur,Elendil,7170 +Isildur,Ohtar,7170 +Valandil,Elendil,7170 +Valandil,Ohtar,7170 +Elendil,Ohtar,7170 +Isildur,Elendil,7179 +Isildur,Gil-galad,7179 +Isildur,Anárion,7179 +Elendil,Gil-galad,7179 +Elendil,Anárion,7179 +Gil-galad,Anárion,7179 +Meneldil,Anárion,7192 +Elrond,Aragorn,7235 +Isildur,Elendil,7241 +Isildur,Elrond,7241 +Isildur,Aragorn,7241 +Elendil,Elrond,7241 +Elendil,Aragorn,7241 +Elrond,Aragorn,7241 +Frodo,Bilbo,7270 +Aragorn,Boromir,7279 +Isildur,Elendil,7281 +Isildur,Denethor,7281 +Elendil,Denethor,7281 +Gollum,Bilbo,7330 +Frodo,Bilbo,7336 +Galdor,Elrond,7353 +Frodo,Gandalf,7356 +Frodo,Galdor,7356 +Frodo,Glóin,7356 +Gandalf,Galdor,7356 +Gandalf,Glóin,7356 +Galdor,Glóin,7356 +Gollum,Gandalf,7391 +Isildur,Aragorn,7404 +Gollum,Aragorn,7405 +Isildur,Gandalf,7407 +Isildur,Saruman,7434 +Isildur,Boromir,7434 +Saruman,Boromir,7434 +Isildur,Elendil,7440 +Gil-galad,Sauron,7448 +Celebrimbor,Sauron,7454 +Gollum,Denethor,7455 +Gollum,Aragorn,7455 +Denethor,Aragorn,7455 +Gandalf,Elrond,7484 +Sméagol,Gollum,7509 +Legolas,Glóin,7525 +Saruman,Radagast,7581 +Gandalf,Saruman,7599 +Saruman,Sauron,7714 +Saruman,Radagast,7739 +Sauron,Aragorn,7769 +Glorfindel,Galdor,7904 +Gandalf,Elrond,7911 +Glorfindel,Saruman,7913 +Glorfindel,Erestor,7932 +Frodo,Boromir,7948 +Balin,Thrór,7992 +Glorfindel,Sauron,8028 +Bilbo,Boromir,8060 +Gandalf,Bilbo,8062 +Frodo,Elrond,8097 +Beren,Túrin,8105 +Beren,Hador,8105 +Beren,Húrin,8105 +Túrin,Hador,8105 +Túrin,Húrin,8105 +Hador,Húrin,8105 +Pippin,Frodo,8117 +Pippin,Merry,8117 +Pippin,Sam,8117 +Frodo,Merry,8117 +Frodo,Sam,8117 +Merry,Sam,8117 +Legolas,Gollum,8145 +Sam,Frodo,8150 +Sam,Bilbo,8150 +Frodo,Bilbo,8150 +Elrond,Thranduil,8157 +Elrond,Aragorn,8158 +Frodo,Gandalf,8183 +Frodo,Gandalf,8190 +Frodo,Bilbo,8190 +Gandalf,Bilbo,8190 +Gimli,Legolas,8255 +Gimli,Glóin,8255 +Legolas,Glóin,8255 +Isildur,Aragorn,8257 +Frodo,Boromir,8262 +Pippin,Frodo,8276 +Pippin,Gandalf,8276 +Frodo,Gandalf,8276 +Pippin,Elrond,8286 +Elendil,Aragorn,8292 +Gandalf,Elrond,8295 +Gandalf,Aragorn,8295 +Elrond,Aragorn,8295 +Frodo,Bilbo,8296 +Pippin,Merry,8297 +Pippin,Frodo,8297 +Pippin,Beren,8297 +Pippin,Bilbo,8297 +Pippin,Sam,8297 +Pippin,Lúthien,8297 +Merry,Frodo,8297 +Merry,Beren,8297 +Merry,Bilbo,8297 +Merry,Sam,8297 +Merry,Lúthien,8297 +Frodo,Beren,8297 +Frodo,Bilbo,8297 +Frodo,Sam,8297 +Frodo,Lúthien,8297 +Beren,Bilbo,8297 +Beren,Sam,8297 +Beren,Lúthien,8297 +Bilbo,Sam,8297 +Bilbo,Lúthien,8297 +Sam,Lúthien,8297 +Frodo,Bilbo,8300 +Frodo,Bilbo,8336 +Frodo,Bilbo,8341 +Elrond,Boromir,8373 +Frodo,Bilbo,8379 +Thorin,Gandalf,8380 +Sam,Bill,8384 +Pippin,Sam,8386 +Frodo,Bilbo,8390 +Elrond,Aragorn,8391 +Sam,Frodo,8397 +Gandalf,Elrond,8405 +Gandalf,Aragorn,8438 +Frodo,Gandalf,8459 +Gimli,Legolas,8473 +Sam,Aragorn,8525 +Sam,Aragorn,8534 +Sam,Aragorn,8538 +Sam,Frodo,8563 +Frodo,Gandalf,8598 +Frodo,Aragorn,8598 +Gandalf,Aragorn,8598 +Gandalf,Aragorn,8600 +Gandalf,Aragorn,8614 +Frodo,Aragorn,8619 +Gandalf,Aragorn,8647 +Gimli,Sauron,8686 +Frodo,Merry,8690 +Frodo,Bilbo,8710 +Gandalf,Boromir,8712 +Legolas,Gandalf,8765 +Aragorn,Boromir,8779 +Aragorn,Boromir,8791 +Aragorn,Boromir,8796 +Merry,Aragorn,8822 +Pippin,Legolas,8827 +Pippin,Merry,8827 +Legolas,Merry,8827 +Sam,Boromir,8828 +Gimli,Gandalf,8829 +Gimli,Bill,8829 +Gandalf,Bill,8829 +Frodo,Aragorn,8830 +Pippin,Sam,8866 +Pippin,Merry,8866 +Sam,Merry,8866 +Aragorn,Boromir,8867 +Frodo,Gandalf,8871 +Gandalf,Boromir,8892 +Gandalf,Sauron,8912 +Balin,Fundin,8919 +Sam,Frodo,8952 +Pippin,Sam,8978 +Pippin,Elrond,8978 +Sam,Elrond,8978 +Pippin,Sam,8982 +Gandalf,Aragorn,9006 +Boromir,Aragorn,9018 +Gandalf,Boromir,9059 +Gimli,Frodo,9098 +Gimli,Gandalf,9098 +Frodo,Gandalf,9098 +Sam,Bill,9134 +Gimli,Legolas,9154 +Gandalf,Bill,9162 +Sam,Frodo,9169 +Sam,Bill,9179 +Gandalf,Fëanor,9210 +Gandalf,Aragorn,9240 +Peregrin,Gandalf,9258 +Sam,Bill,9282 +Gimli,Gandalf,9316 +Sam,Frodo,9329 +Sam,Frodo,9334 +Sam,Frodo,9343 +Sam,Gandalf,9343 +Frodo,Gandalf,9343 +Sam,Frodo,9350 +Gimli,Gandalf,9374 +Legolas,Sam,9402 +Legolas,Frodo,9402 +Legolas,Boromir,9402 +Sam,Frodo,9402 +Sam,Boromir,9402 +Frodo,Boromir,9402 +Gimli,Gandalf,9410 +Gimli,Glóin,9413 +Gimli,Gandalf,9417 +Gimli,Legolas,9444 +Gimli,Boromir,9444 +Gimli,Aragorn,9444 +Legolas,Boromir,9444 +Legolas,Aragorn,9444 +Boromir,Aragorn,9444 +Pippin,Gandalf,9468 +Pippin,Merry,9468 +Gandalf,Merry,9468 +Merry,Aragorn,9478 +Thorin,Bilbo,9605 +Gandalf,Daeron,9670 +Balin,Frodo,9676 +Balin,Bilbo,9676 +Frodo,Bilbo,9676 +Gimli,Frodo,9687 +Ori,Gimli,9717 +Balin,Fundin,9755 +Balin,Gimli,9852 +Aragorn,Boromir,9853 +Pippin,Merry,9861 +Frodo,Aragorn,9864 +Aragorn,Boromir,9870 +Pippin,Frodo,9877 +Pippin,Merry,9877 +Pippin,Aragorn,9877 +Frodo,Merry,9877 +Frodo,Aragorn,9877 +Merry,Aragorn,9877 +Balin,Gimli,9878 +Balin,Legolas,9878 +Gimli,Legolas,9878 +Sam,Frodo,9901 +Frodo,Gandalf,9903 +Gandalf,Bilbo,9982 +Gandalf,Boromir,10098 +Gandalf,Aragorn,10098 +Boromir,Aragorn,10098 +Aragorn,Boromir,10144 +Aragorn,Boromir,10152 +Sam,Frodo,10155 +Sam,Frodo,10235 +Pippin,Sam,10251 +Sam,Frodo,10273 +Sam,Aragorn,10273 +Frodo,Aragorn,10273 +Legolas,Aragorn,10279 +Aragorn,Boromir,10280 +Gimli,Frodo,10298 +Gimli,Sam,10298 +Gimli,Aragorn,10298 +Frodo,Sam,10298 +Frodo,Aragorn,10298 +Sam,Aragorn,10298 +Sam,Aragorn,10299 +Merry,Bilbo,10334 +Sam,Frodo,10356 +Sam,Aragorn,10356 +Frodo,Aragorn,10356 +Gimli,Frodo,10360 +Gimli,Aragorn,10478 +Legolas,Sam,10515 +Legolas,Frodo,10515 +Sam,Frodo,10515 +Legolas,Frodo,10518 +Sam,Frodo,10521 +Orophin,Rúmil,10531 +Haldir,Aragorn,10540 +Frodo,Elrond,10550 +Legolas,Aragorn,10555 +Haldir,Pippin,10568 +Haldir,Legolas,10568 +Haldir,Merry,10568 +Pippin,Legolas,10568 +Pippin,Merry,10568 +Legolas,Merry,10568 +Legolas,Frodo,10591 +Haldir,Rúmil,10645 +Haldir,Amroth,10795 +Frodo,Aragorn,10843 +Frodo,Aragorn,10845 +Frodo,Amroth,10850 +Galadriel,Celeborn,10863 +Haldir,Frodo,10869 +Haldir,Galadriel,10879 +Haldir,Celeborn,10879 +Galadriel,Celeborn,10879 +Legolas,Frodo,10883 +Galadriel,Celeborn,10892 +Haldir,Frodo,10896 +Gimli,Glóin,10909 +Haldir,Legolas,10933 +Balin,Celeborn,10938 +Balin,Aragorn,10938 +Celeborn,Aragorn,10938 +Frodo,Galadriel,10964 +Legolas,Aragorn,10978 +Frodo,Boromir,11009 +Sam,Frodo,11054 +Sam,Frodo,11062 +Sam,Frodo,11095 +Sam,Frodo,11140 +Frodo,Gandalf,11189 +Frodo,Gandalf,11192 +Frodo,Saruman,11192 +Gandalf,Saruman,11192 +Frodo,Galadriel,11248 +Frodo,Gandalf,11279 +Legolas,Boromir,11333 +Legolas,Aragorn,11333 +Boromir,Aragorn,11333 +Celeborn,Aragorn,11344 +Legolas,Galadriel,11359 +Frodo,Aragorn,11362 +Gandalf,Boromir,11363 +Elendil,Sauron,11364 +Frodo,Gandalf,11365 +Frodo,Boromir,11365 +Gandalf,Boromir,11365 +Frodo,Boromir,11367 +Frodo,Boromir,11375 +Frodo,Boromir,11380 +Frodo,Aragorn,11380 +Boromir,Aragorn,11380 +Pippin,Sam,11382 +Pippin,Merry,11382 +Sam,Merry,11382 +Pippin,Merry,11471 +Pippin,Frodo,11471 +Pippin,Boromir,11471 +Pippin,Gimli,11471 +Pippin,Sam,11471 +Pippin,Aragorn,11471 +Pippin,Legolas,11471 +Merry,Frodo,11471 +Merry,Boromir,11471 +Merry,Gimli,11471 +Merry,Sam,11471 +Merry,Aragorn,11471 +Merry,Legolas,11471 +Frodo,Boromir,11471 +Frodo,Gimli,11471 +Frodo,Sam,11471 +Frodo,Aragorn,11471 +Frodo,Legolas,11471 +Boromir,Gimli,11471 +Boromir,Sam,11471 +Boromir,Aragorn,11471 +Boromir,Legolas,11471 +Gimli,Sam,11471 +Gimli,Aragorn,11471 +Gimli,Legolas,11471 +Sam,Aragorn,11471 +Sam,Legolas,11471 +Aragorn,Legolas,11471 +Galadriel,Celeborn,11486 +Aragorn,Boromir,11516 +Galadriel,Celeborn,11528 +Celeborn,Galadriel,11539 +Celeborn,Aragorn,11539 +Galadriel,Aragorn,11539 +Elendil,Elessar,11549 +Arwen,Celebrían,11551 +Pippin,Merry,11553 +Pippin,Boromir,11553 +Merry,Boromir,11553 +Gimli,Galadriel,11566 +Gimli,Glóin,11572 +Gimli,Galadriel,11575 +Gimli,Glóin,11589 +Gimli,Glóin,11649 +Gimli,Glóin,11655 +Sam,Frodo,11680 +Gimli,Legolas,11730 +Pippin,Merry,11731 +Pippin,Boromir,11731 +Pippin,Aragorn,11731 +Merry,Boromir,11731 +Merry,Aragorn,11731 +Boromir,Aragorn,11731 +Pippin,Frodo,11732 +Frodo,Aragorn,11736 +Sam,Frodo,11739 +Sam,Frodo,11744 +Sam,Frodo,11771 +Sam,Frodo,11794 +Gollum,Aragorn,11841 +Merry,Aragorn,11895 +Sam,Frodo,11903 +Legolas,Galadriel,11943 +Frodo,Boromir,11956 +Frodo,Boromir,12025 +Frodo,Aragorn,12025 +Boromir,Aragorn,12025 +Legolas,Aragorn,12030 +Legolas,Frodo,12042 +Legolas,Aragorn,12042 +Frodo,Aragorn,12042 +Pippin,Merry,12063 +Aragorn,Boromir,12066 +Sam,Frodo,12117 +Isildur,Anárion,12126 +Isildur,Valandil,12127 +Isildur,Elendil,12127 +Isildur,Elessar,12127 +Valandil,Elendil,12127 +Valandil,Elessar,12127 +Elendil,Elessar,12127 +Frodo,Aragorn,12205 +Sam,Frodo,12226 +Sam,Boromir,12226 +Frodo,Boromir,12226 +Gandalf,Bilbo,12236 +Frodo,Boromir,12260 +Frodo,Boromir,12321 +Pippin,Sam,12469 +Pippin,Merry,12469 +Sam,Merry,12469 +Denethor,Elrond,12500 +Gimli,Sam,12520 +Legolas,Meriadoc,12521 +Legolas,Peregrin,12521 +Legolas,Boromir,12521 +Meriadoc,Peregrin,12521 +Meriadoc,Boromir,12521 +Peregrin,Boromir,12521 +Sam,Aragorn,12563 +Aragorn,Boromir,12601 +Frodo,Boromir,12615 +Pippin,Frodo,12634 +Pippin,Merry,12634 +Frodo,Merry,12634 +Gimli,Legolas,12637 +Sam,Aragorn,12645 +Sam,Frodo,12702 +Sam,Frodo,12730 +Sam,Frodo,12736 +Sam,Frodo,12769 +Samwise,Boromir,12805 +Samwise,Aragorn,12805 +Boromir,Aragorn,12805 +Gimli,Legolas,12876 +Gimli,Legolas,12878 +Legolas,Aragorn,12881 +Pippin,Frodo,12893 +Pippin,Merry,12893 +Pippin,Sam,12893 +Frodo,Merry,12893 +Frodo,Sam,12893 +Merry,Sam,12893 +Gimli,Sauron,12935 +Gandalf,Saruman,12946 +Gimli,Legolas,12961 +Denethor,Boromir,12986 +Frodo,Aragorn,13052 +Gimli,Frodo,13056 +Legolas,Aragorn,13191 +Gimli,Legolas,13238 +Gimli,Legolas,13309 +Gimli,Legolas,13429 +Gimli,Aragorn,13429 +Legolas,Aragorn,13429 +Saruman,Sauron,13474 +Gimli,Gandalf,13478 +Aragorn,Boromir,13479 +Gimli,Legolas,13530 +Sauron,Aragorn,13562 +Isildur,Elendil,13572 +Isildur,Elessar,13572 +Isildur,Aragorn,13572 +Elendil,Elessar,13572 +Elendil,Aragorn,13572 +Elessar,Aragorn,13572 +Gimli,Legolas,13576 +Legolas,Aragorn,13578 +Denethor,Boromir,13585 +Sauron,Thengel,13588 +Denethor,Sauron,13644 +Gandalf,Saruman,13655 +Gandalf,Shadowfax,13660 +Gandalf,Shadowfax,13663 +Gandalf,Aragorn,13663 +Shadowfax,Aragorn,13663 +Gimli,Legolas,13739 +Gimli,Legolas,13794 +Pippin,Merry,13839 +Legolas,Gandalf,13845 +Gimli,Gandalf,13846 +Gandalf,Aragorn,13848 +Legolas,Aragorn,13870 +Celeborn,Aragorn,13882 +Gimli,Saruman,13908 +Legolas,Aragorn,13909 +Gimli,Legolas,13949 +Gimli,Aragorn,13949 +Legolas,Aragorn,13949 +Pippin,Frodo,13952 +Pippin,Merry,13967 +Gandalf,Elrond,13986 +Pippin,Merry,14151 +Pippin,Merry,14177 +Pippin,Merry,14236 +Pippin,Merry,14331 +Frodo,Gandalf,14364 +Gandalf,Shadowfax,14365 +Pippin,Merry,14394 +Pippin,Merry,14436 +Pippin,Merry,14468 +Pippin,Merry,14544 +Pippin,Merry,14621 +Meriadoc,Merry,14799 +Pippin,Peregrin,14800 +Pippin,Merry,14804 +Pippin,Merry,15032 +Tom,Galadriel,15038 +Tom,Elrond,15038 +Galadriel,Elrond,15038 +Treebeard,Gandalf,15049 +Treebeard,Saruman,15049 +Gandalf,Saruman,15049 +Sam,Gandalf,15050 +Gandalf,Galadriel,15059 +Pippin,Merry,15258 +Treebeard,Merry,15265 +Treebeard,Pippin,15265 +Merry,Pippin,15265 +Pippin,Merry,15267 +Pippin,Merry,15292 +Treebeard,Pippin,15294 +Treebeard,Merry,15298 +Treebeard,Pippin,15298 +Merry,Pippin,15298 +Pippin,Merry,15312 +Pippin,Merry,15338 +Sam,Frodo,15374 +Legolas,Aragorn,15600 +Pippin,Merry,15612 +Pippin,Merry,15619 +Pippin,Boromir,15619 +Merry,Boromir,15619 +Gimli,Legolas,15669 +Gimli,Aragorn,15669 +Legolas,Aragorn,15669 +Pippin,Merry,15672 +Aragorn,Boromir,15901 +Pippin,Merry,15958 +Gandalf,Saruman,16001 +Gandalf,Saruman,16002 +Treebeard,Gandalf,16018 +Pippin,Merry,16036 +Gimli,Saruman,16040 +Legolas,Aragorn,16046 +Gandalf,Aragorn,16109 +Elessar,Aragorn,16187 +Gandalf,Shadowfax,16250 +Legolas,Aragorn,16259 +Gimli,Shadowfax,16260 +Gandalf,Shadowfax,16268 +Gimli,Legolas,16293 +Gimli,Gandalf,16293 +Gimli,Aragorn,16293 +Legolas,Gandalf,16293 +Legolas,Aragorn,16293 +Gandalf,Aragorn,16293 +Gimli,Gandalf,16298 +Gimli,Legolas,16410 +Elendil,Thengel,16455 +Elendil,Aragorn,16455 +Thengel,Aragorn,16455 +Denethor,Aragorn,16456 +Gandalf,Wormtongue,16556 +Gandalf,Thengel,16571 +Gimli,Gandalf,16579 +Gandalf,Wormtongue,16663 +Gandalf,Wormtongue,16851 +Gandalf,Saruman,16910 +Legolas,Aragorn,16949 +Gimli,Gandalf,16951 +Gimli,Legolas,17015 +Gandalf,Shadowfax,17026 +Legolas,Aragorn,17034 +Gimli,Galadriel,17041 +Gandalf,Shadowfax,17059 +Legolas,Gandalf,17094 +Helm,Erkenbrand,17122 +Helm,Erkenbrand,17178 +Gandalf,Wormtongue,17194 +Helm,Gandalf,17196 +Helm,Erkenbrand,17196 +Gandalf,Erkenbrand,17196 +Legolas,Aragorn,17207 +Helm,Erkenbrand,17239 +Helm,Gamling,17258 +Legolas,Aragorn,17457 +Gamling,Saruman,17476 +Gandalf,Aragorn,17581 +Elendil,Aragorn,17650 +Legolas,Erkenbrand,17707 +Legolas,Aragorn,17707 +Erkenbrand,Aragorn,17707 +Gimli,Gamling,17710 +Gandalf,Saruman,17770 +Gimli,Legolas,17785 +Gimli,Gandalf,17785 +Gimli,Aragorn,17785 +Legolas,Gandalf,17785 +Legolas,Aragorn,17785 +Gandalf,Aragorn,17785 +Gimli,Legolas,17820 +Gimli,Gandalf,17820 +Legolas,Gandalf,17820 +Legolas,Gandalf,17821 +Gimli,Legolas,17828 +Legolas,Helm,17848 +Legolas,Galadriel,17857 +Gimli,Legolas,17881 +Gandalf,Shadowfax,17989 +Shadowfax,Gandalf,18002 +Grimbold,Erkenbrand,18006 +Helm,Saruman,18010 +Saruman,Wormtongue,18116 +Saruman,Wormtongue,18157 +Gimli,Legolas,18177 +Pippin,Merry,18196 +Treebeard,Merry,18213 +Treebeard,Gandalf,18230 +Gimli,Legolas,18252 +Gimli,Aragorn,18252 +Legolas,Aragorn,18252 +Gimli,Merry,18299 +Pippin,Aragorn,18431 +Gimli,Gandalf,18452 +Pippin,Saruman,18465 +Treebeard,Saruman,18528 +Gandalf,Galadriel,18554 +Gandalf,Elrond,18554 +Galadriel,Elrond,18554 +Treebeard,Merry,18634 +Treebeard,Gandalf,18654 +Treebeard,Gandalf,18664 +Treebeard,Gandalf,18669 +Sam,Frodo,18689 +Sam,Boromir,18689 +Frodo,Boromir,18689 +Saruman,Aragorn,18825 +Treebeard,Merry,18829 +Pippin,Merry,18936 +Saruman,Wormtongue,18954 +Gandalf,Saruman,18988 +Gandalf,Saruman,18993 +Gandalf,Saruman,19202 +Gandalf,Saruman,19207 +Saruman,Aragorn,19218 +Gandalf,Saruman,19271 +Gimli,Legolas,19275 +Gimli,Aragorn,19275 +Legolas,Aragorn,19275 +Treebeard,Gandalf,19276 +Gimli,Legolas,19292 +Gimli,Legolas,19300 +Pippin,Merry,19326 +Pippin,Gandalf,19362 +Pippin,Merry,19362 +Pippin,Aragorn,19362 +Gandalf,Merry,19362 +Gandalf,Aragorn,19362 +Merry,Aragorn,19362 +Pippin,Merry,19366 +Pippin,Merry,19437 +Pippin,Merry,19453 +Gandalf,Saruman,19471 +Pippin,Gandalf,19475 +Pippin,Sam,19489 +Pippin,Gandalf,19519 +Gandalf,Aragorn,19687 +Pippin,Gandalf,19756 +Pippin,Gandalf,19762 +Pippin,Aragorn,19762 +Gandalf,Aragorn,19762 +Merry,Aragorn,19771 +Shadowfax,Gandalf,19793 +Elendil,Elrond,19848 +Wormtongue,Aragorn,19918 +Peregrin,Shadowfax,19921 +Pippin,Gandalf,19957 +Sam,Frodo,20041 +Sam,Frodo,20047 +Sam,Frodo,20230 +Sam,Frodo,20267 +Frodo,Bilbo,20483 +Frodo,Gollum,20519 +Sam,Frodo,20598 +Sam,Frodo,20607 +Sam,Gollum,20627 +Frodo,Gollum,20653 +Sam,Frodo,20655 +Sam,Gollum,20686 +Frodo,Gollum,20688 +Sam,Frodo,20735 +Sam,Gollum,20769 +Sam,Frodo,20777 +Sam,Frodo,20792 +Sam,Gollum,20838 +Sam,Frodo,20844 +Sam,Gollum,20844 +Frodo,Gollum,20844 +Sam,Frodo,20849 +Sam,Gollum,20859 +Sam,Gollum,20861 +Sam,Frodo,20888 +Frodo,Gollum,20903 +Sam,Samwise,20916 +Sam,Frodo,20985 +Sam,Gollum,20985 +Frodo,Gollum,20985 +Sam,Frodo,21041 +Sam,Frodo,21066 +Sam,Frodo,21137 +Sam,Frodo,21147 +Sam,Frodo,21153 +Sam,Gollum,21167 +Sam,Frodo,21168 +Sam,Gollum,21172 +Sam,Frodo,21214 +Sam,Frodo,21229 +Frodo,Gollum,21299 +Sam,Gollum,21301 +Frodo,Gollum,21369 +Sam,Gollum,21455 +Frodo,Gollum,21466 +Frodo,Gollum,21490 +Sam,Frodo,21492 +Sam,Gollum,21492 +Frodo,Gollum,21492 +Frodo,Gollum,21501 +Frodo,Gandalf,21551 +Frodo,Bilbo,21551 +Gandalf,Bilbo,21551 +Frodo,Gollum,21558 +Isildur,Elendil,21588 +Isildur,Frodo,21588 +Elendil,Frodo,21588 +Sam,Frodo,21626 +Sam,Gollum,21626 +Frodo,Gollum,21626 +Gollum,Aragorn,21673 +Frodo,Gollum,21680 +Frodo,Gollum,21683 +Gollum,Aragorn,21687 +Gandalf,Aragorn,21702 +Gandalf,Saruman,21703 +Gandalf,Aragorn,21703 +Saruman,Aragorn,21703 +Frodo,Samwise,21705 +Frodo,Gandalf,21706 +Sam,Frodo,21727 +Frodo,Gollum,21753 +Sam,Frodo,21765 +Frodo,Gollum,21825 +Sam,Frodo,21860 +Sam,Gollum,21860 +Frodo,Gollum,21860 +Sam,Gollum,21887 +Frodo,Elrond,21938 +Sam,Gollum,21945 +Sam,Frodo,21970 +Sam,Gollum,21970 +Frodo,Gollum,21970 +Sam,Frodo,22061 +Sam,Gollum,22081 +Sam,Frodo,22143 +Frodo,Boromir,22150 +Frodo,Hamfast,22183 +Frodo,Samwise,22183 +Hamfast,Samwise,22183 +Boromir,Aragorn,22187 +Denethor,Boromir,22194 +Mablung,Damrod,22233 +Sam,Frodo,22280 +Sam,Faramir,22341 +Frodo,Faramir,22343 +Sam,Frodo,22349 +Sam,Boromir,22349 +Frodo,Boromir,22349 +Elendil,Boromir,22366 +Isildur,Elendil,22367 +Isildur,Aragorn,22367 +Elendil,Aragorn,22367 +Sam,Frodo,22384 +Sam,Faramir,22384 +Frodo,Faramir,22384 +Frodo,Boromir,22391 +Frodo,Denethor,22441 +Frodo,Faramir,22441 +Denethor,Faramir,22441 +Mablung,Damrod,22556 +Samwise,Frodo,22557 +Samwise,Faramir,22557 +Frodo,Faramir,22557 +Sam,Frodo,22568 +Sam,Mablung,22568 +Sam,Damrod,22568 +Sam,Faramir,22568 +Frodo,Mablung,22568 +Frodo,Damrod,22568 +Frodo,Faramir,22568 +Mablung,Damrod,22568 +Mablung,Faramir,22568 +Damrod,Faramir,22568 +Samwise,Faramir,22571 +Isildur,Frodo,22593 +Boromir,Faramir,22687 +Mablung,Damrod,22692 +Mablung,Damrod,22710 +Sam,Frodo,22718 +Mablung,Damrod,22723 +Sam,Frodo,22824 +Sam,Faramir,22824 +Frodo,Faramir,22824 +Sam,Frodo,22832 +Sam,Frodo,22844 +Frodo,Boromir,22845 +Frodo,Aragorn,22849 +Boromir,Faramir,22852 +Sam,Faramir,22890 +Samwise,Faramir,22891 +Bilbo,Aragorn,22908 +Sam,Frodo,22959 +Frodo,Faramir,22986 +Frodo,Faramir,23030 +Sam,Frodo,23064 +Frodo,Faramir,23077 +Frodo,Faramir,23094 +Anborn,Faramir,23143 +Anborn,Frodo,23166 +Frodo,Gandalf,23212 +Frodo,Gollum,23277 +Frodo,Faramir,23277 +Gollum,Faramir,23277 +Anborn,Gollum,23341 +Frodo,Faramir,23375 +Gollum,Faramir,23389 +Frodo,Gollum,23399 +Frodo,Faramir,23420 +Frodo,Gollum,23435 +Frodo,Faramir,23435 +Gollum,Faramir,23435 +Anborn,Gollum,23477 +Sam,Frodo,23545 +Frodo,Elrond,23572 +Frodo,Gollum,23576 +Frodo,Faramir,23576 +Gollum,Faramir,23576 +Samwise,Faramir,23577 +Frodo,Gollum,23578 +Sam,Frodo,23617 +Sam,Gollum,23624 +Sam,Frodo,23678 +Sam,Gollum,23678 +Frodo,Gollum,23678 +Sam,Frodo,23726 +Sam,Gollum,23726 +Frodo,Gollum,23726 +Sam,Gandalf,23787 +Sam,Gollum,23789 +Sam,Frodo,23816 +Sam,Frodo,23837 +Frodo,Gollum,23855 +Sam,Frodo,23864 +Sam,Gollum,23890 +Sam,Frodo,23896 +Sam,Frodo,23910 +Sam,Frodo,23917 +Sam,Frodo,23940 +Gandalf,Galadriel,24026 +Gandalf,Faramir,24026 +Gandalf,Elrond,24026 +Gandalf,Aragorn,24026 +Galadriel,Faramir,24026 +Galadriel,Elrond,24026 +Galadriel,Aragorn,24026 +Faramir,Elrond,24026 +Faramir,Aragorn,24026 +Elrond,Aragorn,24026 +Sam,Frodo,24050 +Sam,Frodo,24058 +Sam,Frodo,24099 +Sam,Frodo,24138 +Sam,Frodo,24219 +Sam,Frodo,24220 +Sam,Frodo,24253 +Sam,Frodo,24284 +Sam,Frodo,24292 +Sam,Frodo,24334 +Frodo,Gollum,24339 +Sam,Frodo,24355 +Gollum,Shelob,24375 +Sam,Frodo,24413 +Sam,Frodo,24436 +Sam,Frodo,24442 +Sam,Frodo,24515 +Gollum,Faramir,24578 +Ungoliant,Shelob,24618 +Sam,Frodo,24664 +Sam,Gollum,24681 +Sam,Gollum,24687 +Sam,Gollum,24690 +Sam,Gollum,24692 +Sam,Gollum,24693 +Sam,Frodo,24711 +Sam,Frodo,24715 +Sam,Samwise,24715 +Frodo,Samwise,24715 +Sam,Shelob,24736 +Sam,Frodo,24742 +Hamfast,Samwise,24753 +Frodo,Galadriel,24792 +Frodo,Faramir,24816 +Frodo,Bilbo,24817 +Frodo,Bilbo,24857 +Sam,Frodo,24873 +Sam,Shagrat,25066 +Shelob,Shagrat,25096 +Gorbag,Shagrat,25308 +Gorbag,Shagrat,25310 +Pippin,Gandalf,25348 +Pippin,Gandalf,25359 +Pippin,Gandalf,25366 +Pippin,Gandalf,25380 +Pippin,Gandalf,25394 +Gandalf,Denethor,25399 +Gandalf,Boromir,25411 +Amroth,Imrahil,25440 +Pippin,Gandalf,25441 +Pippin,Ecthelion,25446 +Peregrin,Gandalf,25447 +Gandalf,Shadowfax,25472 +Pippin,Gandalf,25480 +Pippin,Gandalf,25485 +Denethor,Ecthelion,25521 +Pippin,Boromir,25523 +Pippin,Aragorn,25523 +Boromir,Aragorn,25523 +Gandalf,Faramir,25533 +Pippin,Boromir,25541 +Pippin,Denethor,25550 +Meriadoc,Pippin,25560 +Pippin,Denethor,25561 +Pippin,Gandalf,25572 +Pippin,Denethor,25573 +Pippin,Peregrin,25574 +Pippin,Denethor,25574 +Pippin,Ecthelion,25574 +Peregrin,Denethor,25574 +Peregrin,Ecthelion,25574 +Denethor,Ecthelion,25574 +Pippin,Gandalf,25578 +Gandalf,Denethor,25579 +Théoden,Saruman,25583 +Pippin,Gandalf,25584 +Gandalf,Denethor,25585 +Pippin,Gandalf,25586 +Treebeard,Gandalf,25589 +Gandalf,Denethor,25592 +Pippin,Gandalf,25599 +Pippin,Denethor,25600 +Pippin,Gandalf,25621 +Pippin,Gandalf,25629 +Isildur,Denethor,25635 +Isildur,Boromir,25635 +Denethor,Boromir,25635 +Pippin,Boromir,25636 +Pippin,Faramir,25636 +Boromir,Faramir,25636 +Denethor,Faramir,25654 +Beregond,Aragorn,25692 +Pippin,Peregrin,25697 +Gandalf,Shadowfax,25715 +Beregond,Shadowfax,25716 +Pippin,Shadowfax,25723 +Pippin,Shadowfax,25726 +Beregond,Pippin,25741 +Beregond,Pippin,25749 +Beregond,Peregrin,25753 +Beregond,Pippin,25780 +Pippin,Peregrin,25817 +Pippin,Peregrin,25867 +Beregond,Gandalf,25868 +Beregond,Pippin,25868 +Gandalf,Pippin,25868 +Pippin,Beregond,25869 +Pippin,Denethor,25872 +Pippin,Boromir,25872 +Denethor,Boromir,25872 +Beregond,Bergil,25917 +Pippin,Merry,25928 +Pippin,Bergil,25928 +Merry,Bergil,25928 +Pippin,Bergil,25930 +Pippin,Bergil,25935 +Pippin,Forlong,25939 +Forlong,Bergil,25941 +Amroth,Imrahil,25961 +Pippin,Gandalf,25987 +Pippin,Faramir,26001 +Gandalf,Merry,26010 +Gandalf,Shadowfax,26010 +Gandalf,Aragorn,26010 +Merry,Shadowfax,26010 +Merry,Aragorn,26010 +Shadowfax,Aragorn,26010 +Gimli,Legolas,26013 +Merry,Aragorn,26031 +Gimli,Legolas,26034 +Gimli,Merry,26034 +Gimli,Aragorn,26034 +Legolas,Merry,26034 +Legolas,Aragorn,26034 +Merry,Aragorn,26034 +Merry,Aragorn,26042 +Théoden,Saruman,26073 +Halbarad,Aragorn,26089 +Gimli,Legolas,26099 +Gimli,Merry,26099 +Legolas,Merry,26099 +Gimli,Legolas,26102 +Gimli,Halbarad,26115 +Legolas,Aragorn,26116 +Legolas,Gandalf,26125 +Legolas,Galadriel,26125 +Gandalf,Galadriel,26125 +Gimli,Gandalf,26126 +Meriadoc,Théoden,26151 +Théoden,Merry,26160 +Théoden,Aragorn,26160 +Merry,Aragorn,26160 +Éomer,Elrond,26169 +Éomer,Halbarad,26169 +Éomer,Aragorn,26169 +Elrond,Halbarad,26169 +Elrond,Aragorn,26169 +Halbarad,Aragorn,26169 +Gimli,Legolas,26170 +Merry,Aragorn,26171 +Éomer,Théoden,26176 +Éomer,Merry,26186 +Éomer,Aragorn,26186 +Merry,Aragorn,26186 +Éomer,Aragorn,26188 +Éomer,Théoden,26190 +Éomer,Aragorn,26190 +Théoden,Aragorn,26190 +Gimli,Legolas,26197 +Gimli,Merry,26197 +Legolas,Merry,26197 +Éomer,Théoden,26202 +Gimli,Legolas,26212 +Gandalf,Aragorn,26219 +Isildur,Elendil,26234 +Isildur,Théoden,26234 +Isildur,Sauron,26234 +Elendil,Théoden,26234 +Elendil,Sauron,26234 +Théoden,Sauron,26234 +Gimli,Aragorn,26236 +Legolas,Elrond,26249 +Legolas,Aragorn,26249 +Elrond,Aragorn,26249 +Arvedui,Aragorn,26250 +Gimli,Aragorn,26257 +Gimli,Legolas,26260 +Gimli,Aragorn,26260 +Legolas,Aragorn,26260 +Isildur,Sauron,26263 +Isildur,Sauron,26266 +Gimli,Legolas,26272 +Gimli,Aragorn,26272 +Legolas,Aragorn,26272 +Gimli,Legolas,26274 +Helm,Halbarad,26276 +Éowyn,Elrond,26278 +Éowyn,Aragorn,26278 +Elrond,Aragorn,26278 +Helm,Théoden,26279 +Éowyn,Aragorn,26283 +Gimli,Legolas,26295 +Gimli,Aragorn,26295 +Legolas,Aragorn,26295 +Gimli,Legolas,26330 +Gimli,Glóin,26361 +Gimli,Aragorn,26362 +Gimli,Glóin,26371 +Gimli,Legolas,26398 +Gimli,Legolas,26400 +Gimli,Amroth,26407 +Isildur,Elessar,26428 +Isildur,Halbarad,26428 +Elessar,Halbarad,26428 +Gimli,Legolas,26433 +Pippin,Amroth,26438 +Gimli,Pippin,26465 +Gimli,Legolas,26465 +Gimli,Aragorn,26465 +Pippin,Legolas,26465 +Pippin,Aragorn,26465 +Legolas,Aragorn,26465 +Sam,Frodo,26466 +Éomer,Théoden,26476 +Gandalf,Shadowfax,26497 +Dúnhere,Gandalf,26500 +Éowyn,Théoden,26544 +Éowyn,Merry,26544 +Théoden,Merry,26544 +Éomer,Aragorn,26549 +Pippin,Gandalf,26571 +Pippin,Frodo,26571 +Pippin,Gimli,26571 +Pippin,Sam,26571 +Pippin,Legolas,26571 +Gandalf,Frodo,26571 +Gandalf,Gimli,26571 +Gandalf,Sam,26571 +Gandalf,Legolas,26571 +Frodo,Gimli,26571 +Frodo,Sam,26571 +Frodo,Legolas,26571 +Gimli,Sam,26571 +Gimli,Legolas,26571 +Sam,Legolas,26571 +Éowyn,Dúnhere,26576 +Éowyn,Éomer,26576 +Éowyn,Théoden,26576 +Dúnhere,Éomer,26576 +Dúnhere,Théoden,26576 +Éomer,Théoden,26576 +Éomer,Aragorn,26584 +Brego,Baldor,26592 +Merry,Aragorn,26598 +Éowyn,Éomer,26599 +Brego,Baldor,26603 +Éomer,Aragorn,26614 +Merry,Boromir,26622 +Hirgon,Denethor,26628 +Théoden,Denethor,26630 +Hirgon,Denethor,26634 +Hirgon,Gandalf,26638 +Hirgon,Denethor,26638 +Gandalf,Denethor,26638 +Éomer,Théoden,26705 +Éomer,Merry,26706 +Éowyn,Merry,26712 +Théoden,Merry,26714 +Éowyn,Aragorn,26721 +Gimli,Merry,26725 +Meriadoc,Merry,26759 +Pippin,Gandalf,26788 +Pippin,Denethor,26800 +Pippin,Gandalf,26801 +Éomer,Gandalf,26819 +Éomer,Denethor,26819 +Gandalf,Denethor,26819 +Pippin,Denethor,26820 +Pippin,Denethor,26821 +Pippin,Gandalf,26862 +Beregond,Pippin,26868 +Beregond,Faramir,26868 +Pippin,Faramir,26868 +Beregond,Faramir,26909 +Beregond,Pippin,26910 +Gandalf,Shadowfax,26929 +Pippin,Faramir,26930 +Pippin,Boromir,26932 +Pippin,Denethor,26949 +Pippin,Faramir,26953 +Pippin,Gandalf,26955 +Pippin,Gandalf,26957 +Pippin,Faramir,26957 +Gandalf,Faramir,26957 +Pippin,Frodo,26958 +Pippin,Gandalf,26958 +Frodo,Gandalf,26958 +Pippin,Gandalf,26959 +Boromir,Denethor,26991 +Boromir,Faramir,26991 +Denethor,Faramir,26991 +Boromir,Denethor,26998 +Boromir,Faramir,26998 +Denethor,Faramir,26998 +Pippin,Gandalf,27035 +Pippin,Denethor,27035 +Gandalf,Denethor,27035 +Pippin,Gandalf,27056 +Pippin,Gandalf,27058 +Pippin,Frodo,27060 +Pippin,Gandalf,27060 +Frodo,Gandalf,27060 +Pippin,Gollum,27090 +Frodo,Gollum,27094 +Boromir,Faramir,27114 +Denethor,Boromir,27125 +Gandalf,Faramir,27128 +Beregond,Faramir,27161 +Nazgûl,Denethor,27184 +Nazgûl,Sauron,27184 +Denethor,Sauron,27184 +Pippin,Gandalf,27187 +Shadowfax,Amroth,27234 +Shadowfax,Faramir,27234 +Amroth,Faramir,27234 +Denethor,Faramir,27255 +Nazgûl,Faramir,27259 +Imrahil,Faramir,27261 +Denethor,Faramir,27265 +Pippin,Gandalf,27269 +Gandalf,Denethor,27322 +Denethor,Faramir,27350 +Denethor,Faramir,27362 +Denethor,Faramir,27368 +Denethor,Faramir,27387 +Pippin,Denethor,27389 +Denethor,Faramir,27404 +Pippin,Denethor,27422 +Beregond,Pippin,27428 +Beregond,Faramir,27428 +Pippin,Faramir,27428 +Beregond,Pippin,27435 +Beregond,Faramir,27435 +Pippin,Faramir,27435 +Gandalf,Shadowfax,27488 +Pippin,Merry,27536 +Éomer,Merry,27539 +Éomer,Théoden,27567 +Théoden,Ghân-buri-Ghân,27665 +Éomer,Denethor,27690 +Éomer,Théoden,27771 +Théoden,Thengel,27796 +Éowyn,Merry,27873 +Éowyn,Meriadoc,27881 +Éomer,Amroth,27905 +Merry,Gandalf,27906 +Éowyn,Éomer,27908 +Éomer,Théoden,27918 +Éowyn,Théoden,27951 +Imrahil,Amroth,27953 +Éowyn,Éomer,27962 +Éomer,Imrahil,27972 +Éomer,Húrin,27972 +Imrahil,Húrin,27972 +Arwen,Elrond,28013 +Isildur,Elessar,28014 +Isildur,Aragorn,28014 +Elessar,Aragorn,28014 +Gimli,Legolas,28019 +Gimli,Halbarad,28019 +Legolas,Halbarad,28019 +Elendil,Aragorn,28020 +Éomer,Aragorn,28021 +Éomer,Imrahil,28034 +Éomer,Aragorn,28034 +Imrahil,Aragorn,28034 +Grimbold,Halbarad,28038 +Grimbold,Dúnhere,28044 +Grimbold,Fastred,28044 +Grimbold,Horn,28044 +Grimbold,Harding,28044 +Dúnhere,Fastred,28044 +Dúnhere,Horn,28044 +Dúnhere,Harding,28044 +Fastred,Horn,28044 +Fastred,Harding,28044 +Horn,Harding,28044 +Gandalf,Denethor,28049 +Gandalf,Shadowfax,28053 +Pippin,Shadowfax,28055 +Pippin,Faramir,28067 +Pippin,Denethor,28070 +Pippin,Faramir,28070 +Denethor,Faramir,28070 +Pippin,Gandalf,28073 +Gandalf,Faramir,28074 +Pippin,Shadowfax,28077 +Pippin,Beregond,28089 +Beregond,Denethor,28102 +Pippin,Gandalf,28104 +Pippin,Denethor,28104 +Gandalf,Denethor,28104 +Gandalf,Denethor,28109 +Denethor,Faramir,28116 +Beregond,Gandalf,28122 +Beregond,Pippin,28122 +Beregond,Denethor,28122 +Gandalf,Pippin,28122 +Gandalf,Denethor,28122 +Pippin,Denethor,28122 +Gandalf,Denethor,28174 +Beregond,Faramir,28177 +Beregond,Gandalf,28191 +Beregond,Denethor,28191 +Beregond,Ecthelion,28191 +Gandalf,Denethor,28191 +Gandalf,Ecthelion,28191 +Denethor,Ecthelion,28191 +Beregond,Faramir,28195 +Beregond,Gandalf,28197 +Beregond,Pippin,28197 +Beregond,Faramir,28197 +Gandalf,Pippin,28197 +Gandalf,Faramir,28197 +Pippin,Faramir,28197 +Gandalf,Amroth,28204 +Beregond,Gandalf,28210 +Beregond,Pippin,28210 +Beregond,Faramir,28210 +Gandalf,Pippin,28210 +Gandalf,Faramir,28210 +Pippin,Faramir,28210 +Pippin,Beregond,28212 +Denethor,Sauron,28225 +Beregond,Faramir,28229 +Éowyn,Théoden,28249 +Pippin,Merry,28276 +Beregond,Bergil,28285 +Éowyn,Meriadoc,28301 +Éowyn,Faramir,28301 +Meriadoc,Faramir,28301 +Éomer,Imrahil,28323 +Éomer,Aragorn,28323 +Imrahil,Aragorn,28323 +Éomer,Elendil,28327 +Éomer,Imrahil,28334 +Éowyn,Éomer,28338 +Éowyn,Imrahil,28339 +Éowyn,Gandalf,28345 +Denethor,Faramir,28346 +Galadriel,Aragorn,28350 +Amroth,Faramir,28352 +Éomer,Imrahil,28365 +Elessar,Aragorn,28366 +Éowyn,Meriadoc,28368 +Éowyn,Gandalf,28368 +Meriadoc,Gandalf,28368 +Éowyn,Merry,28370 +Éowyn,Faramir,28370 +Éowyn,Aragorn,28370 +Merry,Faramir,28370 +Merry,Aragorn,28370 +Faramir,Aragorn,28370 +Éomer,Faramir,28373 +Éomer,Elrond,28373 +Éomer,Aragorn,28373 +Faramir,Elrond,28373 +Faramir,Aragorn,28373 +Elrond,Aragorn,28373 +Gandalf,Faramir,28386 +Shadowfax,Aragorn,28387 +Gandalf,Aragorn,28390 +Aragorn,Faramir,28419 +Faramir,Aragorn,28421 +Faramir,Aragorn,28432 +Faramir,Aragorn,28436 +Beregond,Gandalf,28443 +Beregond,Imrahil,28443 +Gandalf,Imrahil,28443 +Pippin,Gandalf,28444 +Éowyn,Aragorn,28448 +Théoden,Wormtongue,28464 +Saruman,Wormtongue,28468 +Éomer,Aragorn,28472 +Éowyn,Aragorn,28483 +Éowyn,Aragorn,28484 +Éomer,Éowyn,28488 +Éowyn,Éomer,28499 +Éomer,Gandalf,28503 +Pippin,Gandalf,28510 +Pippin,Merry,28510 +Pippin,Aragorn,28510 +Gandalf,Merry,28510 +Gandalf,Aragorn,28510 +Merry,Aragorn,28510 +Pippin,Merry,28511 +Merry,Aragorn,28516 +Pippin,Gandalf,28518 +Pippin,Aragorn,28528 +Meriadoc,Saruman,28532 +Meriadoc,Aragorn,28532 +Saruman,Aragorn,28532 +Merry,Gandalf,28543 +Éowyn,Gandalf,28565 +Éowyn,Faramir,28565 +Éowyn,Aragorn,28565 +Gandalf,Faramir,28565 +Gandalf,Aragorn,28565 +Faramir,Aragorn,28565 +Éowyn,Gandalf,28566 +Éowyn,Faramir,28566 +Éowyn,Aragorn,28566 +Gandalf,Faramir,28566 +Gandalf,Aragorn,28566 +Faramir,Aragorn,28566 +Beregond,Meriadoc,28568 +Elrond,Aragorn,28573 +Gimli,Pippin,28578 +Gimli,Legolas,28578 +Gimli,Merry,28578 +Pippin,Legolas,28578 +Pippin,Merry,28578 +Legolas,Merry,28578 +Gimli,Legolas,28579 +Legolas,Aragorn,28581 +Legolas,Imrahil,28583 +Legolas,Aragorn,28583 +Imrahil,Aragorn,28583 +Legolas,Aragorn,28587 +Meriadoc,Peregrin,28589 +Meriadoc,Imrahil,28589 +Peregrin,Imrahil,28589 +Legolas,Merry,28603 +Legolas,Merry,28619 +Gimli,Glóin,28632 +Éowyn,Merry,28635 +Pippin,Legolas,28639 +Pippin,Merry,28639 +Legolas,Merry,28639 +Angbor,Aragorn,28652 +Lúthien,Sauron,28683 +Isildur,Aragorn,28690 +Gandalf,Aragorn,28715 +Théoden,Imrahil,28719 +Théoden,Gimli,28719 +Théoden,Aragorn,28719 +Théoden,Éomer,28719 +Théoden,Legolas,28719 +Imrahil,Gimli,28719 +Imrahil,Aragorn,28719 +Imrahil,Éomer,28719 +Imrahil,Legolas,28719 +Gimli,Aragorn,28719 +Gimli,Éomer,28719 +Gimli,Legolas,28719 +Aragorn,Éomer,28719 +Aragorn,Legolas,28719 +Éomer,Legolas,28719 +Gandalf,Elrond,28720 +Gandalf,Aragorn,28720 +Elrond,Aragorn,28720 +Amroth,Imrahil,28730 +Gandalf,Sauron,28793 +Imrahil,Aragorn,28799 +Imrahil,Gimli,28818 +Imrahil,Glóin,28818 +Imrahil,Aragorn,28818 +Gimli,Glóin,28818 +Gimli,Aragorn,28818 +Glóin,Aragorn,28818 +Éomer,Elrond,28820 +Éomer,Amroth,28820 +Éomer,Imrahil,28820 +Éomer,Aragorn,28820 +Elrond,Amroth,28820 +Elrond,Imrahil,28820 +Elrond,Aragorn,28820 +Amroth,Imrahil,28820 +Amroth,Aragorn,28820 +Imrahil,Aragorn,28820 +Gimli,Legolas,28836 +Gimli,Gandalf,28836 +Gimli,Elrond,28836 +Gimli,Aragorn,28836 +Legolas,Gandalf,28836 +Legolas,Elrond,28836 +Legolas,Aragorn,28836 +Gandalf,Elrond,28836 +Gandalf,Aragorn,28836 +Elrond,Aragorn,28836 +Gandalf,Imrahil,28872 +Gandalf,Faramir,28872 +Imrahil,Faramir,28872 +Gandalf,Aragorn,28876 +Nazgûl,Aragorn,28894 +Gandalf,Imrahil,28926 +Gandalf,Gimli,28926 +Gandalf,Peregrin,28926 +Gandalf,Elrond,28926 +Gandalf,Aragorn,28926 +Gandalf,Éomer,28926 +Gandalf,Legolas,28926 +Imrahil,Gimli,28926 +Imrahil,Peregrin,28926 +Imrahil,Elrond,28926 +Imrahil,Aragorn,28926 +Imrahil,Éomer,28926 +Imrahil,Legolas,28926 +Gimli,Peregrin,28926 +Gimli,Elrond,28926 +Gimli,Aragorn,28926 +Gimli,Éomer,28926 +Gimli,Legolas,28926 +Peregrin,Elrond,28926 +Peregrin,Aragorn,28926 +Peregrin,Éomer,28926 +Peregrin,Legolas,28926 +Elrond,Aragorn,28926 +Elrond,Éomer,28926 +Elrond,Legolas,28926 +Aragorn,Éomer,28926 +Aragorn,Legolas,28926 +Éomer,Legolas,28926 +Gandalf,Sauron,28956 +Sam,Frodo,28958 +Pippin,Imrahil,28960 +Saruman,Sauron,28982 +Elrond,Amroth,29020 +Elrond,Imrahil,29020 +Amroth,Imrahil,29020 +Beregond,Frodo,29024 +Beregond,Gandalf,29024 +Beregond,Pippin,29024 +Beregond,Imrahil,29024 +Frodo,Gandalf,29024 +Frodo,Pippin,29024 +Frodo,Imrahil,29024 +Gandalf,Pippin,29024 +Gandalf,Imrahil,29024 +Pippin,Imrahil,29024 +Beregond,Pippin,29040 +Pippin,Merry,29069 +Pippin,Denethor,29069 +Pippin,Aragorn,29069 +Merry,Denethor,29069 +Merry,Aragorn,29069 +Denethor,Aragorn,29069 +Sam,Frodo,29070 +Hamfast,Samwise,29072 +Frodo,Shelob,29084 +Frodo,Shagrat,29084 +Shelob,Shagrat,29084 +Gorbag,Shagrat,29101 +Gorbag,Shagrat,29163 +Gorbag,Shagrat,29164 +Gorbag,Sam,29169 +Sam,Frodo,29207 +Gorbag,Shagrat,29246 +Gorbag,Shagrat,29288 +Nazgûl,Shagrat,29309 +Sam,Shagrat,29317 +Sam,Shagrat,29323 +Sam,Shagrat,29344 +Sam,Shagrat,29350 +Sam,Frodo,29427 +Sam,Frodo,29440 +Sam,Frodo,29449 +Sam,Frodo,29461 +Sam,Frodo,29480 +Sam,Frodo,29499 +Sam,Frodo,29503 +Sam,Frodo,29511 +Sam,Frodo,29521 +Sam,Frodo,29532 +Sam,Frodo,29545 +Gorbag,Sam,29578 +Sam,Frodo,29583 +Sam,Frodo,29584 +Sam,Frodo,29593 +Sam,Faramir,29619 +Sam,Frodo,29623 +Sam,Galadriel,29649 +Sam,Frodo,29654 +Sam,Frodo,29679 +Sam,Frodo,29684 +Sam,Frodo,29689 +Sam,Frodo,29694 +Sam,Frodo,29709 +Sam,Shagrat,29717 +Sam,Frodo,29739 +Sam,Frodo,29751 +Sam,Frodo,29764 +Sam,Frodo,29780 +Sam,Frodo,29788 +Gorbag,Shagrat,29792 +Sam,Frodo,29795 +Sam,Frodo,29807 +Sam,Frodo,29836 +Sam,Frodo,29851 +Sam,Faramir,29855 +Sam,Frodo,29863 +Sam,Frodo,29874 +Sam,Frodo,29888 +Sam,Frodo,29972 +Sam,Frodo,29987 +Sam,Shelob,29987 +Sam,Gollum,29987 +Frodo,Shelob,29987 +Frodo,Gollum,29987 +Shelob,Gollum,29987 +Sam,Frodo,29988 +Sam,Frodo,30000 +Sam,Frodo,30011 +Sam,Frodo,30029 +Sam,Frodo,30031 +Sam,Frodo,30035 +Sam,Frodo,30042 +Sam,Frodo,30043 +Frodo,Gollum,30061 +Sam,Frodo,30064 +Sam,Frodo,30085 +Sam,Frodo,30118 +Sam,Frodo,30141 +Sam,Frodo,30153 +Sam,Frodo,30170 +Frodo,Gandalf,30173 +Sam,Frodo,30187 +Sam,Frodo,30276 +Sam,Faramir,30276 +Frodo,Faramir,30276 +Sam,Frodo,30316 +Sam,Frodo,30351 +Sam,Frodo,30365 +Sam,Frodo,30368 +Sam,Frodo,30371 +Sam,Frodo,30379 +Sam,Frodo,30413 +Sam,Frodo,30419 +Frodo,Gollum,30437 +Sam,Gollum,30440 +Sam,Frodo,30453 +Sam,Gollum,30453 +Frodo,Gollum,30453 +Sam,Gollum,30488 +Sam,Frodo,30513 +Sam,Gollum,30536 +Sam,Frodo,30547 +Gollum,Gandalf,30571 +Gandalf,Sauron,30601 +Gandalf,Aragorn,30610 +Meneldor,Gandalf,30615 +Sam,Frodo,30618 +Sam,Frodo,30622 +Sam,Frodo,30628 +Sam,Frodo,30630 +Sam,Gandalf,30668 +Gandalf,Sauron,30681 +Sam,Frodo,30694 +Sam,Galadriel,30694 +Frodo,Galadriel,30694 +Frodo,Samwise,30706 +Sam,Frodo,30716 +Sam,Frodo,30721 +Sam,Aragorn,30723 +Sam,Frodo,30725 +Sam,Frodo,30729 +Sam,Frodo,30736 +Frodo,Gandalf,30737 +Sam,Frodo,30741 +Frodo,Gandalf,30744 +Frodo,Bilbo,30744 +Gandalf,Bilbo,30744 +Éomer,Gimli,30745 +Éomer,Legolas,30745 +Éomer,Gandalf,30745 +Éomer,Imrahil,30745 +Gimli,Legolas,30745 +Gimli,Gandalf,30745 +Gimli,Imrahil,30745 +Legolas,Gandalf,30745 +Legolas,Imrahil,30745 +Gandalf,Imrahil,30745 +Peregrin,Merry,30751 +Pippin,Merry,30759 +Pippin,Gandalf,30759 +Pippin,Frodo,30759 +Pippin,Gimli,30759 +Pippin,Sam,30759 +Pippin,Legolas,30759 +Merry,Gandalf,30759 +Merry,Frodo,30759 +Merry,Gimli,30759 +Merry,Sam,30759 +Merry,Legolas,30759 +Gandalf,Frodo,30759 +Gandalf,Gimli,30759 +Gandalf,Sam,30759 +Gandalf,Legolas,30759 +Frodo,Gimli,30759 +Frodo,Sam,30759 +Frodo,Legolas,30759 +Gimli,Sam,30759 +Gimli,Legolas,30759 +Sam,Legolas,30759 +Sam,Frodo,30760 +Pippin,Frodo,30763 +Pippin,Merry,30763 +Frodo,Merry,30763 +Gandalf,Bilbo,30772 +Gimli,Pippin,30775 +Gimli,Frodo,30775 +Gimli,Sam,30775 +Pippin,Frodo,30775 +Pippin,Sam,30775 +Frodo,Sam,30775 +Sam,Frodo,30798 +Éomer,Théoden,30860 +Éomer,Faramir,30860 +Théoden,Faramir,30860 +Éowyn,Merry,30888 +Éowyn,Faramir,30888 +Merry,Faramir,30888 +Merry,Faramir,30889 +Éowyn,Faramir,30896 +Éowyn,Amroth,30903 +Éowyn,Finduilas,30903 +Amroth,Finduilas,30903 +Éowyn,Faramir,30905 +Merry,Faramir,30945 +Éowyn,Faramir,30949 +Elendil,Aragorn,30950 +Éowyn,Faramir,30960 +Éowyn,Faramir,30981 +Éowyn,Húrin,30991 +Éowyn,Faramir,30991 +Húrin,Faramir,30991 +Éomer,Gandalf,30995 +Éomer,Imrahil,30995 +Gandalf,Imrahil,30995 +Húrin,Faramir,31008 +Faramir,Aragorn,31009 +Isildur,Valandil,31013 +Isildur,Elendil,31013 +Isildur,Elessar,31013 +Isildur,Aragorn,31013 +Valandil,Elendil,31013 +Valandil,Elessar,31013 +Valandil,Aragorn,31013 +Elendil,Elessar,31013 +Elendil,Aragorn,31013 +Elessar,Aragorn,31013 +Eärnur,Faramir,31017 +Faramir,Aragorn,31022 +Frodo,Gandalf,31023 +Frodo,Faramir,31023 +Frodo,Aragorn,31023 +Gandalf,Faramir,31023 +Gandalf,Aragorn,31023 +Faramir,Aragorn,31023 +Húrin,Elessar,31025 +Húrin,Faramir,31025 +Elessar,Faramir,31025 +Beregond,Faramir,31038 +Faramir,Aragorn,31039 +Éomer,Aragorn,31040 +Éowyn,Elrond,31045 +Éowyn,Faramir,31045 +Elrond,Faramir,31045 +Gimli,Legolas,31048 +Gimli,Aragorn,31048 +Legolas,Aragorn,31048 +Frodo,Gandalf,31052 +Frodo,Aragorn,31052 +Gandalf,Aragorn,31052 +Gandalf,Bilbo,31053 +Pippin,Frodo,31054 +Pippin,Gandalf,31054 +Frodo,Gandalf,31054 +Gandalf,Aragorn,31057 +Gandalf,Aragorn,31058 +Gandalf,Aragorn,31067 +Gandalf,Nimloth,31086 +Nimloth,Aragorn,31094 +Glorfindel,Arwen,31102 +Glorfindel,Erestor,31102 +Glorfindel,Elrond,31102 +Glorfindel,Galadriel,31102 +Glorfindel,Celeborn,31102 +Arwen,Erestor,31102 +Arwen,Elrond,31102 +Arwen,Galadriel,31102 +Arwen,Celeborn,31102 +Erestor,Elrond,31102 +Erestor,Galadriel,31102 +Erestor,Celeborn,31102 +Elrond,Galadriel,31102 +Elrond,Celeborn,31102 +Galadriel,Celeborn,31102 +Frodo,Gandalf,31103 +Arwen,Elessar,31106 +Arwen,Aragorn,31106 +Elessar,Aragorn,31106 +Arwen,Frodo,31108 +Frodo,Aragorn,31109 +Arwen,Bilbo,31113 +Arwen,Elrond,31113 +Bilbo,Elrond,31113 +Éomer,Théoden,31119 +Éomer,Gimli,31129 +Éomer,Glóin,31129 +Gimli,Glóin,31129 +Éomer,Gimli,31131 +Théoden,Merry,31141 +Pippin,Gandalf,31142 +Pippin,Shadowfax,31142 +Pippin,Frodo,31142 +Pippin,Gimli,31142 +Pippin,Aragorn,31142 +Pippin,Legolas,31142 +Pippin,Samwise,31142 +Gandalf,Shadowfax,31142 +Gandalf,Frodo,31142 +Gandalf,Gimli,31142 +Gandalf,Aragorn,31142 +Gandalf,Legolas,31142 +Gandalf,Samwise,31142 +Shadowfax,Frodo,31142 +Shadowfax,Gimli,31142 +Shadowfax,Aragorn,31142 +Shadowfax,Legolas,31142 +Shadowfax,Samwise,31142 +Frodo,Gimli,31142 +Frodo,Aragorn,31142 +Frodo,Legolas,31142 +Frodo,Samwise,31142 +Gimli,Aragorn,31142 +Gimli,Legolas,31142 +Gimli,Samwise,31142 +Aragorn,Legolas,31142 +Aragorn,Samwise,31142 +Legolas,Samwise,31142 +Elrond,Arwen,31143 +Elrond,Galadriel,31143 +Elrond,Amroth,31143 +Elrond,Celeborn,31143 +Arwen,Galadriel,31143 +Arwen,Amroth,31143 +Arwen,Celeborn,31143 +Galadriel,Amroth,31143 +Galadriel,Celeborn,31143 +Amroth,Celeborn,31143 +Théoden,Thengel,31144 +Elessar,Aragorn,31146 +Théoden,Thengel,31152 +Helm,Théoden,31153 +Théoden,Merry,31156 +Éowyn,Éomer,31161 +Fréa,Fengel,31162 +Fréa,Helm,31162 +Fréa,Théoden,31162 +Fréa,Baldor,31162 +Fréa,Déor,31162 +Fréa,Brego,31162 +Fréa,Goldwine,31162 +Fréa,Folcwine,31162 +Fréa,Aldor,31162 +Fréa,Walda,31162 +Fréa,Folca,31162 +Fréa,Thengel,31162 +Fréa,Gram,31162 +Fengel,Helm,31162 +Fengel,Théoden,31162 +Fengel,Baldor,31162 +Fengel,Déor,31162 +Fengel,Brego,31162 +Fengel,Goldwine,31162 +Fengel,Folcwine,31162 +Fengel,Aldor,31162 +Fengel,Walda,31162 +Fengel,Folca,31162 +Fengel,Thengel,31162 +Fengel,Gram,31162 +Helm,Théoden,31162 +Helm,Baldor,31162 +Helm,Déor,31162 +Helm,Brego,31162 +Helm,Goldwine,31162 +Helm,Folcwine,31162 +Helm,Aldor,31162 +Helm,Walda,31162 +Helm,Folca,31162 +Helm,Thengel,31162 +Helm,Gram,31162 +Théoden,Baldor,31162 +Théoden,Déor,31162 +Théoden,Brego,31162 +Théoden,Goldwine,31162 +Théoden,Folcwine,31162 +Théoden,Aldor,31162 +Théoden,Walda,31162 +Théoden,Folca,31162 +Théoden,Thengel,31162 +Théoden,Gram,31162 +Baldor,Déor,31162 +Baldor,Brego,31162 +Baldor,Goldwine,31162 +Baldor,Folcwine,31162 +Baldor,Aldor,31162 +Baldor,Walda,31162 +Baldor,Folca,31162 +Baldor,Thengel,31162 +Baldor,Gram,31162 +Déor,Brego,31162 +Déor,Goldwine,31162 +Déor,Folcwine,31162 +Déor,Aldor,31162 +Déor,Walda,31162 +Déor,Folca,31162 +Déor,Thengel,31162 +Déor,Gram,31162 +Brego,Goldwine,31162 +Brego,Folcwine,31162 +Brego,Aldor,31162 +Brego,Walda,31162 +Brego,Folca,31162 +Brego,Thengel,31162 +Brego,Gram,31162 +Goldwine,Folcwine,31162 +Goldwine,Aldor,31162 +Goldwine,Walda,31162 +Goldwine,Folca,31162 +Goldwine,Thengel,31162 +Goldwine,Gram,31162 +Folcwine,Aldor,31162 +Folcwine,Walda,31162 +Folcwine,Folca,31162 +Folcwine,Thengel,31162 +Folcwine,Gram,31162 +Aldor,Walda,31162 +Aldor,Folca,31162 +Aldor,Thengel,31162 +Aldor,Gram,31162 +Walda,Folca,31162 +Walda,Thengel,31162 +Walda,Gram,31162 +Folca,Thengel,31162 +Folca,Gram,31162 +Thengel,Gram,31162 +Éomer,Théoden,31163 +Éowyn,Éomer,31164 +Éowyn,Théoden,31164 +Éomer,Théoden,31164 +Éowyn,Faramir,31166 +Éowyn,Faramir,31167 +Éowyn,Éomer,31168 +Éowyn,Aragorn,31168 +Éomer,Aragorn,31168 +Arwen,Faramir,31170 +Arwen,Imrahil,31170 +Arwen,Aragorn,31170 +Faramir,Imrahil,31170 +Faramir,Aragorn,31170 +Imrahil,Aragorn,31170 +Éowyn,Meriadoc,31172 +Éowyn,Éomer,31172 +Éowyn,Merry,31172 +Meriadoc,Éomer,31172 +Meriadoc,Merry,31172 +Éomer,Merry,31172 +Éowyn,Merry,31174 +Éowyn,Merry,31178 +Gimli,Legolas,31180 +Treebeard,Gandalf,31188 +Treebeard,Gandalf,31192 +Treebeard,Aragorn,31197 +Treebeard,Gandalf,31202 +Treebeard,Merry,31202 +Gandalf,Merry,31202 +Galadriel,Celeborn,31245 +Gimli,Legolas,31247 +Treebeard,Galadriel,31255 +Treebeard,Celeborn,31255 +Galadriel,Celeborn,31255 +Galadriel,Celeborn,31259 +Pippin,Merry,31261 +Pippin,Aragorn,31268 +Pippin,Aragorn,31270 +Celeborn,Galadriel,31274 +Celeborn,Aragorn,31274 +Galadriel,Aragorn,31274 +Gandalf,Saruman,31282 +Saruman,Galadriel,31295 +Saruman,Gandalf,31313 +Saruman,Wormtongue,31313 +Gandalf,Wormtongue,31313 +Saruman,Merry,31330 +Saruman,Wormtongue,31330 +Merry,Wormtongue,31330 +Galadriel,Celeborn,31349 +Gandalf,Elrond,31350 +Frodo,Galadriel,31356 +Sam,Elrond,31357 +Sam,Frodo,31389 +Sam,Elrond,31389 +Frodo,Elrond,31389 +Pippin,Merry,31401 +Pippin,Bilbo,31402 +Frodo,Bilbo,31406 +Sam,Aragorn,31411 +Sam,Frodo,31423 +Gandalf,Bilbo,31436 +Gandalf,Elrond,31436 +Bilbo,Elrond,31436 +Frodo,Elrond,31437 +Frodo,Gandalf,31445 +Pippin,Sam,31474 +Pippin,Sam,31475 +Gandalf,Butterbur,31524 +Gandalf,Barliman,31581 +Butterbur,Merry,31614 +Sam,Bill,31626 +Sam,Bill,31629 +Sam,Bill,31651 +Frodo,Barliman,31652 +Pippin,Gandalf,31656 +Pippin,Lotho,31656 +Gandalf,Lotho,31656 +Merry,Gandalf,31658 +Gandalf,Shadowfax,31684 +Frodo,Lotho,31707 +Pippin,Merry,31716 +Pippin,Merry,31717 +Merry,Bill,31723 +Sam,Bill,31731 +Robin,Sam,31816 +Sam,Frodo,31834 +Frodo,Merry,31844 +Pippin,Sam,31845 +Pippin,Merry,31845 +Sam,Merry,31845 +Sam,Frodo,31863 +Sam,Merry,31875 +Sam,Bill,31880 +Pippin,Frodo,31930 +Pippin,Merry,31930 +Pippin,Sam,31930 +Frodo,Merry,31930 +Frodo,Sam,31930 +Merry,Sam,31930 +Frodo,Lotho,31940 +Pippin,Frodo,31943 +Frodo,Lotho,31950 +Pippin,Lotho,31957 +Sam,Frodo,32044 +Tom,Frodo,32058 +Tom,Sam,32058 +Frodo,Sam,32058 +Frodo,Merry,32075 +Peregrin,Lotho,32080 +Frodo,Merry,32095 +Sam,Frodo,32234 +Sam,Frodo,32238 +Sam,Frodo,32239 +Pippin,Merry,32283 +Meriadoc,Peregrin,32293 +Pippin,Sam,32295 +Pippin,Merry,32295 +Sam,Merry,32295 +Frodo,Merry,32296 +Pippin,Frodo,32301 +Pippin,Merry,32301 +Pippin,Sam,32301 +Frodo,Merry,32301 +Frodo,Sam,32301 +Merry,Sam,32301 +Sam,Frodo,32331 +Merry,Lotho,32348 +Merry,Saruman,32355 +Gandalf,Saruman,32372 +Saruman,Wormtongue,32396 +Saruman,Wormtongue,32399 +Frodo,Saruman,32400 +Sam,Frodo,32404 +Frodo,Saruman,32409 +Frodo,Wormtongue,32422 +Saruman,Wormtongue,32432 +Saruman,Wormtongue,32443 +Frodo,Wormtongue,32444 +Pippin,Fredegar,32459 +Frodo,Lotho,32469 +Pippin,Merry,32473 +Sam,Frodo,32499 +Sam,Frodo,32520 +Sam,Frodo,32523 +Pippin,Merry,32524 +Sam,Frodo,32525 +Sam,Frodo,32527 +Sam,Frodo,32538 +Pippin,Merry,32541 +Sam,Frodo,32544 +Sam,Frodo,32547 +Meriadoc,Peregrin,32548 +Meriadoc,Sam,32548 +Peregrin,Sam,32548 +Sam,Frodo,32552 +Sam,Frodo,32557 +Sam,Frodo,32563 +Sam,Frodo,32565 +Sam,Frodo,32568 +Sam,Frodo,32569 +Sam,Bilbo,32570 +Sam,Frodo,32573 +Sam,Frodo,32579 +Sam,Bilbo,32579 +Frodo,Bilbo,32579 +Sam,Frodo,32584 +Frodo,Bilbo,32586 +Frodo,Bilbo,32595 +Sam,Frodo,32597 +Sam,Frodo,32598 +Sam,Frodo,32599 +Sam,Bill,32599 +Frodo,Bill,32599 +Sam,Frodo,32603 +Sam,Frodo,32613 +Elrond,Sam,32614 +Elrond,Galadriel,32614 +Sam,Galadriel,32614 +Elrond,Galadriel,32619 +Sam,Frodo,32631 +Pippin,Merry,32645 +Pippin,Goldilocks,32645 +Merry,Goldilocks,32645 +Galadriel,Elrond,32650 +Sam,Frodo,32651 +Sam,Bilbo,32651 +Frodo,Bilbo,32651 +Frodo,Gandalf,32656 +Pippin,Merry,32659 +Sam,Gandalf,32663 +Pippin,Frodo,32666 +Pippin,Merry,32666 +Pippin,Galadriel,32666 +Pippin,Sam,32666 +Frodo,Merry,32666 +Frodo,Galadriel,32666 +Frodo,Sam,32666 +Merry,Galadriel,32666 +Merry,Sam,32666 +Galadriel,Sam,32666 +Pippin,Merry,32671 +Pippin,Merry,32674 From 9a4b2e131fbc81abdffae510111565ae40cc7391 Mon Sep 17 00:00:00 2001 From: miratepuffin Date: Thu, 5 Nov 2020 17:38:53 +0000 Subject: [PATCH 20/58] Initial attempt to fix the properties --- .../analysis/API/entityVisitors/EntityVisitor.scala | 2 +- .../core/model/graphentities/MutableProperty.scala | 10 +++++++--- .../scala/com/raphtory/examples/lotr/LOTRRouter.scala | 5 +++-- .../scala/com/raphtory/examples/lotr/LOTRSpout.scala | 4 ++-- .../main/scala/com/raphtory/spouts/FileSpout.scala | 5 +++-- .../raphtory/tests/TemporalTriangleCountExample.scala | 11 ++++++----- 6 files changed, 22 insertions(+), 15 deletions(-) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/EntityVisitor.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/EntityVisitor.scala index e108d96a04..f0dc5284a0 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/EntityVisitor.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/EntityVisitor.scala @@ -18,7 +18,7 @@ abstract class EntityVisitor(entity:Entity,viewJob:ViewJob) { val x =entity.properties.filter(p =>{ p._2.creation()<=viewJob.timestamp }).map(f => (f._1,f._2.valueAt(viewJob.timestamp))) - if(x.isEmpty){ + if(x.isEmpty&&this.isInstanceOf[VertexVisitor]){ println("empty") } x diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/MutableProperty.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/MutableProperty.scala index 5290291b8f..c1f7120097 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/MutableProperty.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/MutableProperty.scala @@ -17,16 +17,20 @@ class MutableProperty(creationTime: Long, value: Any) extends Property { update(creationTime, value) var earliest = creationTime + var earliestval = value override def creation(): Long = earliest def update(msgTime: Long, newValue: Any): Unit = { - if(msgTime Date: Thu, 5 Nov 2020 17:42:09 +0000 Subject: [PATCH 21/58] viewjob print --- .../core/analysis/API/entityVisitors/EntityVisitor.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/EntityVisitor.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/EntityVisitor.scala index f0dc5284a0..75dcc5f60d 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/EntityVisitor.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/EntityVisitor.scala @@ -19,7 +19,7 @@ abstract class EntityVisitor(entity:Entity,viewJob:ViewJob) { p._2.creation()<=viewJob.timestamp }).map(f => (f._1,f._2.valueAt(viewJob.timestamp))) if(x.isEmpty&&this.isInstanceOf[VertexVisitor]){ - println("empty") + println(viewJob) } x } From f2a4387f7808c14398a68f84b3063d41813e854b Mon Sep 17 00:00:00 2001 From: Yanyang Wang Date: Wed, 7 Oct 2020 19:41:08 +0100 Subject: [PATCH 22/58] add sbt-assembly plugin --- mainproject/build.sbt | 30 ++++++++++++++++++++++++++++++ mainproject/project/plugins.sbt | 3 ++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/mainproject/build.sbt b/mainproject/build.sbt index deb58aa57f..25a69af419 100644 --- a/mainproject/build.sbt +++ b/mainproject/build.sbt @@ -1,5 +1,6 @@ import com.typesafe.sbt.packager.archetypes.scripts.AshScriptPlugin import com.typesafe.sbt.packager.docker.Cmd +import sbtassembly.MergeStrategy val Akka = "2.5.26" val Config = "1.2.1" @@ -96,6 +97,31 @@ lazy val dockerStuff = Seq( ) +lazy val mergeStrategy: String => MergeStrategy = { + case PathList(xs @ _*) if xs.last == "io.netty.versions.properties" => MergeStrategy.first + case PathList(xs @ _*) if xs.last == "module-info.class" => MergeStrategy.first + case x if Assembly.isConfigFile(x) => + MergeStrategy.concat + case PathList(ps @ _*) if Assembly.isReadme(ps.last) || Assembly.isLicenseFile(ps.last) => + MergeStrategy.rename + case PathList("META-INF", xs @ _*) => + xs map { _.toLowerCase } match { + case "manifest.mf" :: Nil | "index.list" :: Nil | "dependencies" :: Nil => + MergeStrategy.discard + case ps @ x :: xs if ps.last.endsWith(".sf") || ps.last.endsWith(".dsa") => + MergeStrategy.discard + case "plexus" :: xs => + MergeStrategy.discard + case "services" :: xs => + MergeStrategy.filterDistinctLines + case "spring.schemas" :: Nil | "spring.handlers" :: Nil | + "io.netty.versions.properties" :: Nil => + MergeStrategy.filterDistinctLines + case _ => MergeStrategy.deduplicate + } + case _ => MergeStrategy.deduplicate +} + lazy val root = Project(id = "raphtory", base = file(".")) aggregate (cluster) lazy val cluster = project @@ -158,3 +184,7 @@ lazy val cluster = project javaAgents += "io.kamon" % "kanela-agent" % "1.0.5", javaOptions in Universal += "-Dorg.aspectj.tracing.factory=default" ) + .settings( + assemblyMergeStrategy in assembly := mergeStrategy, + mainClass in assembly := Some("com.raphtory.Go") + ) diff --git a/mainproject/project/plugins.sbt b/mainproject/project/plugins.sbt index f4367b69d5..5621e42726 100644 --- a/mainproject/project/plugins.sbt +++ b/mainproject/project/plugins.sbt @@ -1,4 +1,5 @@ // "2.3.2" is just sbt plugin version addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.3.2") addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.3.1") -addSbtPlugin("com.lightbend.sbt" % "sbt-javaagent" % "0.1.5") \ No newline at end of file +addSbtPlugin("com.lightbend.sbt" % "sbt-javaagent" % "0.1.5") +addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.9") From 7ad06fd507d366e64ea3c085fac48eab0723593a Mon Sep 17 00:00:00 2001 From: miratepuffin Date: Fri, 6 Nov 2020 17:22:55 +0000 Subject: [PATCH 23/58] Removed Kamon for now --- mainproject/build.sbt | 2 +- mainproject/cluster/src/main/scala/com/raphtory/Server.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mainproject/build.sbt b/mainproject/build.sbt index deb58aa57f..f950dc4484 100644 --- a/mainproject/build.sbt +++ b/mainproject/build.sbt @@ -155,6 +155,6 @@ lazy val cluster = project ) .settings( javaAgents += "org.aspectj" % "aspectjweaver" % "1.8.13", - javaAgents += "io.kamon" % "kanela-agent" % "1.0.5", +// javaAgents += "io.kamon" % "kanela-agent" % "1.0.5", javaOptions in Universal += "-Dorg.aspectj.tracing.factory=default" ) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/Server.scala b/mainproject/cluster/src/main/scala/com/raphtory/Server.scala index e9be690baf..e6086ee6ea 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/Server.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/Server.scala @@ -22,7 +22,7 @@ import scala.sys.process._ //main function object Go extends App { - Kamon.init() //start tool logging + //Kamon.init() //start tool logging val conf = ConfigFactory.load() val seedLoc = s"${sys.env("HOST_IP")}:${conf.getInt("settings.bport")}" val root = LoggerFactory.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME).asInstanceOf[ch.qos.logback.classic.Logger] From dd48e0d3f515ee6d13ab55adac7863657ce0e024 Mon Sep 17 00:00:00 2001 From: Miratepuffin Date: Mon, 9 Nov 2020 18:01:37 +0000 Subject: [PATCH 24/58] fixing issues with router worker --- mainproject/Build-Scripts/env-setter.sh | 4 +- mainproject/build.sbt | 3 +- .../src/main/scala/com/raphtory/Server.scala | 2 +- .../analysis/serialisers/JSONSerialiser.scala | 61 +++++++++++++++++++ .../serialisers/NetworkXSerialiser.scala | 2 + .../analysis/serialisers/Serialiser.scala | 24 ++++---- .../Workers/IngestionWorker.scala | 10 +++ .../core/components/Router/RouterWorker.scala | 30 +++++---- .../core/components/Spout/SpoutTrait.scala | 7 ++- .../communication/raphtoryMessages.scala | 1 - .../scala/com/raphtory/core/utils/Utils.scala | 1 - .../scala/com/raphtory/spouts/FileSpout.scala | 31 ++++++---- 12 files changed, 131 insertions(+), 45 deletions(-) create mode 100644 mainproject/cluster/src/main/scala/com/raphtory/core/analysis/serialisers/JSONSerialiser.scala diff --git a/mainproject/Build-Scripts/env-setter.sh b/mainproject/Build-Scripts/env-setter.sh index 6e71e9d6e3..2cbcf1716a 100644 --- a/mainproject/Build-Scripts/env-setter.sh +++ b/mainproject/Build-Scripts/env-setter.sh @@ -10,8 +10,8 @@ echo "HOST_IP = $HOST_IP" echo "HOSTNAME = $HOSTNAME" echo "/////////////////////" -cd /node_exporter-1.0.1.linux-386 -./node_exporter >> /dev/null & +#cd /node_exporter-1.0.1.linux-386 +#./node_exporter >> /dev/null & sleep 20 diff --git a/mainproject/build.sbt b/mainproject/build.sbt index f950dc4484..2f6b25415c 100644 --- a/mainproject/build.sbt +++ b/mainproject/build.sbt @@ -91,7 +91,7 @@ lazy val basicSettings = Seq( lazy val dockerStuff = Seq( maintainer := "Imane Hafnaoui ", dockerBaseImage := "miratepuffin/raphtory-redis:latest", - dockerRepository := Some("miratepuffin"), + dockerRepository := Some("niallroche"), dockerExposedPorts := Seq(2551, 8080, 2552, 1600, 11600,8081,46339,9100), ) @@ -155,6 +155,5 @@ lazy val cluster = project ) .settings( javaAgents += "org.aspectj" % "aspectjweaver" % "1.8.13", -// javaAgents += "io.kamon" % "kanela-agent" % "1.0.5", javaOptions in Universal += "-Dorg.aspectj.tracing.factory=default" ) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/Server.scala b/mainproject/cluster/src/main/scala/com/raphtory/Server.scala index e6086ee6ea..bd0cee9f15 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/Server.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/Server.scala @@ -22,7 +22,7 @@ import scala.sys.process._ //main function object Go extends App { - //Kamon.init() //start tool logging +// Kamon.init() //start tool logging val conf = ConfigFactory.load() val seedLoc = s"${sys.env("HOST_IP")}:${conf.getInt("settings.bport")}" val root = LoggerFactory.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME).asInstanceOf[ch.qos.logback.classic.Logger] diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/serialisers/JSONSerialiser.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/serialisers/JSONSerialiser.scala new file mode 100644 index 0000000000..2dcaa563b4 --- /dev/null +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/serialisers/JSONSerialiser.scala @@ -0,0 +1,61 @@ +package com.raphtory.core.analysis.serialisers + +import com.raphtory.core.analysis.API.entityVisitors.{EdgeVisitor, VertexVisitor} + +import scala.collection.parallel.mutable.ParTrieMap + +class JSONSerialiser extends Serialiser { + + override def startOfFile(): String = "{\"directed\": true, \"multigraph\": false, \"graph\": {}, \"nodes\": [\n" + + override def middleOfFile() :String = "],\n\"links\":[\n" + + override def fileExtension(): String = { + "json" + } + + override def serialiseVertex(v: VertexVisitor): String = { + val properties: String = extractProperties(v.getPropertySet()) + + if(properties.nonEmpty) { + if (v.Type().nonEmpty) + s"""\t{\"id\":${v.ID()},\"doctype\":\"${v.Type()}\",$properties}""" + else + s"""\t{\"id\":${v.ID()},$properties}""" + } + else { + if (v.Type().nonEmpty) + s"""\t{\"id\":${v.ID()},\"doctype\":\"${v.Type()}\"}""" + else + s"""\t{\"id\":${v.ID()}}""" + } + } + + private def extractProperties(ps: ParTrieMap[String,Any]) = { + ps.map(property => + if (property._2.isInstanceOf[Long]) + s"""\"${property._1}\":${property._2}""" + else + s"""\"${property._1}\":\"${property._2}\"""" + ).toArray.mkString(",") + } + + override def serialiseEdge(e: EdgeVisitor): String = { + val properties = extractProperties(e.getPropertySet()) + if(properties.nonEmpty) { + if (e.Type().nonEmpty) + s"""\t{\"source\":${e.src()},\"target\":${e.dst()},\"edgetype\":\"${e.Type()}\",$properties}""" + else + s"""\t{\"source\":${e.src()},\"target\":${e.dst()},$properties}""" + } + else { + if (e.Type().nonEmpty) + s"""\t{\"source\":${e.src()},\"target\":${e.dst()},\"edgetype\":\"${e.Type()}\"}""" + else + s"""\t{\"source\":${e.src()},\"target\":${e.dst()}}""" + } + } + + override def endOfFile(): String = "\n]}\n" + +} diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/serialisers/NetworkXSerialiser.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/serialisers/NetworkXSerialiser.scala index 246e607645..388c7f9c73 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/serialisers/NetworkXSerialiser.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/serialisers/NetworkXSerialiser.scala @@ -5,6 +5,8 @@ class NetworkXSerialiser extends Serialiser { override def startOfFile(): String = "G = nx.Graph()" + override def middleOfFile() :String = "" + override def serialiseVertex(v: VertexVisitor): String = { val properties =v.getPropertySet().map(property => s"${property._1} = ${property._2}").toArray.mkString(",") if(properties.nonEmpty) { diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/serialisers/Serialiser.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/serialisers/Serialiser.scala index 58134cae9e..2bcc49870a 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/serialisers/Serialiser.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/serialisers/Serialiser.scala @@ -15,8 +15,12 @@ abstract class Serialiser extends Analyser(null){ def serialiseVertex(v:VertexVisitor):String def serialiseEdge(e:EdgeVisitor):String def startOfFile():String + def middleOfFile():String def endOfFile():String + def fileExtension(): String = {"txt"} + def rowDelimeter(): String = {",\n"} + override def returnResults(): Any = { val serialisedEntities = view.getVertices().map { vertex => (serialiseVertex(vertex),vertex.getOutEdges.map(e=> serialiseEdge(e)).toArray) @@ -28,30 +32,24 @@ abstract class Serialiser extends Analyser(null){ override def processResults(results: ArrayBuffer[Any], timeStamp: Long, viewCompleteTime: Long): Unit = { val serialisedResults = results.asInstanceOf[ArrayBuffer[(Array[String],Array[String])]] - val file = new File(s"$path/Raphtory_Snapshot_$timeStamp.txt") + val file = new File(s"$path/Raphtory_Snapshot_$timeStamp.${fileExtension()}") write((serialisedResults.flatMap(x=>x._1).toArray,serialisedResults.flatMap(x=>x._2).toArray),file) } override def processWindowResults(results: ArrayBuffer[Any], timestamp: Long, windowSize: Long, viewCompleteTime: Long): Unit = { val serialisedResults = results.asInstanceOf[ArrayBuffer[(Array[String],Array[String])]] - val file = new File(s"$path/Raphtory_Snapshot_${timestamp}_$windowSize.txt") + val file = new File(s"$path/Raphtory_Snapshot_${timestamp}_$windowSize.${fileExtension()}") write((serialisedResults.flatMap(x=>x._1).toArray,serialisedResults.flatMap(x=>x._2).toArray),file) - - } def write(serialisedResults:(Array[String],Array[String]),file:File) = { +// println("write :"+serialisedResults._1.length+":"++serialisedResults._2.length) + println("vertices & edges", serialisedResults._1.length, serialisedResults._2.length) val bw = new BufferedWriter(new FileWriter(file)) bw.write(startOfFile()) - bw.newLine() - for (line <- serialisedResults._1) { - bw.write(line) - bw.newLine() - } - for (line <- serialisedResults._2) { - bw.write(line) - bw.newLine() - } + bw.write(serialisedResults._1.mkString(rowDelimeter())) + bw.write(middleOfFile()) + bw.write(serialisedResults._2.mkString(rowDelimeter())) bw.write(endOfFile()) bw.newLine() bw.close() diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/IngestionWorker.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/IngestionWorker.scala index 80facd6c35..b9ed71a8c4 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/IngestionWorker.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/IngestionWorker.scala @@ -25,6 +25,10 @@ class IngestionWorker(workerId: Int,partitionID:Int, storage: EntityStorage) ext mediator ! DistributedPubSubMediator.Put(self) implicit val executionContext = context.system.dispatchers.lookup("worker-dispatcher") + var increments =0 + var updates = 0 + var updates2 =0 + val synchTime = Kamon.histogram("Raphtory_Wall_Clock").withTag("actor",s"PartitionWriter_$partitionID").withTag("ID",workerId) val routerUpdates = Kamon.counter("Raphtory_Router_Updates").withTag("actor",s"PartitionWriter_$partitionID").withTag("ID",workerId) val interWorkerUpdates = Kamon.counter("Raphtory_Inter_Worker_Updates").withTag("actor",s"PartitionWriter_$partitionID").withTag("ID",workerId) @@ -253,6 +257,7 @@ class IngestionWorker(workerId: Int,partitionID:Int, storage: EntityStorage) ext queue += queueItem(routerTime,msgTime) queuedMessageMap put(routerID,queue) } + updates+=1 synchronisedUpdates.increment() processSynchTime(spoutTime) } @@ -267,6 +272,11 @@ class IngestionWorker(workerId: Int,partitionID:Int, storage: EntityStorage) ext setSafePoint(queue._1, queue._2) }) val timestamps = queueState.map(q => q.timestamp) + //println(s"Writer Worker $partitionID $workerId ${queueState.mkString("[",",","]")} ${storage.vertices.size}") + println(s"$increments Writer Worker $partitionID $workerId ${timestamps.min} ${storage.vertices.size} $updates ${updates-updates2}") + updates2=updates + increments+=1 + val min = timestamps.min if(storage.windowTime + case TimeBroadcast => { Utils.getAllWriterWorkers(managerCount).foreach { workerPath => mediator ! DistributedPubSubMediator.Send( - workerPath, - RouterWorkerTimeSync(newestTime, s"${routerId}_$workerID", getMessageIDForWriter(workerPath)), - false + workerPath, + RouterWorkerTimeSync(newestTime, s"${routerId}_$workerID", getMessageIDForWriter(workerPath)), + false ) } + context.system.scheduler + .scheduleOnce(delay = 5.seconds, receiver = self, message = TimeBroadcast) + println(s"Router $routerId $workerID with $newestTime ${messageIDs.mkString("[",",","]")}") + } case DataFinished => { Utils.getAllRouterWorkers(initialRouterCount).foreach { workerPath => mediator ! DistributedPubSubMediator.Send( @@ -84,6 +87,9 @@ abstract class RouterWorker[In <: SpoutGoing](val routerId: Int, val workerID: I } case DataFinishedSync(time) => { + println(s"Router $routerId $workerID finished") + println(s"Router $routerId $workerID ${update}") + if (time >= newestTime) { Utils.getAllWriterWorkers(managerCount).foreach { workerPath => mediator ! DistributedPubSubMediator.Send( @@ -125,8 +131,8 @@ abstract class RouterWorker[In <: SpoutGoing](val routerId: Int, val workerID: I trackedMessage: Boolean, trackedTime: Long ): Long = { + update+=1 routerWorkerUpdates.increment() - val path = getManager(message.srcID, managerCount) val id = getMessageIDForWriter(path) val trackedTimeToUse = if (trackedMessage) trackedTime else -1L diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/SpoutTrait.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/SpoutTrait.scala index cc7e387ddc..3d39c96ad4 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/SpoutTrait.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/SpoutTrait.scala @@ -11,6 +11,7 @@ import com.raphtory.core.model.communication._ import kamon.Kamon import scala.concurrent.ExecutionContext +import scala.util.Random //import kamon.metric.CounterMetric //import kamon.metric.GaugeMetric @@ -78,6 +79,8 @@ trait SpoutTrait[Domain <: DomainMessage, Out <: SpoutGoing] extends Actor with self ! StartSpout partitionManagers=pmCount routers=roCount + println(s"Number of routers: $routers") + println(s"Number of partitions: $partitionManagers") } else context.system.scheduler.scheduleOnce(delay = 1 second, receiver = self, message = IsSafe) @@ -98,8 +101,8 @@ trait SpoutTrait[Domain <: DomainMessage, Out <: SpoutGoing] extends Actor with AllocateTrackedTuple(System.currentTimeMillis(), command) else AllocateTuple(command) - val mod = count % (routers * 10) - lastRouter=s"/user/router/router_${mod/10}_Worker_${mod%10}" + + lastRouter=s"/user/router/router_${Random.nextInt(routers)}_Worker_${Random.nextInt(10)}" mediator ! DistributedPubSubMediator.Send(lastRouter, message, localAffinity = false) } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/model/communication/raphtoryMessages.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/model/communication/raphtoryMessages.scala index 355b2f520e..7a666f9f34 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/model/communication/raphtoryMessages.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/model/communication/raphtoryMessages.scala @@ -75,7 +75,6 @@ case class EdgeSyncAck(msgTime: Long, routerID: String, routerTime: Int,spoutTim case class VertexRemoveSyncAck(msgTime: Long, routerID: String, routerTime: Int,spoutTime:Long) case class RouterWorkerTimeSync(msgTime:Long,routerID:String,routerTime:Int) - case class UpdatedCounter(newValue: Int) case class AssignedId(id: Int) case class PartitionsCount(count: Int) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/utils/Utils.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/utils/Utils.scala index ba1b634675..424c85061c 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/utils/Utils.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/utils/Utils.scala @@ -52,7 +52,6 @@ object Utils { val mod = srcId.abs % (managerCount * totalWorkers) val manager = mod / totalWorkers val worker = mod % totalWorkers - s"/user/Manager_${manager}_reader_$worker" } def getReaderInt(srcId: Long, managerCount: Int): (Long, Long) = { diff --git a/mainproject/cluster/src/main/scala/com/raphtory/spouts/FileSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/spouts/FileSpout.scala index 2cbc158fce..32b139d8b4 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/spouts/FileSpout.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/spouts/FileSpout.scala @@ -1,8 +1,8 @@ package com.raphtory.spouts -import java.io.BufferedReader -import java.io.File -import java.io.FileReader +import java.io.{BufferedReader, File, FileInputStream, FileReader, InputStreamReader} +import java.nio.charset.StandardCharsets +import java.util.zip.GZIPInputStream import com.raphtory.core.components.Spout.SpoutTrait import com.raphtory.core.components.Spout.SpoutTrait.DomainMessage @@ -26,6 +26,7 @@ final case class FileSpout() extends SpoutTrait[FileDomain, StringSpoutGoing] { private val TIME = System.getenv().getOrDefault("FILE_SPOUT_TIME", "60").trim.toInt private var fileManager = FileManager(directory, fileName, dropHeader, JUMP) + val t0 = System.nanoTime() def startSpout(): Unit = { self ! NextLineBlock @@ -34,8 +35,10 @@ final case class FileSpout() extends SpoutTrait[FileDomain, StringSpoutGoing] { def handleDomainMessage(message: FileDomain): Unit = message match { case Increase => - if (fileManager.allCompleted) - log.info("All files read") + if (fileManager.allCompleted) { + println("All files read1-" + (System.nanoTime() - t0)) + dataFinished() + } else { fileManager = fileManager.increaseBlockSize(INCREMENT) context.system.scheduler.scheduleOnce(TIME.seconds, self, Increase) @@ -43,7 +46,7 @@ final case class FileSpout() extends SpoutTrait[FileDomain, StringSpoutGoing] { case NextLineBlock => if (fileManager.allCompleted) - log.info("All files read") + println("All files read2-" + (System.nanoTime() - t0)) else { val (newFileManager, block) = fileManager.nextLineBlock() fileManager = newFileManager @@ -52,7 +55,7 @@ final case class FileSpout() extends SpoutTrait[FileDomain, StringSpoutGoing] { } case NextFile => if (fileManager.allCompleted) - log.info("All files read") + println("All files read3-" + (System.nanoTime() - t0)) else { fileManager = fileManager.nextFile() self ! NextLineBlock @@ -115,12 +118,18 @@ final case class FileManager private ( private def getFileReader(file: File): BufferedReader = { logger.info(s"Reading file ${file.getCanonicalPath}") + + var br = new BufferedReader(new FileReader(file)) + if (file.getName.endsWith(".gz")) { + val inStream = new FileInputStream(file) + val inGzipStream = new GZIPInputStream(inStream) + val inReader = new InputStreamReader(inGzipStream) //default to UTF-8 + br = new BufferedReader(inReader) + } if (dropHeader) { - val br = new BufferedReader(new FileReader(file)) br.readLine() - br - } else - new BufferedReader(new FileReader(file)) + } + br } } From 1aa51e4d7d2c97c92ef63d6bcfb38f9c005a2234 Mon Sep 17 00:00:00 2001 From: Miratepuffin Date: Tue, 10 Nov 2020 02:36:05 +0000 Subject: [PATCH 25/58] Implemented simple backpressure in spout/router --- .../src/main/resources/application.conf | 2 +- .../Workers/IngestionWorker.scala | 16 +++-- .../core/components/Router/RouterWorker.scala | 17 ++--- .../core/components/Spout/SpoutTrait.scala | 69 ++++++++++++++----- .../communication/raphtoryMessages.scala | 2 +- .../blockchain/routers/BitcoinRouter.scala | 10 +-- .../routers/ChABClus2ClusRouter.scala | 7 +- .../routers/ChABClus2ClusRouterNoprop.scala | 7 +- .../routers/ChainalysisABRouter.scala | 7 +- .../blockchain/routers/DashcoinRouter.scala | 7 +- .../routers/EthereumGethRouter.scala | 7 +- .../routers/EthereumKafkaRouter.scala | 11 +-- .../routers/EthereumTransactionRouter.scala | 7 +- .../routers/FirehoseKafkaRouter.scala | 9 +-- .../blockchain/routers/LitecoinRouter.scala | 7 +- .../citationNetwork/CitationRouter.scala | 7 +- .../gab/routers/GabPostGraphRouter.scala | 7 +- .../examples/gab/routers/GabRawRouter.scala | 7 +- .../gab/routers/GabUserGraphRouter.scala | 7 +- .../examples/ldbc/routers/LDBCOldRouter.scala | 9 +-- .../examples/ldbc/routers/LDBCRouter.scala | 7 +- .../raphtory/examples/lotr/LOTRRouter.scala | 7 +- .../examples/test/actors/RandomRouter.scala | 14 ++-- .../test/actors/TriangleTestRouter.scala | 5 +- .../routers/TrackAndTraceRouter.scala | 7 +- .../raphtory/examples/tsvnet/TSVRouter.scala | 7 +- .../routers/CooccurrenceMatrixRouter.scala | 7 +- .../CooccurrenceMatrixRouterFiltered.scala | 7 +- .../scala/com/raphtory/spouts/FileSpout.scala | 21 ++---- 29 files changed, 183 insertions(+), 121 deletions(-) diff --git a/mainproject/cluster/src/main/resources/application.conf b/mainproject/cluster/src/main/resources/application.conf index 8ab8f88d55..7966b69a1f 100644 --- a/mainproject/cluster/src/main/resources/application.conf +++ b/mainproject/cluster/src/main/resources/application.conf @@ -377,5 +377,5 @@ spout-dispatcher { # Throughput defines the maximum number of messages to be # processed per actor before the thread jumps to the next actor. # Set to 1 for as fair as possible. - throughput = 1 + throughput = 10 } \ No newline at end of file diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/IngestionWorker.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/IngestionWorker.scala index b9ed71a8c4..17dac21d71 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/IngestionWorker.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/IngestionWorker.scala @@ -253,7 +253,7 @@ class IngestionWorker(workerId: Int,partitionID:Int, storage: EntityStorage) ext queuedMessageMap.get(routerID) match { case Some(queue) => queue += queueItem(routerTime,msgTime) case None => - val queue = new mutable.PriorityQueue[queueItem]() + val queue = new mutable.PriorityQueue[queueItem]()(Ordering.by[queueItem, Int](f=>f.routerEpoch).reverse) queue += queueItem(routerTime,msgTime) queuedMessageMap put(routerID,queue) } @@ -272,8 +272,11 @@ class IngestionWorker(workerId: Int,partitionID:Int, storage: EntityStorage) ext setSafePoint(queue._1, queue._2) }) val timestamps = queueState.map(q => q.timestamp) + if(workerId==1) + println(s"$increments Writer Worker $partitionID $workerId $timestamps") + //println(s"Writer Worker $partitionID $workerId ${queueState.mkString("[",",","]")} ${storage.vertices.size}") - println(s"$increments Writer Worker $partitionID $workerId ${timestamps.min} ${storage.vertices.size} $updates ${updates-updates2}") + //println(s"$increments Writer Worker $partitionID $workerId ${timestamps.min} ${storage.vertices.size} $updates ${updates-updates2}") updates2=updates increments+=1 @@ -294,13 +297,18 @@ class IngestionWorker(workerId: Int,partitionID:Int, storage: EntityStorage) ext case None => default } if(messageQueue nonEmpty) - while(messageQueue.headOption.getOrElse(default).routerEpoch==currentSafePoint.routerEpoch+1) - currentSafePoint = messageQueue.dequeue() + while(messageQueue.headOption.getOrElse(default).routerEpoch==currentSafePoint.routerEpoch+1) { + currentSafePoint = messageQueue.dequeue() + if(workerId==1 && (routerName equals "0_3")) + println(currentSafePoint) + } safeMessageMap put (routerName,currentSafePoint) currentSafePoint } + def recursiveDequeue(q:mutable.PriorityQueue[queueItem]) + private def processRouterTimeSync(req:RouterWorkerTimeSync) ={ storage.timings(req.msgTime) addToWatermarkQueue(req.routerID,req.routerTime,req.msgTime,0) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala index 03c84054e1..13c4ed40df 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala @@ -4,12 +4,13 @@ import akka.actor.{Actor, ActorLogging, Cancellable} import akka.cluster.pubsub.DistributedPubSub import akka.cluster.pubsub.DistributedPubSubMediator import com.raphtory.core.components.Router.RouterWorker.CommonMessage.TimeBroadcast +import com.raphtory.core.components.Spout.SpoutTrait.CommonMessage.{NoWork, SpoutOnline, WorkPlease} import com.raphtory.core.model.communication._ import com.raphtory.core.utils.Utils import com.raphtory.core.utils.Utils.getManager import kamon.Kamon -import scala.collection.parallel.mutable.ParTrieMap +import scala.collection.parallel.mutable.{ParArray, ParHashSet, ParTrieMap} import scala.concurrent.ExecutionContext import scala.concurrent.duration._ import scala.util.hashing.MurmurHash3 @@ -29,7 +30,7 @@ abstract class RouterWorker[In <: SpoutGoing](val routerId: Int, val workerID: I var update = 0 // todo: wvv let people know parseTuple will create a list of update message // and this trait will handle logic to send to graph - protected def parseTuple(tuple: In): List[GraphUpdate] + protected def parseTuple(tuple: In): ParHashSet[GraphUpdate] final protected val mediator = DistributedPubSub(context.system).mediator mediator ! DistributedPubSubMediator.Put(self) @@ -45,6 +46,8 @@ abstract class RouterWorker[In <: SpoutGoing](val routerId: Int, val workerID: I private def work(managerCount: Int, trackedTime: Long, newestTime: Long): Receive = { + case SpoutOnline => context.sender() ! WorkPlease + case NoWork => context.system.scheduler.scheduleOnce(delay = 10.second, receiver = context.sender(), message = WorkPlease) case msg: UpdatedCounter => log.debug(s"RouterWorker [$routerId] received [$msg] request.") if (managerCount < msg.newValue) context.become(work(msg.newValue, trackedTime, newestTime)) @@ -55,6 +58,7 @@ abstract class RouterWorker[In <: SpoutGoing](val routerId: Int, val workerID: I if(newNewestTime>newestTime) context.become(work(managerCount, trackedTime, newNewestTime)) ) + context.sender() ! WorkPlease case msg @ AllocateTrackedTuple( wallClock, @@ -63,7 +67,7 @@ abstract class RouterWorker[In <: SpoutGoing](val routerId: Int, val workerID: I log.debug(s"RouterWorker [$routerId] received [$msg] request.") val newNewestTime = parseTupleAndSendGraph(record, managerCount, true, wallClock).getOrElse(newestTime) context.become(work(managerCount, wallClock, newNewestTime)) - + context.sender() ! WorkPlease case TimeBroadcast => { Utils.getAllWriterWorkers(managerCount).foreach { workerPath => mediator ! DistributedPubSubMediator.Send( @@ -74,7 +78,7 @@ abstract class RouterWorker[In <: SpoutGoing](val routerId: Int, val workerID: I } context.system.scheduler .scheduleOnce(delay = 5.seconds, receiver = self, message = TimeBroadcast) - println(s"Router $routerId $workerID with $newestTime ${messageIDs.mkString("[",",","]")}") + //println(s"Router $routerId $workerID with $newestTime ${messageIDs.mkString("[",",","]")}") } case DataFinished => { Utils.getAllRouterWorkers(initialRouterCount).foreach { workerPath => @@ -87,10 +91,8 @@ abstract class RouterWorker[In <: SpoutGoing](val routerId: Int, val workerID: I } case DataFinishedSync(time) => { - println(s"Router $routerId $workerID finished") - println(s"Router $routerId $workerID ${update}") - if (time >= newestTime) { + println(s"Router $routerId $workerID ${time}") Utils.getAllWriterWorkers(managerCount).foreach { workerPath => mediator ! DistributedPubSubMediator.Send( workerPath, @@ -108,7 +110,6 @@ abstract class RouterWorker[In <: SpoutGoing](val routerId: Int, val workerID: I false ) } - } } case unhandled => log.warning(s"RouterWorker received unknown [$unhandled] message.") diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/SpoutTrait.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/SpoutTrait.scala index 3d39c96ad4..0537e42169 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/SpoutTrait.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/SpoutTrait.scala @@ -1,13 +1,12 @@ package com.raphtory.core.components.Spout -import akka.actor.{Actor, ActorLogging, Cancellable, Timers} +import akka.actor.{Actor, ActorLogging, ActorRef, Cancellable, Timers} import akka.cluster.pubsub.DistributedPubSub import akka.cluster.pubsub.DistributedPubSubMediator -import com.raphtory.core.components.Spout.SpoutTrait.CommonMessage.IsSafe -import com.raphtory.core.components.Spout.SpoutTrait.CommonMessage.StartSpout -import com.raphtory.core.components.Spout.SpoutTrait.CommonMessage.StateCheck +import com.raphtory.core.components.Spout.SpoutTrait.CommonMessage._ import com.raphtory.core.components.Spout.SpoutTrait.DomainMessage import com.raphtory.core.model.communication._ +import com.raphtory.core.utils.Utils import kamon.Kamon import scala.concurrent.ExecutionContext @@ -17,14 +16,16 @@ import scala.util.Random import scala.concurrent.duration._ import scala.language.postfixOps +import scala.collection.mutable.Queue + // TODO Add val name which sub classes that extend this trait must overwrite // e.g. BlockChainSpout val name = "Blockchain Spout" // Log.debug that read 'Spout' should then read 'Blockchain Spout' trait SpoutTrait[Domain <: DomainMessage, Out <: SpoutGoing] extends Actor with ActorLogging with Timers { // todo: wvv should assign the dispatcher when create the actor - // implicit val executionContext: ExecutionContext = context.system.dispatchers.lookup("spout-dispatcher") - implicit val executionContext: ExecutionContext = context.system.dispatcher + implicit val executionContext: ExecutionContext = context.system.dispatchers.lookup("spout-dispatcher") + //implicit val executionContext: ExecutionContext = context.system.dispatcher private val spoutTuples = Kamon.counter("Raphtory_Spout_Tuples").withTag("actor", self.path.name) private var count = 0 @@ -32,6 +33,7 @@ trait SpoutTrait[Domain <: DomainMessage, Out <: SpoutGoing] extends Actor with private var partitionManagers = 0 private var routers = 0 private var finish = false + private val workQueue = Queue[SpoutGoing]() private def recordUpdate(): Unit = { spoutTuples.increment() count += 1 @@ -48,13 +50,16 @@ trait SpoutTrait[Domain <: DomainMessage, Out <: SpoutGoing] extends Actor with final override def receive: Receive = work(false,1,1) - private def work(safe: Boolean,pmCounter:Int,rmCounter:Int): Receive = { + + + private def work(safe: Boolean, pmCounter:Int, rmCounter:Int): Receive = { case StateCheck => processStateCheckMessage(safe) case ClusterStatusResponse(clusterUp,pmCounter,rmCounter) => context.become(work(clusterUp,pmCounter,rmCounter)) context.system.scheduler.scheduleOnce(1 second, self, StateCheck) case IsSafe => processIsSafeMessage(safe,pmCounter,rmCounter) case StartSpout => startSpout() + case WorkPlease => sendData(context.sender()) case x: Domain => handleDomainMessage(x) case unhandled => log.error(s"Unable to handle message [$unhandled].") } @@ -79,6 +84,13 @@ trait SpoutTrait[Domain <: DomainMessage, Out <: SpoutGoing] extends Actor with self ! StartSpout partitionManagers=pmCount routers=roCount + Utils.getAllRouterWorkers(roCount).foreach { workerPath => + mediator ! DistributedPubSubMediator.Send( + workerPath, + SpoutOnline, + false + ) + } println(s"Number of routers: $routers") println(s"Number of partitions: $partitionManagers") @@ -88,22 +100,40 @@ trait SpoutTrait[Domain <: DomainMessage, Out <: SpoutGoing] extends Actor with protected def dataFinished():Unit = { if(!finish) { - mediator ! DistributedPubSubMediator.Send(lastRouter, DataFinished, localAffinity = false) + workQueue += DataFinished() + //mediator ! DistributedPubSubMediator.Send(lastRouter, DataFinished, localAffinity = false) finish=true } } protected def sendTuple(command: Out): Unit = { log.debug(s"The command [$command] received for send.") - recordUpdate() - val message = - if (count % 100 == 0) - AllocateTrackedTuple(System.currentTimeMillis(), command) - else - AllocateTuple(command) - - lastRouter=s"/user/router/router_${Random.nextInt(routers)}_Worker_${Random.nextInt(10)}" - mediator ! DistributedPubSubMediator.Send(lastRouter, message, localAffinity = false) + workQueue += command + //lastRouter=s"/user/router/router_${Random.nextInt(routers)}_Worker_${Random.nextInt(10)}" + //mediator ! DistributedPubSubMediator.Send(lastRouter, message, localAffinity = false) + } + + def sendData(sender:ActorRef): Unit = { + //s"/user/router/router_${Random.nextInt(routers)}_Worker_${Random.nextInt(10)}" + try { + val work =workQueue.dequeue() + if(work.isInstanceOf[DataFinished]) { + sender ! DataFinished + println("All data sent") + } + else { + val message = if (count % 100 == 0) + AllocateTrackedTuple(System.currentTimeMillis(),work) + else + AllocateTuple(work) + recordUpdate() + sender ! message + if (count % 100 == 0) println(s"Spout at Message $count, remaining messages ${workQueue.size}") + //println(s"Spout at Message $count, remaining messages ${work}") + }//mediator ! DistributedPubSubMediator.Send(lastRouter, message, localAffinity = false) + }catch { + case e:Exception => sender ! NoWork//mediator ! DistributedPubSubMediator.Send(lastRouter, NoWork, localAffinity = false) + } } def AllocateSpoutTask(duration: FiniteDuration, task: Any): Cancellable = { @@ -118,6 +148,11 @@ object SpoutTrait { case object StateCheck case object IsSafe case object Next extends BasicDomain + case object WorkPlease + case object NoWork + case object SpoutOnline + + } trait DomainMessage sealed trait BasicDomain extends DomainMessage diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/model/communication/raphtoryMessages.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/model/communication/raphtoryMessages.scala index 7a666f9f34..2f0af0844d 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/model/communication/raphtoryMessages.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/model/communication/raphtoryMessages.scala @@ -21,7 +21,7 @@ sealed trait TrackedGraphUpdate{ trait SpoutGoing final case class StringSpoutGoing(value: String) extends SpoutGoing -case class DataFinished() +case class DataFinished() extends SpoutGoing case class DataFinishedSync(time:Long) case class RouterUp(id: Int) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/BitcoinRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/BitcoinRouter.scala index a0e99166bd..2a25abddf7 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/BitcoinRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/BitcoinRouter.scala @@ -4,12 +4,14 @@ import com.raphtory.core.components.Router.RouterWorker import com.raphtory.core.model.communication._ import com.raphtory.spouts.blockchain.BitcoinTransaction import spray.json.JsArray + import scala.collection.mutable.ListBuffer +import scala.collection.parallel.mutable.{ParArray, ParHashSet} class BitcoinRouter(override val routerId: Int, override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) extends RouterWorker[BitcoinTransaction](routerId,workerID,initialManagerCount, initialRouterCount) { - override protected def parseTuple(tuple:BitcoinTransaction): List[GraphUpdate] = { + override protected def parseTuple(tuple:BitcoinTransaction): ParHashSet[GraphUpdate] = { val transaction = tuple.transaction val time = tuple.time @@ -23,7 +25,7 @@ class BitcoinRouter(override val routerId: Int, override val workerID:Int, overr val vouts = transaction.asJsObject.fields("vout") var total: Double = 0 - var commands = new ListBuffer[GraphUpdate]() + var commands = new ParHashSet[GraphUpdate]() for (vout <- vouts.asInstanceOf[JsArray].elements) { val voutOBJ = vout.asJsObject() var value = voutOBJ.fields("value").toString @@ -74,7 +76,7 @@ class BitcoinRouter(override val routerId: Int, override val workerID:Int, overr //creates edge between coingen and the transaction commands += EdgeAdd(msgTime = timeAsLong, srcID = "coingen".hashCode, dstID = txid.hashCode) - commands.toList + commands } else for (vin <- vins.asInstanceOf[JsArray].elements) { val vinOBJ = vin.asJsObject() @@ -89,7 +91,7 @@ class BitcoinRouter(override val routerId: Int, override val workerID:Int, overr properties = Properties(StringProperty("vout", prevVout))) } - commands.toList + commands } } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusRouter.scala index 04e4cad552..3c44137d65 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusRouter.scala @@ -8,11 +8,12 @@ import com.raphtory.core.model.communication.Type import com.raphtory.core.model.communication._ import scala.collection.mutable.ListBuffer +import scala.collection.parallel.mutable.ParHashSet class ChABClus2ClusRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount,initialRouterCount) { - override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { + override protected def parseTuple(tuple: StringSpoutGoing): ParHashSet[GraphUpdate] = { val dp = formatLine(tuple.value.split(",").map(_.trim)) val transactionTime = dp.time val srcClusterId = dp.srcCluster @@ -21,7 +22,7 @@ class ChABClus2ClusRouter(override val routerId: Int,override val workerID:Int, val btcAmount = dp.amount val usdAmount = dp.usd - val commands = new ListBuffer[GraphUpdate]() + val commands = new ParHashSet[GraphUpdate]() commands+= VertexAdd(msgTime = transactionTime, srcID = srcClusterId, Type("Cluster")) commands+=(VertexAdd(msgTime = transactionTime, srcID = dstClusterId, Type("Cluster"))) @@ -36,7 +37,7 @@ class ChABClus2ClusRouter(override val routerId: Int,override val workerID:Int, Type("Transfer") ) ) - commands.toList + commands } //converts the line into a case class which has all of the data via the correct name and type diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusRouterNoprop.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusRouterNoprop.scala index 786e06c8d3..0892c5950e 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusRouterNoprop.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusRouterNoprop.scala @@ -8,20 +8,21 @@ import com.raphtory.core.model.communication.Type import com.raphtory.core.model.communication._ import scala.collection.mutable.ListBuffer +import scala.collection.parallel.mutable.ParHashSet class ChABClus2ClusRouterNoprop(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount,initialRouterCount) { - override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { + override protected def parseTuple(tuple: StringSpoutGoing): ParHashSet[GraphUpdate] = { val dp = formatLine(tuple.value.split(",").map(_.trim)) val transactionTime = dp.time val srcClusterId = dp.srcCluster val dstClusterId = dp.dstCluster - val commands = new ListBuffer[GraphUpdate]() + val commands = new ParHashSet[GraphUpdate]() commands+=(VertexAdd(msgTime = transactionTime, srcID = srcClusterId, Type("Cluster"))) commands+=(VertexAdd(msgTime = transactionTime, srcID = dstClusterId, Type("Cluster"))) commands+=(EdgeAdd(msgTime = transactionTime, srcID = srcClusterId, dstID = dstClusterId, Type("Transfer"))) - commands.toList + commands } //converts the line into a case class which has all of the data via the correct name and type diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChainalysisABRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChainalysisABRouter.scala index 482b442ab5..fb9f673c21 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChainalysisABRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChainalysisABRouter.scala @@ -7,11 +7,12 @@ import com.raphtory.core.model.communication.Type import com.raphtory.core.model.communication._ import scala.collection.mutable.ListBuffer +import scala.collection.parallel.mutable.ParHashSet class ChainalysisABRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount,initialRouterCount) { - override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { + override protected def parseTuple(tuple: StringSpoutGoing): ParHashSet[GraphUpdate] = { val dp = formatLine(tuple.value.split(",").map(_.trim)) val transactionTime = dp.time val srcClusterId = dp.srcCluster @@ -19,7 +20,7 @@ class ChainalysisABRouter(override val routerId: Int,override val workerID:Int, val transactionId = dp.txid val btcAmount = dp.amount val usdAmount = dp.usd - val commands = new ListBuffer[GraphUpdate]() + val commands = new ParHashSet[GraphUpdate]() commands+=(VertexAdd(msgTime = transactionTime, srcID = srcClusterId, Type("Cluster"))) commands+=(VertexAdd(msgTime = transactionTime, srcID = dstClusterId, Type("Cluster"))) @@ -41,7 +42,7 @@ class ChainalysisABRouter(override val routerId: Int,override val workerID:Int, Type("Outgoing Payment") ) ) - commands.toList + commands } //converts the line into a case class which has all of the data via the correct name and type diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/DashcoinRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/DashcoinRouter.scala index 8eee81a2b2..e12068d98a 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/DashcoinRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/DashcoinRouter.scala @@ -6,12 +6,13 @@ import com.raphtory.spouts.blockchain.BitcoinTransaction import spray.json.JsArray import scala.collection.mutable.ListBuffer +import scala.collection.parallel.mutable.ParHashSet import scala.util.hashing.MurmurHash3 class DashcoinRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) extends RouterWorker[BitcoinTransaction](routerId,workerID, initialManagerCount,initialRouterCount) { - override protected def parseTuple(tuple: BitcoinTransaction): List[GraphUpdate] = { + override protected def parseTuple(tuple: BitcoinTransaction): ParHashSet[GraphUpdate] = { val transaction = tuple.transaction val time = tuple.time val blockID = tuple.blockID @@ -25,7 +26,7 @@ class DashcoinRouter(override val routerId: Int,override val workerID:Int, overr val locktime = transaction.asJsObject.fields("locktime") val version = transaction.asJsObject.fields("version") var total: Double = 0 - val commands = new ListBuffer[GraphUpdate]() + val commands = new ParHashSet[GraphUpdate]() for (vout <- vouts.asInstanceOf[JsArray].elements) { val voutOBJ = vout.asJsObject() var value = voutOBJ.fields("value").toString @@ -110,7 +111,7 @@ class DashcoinRouter(override val routerId: Int,override val workerID:Int, overr ) } - commands.toList + commands } } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumGethRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumGethRouter.scala index e760f673fc..cac1981f39 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumGethRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumGethRouter.scala @@ -8,6 +8,7 @@ import akka.stream.ActorMaterializer import spray.json._ import scala.collection.mutable.ListBuffer +import scala.collection.parallel.mutable.ParHashSet import scala.util.hashing.MurmurHash3 class EthereumGethRouter(override val routerId: Int,override val workerID:Int,override val initialManagerCount: Int, override val initialRouterCount: Int) extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount, initialRouterCount) { @@ -15,7 +16,7 @@ class EthereumGethRouter(override val routerId: Int,override val workerID:Int,ov print(routerId) - override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { + override protected def parseTuple(tuple: StringSpoutGoing): ParHashSet[GraphUpdate] = { print(tuple) val transaction = tuple.value.split(",") val blockNumber = hexToInt(transaction(0)) @@ -28,7 +29,7 @@ class EthereumGethRouter(override val routerId: Int,override val workerID:Int,ov print(from) print(to) - val commands = new ListBuffer[GraphUpdate]() + val commands = new ParHashSet[GraphUpdate]() commands+=( VertexAddWithProperties(blockNumber, sourceNode, properties = Properties(ImmutableProperty("id", from))) @@ -44,7 +45,7 @@ class EthereumGethRouter(override val routerId: Int,override val workerID:Int,ov properties = Properties(StringProperty("value", sent)) ) ) - commands.toList + commands } } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumKafkaRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumKafkaRouter.scala index e9f2eb2664..324b4baeee 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumKafkaRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumKafkaRouter.scala @@ -8,15 +8,16 @@ import akka.stream.ActorMaterializer import spray.json._ import scala.collection.mutable.ListBuffer +import scala.collection.parallel.mutable.ParHashSet import scala.util.hashing.MurmurHash3 class EthereumKafkaRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount, initialRouterCount) { def hexToInt(hex: String) = Integer.parseInt(hex.drop(2), 16) - override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { + override protected def parseTuple(tuple: StringSpoutGoing): ParHashSet[GraphUpdate] = { val transaction = tuple.value.split(",") - if(transaction(1).equals("block_number")) return List() + if(transaction(1).equals("block_number")) return ParHashSet() print(transaction) @@ -27,7 +28,7 @@ class EthereumKafkaRouter(override val routerId: Int,override val workerID:Int, // val to = transaction(2).replaceAll("\"", "").toLowerCase // val sent = transaction(5).replaceAll("\"", "") - if(transaction(2).equals("block_number")) return List() + if(transaction(2).equals("block_number")) return ParHashSet() val blockNumber = transaction(2).toInt val from = transaction(4).replaceAll("\"", "").toLowerCase @@ -36,7 +37,7 @@ class EthereumKafkaRouter(override val routerId: Int,override val workerID:Int, val sourceNode = assignID(from) //hash the id to get a vertex ID val destinationNode = assignID(to) //hash the id to get a vertex ID - val commands = new ListBuffer[GraphUpdate]() + val commands = new ParHashSet[GraphUpdate]() commands+=( VertexAddWithProperties(blockNumber, sourceNode, properties = Properties(ImmutableProperty("id", from))) @@ -52,7 +53,7 @@ class EthereumKafkaRouter(override val routerId: Int,override val workerID:Int, properties = Properties(StringProperty("value", sent)) ) ) - commands.toList + commands } } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumTransactionRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumTransactionRouter.scala index f19d4b74db..3b0fa3a405 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumTransactionRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumTransactionRouter.scala @@ -4,16 +4,17 @@ import com.raphtory.core.components.Router.RouterWorker import com.raphtory.core.model.communication.{EdgeAddWithProperties, GraphUpdate, Properties, StringProperty, StringSpoutGoing, VertexAddWithProperties} import scala.collection.mutable.ListBuffer +import scala.collection.parallel.mutable.ParHashSet import scala.util.hashing.MurmurHash3 class EthereumTransactionRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount, initialRouterCount) { - override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { + override protected def parseTuple(tuple: StringSpoutGoing): ParHashSet[GraphUpdate] = { val components = tuple.value.drop(1).dropRight(1).split(",") val creationDate = components(3).toLong * 1000 //seconds to miliseconds val sourceNode = MurmurHash3.stringHash(components(0)) //hash the id to get a vertex ID - val commands = new ListBuffer[GraphUpdate]() + val commands = new ParHashSet[GraphUpdate]() commands+=( VertexAddWithProperties(creationDate, sourceNode, Properties(StringProperty("id", components(0)))) ) //create the source node and add the wallet ID as a property @@ -42,6 +43,6 @@ class EthereumTransactionRouter(override val routerId: Int,override val workerID ) ) } - commands.toList + commands } } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/FirehoseKafkaRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/FirehoseKafkaRouter.scala index 9f9309248d..8419a8fde0 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/FirehoseKafkaRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/FirehoseKafkaRouter.scala @@ -8,6 +8,7 @@ import akka.stream.ActorMaterializer import spray.json._ import scala.collection.mutable.ListBuffer +import scala.collection.parallel.mutable.ParHashSet import scala.util.Random import scala.util.hashing.MurmurHash3 import scala.math.BigInt @@ -18,12 +19,12 @@ class FirehoseKafkaRouter(override val routerId: Int,override val workerID:Int, val random = new Random(DELETESEED) def hexToInt(hex: String) = Integer.parseInt(hex.drop(2), 16) - override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { + override protected def parseTuple(tuple: StringSpoutGoing): ParHashSet[GraphUpdate] = { //if(value.toString.contains("0xa09871aeadf4994ca12f5c0b6056bbd1d343c029")) println(value.toString) val transaction = tuple.value.split(",") - if(transaction(1).equals("block_number")) return List() + if(transaction(1).equals("block_number")) return ParHashSet() val blockNumber = transaction(2).toInt - val commands = new ListBuffer[GraphUpdate]() + val commands = new ParHashSet[GraphUpdate]() val from = transaction(4).replaceAll("\"", "").toLowerCase val to = transaction(5).replaceAll("\"", "").toLowerCase @@ -43,7 +44,7 @@ class FirehoseKafkaRouter(override val routerId: Int,override val workerID:Int, commands+=(EdgeAddWithProperties(blockNumber, sourceNode, destinationNode,properties = Properties(DoubleProperty("value", sent)))) // if(random.nextDouble()<=DELETEPERCENT) // sendGraphUpdate(EdgeDelete(blockNumber+1,sourceNode,destinationNode)) - commands.toList + commands } } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/LitecoinRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/LitecoinRouter.scala index 5eb7f15c40..483e171a27 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/LitecoinRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/LitecoinRouter.scala @@ -6,12 +6,13 @@ import com.raphtory.spouts.blockchain.BitcoinTransaction import spray.json.JsArray import scala.collection.mutable.ListBuffer +import scala.collection.parallel.mutable.ParHashSet import scala.util.hashing.MurmurHash3 class LitecoinRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) extends RouterWorker[BitcoinTransaction](routerId,workerID,initialManagerCount, initialRouterCount) { - override protected def parseTuple(tuple:BitcoinTransaction): List[GraphUpdate] = { + override protected def parseTuple(tuple:BitcoinTransaction): ParHashSet[GraphUpdate] = { val transaction = tuple.transaction val time = tuple.time val blockID = tuple.blockID @@ -25,7 +26,7 @@ class LitecoinRouter(override val routerId: Int,override val workerID:Int, overr val locktime = transaction.asJsObject.fields("locktime") val version = transaction.asJsObject.fields("version") var total: Double = 0 - val commands = new ListBuffer[GraphUpdate]() + val commands = new ParHashSet[GraphUpdate]() for (vout <- vouts.asInstanceOf[JsArray].elements) { val voutOBJ = vout.asJsObject() var value = voutOBJ.fields("value").toString @@ -117,7 +118,7 @@ class LitecoinRouter(override val routerId: Int,override val workerID:Int, overr } } - commands.toList + commands } } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/citationNetwork/CitationRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/citationNetwork/CitationRouter.scala index 02eb85a361..f5b3d9da09 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/citationNetwork/CitationRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/citationNetwork/CitationRouter.scala @@ -6,10 +6,11 @@ import com.raphtory.core.components.Router.RouterWorker import com.raphtory.core.model.communication._ import scala.collection.mutable.ListBuffer +import scala.collection.parallel.mutable.ParHashSet class CitationRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount, initialRouterCount) { - override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { + override protected def parseTuple(tuple: StringSpoutGoing): ParHashSet[GraphUpdate] = { val fileLine = tuple.value.split(",").map(_.trim) //take the tuple and split on , as we are only interested in the first 4 fields // title_paper,year,volume,title,pages,number,journal,author,ENTRYTYPE,ID val sourceTitle = fileLine(0) @@ -19,7 +20,7 @@ class CitationRouter(override val routerId: Int,override val workerID:Int, overr val sourceID = assignID(sourceTitle) val destinationID = assignID(destinationTitle) - val commands = new ListBuffer[GraphUpdate]() + val commands = new ParHashSet[GraphUpdate]() //create sourceNode commands+=(VertexAddWithProperties( sourceYear, //when it happened ?? @@ -43,6 +44,6 @@ class CitationRouter(override val routerId: Int,override val workerID:Int, overr Type("Cited") // edge type )) - commands.toList + commands } } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabPostGraphRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabPostGraphRouter.scala index 215ffbdcc9..0286c3285e 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabPostGraphRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabPostGraphRouter.scala @@ -6,6 +6,7 @@ import com.raphtory.core.components.Router.RouterWorker import com.raphtory.core.model.communication._ import scala.collection.mutable.ListBuffer +import scala.collection.parallel.mutable.ParHashSet // The lines sent by the Gab mining spout are read and processed accordingly. //In this router we needed to transform the data that was sent by the spout by turning it into a epoch value (long value) @@ -17,12 +18,12 @@ import scala.collection.mutable.ListBuffer class GabPostGraphRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount, initialRouterCount) { - override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { + override protected def parseTuple(tuple: StringSpoutGoing): ParHashSet[GraphUpdate] = { val fileLine = tuple.value.split(";").map(_.trim) //user wise // val sourceNode=fileLine(2).toInt // val targetNode=fileLine(5).toInt - val commands = new ListBuffer[GraphUpdate]() + val commands = new ParHashSet[GraphUpdate]() //comment wise val sourceNode = fileLine(1).toInt val targetNode = fileLine(4).toInt @@ -33,7 +34,7 @@ class GabPostGraphRouter(override val routerId: Int,override val workerID:Int, o commands+=(VertexAdd(creationDate, targetNode)) commands+=(EdgeAdd(creationDate, sourceNode, targetNode)) } - commands.toList + commands } def dateToUnixTime(timestamp: => String): Long = { diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabRawRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabRawRouter.scala index 8d13d43c89..fbe78cf8eb 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabRawRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabRawRouter.scala @@ -8,6 +8,7 @@ import com.raphtory.examples.gab.rawgraphmodel.GabPost import spray.json._ import scala.collection.mutable.ListBuffer +import scala.collection.parallel.mutable.ParHashSet /** * The Graph Manager is the top level actor in this system (under the stream) @@ -26,8 +27,8 @@ final class GabRawRouter(override val routerId: Int,override val workerID:Int, o private val nullStr = "null" - override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { - val commands = new ListBuffer[GraphUpdate]() + override protected def parseTuple(tuple: StringSpoutGoing): ParHashSet[GraphUpdate] = { + val commands = new ParHashSet[GraphUpdate]() try { val command = tuple.value val post = command.parseJson.convertTo[GabPost] @@ -133,7 +134,7 @@ final class GabRawRouter(override val routerId: Int,override val workerID:Int, o } - commands.toList + commands } } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabUserGraphRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabUserGraphRouter.scala index 11f34e98e6..66e5c1b579 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabUserGraphRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabUserGraphRouter.scala @@ -7,6 +7,7 @@ import com.raphtory.core.model.communication.Type import com.raphtory.core.model.communication._ import scala.collection.mutable.ListBuffer +import scala.collection.parallel.mutable.ParHashSet import scala.util.Random // The lines sent by the Gab mining spout are read and processed accordingly. @@ -19,12 +20,12 @@ import scala.util.Random class GabUserGraphRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount, initialRouterCount) { - override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { + override protected def parseTuple(tuple: StringSpoutGoing): ParHashSet[GraphUpdate] = { val fileLine = tuple.value.split(";").map(_.trim) //user wise val sourceNode = fileLine(2).toInt val targetNode = fileLine(5).toInt - val commands = new ListBuffer[GraphUpdate]() + val commands = new ParHashSet[GraphUpdate]() //comment wise // val sourceNode=fileLine(1).toInt //val targetNode=fileLine(4).toInt @@ -37,7 +38,7 @@ class GabUserGraphRouter(override val routerId: Int,override val workerID:Int, o // sendGraphUpdate(VertexAddWithProperties(creationDate, targetNode, Properties(StringProperty("test1","value1"),StringProperty("test2","Value2")),Type("User"))) // sendGraphUpdate(EdgeAddWithProperties(creationDate, sourceNode, targetNode, Properties(StringProperty("test1","value1"),StringProperty("test2","Value2")),Type("User To User"))) } - commands.toList + commands } def dateToUnixTime(timestamp: => String): Long = { diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCOldRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCOldRouter.scala index 6af88f2931..3053409d5d 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCOldRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCOldRouter.scala @@ -7,13 +7,14 @@ import com.raphtory.core.components.Router.RouterWorker import com.raphtory.core.model.communication.{EdgeAdd, EdgeDelete, GraphUpdate, StringSpoutGoing, Type, VertexAdd, VertexDelete} import scala.collection.mutable.ListBuffer +import scala.collection.parallel.mutable.ParHashSet class LDBCOldRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount, initialRouterCount) { - override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { + override protected def parseTuple(tuple: StringSpoutGoing): ParHashSet[GraphUpdate] = { val fileLine = tuple.asInstanceOf[String].split("\\|") - val commands = new ListBuffer[GraphUpdate]() + val commands = new ParHashSet[GraphUpdate]() //val deletionDate:Long = new SimpleDateFormat("yyyy-MM-ddHH:mm:ss.SSS").parse(date2).getTime() fileLine(0) match { case "person" => @@ -21,7 +22,7 @@ class LDBCOldRouter(override val routerId: Int,override val workerID:Int, overri //val date2 = fileLine(2).substring(0, 10) + fileLine(1).substring(11, 23); //extract the day of the event val creationDate: Long = new SimpleDateFormat("yyyy-MM-ddHH:mm:ss.SSS").parse(date).getTime() commands+=(VertexAdd(creationDate, assignID("person" + fileLine(1)), Type("person"))) - commands.toList + commands //sendGraphUpdate(VertexAdd(creationDate, fileLine(3).toLong,Type("person"))) // sendGraphUpdate(VertexDelete(deletionDate, assignID("person"+fileLine(3)))) case "person_knows_person" => @@ -37,7 +38,7 @@ class LDBCOldRouter(override val routerId: Int,override val workerID:Int, overri Type("person_knows_person") ) ) - commands.toList + commands //sendGraphUpdate(EdgeDelete(deletionDate, assignID("person"+fileLine(3)),assignID("person"+fileLine(4)))) } } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCRouter.scala index 0744940433..67314e21be 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCRouter.scala @@ -8,11 +8,12 @@ import com.raphtory.core.model.communication.{EdgeAdd, EdgeDelete, GraphUpdate, import com.raphtory.examples.test.actors.RandomSpout import scala.collection.mutable.ListBuffer +import scala.collection.parallel.mutable.ParHashSet class LDBCRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount, initialRouterCount) { - override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { - val commands = new ListBuffer[GraphUpdate]() + override protected def parseTuple(tuple: StringSpoutGoing): ParHashSet[GraphUpdate] = { + val commands = new ParHashSet[GraphUpdate]() val fileLine = tuple.asInstanceOf[String].split("\\|") val date = fileLine(1).substring(0, 10) + fileLine(1).substring(11, 23); //extract the day of the event val date2 = fileLine(2).substring(0, 10) + fileLine(1).substring(11, 23); //extract the day of the event @@ -39,7 +40,7 @@ class LDBCRouter(override val routerId: Int,override val workerID:Int, override if(edgeDeletion) commands+=(EdgeDelete(deletionDate, assignID("person"+fileLine(3)),assignID("person"+fileLine(4)))) } - commands.toList + commands } } //2012-11-01T09:28:01.185+00:00|2019-07-22T11:24:24.362+00:00|35184372093644|Jose|Garcia|female|1988-05-20|111.68.47.44|Firefox diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRRouter.scala index 1b2f91ab58..f0f9f2a821 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRRouter.scala @@ -4,15 +4,16 @@ import com.raphtory.core.components.Router.RouterWorker import com.raphtory.core.model.communication.{EdgeAdd, Type, VertexAdd, _} import scala.collection.mutable.ListBuffer +import scala.collection.parallel.mutable.ParHashSet import scala.util.Random class LOTRRouter(override val routerId: Int, override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount, initialRouterCount) { - override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { + override protected def parseTuple(tuple: StringSpoutGoing): ParHashSet[GraphUpdate] = { val fileLine = tuple.value.split(",").map(_.trim) - val commands = new ListBuffer[GraphUpdate]() + val commands = new ParHashSet[GraphUpdate]() val sourceNode = fileLine(0) val srcID = assignID(sourceNode) @@ -24,6 +25,6 @@ class LOTRRouter(override val routerId: Int, override val workerID:Int, override commands+=(VertexAddWithProperties(timeStamp, srcID, Properties(LongProperty("test",Random.nextLong()),ImmutableProperty("name",sourceNode)),Type("Character"))) commands+=(VertexAddWithProperties(timeStamp, tarID, Properties(LongProperty("test2",Random.nextLong()),ImmutableProperty("name",targetNode)),Type("Character"))) commands+=(EdgeAdd(timeStamp,srcID,tarID, Type("Character Co-occurence"))) - commands.toList + commands } } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomRouter.scala index 201968c93e..1e259e61ab 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomRouter.scala @@ -4,6 +4,8 @@ import com.raphtory.core.components.Router.RouterWorker import com.raphtory.core.model.communication.{GraphUpdate, _} import spray.json._ +import scala.collection.parallel.mutable.ParHashSet + /** * The Graph Manager is the top level actor in this system (under the stream) * which tracks all the graph partitions - passing commands processed by the 'command processor' actors @@ -19,21 +21,21 @@ class RandomRouter(override val routerId: Int,override val workerID:Int, overrid //************* MESSAGE HANDLING BLOCK - override protected def parseTuple(tuple:StringSpoutGoing): List[GraphUpdate] = { + override protected def parseTuple(tuple:StringSpoutGoing): ParHashSet[GraphUpdate] = { val command = tuple.asInstanceOf[String] val parsedOBJ = command.parseJson.asJsObject //get the json object val commandKey = parsedOBJ.fields //get the command type if (commandKey.contains("VertexAdd")) - List(vertexAdd(parsedOBJ.getFields("VertexAdd").head.asJsObject)) + ParHashSet(vertexAdd(parsedOBJ.getFields("VertexAdd").head.asJsObject)) //else if(commandKey.contains("VertexUpdateProperties")) vertexUpdateProperties(parsedOBJ.getFields("VertexUpdateProperties").head.asJsObject) else if (commandKey.contains("VertexRemoval")) - List(vertexRemoval(parsedOBJ.getFields("VertexRemoval").head.asJsObject)) + ParHashSet(vertexRemoval(parsedOBJ.getFields("VertexRemoval").head.asJsObject)) else if (commandKey.contains("EdgeAdd")) - List(edgeAdd(parsedOBJ.getFields("EdgeAdd").head.asJsObject)) //if addVertex, parse to handling function + ParHashSet(edgeAdd(parsedOBJ.getFields("EdgeAdd").head.asJsObject)) //if addVertex, parse to handling function // else if(commandKey.contains("EdgeUpdateProperties")) edgeUpdateProperties(parsedOBJ.getFields("EdgeUpdateProperties").head.asJsObject) else if (commandKey.contains("EdgeRemoval")) - List(edgeRemoval(parsedOBJ.getFields("EdgeRemoval").head.asJsObject)) - else List() + ParHashSet(edgeRemoval(parsedOBJ.getFields("EdgeRemoval").head.asJsObject)) + else ParHashSet() } def vertexAdd(command: JsObject) = { diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleTestRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleTestRouter.scala index 5ff7a38ed8..db8ecc2e64 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleTestRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleTestRouter.scala @@ -5,14 +5,15 @@ import com.raphtory.core.components.Spout.SpoutTrait import com.raphtory.core.model.communication.{EdgeAdd, GraphUpdate, StringSpoutGoing, VertexAddWithProperties} import scala.collection.mutable.ListBuffer +import scala.collection.parallel.mutable.ParHashSet class TriangleTestRouter (override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount, initialRouterCount) { - override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { + override protected def parseTuple(tuple: StringSpoutGoing): ParHashSet[GraphUpdate] = { val command = tuple.value.split(",") val msgTime = command(0).toLong val sourceID = command(1).toLong val destID = command(2).toLong - List((EdgeAdd(msgTime, sourceID, destID))) + ParHashSet((EdgeAdd(msgTime, sourceID, destID))) } } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/trackAndTrace/routers/TrackAndTraceRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/trackAndTrace/routers/TrackAndTraceRouter.scala index 886d2a6ea7..733e026a7e 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/trackAndTrace/routers/TrackAndTraceRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/trackAndTrace/routers/TrackAndTraceRouter.scala @@ -7,6 +7,7 @@ import com.raphtory.core.model.communication.Type import com.raphtory.core.model.communication._ import scala.collection.mutable.ListBuffer +import scala.collection.parallel.mutable.ParHashSet import scala.util.control.Breaks._ class TrackAndTraceRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) @@ -15,14 +16,14 @@ class TrackAndTraceRouter(override val routerId: Int,override val workerID:Int, val EARTH_POL = 6356752.3142 //m val STEPSIZE = System.getenv().getOrDefault("MAP_GRID_SIZE", "100").trim.toDouble //m - override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { + override protected def parseTuple(tuple: StringSpoutGoing): ParHashSet[GraphUpdate] = { val datapoint = lineToDatapoint(tuple.value.split(",").map(_.trim)) val eventTime = datapoint.time val userID = datapoint.userId val latitude = datapoint.latitude val longitude = datapoint.longitude val locationID = locationIDGenerator(latitude, longitude) - val commands = new ListBuffer[GraphUpdate]() + val commands = new ParHashSet[GraphUpdate]() commands+=(VertexAdd(eventTime, userID, Type("User"))) commands+=( @@ -35,7 +36,7 @@ class TrackAndTraceRouter(override val routerId: Int,override val workerID:Int, ) commands+=(EdgeAdd(eventTime, userID, locationID, Type("User Visted Location"))) - commands.toList + commands } //converts the line into a case class which has all of the data via the correct name and type diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/TSVRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/TSVRouter.scala index c68d633ced..3a75601c64 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/TSVRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/TSVRouter.scala @@ -4,12 +4,13 @@ import com.raphtory.core.components.Router.RouterWorker import com.raphtory.core.model.communication.{EdgeAdd, GraphUpdate, StringSpoutGoing, Type, VertexAdd} import scala.collection.mutable.ListBuffer +import scala.collection.parallel.mutable.ParHashSet /** Spout for network datasets of the form SRC_NODE_ID DEST_NODE_ID TIMESTAMP */ class TSVRouter(override val routerId: Int, override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount, initialRouterCount) { - val commands = new ListBuffer[GraphUpdate]() - override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { + val commands = new ParHashSet[GraphUpdate]() + override protected def parseTuple(tuple: StringSpoutGoing): ParHashSet[GraphUpdate] = { val fileLine = tuple.value.split(" ").map(_.trim) //user wise val sourceNode = fileLine(0).toInt @@ -28,6 +29,6 @@ class TSVRouter(override val routerId: Int, override val workerID:Int, override commands+=(EdgeAdd(creationDate, sourceNode, targetNode, Type("User to User"))) } - commands.toList + commands } } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixRouter.scala index af38f931a7..f83a114c5a 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixRouter.scala @@ -4,14 +4,15 @@ import com.raphtory.core.components.Router.RouterWorker import com.raphtory.core.model.communication.{Type, _} import scala.collection.mutable.ListBuffer +import scala.collection.parallel.mutable.ParHashSet class CooccurrenceMatrixRouter(override val routerId: Int, override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount, initialRouterCount) { - override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { + override protected def parseTuple(tuple: StringSpoutGoing): ParHashSet[GraphUpdate] = { //println(record) var dp = tuple.value.split(" ").map(_.trim) - val commands = new ListBuffer[GraphUpdate]() + val commands = new ParHashSet[GraphUpdate]() val occurenceTime = dp.head.toLong//DateFormatting(dp.head) //.slice(4, dp.head.length) try { dp = dp.last.split("\t") @@ -35,7 +36,7 @@ class CooccurrenceMatrixRouter(override val routerId: Int, override val workerID }catch { case e: Exception => println(e, dp.length, tuple.value) } - commands.toList + commands } def DateFormatting(date: String): Long = { diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixRouterFiltered.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixRouterFiltered.scala index 32db76b97b..cc9d3795b7 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixRouterFiltered.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixRouterFiltered.scala @@ -4,17 +4,18 @@ import com.raphtory.core.components.Router.RouterWorker import com.raphtory.core.model.communication._ import scala.collection.mutable.ListBuffer +import scala.collection.parallel.mutable.ParHashSet class CooccurrenceMatrixRouterFiltered(override val routerId: Int, override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount, initialRouterCount) { val THR = System.getenv().getOrDefault("COOC_FREQ_THRESHOLD ", "0.05").trim.toDouble - override protected def parseTuple(tuple: StringSpoutGoing): List[GraphUpdate] = { + override protected def parseTuple(tuple: StringSpoutGoing): ParHashSet[GraphUpdate] = { //println(record) var dp =tuple.value.split(" ").map(_.trim) val occurenceTime = dp.head.toLong//DateFormatting(dp.head) //.slice(4, dp.head.length) val scale = dp(1).toDouble - val commands = new ListBuffer[GraphUpdate]() + val commands = new ParHashSet[GraphUpdate]() try { dp = dp.last.split("\t") val srcClusterId = assignID(dp.head) @@ -38,6 +39,6 @@ class CooccurrenceMatrixRouterFiltered(override val routerId: Int, override val }catch { case e: Exception => println(e, dp.length, tuple.asInstanceOf[String]) } - commands.toList + commands } } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/spouts/FileSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/spouts/FileSpout.scala index 32b139d8b4..3ece489841 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/spouts/FileSpout.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/spouts/FileSpout.scala @@ -21,7 +21,7 @@ final case class FileSpout() extends SpoutTrait[FileDomain, StringSpoutGoing] { private val directory = System.getenv().getOrDefault("FILE_SPOUT_DIRECTORY", "/app").trim private val fileName = System.getenv().getOrDefault("FILE_SPOUT_FILENAME", "").trim //gabNetwork500.csv private val dropHeader = System.getenv().getOrDefault("FILE_SPOUT_DROP_HEADER", "false").trim.toBoolean - private val JUMP = System.getenv().getOrDefault("FILE_SPOUT_BLOCK_SIZE", "10").trim.toInt + private val JUMP = System.getenv().getOrDefault("FILE_SPOUT_BLOCK_SIZE", "1000").trim.toInt private val INCREMENT = System.getenv().getOrDefault("FILE_SPOUT_INCREMENT", "0").trim.toInt private val TIME = System.getenv().getOrDefault("FILE_SPOUT_TIME", "60").trim.toInt @@ -30,23 +30,14 @@ final case class FileSpout() extends SpoutTrait[FileDomain, StringSpoutGoing] { def startSpout(): Unit = { self ! NextLineBlock - context.system.scheduler.scheduleOnce(TIME.seconds, self, Increase) } def handleDomainMessage(message: FileDomain): Unit = message match { - case Increase => - if (fileManager.allCompleted) { - println("All files read1-" + (System.nanoTime() - t0)) - dataFinished() - } - else { - fileManager = fileManager.increaseBlockSize(INCREMENT) - context.system.scheduler.scheduleOnce(TIME.seconds, self, Increase) - } - case NextLineBlock => - if (fileManager.allCompleted) + if (fileManager.allCompleted){ + dataFinished() println("All files read2-" + (System.nanoTime() - t0)) + } else { val (newFileManager, block) = fileManager.nextLineBlock() fileManager = newFileManager @@ -54,8 +45,10 @@ final case class FileSpout() extends SpoutTrait[FileDomain, StringSpoutGoing] { self ! NextLineBlock } case NextFile => - if (fileManager.allCompleted) + if (fileManager.allCompleted){ println("All files read3-" + (System.nanoTime() - t0)) + dataFinished() + } else { fileManager = fileManager.nextFile() self ! NextLineBlock From 36c5dec0e5e75d0efa74f576c83f5a3e65f8683b Mon Sep 17 00:00:00 2001 From: Miratepuffin Date: Tue, 10 Nov 2020 03:22:48 +0000 Subject: [PATCH 26/58] Fixed watermarking for now, but having strnage double update in queue --- .../Workers/IngestionWorker.scala | 30 ++++++++++++------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/IngestionWorker.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/IngestionWorker.scala index 17dac21d71..8469d3018d 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/IngestionWorker.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/IngestionWorker.scala @@ -291,24 +291,32 @@ class IngestionWorker(workerId: Int,partitionID:Int, storage: EntityStorage) ext } private def setSafePoint(routerName:String,messageQueue:mutable.PriorityQueue[queueItem]) = { - val default = queueItem(-1,0) + var currentSafePoint = safeMessageMap.get(routerName) match { case Some(value) => value - case None => default + case None => queueItem(-1,0) } + recursiveDequeue(routerName,messageQueue,currentSafePoint) + } + + def recursiveDequeue(routerName:String,messageQueue:mutable.PriorityQueue[queueItem],currentSafePoint:queueItem): queueItem ={ if(messageQueue nonEmpty) - while(messageQueue.headOption.getOrElse(default).routerEpoch==currentSafePoint.routerEpoch+1) { - currentSafePoint = messageQueue.dequeue() - if(workerId==1 && (routerName equals "0_3")) - println(currentSafePoint) + if(messageQueue.head.routerEpoch==currentSafePoint.routerEpoch+1) + recursiveDequeue(routerName,messageQueue,messageQueue.dequeue()) + else if(messageQueue.head.routerEpoch==currentSafePoint.routerEpoch) + recursiveDequeue(routerName,messageQueue,messageQueue.dequeue()) + else { + if(workerId==1) + println(s"$increments Writer Worker $partitionID $workerId --- ${safeMessageMap.get(routerName).get} $currentSafePoint ${messageQueue.head.routerEpoch}") + safeMessageMap put(routerName, currentSafePoint) + currentSafePoint } - safeMessageMap put (routerName,currentSafePoint) - - currentSafePoint + else { + safeMessageMap put(routerName, currentSafePoint) + currentSafePoint + } } - def recursiveDequeue(q:mutable.PriorityQueue[queueItem]) - private def processRouterTimeSync(req:RouterWorkerTimeSync) ={ storage.timings(req.msgTime) addToWatermarkQueue(req.routerID,req.routerTime,req.msgTime,0) From 91231d4b70df45847b6062f2df5c5fa55f8c6214 Mon Sep 17 00:00:00 2001 From: Miratepuffin Date: Tue, 10 Nov 2020 04:25:58 +0000 Subject: [PATCH 27/58] Fixed watermarking a little bit more, who knows its 4am --- .../API/entityVisitors/EntityVisitor.scala | 6 ++-- .../Workers/IngestionWorker.scala | 33 ++++++++----------- 2 files changed, 16 insertions(+), 23 deletions(-) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/EntityVisitor.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/EntityVisitor.scala index 75dcc5f60d..1e18548075 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/EntityVisitor.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/EntityVisitor.scala @@ -18,9 +18,9 @@ abstract class EntityVisitor(entity:Entity,viewJob:ViewJob) { val x =entity.properties.filter(p =>{ p._2.creation()<=viewJob.timestamp }).map(f => (f._1,f._2.valueAt(viewJob.timestamp))) - if(x.isEmpty&&this.isInstanceOf[VertexVisitor]){ - println(viewJob) - } +// if(x.isEmpty&&this.isInstanceOf[VertexVisitor]){ +// println(viewJob) +// } x } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/IngestionWorker.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/IngestionWorker.scala index 8469d3018d..6b046e38e7 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/IngestionWorker.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/IngestionWorker.scala @@ -272,7 +272,6 @@ class IngestionWorker(workerId: Int,partitionID:Int, storage: EntityStorage) ext setSafePoint(queue._1, queue._2) }) val timestamps = queueState.map(q => q.timestamp) - if(workerId==1) println(s"$increments Writer Worker $partitionID $workerId $timestamps") //println(s"Writer Worker $partitionID $workerId ${queueState.mkString("[",",","]")} ${storage.vertices.size}") @@ -289,32 +288,26 @@ class IngestionWorker(workerId: Int,partitionID:Int, storage: EntityStorage) ext safeTime.update(storage.windowTime) } } - + import scala.util.control.Breaks._ private def setSafePoint(routerName:String,messageQueue:mutable.PriorityQueue[queueItem]) = { var currentSafePoint = safeMessageMap.get(routerName) match { case Some(value) => value case None => queueItem(-1,0) } - recursiveDequeue(routerName,messageQueue,currentSafePoint) - } - - def recursiveDequeue(routerName:String,messageQueue:mutable.PriorityQueue[queueItem],currentSafePoint:queueItem): queueItem ={ - if(messageQueue nonEmpty) - if(messageQueue.head.routerEpoch==currentSafePoint.routerEpoch+1) - recursiveDequeue(routerName,messageQueue,messageQueue.dequeue()) - else if(messageQueue.head.routerEpoch==currentSafePoint.routerEpoch) - recursiveDequeue(routerName,messageQueue,messageQueue.dequeue()) - else { - if(workerId==1) - println(s"$increments Writer Worker $partitionID $workerId --- ${safeMessageMap.get(routerName).get} $currentSafePoint ${messageQueue.head.routerEpoch}") - safeMessageMap put(routerName, currentSafePoint) - currentSafePoint - } - else { - safeMessageMap put(routerName, currentSafePoint) - currentSafePoint + breakable { + while (messageQueue nonEmpty) + if (messageQueue.head.routerEpoch == currentSafePoint.routerEpoch + 1) + currentSafePoint = messageQueue.dequeue() + else if (messageQueue.head.routerEpoch == currentSafePoint.routerEpoch) + currentSafePoint = messageQueue.dequeue() + else { + //println(s"$increments Writer Worker $partitionID $workerId --- ${safeMessageMap.get(routerName).get} $currentSafePoint ${messageQueue.head.routerEpoch}") + break + } } + safeMessageMap put(routerName, currentSafePoint) + currentSafePoint } private def processRouterTimeSync(req:RouterWorkerTimeSync) ={ From 780d4b3afb7a83a5223996062537a4cbd3fe64a6 Mon Sep 17 00:00:00 2001 From: Miratepuffin Date: Wed, 11 Nov 2020 22:29:34 +0000 Subject: [PATCH 28/58] Syncing for Yanyang --- mainproject/build.sbt | 7 +++- .../Algorithms/ConnectedComponents.scala | 5 +++ .../analysis/Algorithms/DegreeBasic.scala | 7 ++-- .../analysis/serialisers/S3Serialiser.scala | 36 +++++++++++++++++++ .../Workers/IngestionWorker.scala | 3 +- .../core/components/Router/RouterWorker.scala | 7 ++-- .../core/components/Spout/SpoutTrait.scala | 4 +-- .../scala/com/raphtory/spouts/FileSpout.scala | 2 +- 8 files changed, 60 insertions(+), 11 deletions(-) create mode 100644 mainproject/cluster/src/main/scala/com/raphtory/core/analysis/serialisers/S3Serialiser.scala diff --git a/mainproject/build.sbt b/mainproject/build.sbt index 2f6b25415c..7160ab72e7 100644 --- a/mainproject/build.sbt +++ b/mainproject/build.sbt @@ -66,6 +66,10 @@ val lift = "net.liftweb" %% "lift-json" % "3.3.0" val bitcoin = "org.scalaj" %% "scalaj-http" % "2.3.0" val twitter_eval = "com.twitter" %% "util-eval" % "6.43.0" +// https://mvnrepository.com/artifact/com.amazonaws/aws-java-sdk +val aws = "com.amazonaws" % "aws-java-sdk" % "1.11.897" + + val IP = java.net.InetAddress.getLocalHost.getHostAddress lazy val basicSettings = Seq( @@ -150,7 +154,8 @@ lazy val cluster = project doobiepostgres, joda, casbah, - mongo + mongo, + aws ) ) .settings( diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/ConnectedComponents.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/ConnectedComponents.scala index 7154931564..25957260f8 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/ConnectedComponents.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/ConnectedComponents.scala @@ -33,7 +33,12 @@ class ConnectedComponents(args:Array[String]) extends Analyser(args){ override def processResults(results: ArrayBuffer[Any], timestamp: Long, viewCompleteTime: Long): Unit = { val er = extractData(results) val text = s"""{"time":$timestamp,"top5":[${er.top5.mkString(",")}],"total":${er.total},"totalIslands":${er.totalIslands},"proportion":${er.proportion},"clustersGT2":${er.totalGT2},"viewTime":$viewCompleteTime}""" + var output_folder = System.getenv().getOrDefault("OUTPUT_FOLDER", "/app").trim + var output_file = output_folder + "/" + System.getenv().getOrDefault("OUTPUT_FILE","ConnectedComponents.json").trim + println(text) + Utils.writeLines(output_file, text, "{\"views\":[") + publishData(text) } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/DegreeBasic.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/DegreeBasic.scala index 0592ce8cb8..cf119aadf2 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/DegreeBasic.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/DegreeBasic.scala @@ -42,8 +42,9 @@ class DegreeBasic(args:Array[String]) extends Analyser(args){ val startTime = System.currentTimeMillis() val text = s"""{"time":$timestamp,"vertices":$totalVert,"edges":$totalEdge,"degree":$degree}""" - - // Utils.writeLines(output_file, text, "") + var output_folder = System.getenv().getOrDefault("OUTPUT_FOLDER", "/app").trim + var output_file = output_folder + "/" + System.getenv().getOrDefault("OUTPUT_FILE","DegreeBasic.json").trim + Utils.writeLines(output_file, text, "") println(text) publishData(text) } @@ -64,7 +65,7 @@ val startTime = System.currentTimeMillis() catch { case e: ArithmeticException => 0 } - val text = s"""{"time":$timestamp,"windowsize":$windowSize,"vertices":$totalVert,"edges":$totalEdge,"degree":$degree}""" + val text = s"""{"time":$timestamp,"windowsize":$windowSize,"vertices":$totalVert,"edges":$totalEdge,"degree":$degree},""" Utils.writeLines(output_file, text, "") println(text) publishData(text) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/serialisers/S3Serialiser.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/serialisers/S3Serialiser.scala new file mode 100644 index 0000000000..7b49e117f1 --- /dev/null +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/serialisers/S3Serialiser.scala @@ -0,0 +1,36 @@ +package com.raphtory.core.analysis.serialisers +import com.amazonaws.auth.BasicAWSCredentials +import java.io.{BufferedWriter, File, FileWriter} +import com.amazonaws.auth.AWSStaticCredentialsProvider +import com.amazonaws.services.s3.AmazonS3 +import com.amazonaws.services.s3.AmazonS3ClientBuilder +import com.amazonaws.regions.Regions + +abstract class S3Serialiser extends Serialiser { + val AWS_ACCESS_KEY = s"${sys.env.getOrElse("AWS_ACCESS_KEY", "")}" + val AWS_SECRET_KEY = s"${sys.env.getOrElse("AWS_SECRET_KEY", "")}" + val bucketName = s"${sys.env.getOrElse("AWS_BUCKET_NAME", "")}" + val yourAWSCredentials = new BasicAWSCredentials(AWS_ACCESS_KEY, AWS_SECRET_KEY) + //val amazonS3Client = new AmazonS3Client(yourAWSCredentials) + + + val s3Client: AmazonS3 = AmazonS3ClientBuilder.standard.withCredentials(new AWSStaticCredentialsProvider(yourAWSCredentials)).build + //s3Client.setRegion(com.amazonaws.regions.Region.getRegion(Regions.EU_WEST_2)) + try{s3Client.createBucket(bucketName)}catch {case e:Exception =>} + + override def write(serialisedResults:(Array[String],Array[String]),file:File) = { + println("vertices & edges", serialisedResults._1.length, serialisedResults._2.length) + val bw = new BufferedWriter(new FileWriter(file)) + bw.write(startOfFile()) + bw.write(serialisedResults._1.mkString(rowDelimeter())) + bw.write(middleOfFile()) + bw.write(serialisedResults._2.mkString(rowDelimeter())) + bw.write(endOfFile()) + bw.newLine() + bw.close() + new Thread(() => { + s3Client.putObject(bucketName, file.getName,file) + file.delete() + }).start() + } +} diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/IngestionWorker.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/IngestionWorker.scala index 6b046e38e7..b942463476 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/IngestionWorker.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/IngestionWorker.scala @@ -302,8 +302,9 @@ class IngestionWorker(workerId: Int,partitionID:Int, storage: EntityStorage) ext else if (messageQueue.head.routerEpoch == currentSafePoint.routerEpoch) currentSafePoint = messageQueue.dequeue() else { + currentSafePoint = messageQueue.dequeue() //println(s"$increments Writer Worker $partitionID $workerId --- ${safeMessageMap.get(routerName).get} $currentSafePoint ${messageQueue.head.routerEpoch}") - break + //break } } safeMessageMap put(routerName, currentSafePoint) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala index 13c4ed40df..29df22e8e4 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala @@ -47,7 +47,7 @@ abstract class RouterWorker[In <: SpoutGoing](val routerId: Int, val workerID: I private def work(managerCount: Int, trackedTime: Long, newestTime: Long): Receive = { case SpoutOnline => context.sender() ! WorkPlease - case NoWork => context.system.scheduler.scheduleOnce(delay = 10.second, receiver = context.sender(), message = WorkPlease) + case NoWork => context.system.scheduler.scheduleOnce(delay = 1.second, receiver = context.sender(), message = WorkPlease) case msg: UpdatedCounter => log.debug(s"RouterWorker [$routerId] received [$msg] request.") if (managerCount < msg.newValue) context.become(work(msg.newValue, trackedTime, newestTime)) @@ -123,8 +123,9 @@ abstract class RouterWorker[In <: SpoutGoing](val routerId: Int, val workerID: I managerCount: Int, trackedMessage: Boolean, trackedTime: Long - ): Option[Long] = - parseTuple(record).map(update => sendGraphUpdate(update, managerCount, trackedMessage, trackedTime)).lastOption + ): Option[Long] =try{ + parseTuple(record).map(update => sendGraphUpdate(update, managerCount, trackedMessage, trackedTime)).lastOption + }catch {case e:Exception => None} private def sendGraphUpdate( message: GraphUpdate, diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/SpoutTrait.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/SpoutTrait.scala index 0537e42169..3ac87bf894 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/SpoutTrait.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/SpoutTrait.scala @@ -122,13 +122,13 @@ trait SpoutTrait[Domain <: DomainMessage, Out <: SpoutGoing] extends Actor with println("All data sent") } else { - val message = if (count % 100 == 0) + val message = if (count % 10000 == 0) AllocateTrackedTuple(System.currentTimeMillis(),work) else AllocateTuple(work) recordUpdate() sender ! message - if (count % 100 == 0) println(s"Spout at Message $count, remaining messages ${workQueue.size}") + if (count % 10000 == 0) println(s"Spout at Message $count, remaining messages ${workQueue.size}") //println(s"Spout at Message $count, remaining messages ${work}") }//mediator ! DistributedPubSubMediator.Send(lastRouter, message, localAffinity = false) }catch { diff --git a/mainproject/cluster/src/main/scala/com/raphtory/spouts/FileSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/spouts/FileSpout.scala index 3ece489841..9a33d6bba0 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/spouts/FileSpout.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/spouts/FileSpout.scala @@ -21,7 +21,7 @@ final case class FileSpout() extends SpoutTrait[FileDomain, StringSpoutGoing] { private val directory = System.getenv().getOrDefault("FILE_SPOUT_DIRECTORY", "/app").trim private val fileName = System.getenv().getOrDefault("FILE_SPOUT_FILENAME", "").trim //gabNetwork500.csv private val dropHeader = System.getenv().getOrDefault("FILE_SPOUT_DROP_HEADER", "false").trim.toBoolean - private val JUMP = System.getenv().getOrDefault("FILE_SPOUT_BLOCK_SIZE", "1000").trim.toInt + private val JUMP = System.getenv().getOrDefault("FILE_SPOUT_BLOCK_SIZE", "10").trim.toInt private val INCREMENT = System.getenv().getOrDefault("FILE_SPOUT_INCREMENT", "0").trim.toInt private val TIME = System.getenv().getOrDefault("FILE_SPOUT_TIME", "60").trim.toInt From 8f0f523202a590088d9c156b2be895f8395122d4 Mon Sep 17 00:00:00 2001 From: Miratepuffin Date: Wed, 25 Nov 2020 01:46:21 +0000 Subject: [PATCH 29/58] Converted Spouts to Data Sources, removing all akka logic and simplfying user API. Should be fixing issues #81 and #107. Needs some testing from Imane on her spouts, but otherwise good to go. --- .../src/main/scala/com/raphtory/Server.scala | 3 +- .../core/components/Router/RouterWorker.scala | 2 +- .../core/components/Spout/DataSource.scala | 16 ++ .../Spout/{SpoutTrait.scala => Spout.scala} | 78 ++---- .../blockchain/routers/BitcoinRouter.scala | 1 - .../blockchain/routers/DashcoinRouter.scala | 6 +- .../blockchain/routers/LitecoinRouter.scala | 6 +- .../GabMongoSource.scala} | 48 ++-- .../examples/gab/spouts/GabSampledSpout.scala | 57 ---- .../examples/ldbc/routers/LDBCRouter.scala | 1 - .../examples/lotr/LOTRDataSource.scala | 36 +++ .../raphtory/examples/lotr/LOTRSpout.scala | 36 --- ...ndomSpout.scala => RandomDataSource.scala} | 68 ++--- .../test/actors/TriangleDataSource.scala | 22 ++ .../test/actors/TriangleTestRouter.scala | 2 +- .../test/actors/TriangleTestSpout.scala | 28 -- .../examples/tsvnet/SamplerSpout.scala | 112 ++++---- .../spouts/CooccurrenceMatrixSpout.scala | 252 +++++++++--------- .../CooccurrenceMatrixSpoutFiltered.scala | 106 ++++---- .../com/raphtory/sources/FileSource.scala | 122 +++++++++ .../com/raphtory/sources/KafkaSource.scala | 72 +++++ .../blockchain/BitcoinNodeSource.scala} | 63 +++-- .../blockchain/EthereumGethSource.scala} | 43 +-- .../blockchain/EthereumPostgresSource.scala} | 38 +-- .../scala/com/raphtory/spouts/FileSpout.scala | 156 ----------- .../com/raphtory/spouts/KafkaSpout.scala | 72 ----- .../tests/TemporalTriangleCountExample.scala | 6 +- .../main/scala/com/raphtory/tests/curltest.sc | 2 +- 28 files changed, 661 insertions(+), 793 deletions(-) create mode 100644 mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/DataSource.scala rename mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/{SpoutTrait.scala => Spout.scala} (58%) rename mainproject/cluster/src/main/scala/com/raphtory/examples/gab/{spouts/GabRawSpout.scala => datasources/GabMongoSource.scala} (56%) delete mode 100644 mainproject/cluster/src/main/scala/com/raphtory/examples/gab/spouts/GabSampledSpout.scala create mode 100644 mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRDataSource.scala delete mode 100644 mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRSpout.scala rename mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/{RandomSpout.scala => RandomDataSource.scala} (62%) create mode 100644 mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleDataSource.scala delete mode 100644 mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleTestSpout.scala create mode 100644 mainproject/cluster/src/main/scala/com/raphtory/sources/FileSource.scala create mode 100644 mainproject/cluster/src/main/scala/com/raphtory/sources/KafkaSource.scala rename mainproject/cluster/src/main/scala/com/raphtory/{spouts/blockchain/BitcoinSpout.scala => sources/blockchain/BitcoinNodeSource.scala} (75%) rename mainproject/cluster/src/main/scala/com/raphtory/{spouts/blockchain/EthereumGethSpout.scala => sources/blockchain/EthereumGethSource.scala} (74%) rename mainproject/cluster/src/main/scala/com/raphtory/{spouts/blockchain/EthereumPostgresSpout.scala => sources/blockchain/EthereumPostgresSource.scala} (65%) delete mode 100644 mainproject/cluster/src/main/scala/com/raphtory/spouts/FileSpout.scala delete mode 100644 mainproject/cluster/src/main/scala/com/raphtory/spouts/KafkaSpout.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/Server.scala b/mainproject/cluster/src/main/scala/com/raphtory/Server.scala index bd0cee9f15..cbd08acafd 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/Server.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/Server.scala @@ -8,7 +8,6 @@ import com.raphtory.core.analysis.Tasks.AnalysisTask import com.raphtory.core.clustersetup._ import com.raphtory.core.clustersetup.singlenode.SingleNodeSetup import com.raphtory.examples.test.actors.RandomRouter -import com.raphtory.examples.test.actors.RandomSpout import com.typesafe.config.Config import com.typesafe.config.ConfigFactory import kamon.metric.PeriodSnapshot @@ -30,7 +29,7 @@ object Go extends App { // debug should give timing //root.setLevel(Level.DEBUG) val routerName = s"${sys.env.getOrElse("ROUTERCLASS", classOf[RandomRouter].getClass.getName)}" - val updaterName = s"${sys.env.getOrElse("SPOUTCLASS", classOf[RandomSpout].getClass.getName)}" + val updaterName = s"${sys.env.getOrElse("SPOUTCLASS", "")}" val docker = System.getenv().getOrDefault("DOCKER", "false").trim.toBoolean val runtimeMxBean = ManagementFactory.getRuntimeMXBean diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala index 29df22e8e4..45f55db7a0 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala @@ -4,7 +4,7 @@ import akka.actor.{Actor, ActorLogging, Cancellable} import akka.cluster.pubsub.DistributedPubSub import akka.cluster.pubsub.DistributedPubSubMediator import com.raphtory.core.components.Router.RouterWorker.CommonMessage.TimeBroadcast -import com.raphtory.core.components.Spout.SpoutTrait.CommonMessage.{NoWork, SpoutOnline, WorkPlease} +import com.raphtory.core.components.Spout.Spout.CommonMessage.{NoWork, SpoutOnline, WorkPlease} import com.raphtory.core.model.communication._ import com.raphtory.core.utils.Utils import com.raphtory.core.utils.Utils.getManager diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/DataSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/DataSource.scala new file mode 100644 index 0000000000..9d0b31460f --- /dev/null +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/DataSource.scala @@ -0,0 +1,16 @@ +package com.raphtory.core.components.Spout + +import com.raphtory.core.model.communication.SpoutGoing + +class NoDataAvailable extends Exception +class DataSourceComplete extends Exception + +trait DataSource{ + def setupDataSource():Unit + + @throws(classOf[NoDataAvailable]) + @throws(classOf[DataSourceComplete]) + def generateData():SpoutGoing + + def closeDataSource():Unit +} diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/SpoutTrait.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/Spout.scala similarity index 58% rename from mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/SpoutTrait.scala rename to mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/Spout.scala index 3ac87bf894..22465ea15a 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/SpoutTrait.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/Spout.scala @@ -3,37 +3,29 @@ package com.raphtory.core.components.Spout import akka.actor.{Actor, ActorLogging, ActorRef, Cancellable, Timers} import akka.cluster.pubsub.DistributedPubSub import akka.cluster.pubsub.DistributedPubSubMediator -import com.raphtory.core.components.Spout.SpoutTrait.CommonMessage._ -import com.raphtory.core.components.Spout.SpoutTrait.DomainMessage +import com.raphtory.core.components.Spout.Spout.CommonMessage._ import com.raphtory.core.model.communication._ import com.raphtory.core.utils.Utils import kamon.Kamon import scala.concurrent.ExecutionContext -import scala.util.Random -//import kamon.metric.CounterMetric -//import kamon.metric.GaugeMetric + import scala.concurrent.duration._ import scala.language.postfixOps -import scala.collection.mutable.Queue -// TODO Add val name which sub classes that extend this trait must overwrite -// e.g. BlockChainSpout val name = "Blockchain Spout" -// Log.debug that read 'Spout' should then read 'Blockchain Spout' -trait SpoutTrait[Domain <: DomainMessage, Out <: SpoutGoing] extends Actor with ActorLogging with Timers { + +class Spout(datasource:DataSource) extends Actor with ActorLogging with Timers { // todo: wvv should assign the dispatcher when create the actor implicit val executionContext: ExecutionContext = context.system.dispatchers.lookup("spout-dispatcher") //implicit val executionContext: ExecutionContext = context.system.dispatcher private val spoutTuples = Kamon.counter("Raphtory_Spout_Tuples").withTag("actor", self.path.name) private var count = 0 - private var lastRouter = "" private var partitionManagers = 0 private var routers = 0 - private var finish = false - private val workQueue = Queue[SpoutGoing]() + private def recordUpdate(): Unit = { spoutTuples.increment() count += 1 @@ -51,22 +43,16 @@ trait SpoutTrait[Domain <: DomainMessage, Out <: SpoutGoing] extends Actor with final override def receive: Receive = work(false,1,1) - private def work(safe: Boolean, pmCounter:Int, rmCounter:Int): Receive = { case StateCheck => processStateCheckMessage(safe) case ClusterStatusResponse(clusterUp,pmCounter,rmCounter) => context.become(work(clusterUp,pmCounter,rmCounter)) context.system.scheduler.scheduleOnce(1 second, self, StateCheck) case IsSafe => processIsSafeMessage(safe,pmCounter,rmCounter) - case StartSpout => startSpout() case WorkPlease => sendData(context.sender()) - case x: Domain => handleDomainMessage(x) case unhandled => log.error(s"Unable to handle message [$unhandled].") } - def startSpout(): Unit - - def handleDomainMessage(message: Domain): Unit private def processStateCheckMessage(safe: Boolean): Unit = { log.debug(s"Spout is handling [StateCheck] message.") @@ -81,7 +67,7 @@ trait SpoutTrait[Domain <: DomainMessage, Out <: SpoutGoing] extends Actor with private def processIsSafeMessage(safe: Boolean,pmCount:Int,roCount:Int): Unit = { log.debug(s"Spout is handling [IsSafe] message.") if (safe) { - self ! StartSpout + datasource.setupDataSource() partitionManagers=pmCount routers=roCount Utils.getAllRouterWorkers(roCount).foreach { workerPath => @@ -98,41 +84,21 @@ trait SpoutTrait[Domain <: DomainMessage, Out <: SpoutGoing] extends Actor with context.system.scheduler.scheduleOnce(delay = 1 second, receiver = self, message = IsSafe) } - protected def dataFinished():Unit = { - if(!finish) { - workQueue += DataFinished() - //mediator ! DistributedPubSubMediator.Send(lastRouter, DataFinished, localAffinity = false) - finish=true - } - } - - protected def sendTuple(command: Out): Unit = { - log.debug(s"The command [$command] received for send.") - workQueue += command - //lastRouter=s"/user/router/router_${Random.nextInt(routers)}_Worker_${Random.nextInt(10)}" - //mediator ! DistributedPubSubMediator.Send(lastRouter, message, localAffinity = false) - } def sendData(sender:ActorRef): Unit = { - //s"/user/router/router_${Random.nextInt(routers)}_Worker_${Random.nextInt(10)}" try { - val work =workQueue.dequeue() - if(work.isInstanceOf[DataFinished]) { - sender ! DataFinished - println("All data sent") - } - else { - val message = if (count % 10000 == 0) - AllocateTrackedTuple(System.currentTimeMillis(),work) - else - AllocateTuple(work) - recordUpdate() - sender ! message - if (count % 10000 == 0) println(s"Spout at Message $count, remaining messages ${workQueue.size}") - //println(s"Spout at Message $count, remaining messages ${work}") - }//mediator ! DistributedPubSubMediator.Send(lastRouter, message, localAffinity = false) - }catch { - case e:Exception => sender ! NoWork//mediator ! DistributedPubSubMediator.Send(lastRouter, NoWork, localAffinity = false) + val work =datasource.generateData() + val message = if (count % 10000 == 0) + AllocateTrackedTuple(System.currentTimeMillis(),work) + else + AllocateTuple(work) + sender ! message + recordUpdate() + if (count % 10000 == 0) println(s"Spout at Message $count") + }//mediator ! DistributedPubSubMediator.Send(lastRouter, message, localAffinity = false) + catch { + case e:NoDataAvailable => sender ! NoWork + case e:DataSourceComplete => sender ! DataFinished; println("All data sent") } } @@ -142,18 +108,12 @@ trait SpoutTrait[Domain <: DomainMessage, Out <: SpoutGoing] extends Actor with } } -object SpoutTrait { +object Spout { object CommonMessage { - case object StartSpout case object StateCheck case object IsSafe - case object Next extends BasicDomain case object WorkPlease case object NoWork case object SpoutOnline - - } - trait DomainMessage - sealed trait BasicDomain extends DomainMessage } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/BitcoinRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/BitcoinRouter.scala index 2a25abddf7..3fe72f9524 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/BitcoinRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/BitcoinRouter.scala @@ -2,7 +2,6 @@ package com.raphtory.examples.blockchain.routers import com.raphtory.core.components.Router.RouterWorker import com.raphtory.core.model.communication._ -import com.raphtory.spouts.blockchain.BitcoinTransaction import spray.json.JsArray import scala.collection.mutable.ListBuffer diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/DashcoinRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/DashcoinRouter.scala index e12068d98a..41a7cecc5f 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/DashcoinRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/DashcoinRouter.scala @@ -1,9 +1,9 @@ package com.raphtory.examples.blockchain.routers import com.raphtory.core.components.Router.RouterWorker -import com.raphtory.core.model.communication.{EdgeAdd, EdgeAddWithProperties, GraphUpdate, Properties, StringProperty, StringSpoutGoing, VertexAdd, VertexAddWithProperties} -import com.raphtory.spouts.blockchain.BitcoinTransaction -import spray.json.JsArray +import com.raphtory.core.model.communication._ +import com.raphtory.examples.blockchain.routers +import spray.json.{JsArray, JsValue} import scala.collection.mutable.ListBuffer import scala.collection.parallel.mutable.ParHashSet diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/LitecoinRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/LitecoinRouter.scala index 483e171a27..4dbdb1b118 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/LitecoinRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/LitecoinRouter.scala @@ -1,13 +1,13 @@ package com.raphtory.examples.blockchain.routers import com.raphtory.core.components.Router.RouterWorker -import com.raphtory.core.model.communication.{EdgeAdd, EdgeAddWithProperties, GraphUpdate, Properties, StringProperty, VertexAddWithProperties} -import com.raphtory.spouts.blockchain.BitcoinTransaction -import spray.json.JsArray +import com.raphtory.core.model.communication._ +import spray.json.{JsArray, JsValue} import scala.collection.mutable.ListBuffer import scala.collection.parallel.mutable.ParHashSet import scala.util.hashing.MurmurHash3 +case class BitcoinTransaction(time: JsValue, block: Int, blockID: JsValue, transaction: JsValue) extends SpoutGoing class LitecoinRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) extends RouterWorker[BitcoinTransaction](routerId,workerID,initialManagerCount, initialRouterCount) { diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/spouts/GabRawSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/datasources/GabMongoSource.scala similarity index 56% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/gab/spouts/GabRawSpout.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/gab/datasources/GabMongoSource.scala index 2956b5c886..230ca2e241 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/spouts/GabRawSpout.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/datasources/GabMongoSource.scala @@ -1,17 +1,16 @@ -package com.raphtory.examples.gab.spouts +package com.raphtory.examples.gab.datasources import akka.actor.Cancellable import ch.qos.logback.classic.Level import com.mongodb.casbah.Imports.{MongoConnection, _} -import com.raphtory.core.components.Spout.SpoutTrait -import com.raphtory.core.components.Spout.SpoutTrait.BasicDomain -import com.raphtory.core.components.Spout.SpoutTrait.CommonMessage.Next -import com.raphtory.core.model.communication.StringSpoutGoing +import com.raphtory.core.components.Spout.DataSource +import com.raphtory.core.model.communication.{SpoutGoing, StringSpoutGoing} import org.slf4j.LoggerFactory +import scala.collection.mutable import scala.language.postfixOps -final class GabRawSpout extends SpoutTrait[BasicDomain,StringSpoutGoing] { +final class GabMongoDataSource extends DataSource { //private val redis = new RedisClient("moe", 6379) //private val redisKey = "gab-posts" @@ -25,41 +24,42 @@ final class GabRawSpout extends SpoutTrait[BasicDomain,StringSpoutGoing] { private var postMin = 0 private var postMax = 1001 + + val root = LoggerFactory.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME).asInstanceOf[ch.qos.logback.classic.Logger] root.setLevel(Level.ERROR) - // private val mongoLogger = Logger.getLogger("org.mongodb.driver.cluster") - // mongoLogger.setLevel(Level.OFF) - override def handleDomainMessage(message: BasicDomain): Unit = message match { - case Next => running() - } + var queue = mutable.Queue[StringSpoutGoing]() - def running(): Unit = { - val count = getNextPosts() - postMin += window - postMax += window - println(s"Current min post is $postMin, max post is $postMax, last call retrieved $count posts") - self ! Next + override def generateData(): SpoutGoing = { + if(queue.isEmpty) { + getNextPosts() + queue.dequeue() + } + else + queue.dequeue() } - private def getNextPosts(): Int = { - var count = 0 + + private def getNextPosts() = { + for (x <- mongoColl.find("_id" $lt postMax $gt postMin)) try { val data = x.get("data").toString.drop(2).dropRight(1).replaceAll("""\\"""", "").replaceAll("""\\""", "") - count += 1 - //println(data) - sendTuple(StringSpoutGoing(data)) + queue += StringSpoutGoing(data) } catch { case e: Throwable => println("Cannot parse record") } - return count + postMin += window + postMax += window } - override def startSpout(): Unit = self ! Next + override def setupDataSource(): Unit = {} + + override def closeDataSource(): Unit = {} } //redis-server --dir /home/moe/ben/gab --dbfilename gab.rdb --daemonize yes diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/spouts/GabSampledSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/spouts/GabSampledSpout.scala deleted file mode 100644 index 17e97d28dd..0000000000 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/spouts/GabSampledSpout.scala +++ /dev/null @@ -1,57 +0,0 @@ -package com.raphtory.examples.gab.spouts - -import java.time.LocalDateTime - -import com.raphtory.core.components.Spout.SpoutTrait -import com.raphtory.core.components.Spout.SpoutTrait.BasicDomain -import com.raphtory.core.components.Spout.SpoutTrait.CommonMessage.Next -import com.raphtory.core.model.communication.StringSpoutGoing - -import scala.io.Source -import scala.util.Random -import scala.util.control.Breaks.{break, breakable} - -class GabSampledSpout extends SpoutTrait[BasicDomain,StringSpoutGoing] { - - val prop = 1.0/10 - val r = new Random() - - val directory = System.getenv().getOrDefault("GAB_DIRECTORY", "/app").trim - val file_name = System.getenv().getOrDefault("GAB_FILE_NAME", "gab.csv").trim - val fileLines = Source.fromFile(directory + "/" + file_name).getLines.drop(1).toArray - // upstream/master - var position = 0 - var linesNumber = fileLines.length - println("Start: " + LocalDateTime.now()) - - override def handleDomainMessage(message: BasicDomain): Unit = message match { - case Next => - try { - if (position < linesNumber) { - for (i <- 1 to 100) { - val randFloat = r.nextFloat() - breakable { - if (randFloat > prop) { - position +=1 - break - } else { - val line = fileLines(position) - sendTuple(StringSpoutGoing(line)) - position += 1 - } - } - } - self ! Next - } - else - { - println("Finished ingestion") - } - } - catch {case e:Exception => println("Finished ingestion")} - case _ => println("message not recognized!") - } - - override def startSpout(): Unit = self ! Next - -} diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCRouter.scala index 67314e21be..94e72e94d6 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCRouter.scala @@ -5,7 +5,6 @@ import java.util.Date import com.raphtory.core.components.Router.RouterWorker import com.raphtory.core.model.communication.{EdgeAdd, EdgeDelete, GraphUpdate, StringSpoutGoing, Type, VertexAdd, VertexDelete} -import com.raphtory.examples.test.actors.RandomSpout import scala.collection.mutable.ListBuffer import scala.collection.parallel.mutable.ParHashSet diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRDataSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRDataSource.scala new file mode 100644 index 0000000000..ddfa05abb4 --- /dev/null +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRDataSource.scala @@ -0,0 +1,36 @@ +package com.raphtory.examples.lotr + +import java.time.LocalDateTime + +import com.raphtory.core.components.Spout.{DataSource, DataSourceComplete, Spout} +import com.raphtory.core.model.communication.{SpoutGoing, StringSpoutGoing} + +import scala.concurrent.duration.{Duration, MILLISECONDS, NANOSECONDS} +import scala.io + +class LOTRDataSource extends DataSource { + + // Relating to where the file is + val directory = System.getenv().getOrDefault("LOTR_DIRECTORY", "com/raphtory/example/lotr").trim + val file_name = System.getenv().getOrDefault("LOTR_FILE_NAME", "lotr.csv").trim + val fileLines = scala.io.Source.fromFile(directory + "/" + file_name).getLines.toArray + + // Initialise ready to read + var position = 0 + var linesNumber = fileLines.length + println("Start: " + LocalDateTime.now()) + + override def setupDataSource(): Unit = {} + + override def generateData(): SpoutGoing = { + if(position==linesNumber) + throw new DataSourceComplete() + else { + val line =fileLines(position) + position+=1 + StringSpoutGoing(line) + } + } + + override def closeDataSource(): Unit = {} +} diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRSpout.scala deleted file mode 100644 index 500606a091..0000000000 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRSpout.scala +++ /dev/null @@ -1,36 +0,0 @@ -package com.raphtory.examples.lotr - -import java.time.LocalDateTime - -import com.raphtory.core.components.Spout.SpoutTrait -import com.raphtory.core.components.Spout.SpoutTrait.BasicDomain -import com.raphtory.core.components.Spout.SpoutTrait.CommonMessage.{Next} -import com.raphtory.core.model.communication.StringSpoutGoing - -import scala.concurrent.duration.{Duration, MILLISECONDS, NANOSECONDS} -import scala.io - -class LOTRSpout extends SpoutTrait[BasicDomain,StringSpoutGoing] { - - // Relating to where the file is - val directory = System.getenv().getOrDefault("LOTR_DIRECTORY", "com/raphtory/example/lotr").trim - val file_name = System.getenv().getOrDefault("LOTR_FILE_NAME", "lotr.csv").trim - val fileLines = scala.io.Source.fromFile(directory + "/" + file_name).getLines.toArray - - // Initialise ready to read - var position = 0 - var linesNumber = fileLines.length - println("Start: " + LocalDateTime.now()) - - override def handleDomainMessage(message: BasicDomain): Unit = message match { - case Next => - if (position < linesNumber) { - var line = fileLines(position) - sendTuple(StringSpoutGoing(line)) - position += 1 - self ! Next - } - case _ => println("message not recognized!") - } - override def startSpout(): Unit = self ! Next -} diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomDataSource.scala similarity index 62% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomSpout.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomDataSource.scala index e9d15bf389..c3847d0b2c 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomSpout.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomDataSource.scala @@ -1,60 +1,43 @@ package com.raphtory.examples.test.actors -import com.raphtory.core.components.Spout.SpoutTrait -import com.raphtory.core.components.Spout.SpoutTrait.DomainMessage -import com.raphtory.core.model.communication.StringSpoutGoing +import com.raphtory.core.components.Spout.DataSource +import com.raphtory.core.model.communication.{SpoutGoing, StringSpoutGoing} import com.raphtory.core.utils.Utils -import com.raphtory.examples.test.actors.RandomSpout.Message.{Increase, RandomDomain, Required,Next} -import scala.concurrent.duration._ import scala.language.postfixOps import scala.util.Random -class RandomSpout extends SpoutTrait[RandomDomain,StringSpoutGoing] { +class RandomDataSource extends DataSource { var totalCount = 100 - var freq = 1000 - var increase = System.getenv().getOrDefault("RAMP_FLAG", "false").toBoolean // (Updates/s) - Hz - var pool = System.getenv().getOrDefault("ENTITY_POOL", "1000000").toInt - var msgID = 0 - - - override def handleDomainMessage(message: RandomDomain): Unit = message match { - case Required => - freq = System.getenv().getOrDefault("UPDATES_FREQ", "10000").toInt; - println(s"Full start ($freq Hz) Entity pool = $pool Ramp flag = $increase") - // (Updates/s) - Hz - case Increase => - if (increase) { - freq += 1000 - println(s"Frequency increased, new frequency: $freq at ${Utils.nowTimeStamp()}") - } - case Next => genRandomCommands(freq / 1000) - case _ => println("message not recognized!") - } + var freq = 1000 + var pool = System.getenv().getOrDefault("ENTITY_POOL", "1000000").toInt + var msgID = 0 + + override def setupDataSource(): Unit = {} + + override def generateData(): SpoutGoing = StringSpoutGoing(distribution()) + + override def closeDataSource(): Unit = {} + def distribution(): String = { val random = Random.nextFloat() if (random <= 0.3) genVertexAdd() - else if(random <=0.7) genEdgeAdd() + else if (random <= 0.7) genEdgeAdd() else if (random <= 0.8) genVertexRemoval() - else genEdgeRemoval() - } - - def genRandomCommands(number: Int): Unit = { - (1 to number) foreach (_ => { - sendTuple(StringSpoutGoing(distribution())) - }) - self ! Next + else genEdgeRemoval() } def genVertexAdd(): String = s""" {"VertexAdd":{${getMessageID()}, ${genSrcID()}, ${genProperties(2)}}}""" + def genVertexAdd(src: Int): String = //overloaded method if you want to specify src ID s""" {"VertexAdd":{${getMessageID()}, ${genSrcID(src)}, ${genProperties(2)}}}""" def genVertexUpdateProperties(): String = s""" {"VertexUpdateProperties":{${getMessageID()}, ${genSrcID(1)}}}""" + def genVertexUpdateProperties(src: Int): String = //overloaded to mass src s""" {"VertexUpdateProperties":{${getMessageID()}, ${genSrcID(src)}, ${genProperties(2)}}}""" @@ -66,26 +49,32 @@ class RandomSpout extends SpoutTrait[RandomDomain,StringSpoutGoing] { def genEdgeAdd(): String = s""" {"EdgeAdd":{${getMessageID()}, ${genSrcID()}, ${genDstID()}, ${genProperties(2)}}}""" + def genEdgeAdd(src: Int, dst: Int): String = s""" {"EdgeAdd":{${getMessageID()}, ${genSrcID(src)}, ${genDstID(dst)}}}""" def genEdgeUpdateProperties(): String = s""" {"EdgeUpdateProperties":{${getMessageID()}, ${genSrcID()}, ${genDstID()}, ${genProperties(2)}}}""" + def genEdgeUpdateProperties(src: Int, dst: Int): String = s""" {"EdgeUpdateProperties":{${getMessageID()}, ${genSrcID(src)}, ${genDstID(dst)}, ${genProperties(2)}}}""" def genEdgeRemoval(): String = s""" {"EdgeRemoval":{${getMessageID()}, ${genSrcID()}, ${genDstID()}}}""" + def genEdgeRemoval(src: Int, dst: Int): String = s""" {"EdgeRemoval":{${getMessageID()}, ${genSrcID(src)}, ${genDstID(dst)}}}""" def genSetSrcID(): String = s""" "srcID":9 """ + def genSetDstID(): String = s""" "dstID":10 """ def genSrcID(): String = s""" "srcID":${Random.nextInt(pool)} """ + def genDstID(): String = s""" "dstID":${Random.nextInt(pool)} """ def genSrcID(src: Int): String = s""" "srcID":$src """ + def genDstID(dst: Int): String = s""" "dstID":$dst """ def getMessageID(): String = { @@ -104,15 +93,4 @@ class RandomSpout extends SpoutTrait[RandomDomain,StringSpoutGoing] { properties } - override def startSpout(): Unit = ??? - } - -object RandomSpout { - object Message { - sealed trait RandomDomain extends DomainMessage - case object Increase extends RandomDomain - case object Required extends RandomDomain - case object Next extends RandomDomain - } -} \ No newline at end of file diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleDataSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleDataSource.scala new file mode 100644 index 0000000000..3c4c5d8b57 --- /dev/null +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleDataSource.scala @@ -0,0 +1,22 @@ +package com.raphtory.examples.test.actors + +import com.raphtory.core.components.Spout.{DataSource, DataSourceComplete, Spout} +import com.raphtory.core.model.communication.{SpoutGoing, StringSpoutGoing} + +import scala.collection.mutable.Queue + +class TriangleDataSource extends DataSource { + + val edges = Queue[StringSpoutGoing](StringSpoutGoing("3,3,1"), StringSpoutGoing("4,3,4"), StringSpoutGoing("2,2,3"), StringSpoutGoing("5,4,1"), StringSpoutGoing("6,1,3"), StringSpoutGoing("1,1,2")) + + override def setupDataSource(): Unit = {} + + override def generateData(): SpoutGoing = { + if(edges.isEmpty) + throw new DataSourceComplete() + else + edges.dequeue() + } + + override def closeDataSource(): Unit = {} +} diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleTestRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleTestRouter.scala index db8ecc2e64..26249dc062 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleTestRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleTestRouter.scala @@ -1,7 +1,7 @@ package com.raphtory.examples.test.actors import com.raphtory.core.components.Router.RouterWorker -import com.raphtory.core.components.Spout.SpoutTrait +import com.raphtory.core.components.Spout.Spout import com.raphtory.core.model.communication.{EdgeAdd, GraphUpdate, StringSpoutGoing, VertexAddWithProperties} import scala.collection.mutable.ListBuffer diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleTestSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleTestSpout.scala deleted file mode 100644 index b3bca14961..0000000000 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleTestSpout.scala +++ /dev/null @@ -1,28 +0,0 @@ -package com.raphtory.examples.test.actors - -import com.raphtory.core.components.Spout.SpoutTrait -import com.raphtory.core.components.Spout.SpoutTrait.BasicDomain -import com.raphtory.core.components.Spout.SpoutTrait.CommonMessage.Next -import com.raphtory.core.model.communication.StringSpoutGoing - -class TriangleTestSpout extends SpoutTrait[BasicDomain,StringSpoutGoing] { - - override def handleDomainMessage(message: BasicDomain): Unit = message match { - case Next => - - - - - } - - override def startSpout(): Unit = { - sendTuple(StringSpoutGoing("3,3,1")) - sendTuple(StringSpoutGoing("4,3,4")) - sendTuple(StringSpoutGoing("2,2,3")) - sendTuple(StringSpoutGoing("5,4,1")) - sendTuple(StringSpoutGoing("6,1,3")) - sendTuple(StringSpoutGoing("1,1,2")) - dataFinished() - } - -} diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/SamplerSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/SamplerSpout.scala index 14577bde87..4ba3214a5e 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/SamplerSpout.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/SamplerSpout.scala @@ -1,56 +1,56 @@ -package com.raphtory.examples.tsvnet - -import java.time.LocalDateTime - -import com.raphtory.core.components.Spout.SpoutTrait -import com.raphtory.core.components.Spout.SpoutTrait.BasicDomain -import com.raphtory.core.components.Spout.SpoutTrait.CommonMessage.Next -import com.raphtory.core.model.communication.StringSpoutGoing - -import scala.concurrent.duration.{Duration, NANOSECONDS} -import scala.util.Random -import scala.util.control.Breaks._ - -class SamplerSpout extends SpoutTrait[BasicDomain,StringSpoutGoing] { - - val prop = 1.0/10 - val r = new Random() - val directory = System.getenv().getOrDefault("TSV_DIRECTORY", "/app").trim - val file_name = System.getenv().getOrDefault("TSV_FILE_NAME", "sx_reordered.txt").trim - val fileLines = scala.io.Source.fromFile(directory + "/" + file_name).getLines.drop(1).toArray - // upstream/master - var position = 0 - var linesNumber = fileLines.length - println("Start: " + LocalDateTime.now()) - - override def handleDomainMessage(message: BasicDomain): Unit = message match { - case Next => - try { - if (position < linesNumber) { - for (i <- 1 to 100) { - val randFloat = r.nextFloat() - breakable { - if (randFloat > prop) { - position +=1 - break - } else { - val line = fileLines(position) - sendTuple(StringSpoutGoing(line)) - position += 1 - } - } - } - AllocateSpoutTask(Duration(1, NANOSECONDS), "newLine") - } - else - { - println("Finished ingestion") - } - } - catch {case e:Exception => println("Finished ingestion")} - case _ => println("message not recognized!") - } - - override def startSpout(): Unit = self ! Next - -} +//package com.raphtory.examples.tsvnet +// +//import java.time.LocalDateTime +// +//import com.raphtory.core.components.Spout.Spout +//import com.raphtory.core.components.Spout.Spout.BasicDomain +//import com.raphtory.core.components.Spout.Spout.CommonMessage.Next +//import com.raphtory.core.model.communication.StringSpoutGoing +// +//import scala.concurrent.duration.{Duration, NANOSECONDS} +//import scala.util.Random +//import scala.util.control.Breaks._ +// +//class SamplerSpout extends Spout[BasicDomain,StringSpoutGoing] { +// +// val prop = 1.0/10 +// val r = new Random() +// val directory = System.getenv().getOrDefault("TSV_DIRECTORY", "/app").trim +// val file_name = System.getenv().getOrDefault("TSV_FILE_NAME", "sx_reordered.txt").trim +// val fileLines = scala.io.Source.fromFile(directory + "/" + file_name).getLines.drop(1).toArray +// // upstream/master +// var position = 0 +// var linesNumber = fileLines.length +// println("Start: " + LocalDateTime.now()) +// +// override def handleDomainMessage(message: BasicDomain): Unit = message match { +// case Next => +// try { +// if (position < linesNumber) { +// for (i <- 1 to 100) { +// val randFloat = r.nextFloat() +// breakable { +// if (randFloat > prop) { +// position +=1 +// break +// } else { +// val line = fileLines(position) +// sendTuple(StringSpoutGoing(line)) +// position += 1 +// } +// } +// } +// AllocateSpoutTask(Duration(1, NANOSECONDS), "newLine") +// } +// else +// { +// println("Finished ingestion") +// } +// } +// catch {case e:Exception => println("Finished ingestion")} +// case _ => println("message not recognized!") +// } +// +// override def startSpout(): Unit = self ! Next +// +//} diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/spouts/CooccurrenceMatrixSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/spouts/CooccurrenceMatrixSpout.scala index 21729334cc..a28e723c6c 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/spouts/CooccurrenceMatrixSpout.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/spouts/CooccurrenceMatrixSpout.scala @@ -1,126 +1,126 @@ -package com.raphtory.examples.wordSemantic.spouts - - -import java.io.{BufferedReader, File, FileReader} -import java.time.LocalDateTime - -import com.raphtory.core.components.Spout.SpoutTrait -import com.raphtory.core.components.Spout.SpoutTrait.{BasicDomain, DomainMessage} -import com.raphtory.core.model.communication.StringSpoutGoing -import com.raphtory.examples.wordSemantic.spouts.CooccurrenceMatrixSpout.Message.{CooccuranceDomain, NextFile, NextLineBlock, NextLineSlice} - -import scala.concurrent.duration._ -import scala.language.postfixOps - -class CooccurrenceMatrixSpout extends SpoutTrait[CooccuranceDomain,StringSpoutGoing] { - - println("Start: " + LocalDateTime.now()) - val directory = System.getenv().getOrDefault("FILE_SPOUT_DIRECTORY", "/app").trim - val fileName = System.getenv().getOrDefault("FILE_SPOUT_FILENAME", "").trim //gabNetwork500.csv - val dropHeader = System.getenv().getOrDefault("FILE_SPOUT_DROP_HEADER", "false").trim.toBoolean - var JUMP = System.getenv().getOrDefault("FILE_SPOUT_BLOCK_SIZE", "100").trim.toInt - var INCREMENT = System.getenv().getOrDefault("FILE_SPOUT_INCREMENT", "1").trim.toInt - var TIME = System.getenv().getOrDefault("FILE_SPOUT_TIME", "60").trim.toInt - - var directoryPosition = 0 - - val filesToRead = if(fileName.isEmpty) - getListOfFiles(directory) - else - Array(directory + "/" + fileName) - - var currentFile = fileToArray(directoryPosition) - val JUMP2 = 20 - var posSlice = 1 - var cline = currentFile.readLine() - var currentLine = cline.split("\t") - var filename = filesToRead(directoryPosition) //D-200001_merge_occ - var time = filename.split('/').last.stripPrefix("D-").stripSuffix("_merge_occ").toLong * 1000000000L - var cnt = time + 1 - - - - def handleDomainMessage(message: CooccuranceDomain): Unit = message match { - - case NextLineSlice => nextLineSlice() - case NextLineBlock => nextLineBlock() - case NextFile => nextFile() - case _ => println("message not recognized!") -} - def nextLineSlice() = { - try { - if (posSlice <= currentLine.length-1) { - val head = currentLine(0) - for (i<- 1 to Set(JUMP, currentLine.length-posSlice/JUMP2).min) { - val currentSlice = currentLine.slice(posSlice, posSlice + JUMP2) - sendTuple(StringSpoutGoing(cnt.toString + ' ' + head + "\t" + currentSlice.mkString("\t"))) - posSlice += JUMP2 - } - AllocateSpoutTask(Duration(1, MILLISECONDS), "nextLineSlice") - } - else { - posSlice = 1 - AllocateSpoutTask(Duration(1, NANOSECONDS), "nextLineBLock") - } - }catch { - case e: Exception => println(e, posSlice) - } - } - - def nextLineBlock() = { - try { - cnt += 1 - cline = currentFile.readLine() - currentLine = cline.split("\t") - AllocateSpoutTask(Duration(1, NANOSECONDS), "nextLineSlice") - } - catch { - case e:Exception => AllocateSpoutTask(Duration(1, NANOSECONDS), "nextFile") - } - } - - def nextFile() = { - directoryPosition += 1 - if (filesToRead.length > directoryPosition) { - currentFile = fileToArray(directoryPosition) - filename = filesToRead(directoryPosition) //D-200001_merge_occ - time = filename.split('/').last.stripPrefix("D-").stripSuffix("_merge_occ").toLong * 1000000000L - cnt = time + 1 - AllocateSpoutTask(Duration(1, NANOSECONDS), "nextLineBLock") - } - else { - println("All files read "+ LocalDateTime.now()) - } - } - - def fileToArray(pos:Int) ={ - println(s"Now reading ${filesToRead(pos)}") - if(dropHeader){ - val br = new BufferedReader(new FileReader(filesToRead(pos))) - br.readLine() - br - } - else - new BufferedReader(new FileReader(filesToRead(pos))) - } - - def getListOfFiles(dir: String):Array[String] = { - val d = new File(dir) - if (d.exists && d.isDirectory) { - d.listFiles.filter(f=> f.isFile && !f.isHidden).map(f=> f.getCanonicalPath).sorted - } else { - Array[String]() - } - } - - override def startSpout(): Unit = self ! NextLineSlice -} - -object CooccurrenceMatrixSpout { - object Message { - sealed trait CooccuranceDomain extends DomainMessage - case object NextLineSlice extends CooccuranceDomain - case object NextLineBlock extends CooccuranceDomain - case object NextFile extends CooccuranceDomain - } -} \ No newline at end of file +//package com.raphtory.examples.wordSemantic.spouts +// +// +//import java.io.{BufferedReader, File, FileReader} +//import java.time.LocalDateTime +// +//import com.raphtory.core.components.Spout.Spout +//import com.raphtory.core.components.Spout.Spout.{BasicDomain, DomainMessage} +//import com.raphtory.core.model.communication.StringSpoutGoing +//import com.raphtory.examples.wordSemantic.spouts.CooccurrenceMatrixSpout.Message.{CooccuranceDomain, NextFile, NextLineBlock, NextLineSlice} +// +//import scala.concurrent.duration._ +//import scala.language.postfixOps +// +//class CooccurrenceMatrixSpout extends Spout[CooccuranceDomain,StringSpoutGoing] { +// +// println("Start: " + LocalDateTime.now()) +// val directory = System.getenv().getOrDefault("FILE_SPOUT_DIRECTORY", "/app").trim +// val fileName = System.getenv().getOrDefault("FILE_SPOUT_FILENAME", "").trim //gabNetwork500.csv +// val dropHeader = System.getenv().getOrDefault("FILE_SPOUT_DROP_HEADER", "false").trim.toBoolean +// var JUMP = System.getenv().getOrDefault("FILE_SPOUT_BLOCK_SIZE", "100").trim.toInt +// var INCREMENT = System.getenv().getOrDefault("FILE_SPOUT_INCREMENT", "1").trim.toInt +// var TIME = System.getenv().getOrDefault("FILE_SPOUT_TIME", "60").trim.toInt +// +// var directoryPosition = 0 +// +// val filesToRead = if(fileName.isEmpty) +// getListOfFiles(directory) +// else +// Array(directory + "/" + fileName) +// +// var currentFile = fileToArray(directoryPosition) +// val JUMP2 = 20 +// var posSlice = 1 +// var cline = currentFile.readLine() +// var currentLine = cline.split("\t") +// var filename = filesToRead(directoryPosition) //D-200001_merge_occ +// var time = filename.split('/').last.stripPrefix("D-").stripSuffix("_merge_occ").toLong * 1000000000L +// var cnt = time + 1 +// +// +// +// def handleDomainMessage(message: CooccuranceDomain): Unit = message match { +// +// case NextLineSlice => nextLineSlice() +// case NextLineBlock => nextLineBlock() +// case NextFile => nextFile() +// case _ => println("message not recognized!") +//} +// def nextLineSlice() = { +// try { +// if (posSlice <= currentLine.length-1) { +// val head = currentLine(0) +// for (i<- 1 to Set(JUMP, currentLine.length-posSlice/JUMP2).min) { +// val currentSlice = currentLine.slice(posSlice, posSlice + JUMP2) +// sendTuple(StringSpoutGoing(cnt.toString + ' ' + head + "\t" + currentSlice.mkString("\t"))) +// posSlice += JUMP2 +// } +// AllocateSpoutTask(Duration(1, MILLISECONDS), "nextLineSlice") +// } +// else { +// posSlice = 1 +// AllocateSpoutTask(Duration(1, NANOSECONDS), "nextLineBLock") +// } +// }catch { +// case e: Exception => println(e, posSlice) +// } +// } +// +// def nextLineBlock() = { +// try { +// cnt += 1 +// cline = currentFile.readLine() +// currentLine = cline.split("\t") +// AllocateSpoutTask(Duration(1, NANOSECONDS), "nextLineSlice") +// } +// catch { +// case e:Exception => AllocateSpoutTask(Duration(1, NANOSECONDS), "nextFile") +// } +// } +// +// def nextFile() = { +// directoryPosition += 1 +// if (filesToRead.length > directoryPosition) { +// currentFile = fileToArray(directoryPosition) +// filename = filesToRead(directoryPosition) //D-200001_merge_occ +// time = filename.split('/').last.stripPrefix("D-").stripSuffix("_merge_occ").toLong * 1000000000L +// cnt = time + 1 +// AllocateSpoutTask(Duration(1, NANOSECONDS), "nextLineBLock") +// } +// else { +// println("All files read "+ LocalDateTime.now()) +// } +// } +// +// def fileToArray(pos:Int) ={ +// println(s"Now reading ${filesToRead(pos)}") +// if(dropHeader){ +// val br = new BufferedReader(new FileReader(filesToRead(pos))) +// br.readLine() +// br +// } +// else +// new BufferedReader(new FileReader(filesToRead(pos))) +// } +// +// def getListOfFiles(dir: String):Array[String] = { +// val d = new File(dir) +// if (d.exists && d.isDirectory) { +// d.listFiles.filter(f=> f.isFile && !f.isHidden).map(f=> f.getCanonicalPath).sorted +// } else { +// Array[String]() +// } +// } +// +// override def startSpout(): Unit = self ! NextLineSlice +//} +// +//object CooccurrenceMatrixSpout { +// object Message { +// sealed trait CooccuranceDomain extends DomainMessage +// case object NextLineSlice extends CooccuranceDomain +// case object NextLineBlock extends CooccuranceDomain +// case object NextFile extends CooccuranceDomain +// } +//} \ No newline at end of file diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/spouts/CooccurrenceMatrixSpoutFiltered.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/spouts/CooccurrenceMatrixSpoutFiltered.scala index 783a2fdab6..eb73e266b8 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/spouts/CooccurrenceMatrixSpoutFiltered.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/spouts/CooccurrenceMatrixSpoutFiltered.scala @@ -1,53 +1,53 @@ -package com.raphtory.examples.wordSemantic.spouts - -import java.io.{BufferedReader, File, FileReader} -import java.time.LocalDateTime - -import com.raphtory.core.components.Spout.SpoutTrait -import com.raphtory.core.model.communication.StringSpoutGoing - -import scala.concurrent.duration._ -import scala.language.postfixOps - -class CooccurrenceMatrixSpoutFiltered extends CooccurrenceMatrixSpout { - var freq = currentLine.drop(2).grouped(2).map(_.head.toInt).toArray - var scale = scalling(freq) - - override def nextLineSlice() = { - try { - if (posSlice <= currentLine.length-1) { - val head = currentLine(0) - for (i<- 1 to Set(JUMP, currentLine.length-posSlice/JUMP2).min) { - val currentSlice = currentLine.slice(posSlice, posSlice + JUMP2) - sendTuple(StringSpoutGoing(cnt.toString + ' ' + scale.toString + ' ' + head + "\t" + currentSlice.mkString("\t"))) - posSlice += JUMP2 - } - AllocateSpoutTask(Duration(1, MILLISECONDS), "nextLineSlice") - } - else { - posSlice = 1 - AllocateSpoutTask(Duration(1, NANOSECONDS), "nextLineBLock") - } - }catch { - case e: Exception => println(e, posSlice) - } - } - - override def nextLineBlock() = { - try { - cnt += 1 - cline = currentFile.readLine() - currentLine = cline.split("\t") - freq = currentLine.drop(2).grouped(2).map(_.head.toInt).toArray - scale = scalling(freq) - AllocateSpoutTask(Duration(1, NANOSECONDS), "nextLineSlice") - } - catch { - case e:Exception => AllocateSpoutTask(Duration(1, NANOSECONDS), "nextFile") - } - } - - def scalling(freq: Array[Int]): Double = { - math.sqrt(freq.map(math.pow(_, 2)).sum) - } -} \ No newline at end of file +//package com.raphtory.examples.wordSemantic.spouts +// +//import java.io.{BufferedReader, File, FileReader} +//import java.time.LocalDateTime +// +//import com.raphtory.core.components.Spout.Spout +//import com.raphtory.core.model.communication.StringSpoutGoing +// +//import scala.concurrent.duration._ +//import scala.language.postfixOps +// +//class CooccurrenceMatrixSpoutFiltered extends CooccurrenceMatrixSpout { +// var freq = currentLine.drop(2).grouped(2).map(_.head.toInt).toArray +// var scale = scalling(freq) +// +// override def nextLineSlice() = { +// try { +// if (posSlice <= currentLine.length-1) { +// val head = currentLine(0) +// for (i<- 1 to Set(JUMP, currentLine.length-posSlice/JUMP2).min) { +// val currentSlice = currentLine.slice(posSlice, posSlice + JUMP2) +// sendTuple(StringSpoutGoing(cnt.toString + ' ' + scale.toString + ' ' + head + "\t" + currentSlice.mkString("\t"))) +// posSlice += JUMP2 +// } +// AllocateSpoutTask(Duration(1, MILLISECONDS), "nextLineSlice") +// } +// else { +// posSlice = 1 +// AllocateSpoutTask(Duration(1, NANOSECONDS), "nextLineBLock") +// } +// }catch { +// case e: Exception => println(e, posSlice) +// } +// } +// +// override def nextLineBlock() = { +// try { +// cnt += 1 +// cline = currentFile.readLine() +// currentLine = cline.split("\t") +// freq = currentLine.drop(2).grouped(2).map(_.head.toInt).toArray +// scale = scalling(freq) +// AllocateSpoutTask(Duration(1, NANOSECONDS), "nextLineSlice") +// } +// catch { +// case e:Exception => AllocateSpoutTask(Duration(1, NANOSECONDS), "nextFile") +// } +// } +// +// def scalling(freq: Array[Int]): Double = { +// math.sqrt(freq.map(math.pow(_, 2)).sum) +// } +//} \ No newline at end of file diff --git a/mainproject/cluster/src/main/scala/com/raphtory/sources/FileSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/sources/FileSource.scala new file mode 100644 index 0000000000..8efe32b6f0 --- /dev/null +++ b/mainproject/cluster/src/main/scala/com/raphtory/sources/FileSource.scala @@ -0,0 +1,122 @@ +package com.raphtory.sources + +import java.io.{BufferedReader, File, FileInputStream, FileReader, InputStreamReader} +import java.nio.charset.StandardCharsets +import java.util.zip.GZIPInputStream + +import com.raphtory.core.components.Spout.{DataSource, DataSourceComplete, NoDataAvailable, Spout} +import com.raphtory.core.model.communication.StringSpoutGoing +import com.typesafe.scalalogging.LazyLogging + +import scala.annotation.tailrec +import scala.concurrent.duration._ + +class FileSpout extends DataSource { + //TODO work out loggging here + //log.info("initialise FileSpout") + private val directory = System.getenv().getOrDefault("FILE_SPOUT_DIRECTORY", "/app").trim + private val fileName = System.getenv().getOrDefault("FILE_SPOUT_FILENAME", "").trim //gabNetwork500.csv + private val dropHeader = System.getenv().getOrDefault("FILE_SPOUT_DROP_HEADER", "false").trim.toBoolean + // private val JUMP = System.getenv().getOrDefault("FILE_SPOUT_BLOCK_SIZE", "10").trim.toInt + private val INCREMENT = System.getenv().getOrDefault("FILE_SPOUT_INCREMENT", "0").trim.toInt + private val TIME = System.getenv().getOrDefault("FILE_SPOUT_TIME", "60").trim.toInt + + private var fileManager = FileManager(directory, fileName, dropHeader) + + override def generateData(): StringSpoutGoing = { + if (fileManager.allCompleted) { + throw new DataSourceComplete() + } + else { + val (newFileManager, line) = fileManager.nextLine() + fileManager = newFileManager + StringSpoutGoing(line) + } + } + + override def setupDataSource(): Unit = {} + + override def closeDataSource(): Unit = {} +} + +final case class FileManager private ( + currentFileReader: Option[BufferedReader], + restFiles: List[File], + dropHeader: Boolean +) extends LazyLogging { + def nextFile(): FileManager = this.copy(currentFileReader = None) + + lazy val allCompleted: Boolean = currentFileReader.isEmpty && restFiles.isEmpty + + def nextLine(): (FileManager, String) = currentFileReader match { + case None => + restFiles match { + case Nil => (this, "") + case head :: tail => + val reader = getFileReader(head) + val (block, endOfFile) = readBlockAndIsEnd(reader) + val currentReader = if (endOfFile) None else Some(reader) + (this.copy(currentFileReader = currentReader, restFiles = tail), block) + } + case Some(reader) => + val (block, endOfFile) = readBlockAndIsEnd(reader) + if (endOfFile) (this.copy(currentFileReader = None), block) + else (this, block) + + } + + private def readBlockAndIsEnd(reader: BufferedReader): (String, Boolean) = { + val line = reader.readLine() + if (line != null) + (line,false) + else{ + reader.close() + ("",true) + } + } + + private def getFileReader(file: File): BufferedReader = { + logger.info(s"Reading file ${file.getCanonicalPath}") + + var br = new BufferedReader(new FileReader(file)) + if (file.getName.endsWith(".gz")) { + val inStream = new FileInputStream(file) + val inGzipStream = new GZIPInputStream(inStream) + val inReader = new InputStreamReader(inGzipStream) //default to UTF-8 + br = new BufferedReader(inReader) + } + if (dropHeader) { + br.readLine() + } + br + } +} + +object FileManager extends LazyLogging { + private val joiner = System.getenv().getOrDefault("FILE_SPOUT_JOINER", "/").trim //gabNetwork500.csv + def apply(dir: String, fileName: String, dropHeader: Boolean): FileManager = { + val filesToRead = + if (fileName.isEmpty) + getListOfFiles(dir) + else { + val file = new File(dir + joiner + fileName) + if (file.exists && file.isFile) + List(file) + else { + logger.error(s"File $dir$joiner$fileName does not exist or is not file ") + List.empty + } + } + FileManager(None, filesToRead, dropHeader) + } + + private def getListOfFiles(dir: String): List[File] = { + val d = new File(dir) + if (d.exists && d.isDirectory) + d.listFiles.toList.filter(f => f.isFile && !f.isHidden) + else { + logger.error(s"Directory $dir does not exist or is not directory") + List.empty + } + } +} diff --git a/mainproject/cluster/src/main/scala/com/raphtory/sources/KafkaSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/sources/KafkaSource.scala new file mode 100644 index 0000000000..3702ba8c53 --- /dev/null +++ b/mainproject/cluster/src/main/scala/com/raphtory/sources/KafkaSource.scala @@ -0,0 +1,72 @@ +//package com.raphtory.spouts +// +//import java.util +//import java.util.Properties +// +//import com.raphtory.core.components.Spout.Spout +//import com.raphtory.core.components.Spout.Spout.CommonMessage.Next +//import com.raphtory.core.components.Spout.Spout.{BasicDomain, DomainMessage} +//import com.raphtory.core.model.communication.StringSpoutGoing +//import org.apache.kafka.clients.consumer.KafkaConsumer +// +//import scala.annotation.tailrec +//import scala.collection.JavaConverters._ +//import scala.concurrent.duration._ +//import scala.util.Random +// +//final case class KafkaSpout() extends Spout[BasicDomain, StringSpoutGoing] { +// log.info("initialising KafkaSpout") +// private val kafkaServer = System.getenv().getOrDefault("KAFKA_ADDRESS", "127.0.0.1").trim +// private val kafkaIp = System.getenv().getOrDefault("KAFKA_PORT", "9092").trim +// private val offset = System.getenv().getOrDefault("KAFKA_OFFSET", "earliest").trim +// private val groupId = System.getenv().getOrDefault("KAFKA_GROUP", "group" + Random.nextLong()).trim +// private val topic = System.getenv().getOrDefault("KAFKA_TOPIC", "sample_topic").trim +// private val restart = System.getenv().getOrDefault("RESTART_RATE", "10").trim.toInt +// private val startingSpeed = System.getenv().getOrDefault("STARTING_SPEED", "1000").trim.toInt +// +// private var kafkaManager = KafkaManager(kafkaServer, kafkaIp, groupId, topic, offset) +// +// override def startSpout(): Unit = +// self ! Next +// +// def handleDomainMessage(message: BasicDomain): Unit = message match { +// case Next => +// val (newManager, block) = kafkaManager.nextNLine(startingSpeed / 100) +// kafkaManager = newManager +// block.foreach(str => sendTuple(StringSpoutGoing(str))) +// context.system.scheduler.scheduleOnce(restart.millis, self, Next) +// } +//} +// +// +//final case class KafkaManager private (buffer: Stream[String], consumer: KafkaConsumer[String, String]) { +// private def poll(): KafkaManager = { +// // this is blocking operation which may waster some resource. +// // But it should only block when no data which may still make sense. +// val incoming = consumer.poll(java.time.Duration.ofMillis(3000)).asScala.toList.map(_.value()) +// this.copy(buffer = buffer ++ incoming) +// } +// +// @tailrec +// def nextNLine(blockSize: Int): (KafkaManager, List[String]) = +// if (buffer.isEmpty) +// poll().nextNLine(blockSize) +// else { +// val (take, rest) = buffer.splitAt(blockSize) +// (this.copy(buffer = rest), take.toList) +// } +//} +// +//object KafkaManager { +// def apply(server: String, ip: String, groupId: String, topic: String, offset: String): KafkaManager = { +// val props = new Properties() +// props.put("bootstrap.servers", s"$server:$ip") +// props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer") +// props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer") +// props.put("auto.offset.reset", offset) +// props.put("group.id", groupId) +// val consumer: KafkaConsumer[String, String] = new KafkaConsumer[String, String](props) +// consumer.subscribe(util.Arrays.asList(topic)) +// KafkaManager(Stream.empty, consumer) +// } +//} diff --git a/mainproject/cluster/src/main/scala/com/raphtory/spouts/blockchain/BitcoinSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/BitcoinNodeSource.scala similarity index 75% rename from mainproject/cluster/src/main/scala/com/raphtory/spouts/blockchain/BitcoinSpout.scala rename to mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/BitcoinNodeSource.scala index 518dfec7e3..09bc0759ca 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/spouts/blockchain/BitcoinSpout.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/BitcoinNodeSource.scala @@ -1,20 +1,21 @@ -package com.raphtory.spouts.blockchain - +package com.raphtory.sources.blockchain +// import java.io.File import java.io.PrintWriter -import com.raphtory.core.components.Spout.SpoutTrait -import com.raphtory.core.components.Spout.SpoutTrait.BasicDomain -import com.raphtory.core.components.Spout.SpoutTrait.CommonMessage.Next +import com.raphtory.core.components.Spout.{DataSource, NoDataAvailable} import com.raphtory.core.model.communication.SpoutGoing import scalaj.http.Http import scalaj.http.HttpRequest import spray.json._ +import scala.collection.mutable import scala.language.postfixOps import scala.sys.process._ +// +case class BitcoinTransaction(time: JsValue, block: Int, blockID: JsValue, transaction: JsValue) extends SpoutGoing -class BitcoinSpout extends SpoutTrait[BasicDomain, BitcoinTransaction] { +class BitcoinSpout extends DataSource{ var blockcount = 1 val rpcuser = System.getenv().getOrDefault("BITCOIN_USERNAME", "").trim @@ -23,18 +24,36 @@ class BitcoinSpout extends SpoutTrait[BasicDomain, BitcoinTransaction] { val id = "scala-jsonrpc" val baseRequest = Http(serverAddress).auth(rpcuser, rpcpassword).header("content-type", "text/plain") + override def setupDataSource(): Unit = {} + override def closeDataSource(): Unit = {} + + val queue = mutable.Queue[BitcoinTransaction]() + + override def generateData(): SpoutGoing = { + if(queue.isEmpty) + getTransactions() + queue.dequeue() + } + def getTransactions(): Unit = { + try { + + val re = request("getblockhash", blockcount.toString).execute().body.toString.parseJson.asJsObject + val blockID = re.fields("result") + val blockData = request("getblock", s"$blockID,2").execute().body.toString.parseJson.asJsObject + val result = blockData.fields("result") + val time = result.asJsObject.fields("time") + for (transaction <- result.asJsObject().fields("tx").asInstanceOf[JsArray].elements) + queue += BitcoinTransaction(time, blockcount, blockID, transaction) + //val time = transaction.asJsObject.fields("time") + blockcount += 1 + } catch { + case e: java.net.SocketTimeoutException => throw new NoDataAvailable + } + } //************* MESSAGE HANDLING BLOCK - def handleDomainMessage(message: BasicDomain): Unit = message match { - case Next => - try { - getTransactions() - blockcount += 1 - self ! Next - } catch { - case e: java.net.SocketTimeoutException => self ! Next - } - case _ => println("message not recognized!") + def handleDomainMessage(): Unit = { + } def outputScript() = { @@ -53,22 +72,10 @@ class BitcoinSpout extends SpoutTrait[BasicDomain, BitcoinTransaction] { def request(command: String, params: String = ""): HttpRequest = baseRequest.postData(s"""{"jsonrpc": "1.0", "id":"$id", "method": "$command", "params": [$params] }""") - def getTransactions(): Unit = { - val re = request("getblockhash", blockcount.toString).execute().body.toString.parseJson.asJsObject - val blockID = re.fields("result") - val blockData = request("getblock", s"$blockID,2").execute().body.toString.parseJson.asJsObject - val result = blockData.fields("result") - val time = result.asJsObject.fields("time") - for (transaction <- result.asJsObject().fields("tx").asInstanceOf[JsArray].elements) - sendTuple(BitcoinTransaction(time, blockcount, blockID, transaction)) - //val time = transaction.asJsObject.fields("time") - } - override def startSpout(): Unit = self ! Next } -case class BitcoinTransaction(time: JsValue, block: Int, blockID: JsValue, transaction: JsValue) extends SpoutGoing //def request(command: String, params: String = ""): HttpRequest = baseRequest.postData(s"""{"jsonrpc": "1.0", "id":"$id", "method": "$command", "params": [$params] }""") diff --git a/mainproject/cluster/src/main/scala/com/raphtory/spouts/blockchain/EthereumGethSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumGethSource.scala similarity index 74% rename from mainproject/cluster/src/main/scala/com/raphtory/spouts/blockchain/EthereumGethSpout.scala rename to mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumGethSource.scala index 3f72c817b1..c5afea0900 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/spouts/blockchain/EthereumGethSpout.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumGethSource.scala @@ -1,32 +1,27 @@ -package com.raphtory.spouts.blockchain +package com.raphtory.sources.blockchain import java.net.InetAddress import java.util.NoSuchElementException -import com.raphtory.core.components.Spout.SpoutTrait +import com.raphtory.core.components.Spout.{DataSource, Spout} import com.raphtory.core.utils.Utils import com.raphtory.tests.EtherAPITest.baseRequest import com.raphtory.tests.EtherAPITest.currentBlock import com.raphtory.tests.EtherAPITest.request -import scala.concurrent.duration.Duration -import scala.concurrent.duration.MILLISECONDS -import scala.concurrent.duration.NANOSECONDS -import scala.concurrent.duration.SECONDS import scala.language.postfixOps import scala.sys.process._ import scalaj.http.Http import scalaj.http.HttpRequest import spray.json._ -import akka.stream.ActorMaterializer -import com.raphtory.core.components.Spout.SpoutTrait.CommonMessage.Next -import com.raphtory.core.components.Spout.SpoutTrait.{BasicDomain, DomainMessage} -import com.raphtory.core.model.communication.StringSpoutGoing +import com.raphtory.core.model.communication.{SpoutGoing, StringSpoutGoing} import spray.json.DefaultJsonProtocol._ +import scala.collection.mutable -class EthereumGethSpout extends SpoutTrait[BasicDomain, StringSpoutGoing] { + +class EthereumGethSpout extends DataSource { var currentBlock = System.getenv().getOrDefault("SPOUT_ETHEREUM_START_BLOCK_INDEX", "9014194").trim.toInt var highestBlock = System.getenv().getOrDefault("SPOUT_ETHEREUM_MAXIMUM_BLOCK_INDEX", "10026447").trim.toInt @@ -36,10 +31,22 @@ class EthereumGethSpout extends SpoutTrait[BasicDomain, StringSpoutGoing] { print(currentBlock) + val queue = mutable.Queue[StringSpoutGoing]() val baseRequest = requestBuilder() - implicit val materializer = ActorMaterializer() + override def setupDataSource(): Unit = {} + override def closeDataSource(): Unit = {} + override def generateData(): SpoutGoing = { + if(queue.isEmpty) + pullNextBlock() + queue.dequeue() + } + + + + + implicit val EthFormat = jsonFormat14(EthResult) implicit val EthTransactionFormat = jsonFormat3(EthTransaction) if (nodeIP.matches(Utils.IPRegex)) @@ -47,15 +54,11 @@ class EthereumGethSpout extends SpoutTrait[BasicDomain, StringSpoutGoing] { else println(s"Connecting to Ethereum RPC \n Address:${hostname2Ip(nodeIP)} \n Port:$nodePort") - override def handleDomainMessage(message: BasicDomain): Unit = message match { - case Next => pullNextBlock() - } def pullNextBlock(): Unit = { if (currentBlock > highestBlock) return try { - log.debug(s"Trying block $currentBlock") val transactionCountHex = executeRequest("eth_getBlockTransactionCountByNumber", "\"0x" + currentBlock.toHexString + "\""); val transactionCount = Integer.parseInt(transactionCountHex.fields("result").toString().drop(3).dropRight(1), 16) if(transactionCount>0){ @@ -65,17 +68,16 @@ class EthereumGethSpout extends SpoutTrait[BasicDomain, StringSpoutGoing] { val trasnactionBlock = executeBatchRequest(transactions.dropRight(1)+"]") val transList = trasnactionBlock.parseJson.convertTo[List[EthTransaction]] transList.foreach(t => { //try needed to ignore contracts //todo include them - try{sendTuple(StringSpoutGoing(s"${t.result.blockNumber.get},${t.result.from.get},${t.result.to.get},${t.result.value.get}"))} + try{queue +=(StringSpoutGoing(s"${t.result.blockNumber.get},${t.result.from.get},${t.result.to.get},${t.result.value.get}"))} catch {case e:NoSuchElementException =>} }) } currentBlock += 1 - AllocateSpoutTask(Duration(1, NANOSECONDS), "nextBlock") } catch { - case e: NumberFormatException => AllocateSpoutTask(Duration(1, SECONDS), "nextBlock") - case e: Exception => e.printStackTrace(); AllocateSpoutTask(Duration(1, SECONDS), "nextBlock") + case e: NumberFormatException => + case e: Exception => e.printStackTrace(); } } @@ -97,7 +99,6 @@ class EthereumGethSpout extends SpoutTrait[BasicDomain, StringSpoutGoing] { def hostname2Ip(hostname: String): String = InetAddress.getByName(hostname).getHostAddress() - override def startSpout(): Unit = self ! Next } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/spouts/blockchain/EthereumPostgresSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumPostgresSource.scala similarity index 65% rename from mainproject/cluster/src/main/scala/com/raphtory/spouts/blockchain/EthereumPostgresSpout.scala rename to mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumPostgresSource.scala index c302200677..a86941a332 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/spouts/blockchain/EthereumPostgresSpout.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumPostgresSource.scala @@ -1,15 +1,15 @@ -package com.raphtory.spouts.blockchain +package com.raphtory.sources.blockchain import cats.effect.{Blocker, IO} -import com.raphtory.core.components.Spout.SpoutTrait -import com.raphtory.core.components.Spout.SpoutTrait.BasicDomain -import com.raphtory.core.components.Spout.SpoutTrait.CommonMessage.Next -import com.raphtory.core.model.communication.StringSpoutGoing +import com.raphtory.core.components.Spout.{DataSource, DataSourceComplete, Spout} +import com.raphtory.core.model.communication.{SpoutGoing, StringSpoutGoing} import doobie.implicits._ import doobie.util.ExecutionContexts import doobie.util.transactor.Transactor -class EthereumPostgresSpout extends SpoutTrait[BasicDomain, StringSpoutGoing] { +import scala.collection.mutable + +class EthereumPostgresSource extends DataSource{ var startBlock = System.getenv().getOrDefault("STARTING_BLOCK", "46147").trim.toInt //first block to have a transaction by default val batchSize = System.getenv().getOrDefault("BLOCK_BATCH_SIZE", "100").trim.toInt //number of blocks to pull each query val maxblock = System.getenv().getOrDefault("MAX_BLOCK", "8828337").trim.toInt //Maximum block in database to stop querying once this is reached @@ -27,14 +27,15 @@ class EthereumPostgresSpout extends SpoutTrait[BasicDomain, StringSpoutGoing] { dbPASSWORD, Blocker.liftExecutionContext(ExecutionContexts.synchronous) ) + val queue = mutable.Queue[StringSpoutGoing]() - override def handleDomainMessage(message: BasicDomain): Unit = message match { - case Next => running() - case _ => println("message not recognized!") + override def generateData(): SpoutGoing = { + if(queue isEmpty) + pullBlocks() + queue.dequeue() } - - protected def running(): Unit = { + protected def pullBlocks(): Unit = { sql"select from_address, to_address, value,block_timestamp from transactions where block_number >= $startBlock AND block_number < ${startBlock + batchSize} " .query[ (String, String, String, String) @@ -42,13 +43,16 @@ class EthereumPostgresSpout extends SpoutTrait[BasicDomain, StringSpoutGoing] { .to[List] // ConnectionIO[List[String]] .transact(dbconnector) // IO[List[String]] .unsafeRunSync // List[String] - .foreach(x => sendTuple(StringSpoutGoing(x.toString()))) //send each transaction to the routers + .foreach(x => queue+=(StringSpoutGoing(x.toString()))) //send each transaction to the routers - startBlock += batchSize //increment batch for the next query - if (startBlock <= maxblock) //if we have reached the max block we stop querying the database - self ! Next // line up the next batch - } + startBlock += batchSize//increment batch for the next query + if (startBlock <= maxblock) + throw new DataSourceComplete()//if we have reached the max block we stop querying the database - override def startSpout(): Unit = self ! Next + + } + + override def setupDataSource(): Unit = {} + override def closeDataSource(): Unit = {} } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/spouts/FileSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/spouts/FileSpout.scala deleted file mode 100644 index 9a33d6bba0..0000000000 --- a/mainproject/cluster/src/main/scala/com/raphtory/spouts/FileSpout.scala +++ /dev/null @@ -1,156 +0,0 @@ -package com.raphtory.spouts - -import java.io.{BufferedReader, File, FileInputStream, FileReader, InputStreamReader} -import java.nio.charset.StandardCharsets -import java.util.zip.GZIPInputStream - -import com.raphtory.core.components.Spout.SpoutTrait -import com.raphtory.core.components.Spout.SpoutTrait.DomainMessage -import com.raphtory.core.model.communication.StringSpoutGoing -import com.raphtory.spouts.FileSpout.Message.FileDomain -import com.raphtory.spouts.FileSpout.Message.Increase -import com.raphtory.spouts.FileSpout.Message.NextFile -import com.raphtory.spouts.FileSpout.Message.NextLineBlock -import com.typesafe.scalalogging.LazyLogging - -import scala.annotation.tailrec -import scala.concurrent.duration._ - -final case class FileSpout() extends SpoutTrait[FileDomain, StringSpoutGoing] { - log.info("initialise FileSpout") - private val directory = System.getenv().getOrDefault("FILE_SPOUT_DIRECTORY", "/app").trim - private val fileName = System.getenv().getOrDefault("FILE_SPOUT_FILENAME", "").trim //gabNetwork500.csv - private val dropHeader = System.getenv().getOrDefault("FILE_SPOUT_DROP_HEADER", "false").trim.toBoolean - private val JUMP = System.getenv().getOrDefault("FILE_SPOUT_BLOCK_SIZE", "10").trim.toInt - private val INCREMENT = System.getenv().getOrDefault("FILE_SPOUT_INCREMENT", "0").trim.toInt - private val TIME = System.getenv().getOrDefault("FILE_SPOUT_TIME", "60").trim.toInt - - private var fileManager = FileManager(directory, fileName, dropHeader, JUMP) - val t0 = System.nanoTime() - - def startSpout(): Unit = { - self ! NextLineBlock - } - - def handleDomainMessage(message: FileDomain): Unit = message match { - case NextLineBlock => - if (fileManager.allCompleted){ - dataFinished() - println("All files read2-" + (System.nanoTime() - t0)) - } - else { - val (newFileManager, block) = fileManager.nextLineBlock() - fileManager = newFileManager - block.foreach(str => sendTuple(StringSpoutGoing(str))) - self ! NextLineBlock - } - case NextFile => - if (fileManager.allCompleted){ - println("All files read3-" + (System.nanoTime() - t0)) - dataFinished() - } - else { - fileManager = fileManager.nextFile() - self ! NextLineBlock - } - } -} - -object FileSpout { - object Message { - sealed trait FileDomain extends DomainMessage - case object Increase extends FileDomain - case object NextLineBlock extends FileDomain - case object NextFile extends FileDomain - } -} - -final case class FileManager private ( - currentFileReader: Option[BufferedReader], - restFiles: List[File], - dropHeader: Boolean, - blockSize: Int -) extends LazyLogging { - def nextFile(): FileManager = this.copy(currentFileReader = None) - - lazy val allCompleted: Boolean = currentFileReader.isEmpty && restFiles.isEmpty - - def increaseBlockSize(inc: Int): FileManager = this.copy(blockSize = blockSize + inc) - - def nextLineBlock(): (FileManager, List[String]) = currentFileReader match { - case None => - restFiles match { - case Nil => (this, List.empty) - case head :: tail => - val reader = getFileReader(head) - val (block, endOfFile) = readBlockAndIsEnd(reader) - val currentReader = if (endOfFile) None else Some(reader) - (this.copy(currentFileReader = currentReader, restFiles = tail), block) - } - case Some(reader) => - val (block, endOfFile) = readBlockAndIsEnd(reader) - if (endOfFile) (this.copy(currentFileReader = None), block) - else (this, block) - - } - - private def readBlockAndIsEnd(reader: BufferedReader): (List[String], Boolean) = { - @tailrec - def rec(count: Int, result: List[String]): (List[String], Boolean) = - if (count > 0) { - val line = reader.readLine() - if (line != null) - rec(count - 1, result :+ line) - else { - reader.close() - (result, true) - } - } else (result, false) - rec(blockSize, List.empty) - } - - private def getFileReader(file: File): BufferedReader = { - logger.info(s"Reading file ${file.getCanonicalPath}") - - var br = new BufferedReader(new FileReader(file)) - if (file.getName.endsWith(".gz")) { - val inStream = new FileInputStream(file) - val inGzipStream = new GZIPInputStream(inStream) - val inReader = new InputStreamReader(inGzipStream) //default to UTF-8 - br = new BufferedReader(inReader) - } - if (dropHeader) { - br.readLine() - } - br - } -} - -object FileManager extends LazyLogging { - private val joiner = System.getenv().getOrDefault("FILE_SPOUT_JOINER", "/").trim //gabNetwork500.csv - def apply(dir: String, fileName: String, dropHeader: Boolean, blockSize: Int): FileManager = { - val filesToRead = - if (fileName.isEmpty) - getListOfFiles(dir) - else { - val file = new File(dir + joiner + fileName) - if (file.exists && file.isFile) - List(file) - else { - logger.error(s"File $dir$joiner$fileName does not exist or is not file ") - List.empty - } - } - FileManager(None, filesToRead, dropHeader, blockSize) - } - - private def getListOfFiles(dir: String): List[File] = { - val d = new File(dir) - if (d.exists && d.isDirectory) - d.listFiles.toList.filter(f => f.isFile && !f.isHidden) - else { - logger.error(s"Directory $dir does not exist or is not directory") - List.empty - } - } -} diff --git a/mainproject/cluster/src/main/scala/com/raphtory/spouts/KafkaSpout.scala b/mainproject/cluster/src/main/scala/com/raphtory/spouts/KafkaSpout.scala deleted file mode 100644 index 9a2b5bfd17..0000000000 --- a/mainproject/cluster/src/main/scala/com/raphtory/spouts/KafkaSpout.scala +++ /dev/null @@ -1,72 +0,0 @@ -package com.raphtory.spouts - -import java.util -import java.util.Properties - -import com.raphtory.core.components.Spout.SpoutTrait -import com.raphtory.core.components.Spout.SpoutTrait.CommonMessage.Next -import com.raphtory.core.components.Spout.SpoutTrait.{BasicDomain, DomainMessage} -import com.raphtory.core.model.communication.StringSpoutGoing -import org.apache.kafka.clients.consumer.KafkaConsumer - -import scala.annotation.tailrec -import scala.collection.JavaConverters._ -import scala.concurrent.duration._ -import scala.util.Random - -final case class KafkaSpout() extends SpoutTrait[BasicDomain, StringSpoutGoing] { - log.info("initialising KafkaSpout") - private val kafkaServer = System.getenv().getOrDefault("KAFKA_ADDRESS", "127.0.0.1").trim - private val kafkaIp = System.getenv().getOrDefault("KAFKA_PORT", "9092").trim - private val offset = System.getenv().getOrDefault("KAFKA_OFFSET", "earliest").trim - private val groupId = System.getenv().getOrDefault("KAFKA_GROUP", "group" + Random.nextLong()).trim - private val topic = System.getenv().getOrDefault("KAFKA_TOPIC", "sample_topic").trim - private val restart = System.getenv().getOrDefault("RESTART_RATE", "10").trim.toInt - private val startingSpeed = System.getenv().getOrDefault("STARTING_SPEED", "1000").trim.toInt - - private var kafkaManager = KafkaManager(kafkaServer, kafkaIp, groupId, topic, offset) - - override def startSpout(): Unit = - self ! Next - - def handleDomainMessage(message: BasicDomain): Unit = message match { - case Next => - val (newManager, block) = kafkaManager.nextNLine(startingSpeed / 100) - kafkaManager = newManager - block.foreach(str => sendTuple(StringSpoutGoing(str))) - context.system.scheduler.scheduleOnce(restart.millis, self, Next) - } -} - - -final case class KafkaManager private (buffer: Stream[String], consumer: KafkaConsumer[String, String]) { - private def poll(): KafkaManager = { - // this is blocking operation which may waster some resource. - // But it should only block when no data which may still make sense. - val incoming = consumer.poll(java.time.Duration.ofMillis(3000)).asScala.toList.map(_.value()) - this.copy(buffer = buffer ++ incoming) - } - - @tailrec - def nextNLine(blockSize: Int): (KafkaManager, List[String]) = - if (buffer.isEmpty) - poll().nextNLine(blockSize) - else { - val (take, rest) = buffer.splitAt(blockSize) - (this.copy(buffer = rest), take.toList) - } -} - -object KafkaManager { - def apply(server: String, ip: String, groupId: String, topic: String, offset: String): KafkaManager = { - val props = new Properties() - props.put("bootstrap.servers", s"$server:$ip") - props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer") - props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer") - props.put("auto.offset.reset", offset) - props.put("group.id", groupId) - val consumer: KafkaConsumer[String, String] = new KafkaConsumer[String, String](props) - consumer.subscribe(util.Arrays.asList(topic)) - KafkaManager(Stream.empty, consumer) - } -} diff --git a/mainproject/cluster/src/main/scala/com/raphtory/tests/TemporalTriangleCountExample.scala b/mainproject/cluster/src/main/scala/com/raphtory/tests/TemporalTriangleCountExample.scala index b65f7ffb4d..15bb0e27b9 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/tests/TemporalTriangleCountExample.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/tests/TemporalTriangleCountExample.scala @@ -4,6 +4,8 @@ import akka.actor.{ActorSystem, Props} import ch.qos.logback.classic.Level import com.raphtory.core.analysis.{AnalysisManager, AnalysisRestApi} import com.raphtory.core.components.ClusterManagement.{RaphtoryReplicator, WatchDog, WatermarkManager} +import com.raphtory.core.components.Spout.Spout +import com.raphtory.sources.FileSpout import kamon.Kamon import org.slf4j.LoggerFactory @@ -28,8 +30,8 @@ object TemporalTriangleCountExample extends App{ //var SpoutName ="com.raphtory.examples.test.actors.TriangleTestSpout" //var SpoutName = "com.raphtory.examples.gab.actors.GabExampleSpout" - var SpoutName = "com.raphtory.spouts.FileSpout" - system.actorOf(Props(Class.forName(SpoutName)), "Spout") + //var SpoutName = "com.raphtory.spouts.FileSpout" + system.actorOf(Props(new Spout(new FileSpout())), "Spout") //var routerClassName = "com.raphtory.examples.test.actors.TriangleTestRouter" var routerClassName = "com.raphtory.examples.lotr.LOTRRouter" diff --git a/mainproject/cluster/src/main/scala/com/raphtory/tests/curltest.sc b/mainproject/cluster/src/main/scala/com/raphtory/tests/curltest.sc index 948117bd65..c46ee59cef 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/tests/curltest.sc +++ b/mainproject/cluster/src/main/scala/com/raphtory/tests/curltest.sc @@ -1,6 +1,6 @@ val nodeIP = System.getenv().getOrDefault("SPOUT_ETHEREUM_IP_ADDRESS", "54.184.149.221").trim val nodePort = System.getenv().getOrDefault("SPOUT_ETHEREUM_PORT", "8545").trim -import com.raphtory.core.components.Spout.SpoutTrait +import com.raphtory.core.components.Spout.Spout import com.raphtory.core.utils.Utils import com.raphtory.tests.EtherAPITest.baseRequest import com.raphtory.tests.EtherAPITest.currentBlock From e9c7f00733cf25f02c1d705f2d38c8702495f859 Mon Sep 17 00:00:00 2001 From: Miratepuffin Date: Wed, 25 Nov 2020 02:24:30 +0000 Subject: [PATCH 30/58] Finished tidying up LOTR data source and the KafkaSource --- .../examples/lotr/LOTRDataSource.scala | 29 ++-- .../com/raphtory/sources/FileSource.scala | 2 +- .../com/raphtory/sources/KafkaSource.scala | 147 +++++++++--------- .../blockchain/BitcoinNodeSource.scala | 1 + .../tests/TemporalTriangleCountExample.scala | 4 +- 5 files changed, 88 insertions(+), 95 deletions(-) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRDataSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRDataSource.scala index ddfa05abb4..cae4eb770c 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRDataSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRDataSource.scala @@ -1,36 +1,25 @@ package com.raphtory.examples.lotr -import java.time.LocalDateTime - -import com.raphtory.core.components.Spout.{DataSource, DataSourceComplete, Spout} +import com.raphtory.core.components.Spout.{DataSource, DataSourceComplete} import com.raphtory.core.model.communication.{SpoutGoing, StringSpoutGoing} -import scala.concurrent.duration.{Duration, MILLISECONDS, NANOSECONDS} -import scala.io +import scala.collection.mutable + class LOTRDataSource extends DataSource { - // Relating to where the file is val directory = System.getenv().getOrDefault("LOTR_DIRECTORY", "com/raphtory/example/lotr").trim val file_name = System.getenv().getOrDefault("LOTR_FILE_NAME", "lotr.csv").trim - val fileLines = scala.io.Source.fromFile(directory + "/" + file_name).getLines.toArray - - // Initialise ready to read - var position = 0 - var linesNumber = fileLines.length - println("Start: " + LocalDateTime.now()) + val fileQueue = mutable.Queue[String]() ++= scala.io.Source.fromFile(directory + "/" + file_name).getLines.toArray - override def setupDataSource(): Unit = {} + override def setupDataSource(): Unit = {}//no setup override def generateData(): SpoutGoing = { - if(position==linesNumber) + if(fileQueue isEmpty) throw new DataSourceComplete() - else { - val line =fileLines(position) - position+=1 - StringSpoutGoing(line) - } + else + StringSpoutGoing(fileQueue.dequeue()) } - override def closeDataSource(): Unit = {} + override def closeDataSource(): Unit = {}//no file closure already done } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/sources/FileSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/sources/FileSource.scala index 8efe32b6f0..617cfb5cea 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/sources/FileSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/sources/FileSource.scala @@ -11,7 +11,7 @@ import com.typesafe.scalalogging.LazyLogging import scala.annotation.tailrec import scala.concurrent.duration._ -class FileSpout extends DataSource { +class FileSource extends DataSource { //TODO work out loggging here //log.info("initialise FileSpout") private val directory = System.getenv().getOrDefault("FILE_SPOUT_DIRECTORY", "/app").trim diff --git a/mainproject/cluster/src/main/scala/com/raphtory/sources/KafkaSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/sources/KafkaSource.scala index 3702ba8c53..d1e4a2f4d4 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/sources/KafkaSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/sources/KafkaSource.scala @@ -1,72 +1,75 @@ -//package com.raphtory.spouts -// -//import java.util -//import java.util.Properties -// -//import com.raphtory.core.components.Spout.Spout -//import com.raphtory.core.components.Spout.Spout.CommonMessage.Next -//import com.raphtory.core.components.Spout.Spout.{BasicDomain, DomainMessage} -//import com.raphtory.core.model.communication.StringSpoutGoing -//import org.apache.kafka.clients.consumer.KafkaConsumer -// -//import scala.annotation.tailrec -//import scala.collection.JavaConverters._ -//import scala.concurrent.duration._ -//import scala.util.Random -// -//final case class KafkaSpout() extends Spout[BasicDomain, StringSpoutGoing] { -// log.info("initialising KafkaSpout") -// private val kafkaServer = System.getenv().getOrDefault("KAFKA_ADDRESS", "127.0.0.1").trim -// private val kafkaIp = System.getenv().getOrDefault("KAFKA_PORT", "9092").trim -// private val offset = System.getenv().getOrDefault("KAFKA_OFFSET", "earliest").trim -// private val groupId = System.getenv().getOrDefault("KAFKA_GROUP", "group" + Random.nextLong()).trim -// private val topic = System.getenv().getOrDefault("KAFKA_TOPIC", "sample_topic").trim -// private val restart = System.getenv().getOrDefault("RESTART_RATE", "10").trim.toInt -// private val startingSpeed = System.getenv().getOrDefault("STARTING_SPEED", "1000").trim.toInt -// -// private var kafkaManager = KafkaManager(kafkaServer, kafkaIp, groupId, topic, offset) -// -// override def startSpout(): Unit = -// self ! Next -// -// def handleDomainMessage(message: BasicDomain): Unit = message match { -// case Next => -// val (newManager, block) = kafkaManager.nextNLine(startingSpeed / 100) -// kafkaManager = newManager -// block.foreach(str => sendTuple(StringSpoutGoing(str))) -// context.system.scheduler.scheduleOnce(restart.millis, self, Next) -// } -//} -// -// -//final case class KafkaManager private (buffer: Stream[String], consumer: KafkaConsumer[String, String]) { -// private def poll(): KafkaManager = { -// // this is blocking operation which may waster some resource. -// // But it should only block when no data which may still make sense. -// val incoming = consumer.poll(java.time.Duration.ofMillis(3000)).asScala.toList.map(_.value()) -// this.copy(buffer = buffer ++ incoming) -// } -// -// @tailrec -// def nextNLine(blockSize: Int): (KafkaManager, List[String]) = -// if (buffer.isEmpty) -// poll().nextNLine(blockSize) -// else { -// val (take, rest) = buffer.splitAt(blockSize) -// (this.copy(buffer = rest), take.toList) -// } -//} -// -//object KafkaManager { -// def apply(server: String, ip: String, groupId: String, topic: String, offset: String): KafkaManager = { -// val props = new Properties() -// props.put("bootstrap.servers", s"$server:$ip") -// props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer") -// props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer") -// props.put("auto.offset.reset", offset) -// props.put("group.id", groupId) -// val consumer: KafkaConsumer[String, String] = new KafkaConsumer[String, String](props) -// consumer.subscribe(util.Arrays.asList(topic)) -// KafkaManager(Stream.empty, consumer) -// } -//} +package com.raphtory.spouts + +import java.util +import java.util.Properties + +import com.raphtory.core.components.Spout.{DataSource, NoDataAvailable} +import com.raphtory.core.model.communication.{SpoutGoing, StringSpoutGoing} +import org.apache.kafka.clients.consumer.KafkaConsumer + +import scala.annotation.tailrec +import scala.collection.JavaConverters._ +import scala.collection.mutable +import scala.util.Random + +final case class KafkaSpout() extends DataSource { + private val kafkaServer = System.getenv().getOrDefault("KAFKA_ADDRESS", "127.0.0.1").trim + private val kafkaIp = System.getenv().getOrDefault("KAFKA_PORT", "9092").trim + private val offset = System.getenv().getOrDefault("KAFKA_OFFSET", "earliest").trim + private val groupId = System.getenv().getOrDefault("KAFKA_GROUP", "group" + Random.nextLong()).trim + private val topic = System.getenv().getOrDefault("KAFKA_TOPIC", "sample_topic").trim + private val restart = System.getenv().getOrDefault("RESTART_RATE", "10").trim.toInt + private val startingSpeed = System.getenv().getOrDefault("STARTING_SPEED", "1000").trim.toInt + + private var kafkaManager = KafkaManager(kafkaServer, kafkaIp, groupId, topic, offset) + val messageQueue = mutable.Queue[StringSpoutGoing]() + override def setupDataSource(): Unit = {} + + override def generateData(): SpoutGoing = { + if(messageQueue isEmpty) { + val (newManager, block) = kafkaManager.nextNLine(startingSpeed / 100) + kafkaManager = newManager + block.foreach(str => messageQueue += StringSpoutGoing(str)) + if(messageQueue isEmpty) //still empty + throw new NoDataAvailable() + } + messageQueue.dequeue() + } + + override def closeDataSource(): Unit = {} + + +} + + +final case class KafkaManager private (buffer: Stream[String], consumer: KafkaConsumer[String, String]) { + private def poll(): KafkaManager = { + // this is blocking operation which may waster some resource. + // But it should only block when no data which may still make sense. + val incoming = consumer.poll(java.time.Duration.ofMillis(3000)).asScala.toList.map(_.value()) + this.copy(buffer = buffer ++ incoming) + } + + @tailrec + def nextNLine(blockSize: Int): (KafkaManager, List[String]) = + if (buffer.isEmpty) + poll().nextNLine(blockSize) + else { + val (take, rest) = buffer.splitAt(blockSize) + (this.copy(buffer = rest), take.toList) + } +} + +object KafkaManager { + def apply(server: String, ip: String, groupId: String, topic: String, offset: String): KafkaManager = { + val props = new Properties() + props.put("bootstrap.servers", s"$server:$ip") + props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer") + props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer") + props.put("auto.offset.reset", offset) + props.put("group.id", groupId) + val consumer: KafkaConsumer[String, String] = new KafkaConsumer[String, String](props) + consumer.subscribe(util.Arrays.asList(topic)) + KafkaManager(Stream.empty, consumer) + } +} diff --git a/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/BitcoinNodeSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/BitcoinNodeSource.scala index 09bc0759ca..cc7e98c693 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/BitcoinNodeSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/BitcoinNodeSource.scala @@ -34,6 +34,7 @@ class BitcoinSpout extends DataSource{ getTransactions() queue.dequeue() } + def getTransactions(): Unit = { try { diff --git a/mainproject/cluster/src/main/scala/com/raphtory/tests/TemporalTriangleCountExample.scala b/mainproject/cluster/src/main/scala/com/raphtory/tests/TemporalTriangleCountExample.scala index 15bb0e27b9..c120644ad2 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/tests/TemporalTriangleCountExample.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/tests/TemporalTriangleCountExample.scala @@ -5,7 +5,7 @@ import ch.qos.logback.classic.Level import com.raphtory.core.analysis.{AnalysisManager, AnalysisRestApi} import com.raphtory.core.components.ClusterManagement.{RaphtoryReplicator, WatchDog, WatermarkManager} import com.raphtory.core.components.Spout.Spout -import com.raphtory.sources.FileSpout +import com.raphtory.sources.FileSource import kamon.Kamon import org.slf4j.LoggerFactory @@ -31,7 +31,7 @@ object TemporalTriangleCountExample extends App{ //var SpoutName ="com.raphtory.examples.test.actors.TriangleTestSpout" //var SpoutName = "com.raphtory.examples.gab.actors.GabExampleSpout" //var SpoutName = "com.raphtory.spouts.FileSpout" - system.actorOf(Props(new Spout(new FileSpout())), "Spout") + system.actorOf(Props(new Spout(new FileSource())), "Spout") //var routerClassName = "com.raphtory.examples.test.actors.TriangleTestRouter" var routerClassName = "com.raphtory.examples.lotr.LOTRRouter" From 052aae03fb09169b535b21aa7c66abc9f80942c1 Mon Sep 17 00:00:00 2001 From: niallproche Date: Wed, 25 Nov 2020 15:31:47 +0000 Subject: [PATCH 31/58] added recursive directory checking when searching for files --- .../main/scala/com/raphtory/sources/FileSource.scala | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/sources/FileSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/sources/FileSource.scala index 617cfb5cea..5671e975b6 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/sources/FileSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/sources/FileSource.scala @@ -112,11 +112,18 @@ object FileManager extends LazyLogging { private def getListOfFiles(dir: String): List[File] = { val d = new File(dir) - if (d.exists && d.isDirectory) - d.listFiles.toList.filter(f => f.isFile && !f.isHidden) + if (d.exists && d.isDirectory) { + val files = getRecursiveListOfFiles(d) + files.filter(f => f.isFile && !f.isHidden) + } else { logger.error(s"Directory $dir does not exist or is not directory") List.empty } } + + private def getRecursiveListOfFiles(dir: File): List[File] = { + val these = dir.listFiles.toList + these ++ these.filter(_.isDirectory).flatMap(getRecursiveListOfFiles) + } } From 457118e5fe731c10693e0f5470153e3adf1b40aa Mon Sep 17 00:00:00 2001 From: Miratepuffin Date: Wed, 25 Nov 2020 15:40:59 +0000 Subject: [PATCH 32/58] Changed from exceptions to Options for the new Spout API --- .../core/components/Spout/DataSource.scala | 12 +++--- .../core/components/Spout/Spout.scala | 41 ++++++++++++------- .../gab/datasources/GabMongoSource.scala | 13 +++--- .../examples/lotr/LOTRDataSource.scala | 14 ++++--- .../test/actors/RandomDataSource.scala | 2 +- .../test/actors/TriangleDataSource.scala | 12 +++--- .../com/raphtory/sources/FileSource.scala | 9 ++-- .../com/raphtory/sources/KafkaSource.scala | 8 ++-- .../blockchain/BitcoinNodeSource.scala | 10 ++--- .../blockchain/EthereumGethSource.scala | 6 +-- .../blockchain/EthereumPostgresSource.scala | 10 ++--- 11 files changed, 77 insertions(+), 60 deletions(-) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/DataSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/DataSource.scala index 9d0b31460f..88f99c2457 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/DataSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/DataSource.scala @@ -2,15 +2,17 @@ package com.raphtory.core.components.Spout import com.raphtory.core.model.communication.SpoutGoing -class NoDataAvailable extends Exception -class DataSourceComplete extends Exception + trait DataSource{ + private var dataComplete = false + def setupDataSource():Unit - @throws(classOf[NoDataAvailable]) - @throws(classOf[DataSourceComplete]) - def generateData():SpoutGoing + def generateData():Option[SpoutGoing] def closeDataSource():Unit + + def dataSourceComplete():Unit = dataComplete=true + def isComplete():Boolean = dataComplete } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/Spout.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/Spout.scala index 22465ea15a..0a6972a97e 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/Spout.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/Spout.scala @@ -25,6 +25,7 @@ class Spout(datasource:DataSource) extends Actor with ActorLogging with Timers { private var count = 0 private var partitionManagers = 0 private var routers = 0 + private var datafinishedSent = false private def recordUpdate(): Unit = { spoutTuples.increment() @@ -49,10 +50,20 @@ class Spout(datasource:DataSource) extends Actor with ActorLogging with Timers { context.become(work(clusterUp,pmCounter,rmCounter)) context.system.scheduler.scheduleOnce(1 second, self, StateCheck) case IsSafe => processIsSafeMessage(safe,pmCounter,rmCounter) - case WorkPlease => sendData(context.sender()) + case WorkPlease => processWorkPlease() case unhandled => log.error(s"Unable to handle message [$unhandled].") } + private def processWorkPlease():Unit = { + if(datasource.isComplete()) + if(!datafinishedSent) { + sender ! DataFinished + datafinishedSent=true + }else + sender ! NoWork + else + sendData(context.sender()) + } private def processStateCheckMessage(safe: Boolean): Unit = { log.debug(s"Spout is handling [StateCheck] message.") @@ -86,20 +97,22 @@ class Spout(datasource:DataSource) extends Actor with ActorLogging with Timers { def sendData(sender:ActorRef): Unit = { - try { - val work =datasource.generateData() - val message = if (count % 10000 == 0) - AllocateTrackedTuple(System.currentTimeMillis(),work) - else - AllocateTuple(work) - sender ! message - recordUpdate() - if (count % 10000 == 0) println(s"Spout at Message $count") + + datasource.generateData() match { + case Some(work) => + val message = if (count % 10000 == 0) + AllocateTrackedTuple(System.currentTimeMillis(),work) + else + AllocateTuple(work) + sender ! message + recordUpdate() + if (count % 10000 == 0) println(s"Spout at Message $count") + case None => sender ! NoWork }//mediator ! DistributedPubSubMediator.Send(lastRouter, message, localAffinity = false) - catch { - case e:NoDataAvailable => sender ! NoWork - case e:DataSourceComplete => sender ! DataFinished; println("All data sent") - } + if(datasource isComplete) { + sender ! DataFinished + datafinishedSent=true + } } def AllocateSpoutTask(duration: FiniteDuration, task: Any): Cancellable = { diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/datasources/GabMongoSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/datasources/GabMongoSource.scala index 230ca2e241..545f5baab6 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/datasources/GabMongoSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/datasources/GabMongoSource.scala @@ -29,15 +29,12 @@ final class GabMongoDataSource extends DataSource { val root = LoggerFactory.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME).asInstanceOf[ch.qos.logback.classic.Logger] root.setLevel(Level.ERROR) - var queue = mutable.Queue[StringSpoutGoing]() + var queue = mutable.Queue[Option[StringSpoutGoing]]() - override def generateData(): SpoutGoing = { - if(queue.isEmpty) { + override def generateData(): Option[SpoutGoing] = { + if(queue.isEmpty) getNextPosts() - queue.dequeue() - } - else - queue.dequeue() + queue.dequeue() } @@ -47,7 +44,7 @@ final class GabMongoDataSource extends DataSource { for (x <- mongoColl.find("_id" $lt postMax $gt postMin)) try { val data = x.get("data").toString.drop(2).dropRight(1).replaceAll("""\\"""", "").replaceAll("""\\""", "") - queue += StringSpoutGoing(data) + queue += Some(StringSpoutGoing(data)) } catch { case e: Throwable => println("Cannot parse record") diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRDataSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRDataSource.scala index cae4eb770c..27e8178955 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRDataSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRDataSource.scala @@ -1,6 +1,6 @@ package com.raphtory.examples.lotr -import com.raphtory.core.components.Spout.{DataSource, DataSourceComplete} +import com.raphtory.core.components.Spout.{DataSource} import com.raphtory.core.model.communication.{SpoutGoing, StringSpoutGoing} import scala.collection.mutable @@ -10,15 +10,17 @@ class LOTRDataSource extends DataSource { val directory = System.getenv().getOrDefault("LOTR_DIRECTORY", "com/raphtory/example/lotr").trim val file_name = System.getenv().getOrDefault("LOTR_FILE_NAME", "lotr.csv").trim - val fileQueue = mutable.Queue[String]() ++= scala.io.Source.fromFile(directory + "/" + file_name).getLines.toArray + val fileQueue = mutable.Queue[String]() ++= scala.io.Source.fromFile(directory + "/" + file_name).getLines override def setupDataSource(): Unit = {}//no setup - override def generateData(): SpoutGoing = { - if(fileQueue isEmpty) - throw new DataSourceComplete() + override def generateData(): Option[SpoutGoing] = { + if(fileQueue isEmpty){ + dataSourceComplete() + None + } else - StringSpoutGoing(fileQueue.dequeue()) + Some(StringSpoutGoing(fileQueue.dequeue())) } override def closeDataSource(): Unit = {}//no file closure already done diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomDataSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomDataSource.scala index c3847d0b2c..0c2fd37cc6 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomDataSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomDataSource.scala @@ -16,7 +16,7 @@ class RandomDataSource extends DataSource { override def setupDataSource(): Unit = {} - override def generateData(): SpoutGoing = StringSpoutGoing(distribution()) + override def generateData(): Option[SpoutGoing] = Some(StringSpoutGoing(distribution())) override def closeDataSource(): Unit = {} diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleDataSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleDataSource.scala index 3c4c5d8b57..93b9321593 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleDataSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleDataSource.scala @@ -1,6 +1,6 @@ package com.raphtory.examples.test.actors -import com.raphtory.core.components.Spout.{DataSource, DataSourceComplete, Spout} +import com.raphtory.core.components.Spout.{DataSource} import com.raphtory.core.model.communication.{SpoutGoing, StringSpoutGoing} import scala.collection.mutable.Queue @@ -11,11 +11,13 @@ class TriangleDataSource extends DataSource { override def setupDataSource(): Unit = {} - override def generateData(): SpoutGoing = { - if(edges.isEmpty) - throw new DataSourceComplete() + override def generateData(): Option[SpoutGoing] = { + if(edges.isEmpty) { + dataSourceComplete() + None + } else - edges.dequeue() + Some(edges.dequeue()) } override def closeDataSource(): Unit = {} diff --git a/mainproject/cluster/src/main/scala/com/raphtory/sources/FileSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/sources/FileSource.scala index 617cfb5cea..00d11e9624 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/sources/FileSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/sources/FileSource.scala @@ -4,7 +4,7 @@ import java.io.{BufferedReader, File, FileInputStream, FileReader, InputStreamRe import java.nio.charset.StandardCharsets import java.util.zip.GZIPInputStream -import com.raphtory.core.components.Spout.{DataSource, DataSourceComplete, NoDataAvailable, Spout} +import com.raphtory.core.components.Spout.{DataSource} import com.raphtory.core.model.communication.StringSpoutGoing import com.typesafe.scalalogging.LazyLogging @@ -23,14 +23,15 @@ class FileSource extends DataSource { private var fileManager = FileManager(directory, fileName, dropHeader) - override def generateData(): StringSpoutGoing = { + override def generateData(): Option[StringSpoutGoing] = { if (fileManager.allCompleted) { - throw new DataSourceComplete() + dataSourceComplete() + None } else { val (newFileManager, line) = fileManager.nextLine() fileManager = newFileManager - StringSpoutGoing(line) + Some(StringSpoutGoing(line)) } } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/sources/KafkaSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/sources/KafkaSource.scala index d1e4a2f4d4..7d98a445a7 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/sources/KafkaSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/sources/KafkaSource.scala @@ -3,7 +3,7 @@ package com.raphtory.spouts import java.util import java.util.Properties -import com.raphtory.core.components.Spout.{DataSource, NoDataAvailable} +import com.raphtory.core.components.Spout.{DataSource} import com.raphtory.core.model.communication.{SpoutGoing, StringSpoutGoing} import org.apache.kafka.clients.consumer.KafkaConsumer @@ -25,15 +25,15 @@ final case class KafkaSpout() extends DataSource { val messageQueue = mutable.Queue[StringSpoutGoing]() override def setupDataSource(): Unit = {} - override def generateData(): SpoutGoing = { + override def generateData(): Option[SpoutGoing] = { if(messageQueue isEmpty) { val (newManager, block) = kafkaManager.nextNLine(startingSpeed / 100) kafkaManager = newManager block.foreach(str => messageQueue += StringSpoutGoing(str)) if(messageQueue isEmpty) //still empty - throw new NoDataAvailable() + None } - messageQueue.dequeue() + Some(messageQueue.dequeue()) } override def closeDataSource(): Unit = {} diff --git a/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/BitcoinNodeSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/BitcoinNodeSource.scala index cc7e98c693..41b73425a4 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/BitcoinNodeSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/BitcoinNodeSource.scala @@ -3,7 +3,7 @@ package com.raphtory.sources.blockchain import java.io.File import java.io.PrintWriter -import com.raphtory.core.components.Spout.{DataSource, NoDataAvailable} +import com.raphtory.core.components.Spout.DataSource import com.raphtory.core.model.communication.SpoutGoing import scalaj.http.Http import scalaj.http.HttpRequest @@ -27,9 +27,9 @@ class BitcoinSpout extends DataSource{ override def setupDataSource(): Unit = {} override def closeDataSource(): Unit = {} - val queue = mutable.Queue[BitcoinTransaction]() + val queue = mutable.Queue[Option[BitcoinTransaction]]() - override def generateData(): SpoutGoing = { + override def generateData(): Option[SpoutGoing] = { if(queue.isEmpty) getTransactions() queue.dequeue() @@ -44,11 +44,11 @@ class BitcoinSpout extends DataSource{ val result = blockData.fields("result") val time = result.asJsObject.fields("time") for (transaction <- result.asJsObject().fields("tx").asInstanceOf[JsArray].elements) - queue += BitcoinTransaction(time, blockcount, blockID, transaction) + queue += Some(BitcoinTransaction(time, blockcount, blockID, transaction)) //val time = transaction.asJsObject.fields("time") blockcount += 1 } catch { - case e: java.net.SocketTimeoutException => throw new NoDataAvailable + case e: java.net.SocketTimeoutException => queue += None } } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumGethSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumGethSource.scala index c5afea0900..323b43d258 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumGethSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumGethSource.scala @@ -31,13 +31,13 @@ class EthereumGethSpout extends DataSource { print(currentBlock) - val queue = mutable.Queue[StringSpoutGoing]() + val queue = mutable.Queue[Option[StringSpoutGoing]]() val baseRequest = requestBuilder() override def setupDataSource(): Unit = {} override def closeDataSource(): Unit = {} - override def generateData(): SpoutGoing = { + override def generateData(): Option[SpoutGoing] = { if(queue.isEmpty) pullNextBlock() queue.dequeue() @@ -68,7 +68,7 @@ class EthereumGethSpout extends DataSource { val trasnactionBlock = executeBatchRequest(transactions.dropRight(1)+"]") val transList = trasnactionBlock.parseJson.convertTo[List[EthTransaction]] transList.foreach(t => { //try needed to ignore contracts //todo include them - try{queue +=(StringSpoutGoing(s"${t.result.blockNumber.get},${t.result.from.get},${t.result.to.get},${t.result.value.get}"))} + try{queue +=(Some(StringSpoutGoing(s"${t.result.blockNumber.get},${t.result.from.get},${t.result.to.get},${t.result.value.get}")))} catch {case e:NoSuchElementException =>} }) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumPostgresSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumPostgresSource.scala index a86941a332..bda843675c 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumPostgresSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumPostgresSource.scala @@ -1,7 +1,7 @@ package com.raphtory.sources.blockchain import cats.effect.{Blocker, IO} -import com.raphtory.core.components.Spout.{DataSource, DataSourceComplete, Spout} +import com.raphtory.core.components.Spout.{DataSource} import com.raphtory.core.model.communication.{SpoutGoing, StringSpoutGoing} import doobie.implicits._ import doobie.util.ExecutionContexts @@ -27,9 +27,9 @@ class EthereumPostgresSource extends DataSource{ dbPASSWORD, Blocker.liftExecutionContext(ExecutionContexts.synchronous) ) - val queue = mutable.Queue[StringSpoutGoing]() + val queue = mutable.Queue[Option[StringSpoutGoing]]() - override def generateData(): SpoutGoing = { + override def generateData(): Option[SpoutGoing] = { if(queue isEmpty) pullBlocks() queue.dequeue() @@ -43,11 +43,11 @@ class EthereumPostgresSource extends DataSource{ .to[List] // ConnectionIO[List[String]] .transact(dbconnector) // IO[List[String]] .unsafeRunSync // List[String] - .foreach(x => queue+=(StringSpoutGoing(x.toString()))) //send each transaction to the routers + .foreach(x => queue+=(Some(StringSpoutGoing(x.toString())))) //send each transaction to the routers startBlock += batchSize//increment batch for the next query if (startBlock <= maxblock) - throw new DataSourceComplete()//if we have reached the max block we stop querying the database + dataSourceComplete()//if we have reached the max block we stop querying the database From b16018c6fd6d6bdadce9c9683af2f3dcf648f85f Mon Sep 17 00:00:00 2001 From: Miratepuffin Date: Thu, 26 Nov 2020 17:10:01 +0000 Subject: [PATCH 33/58] Removed the SpoutGoing Trait to simplify the datasource from the users perspective --- .../core/components/Router/GraphBuilder.scala | 15 +++++++++++++++ .../core/components/Router/RouterWorker.scala | 2 +- .../core/components/Spout/DataSource.scala | 7 ++----- .../raphtory/core/components/Spout/Spout.scala | 4 +++- .../model/communication/raphtoryMessages.scala | 4 +--- .../blockchain/routers/BitcoinRouter.scala | 1 + .../blockchain/routers/ChABClus2ClusRouter.scala | 6 +++--- .../routers/ChABClus2ClusRouterNoprop.scala | 6 +++--- .../blockchain/routers/ChainalysisABRouter.scala | 6 +++--- .../blockchain/routers/DashcoinRouter.scala | 1 + .../blockchain/routers/EthereumGethRouter.scala | 8 ++++---- .../blockchain/routers/EthereumKafkaRouter.scala | 8 ++++---- .../routers/EthereumTransactionRouter.scala | 8 ++++---- .../blockchain/routers/FirehoseKafkaRouter.scala | 8 ++++---- .../blockchain/routers/LitecoinRouter.scala | 3 ++- .../examples/citationNetwork/CitationRouter.scala | 6 +++--- .../examples/gab/datasources/GabMongoSource.scala | 8 ++++---- .../examples/gab/routers/GabPostGraphRouter.scala | 6 +++--- .../examples/gab/routers/GabRawRouter.scala | 8 ++++---- .../examples/gab/routers/GabUserGraphRouter.scala | 6 +++--- .../examples/ldbc/routers/LDBCOldRouter.scala | 6 +++--- .../examples/ldbc/routers/LDBCRouter.scala | 6 +++--- .../raphtory/examples/lotr/LOTRDataSource.scala | 6 +++--- .../com/raphtory/examples/lotr/LOTRRouter.scala | 6 +++--- .../examples/test/actors/RandomDataSource.scala | 4 ++-- .../examples/test/actors/RandomRouter.scala | 4 ++-- .../examples/test/actors/TriangleDataSource.scala | 6 +++--- .../examples/test/actors/TriangleTestRouter.scala | 8 ++++---- .../routers/TrackAndTraceRouter.scala | 6 +++--- .../com/raphtory/examples/tsvnet/TSVRouter.scala | 8 ++++---- .../routers/CooccurrenceMatrixRouter.scala | 8 ++++---- .../CooccurrenceMatrixRouterFiltered.scala | 6 +++--- .../scala/com/raphtory/sources/FileSource.scala | 6 +++--- .../scala/com/raphtory/sources/KafkaSource.scala | 8 ++++---- .../sources/blockchain/BitcoinNodeSource.scala | 6 +++--- .../sources/blockchain/EthereumGethSource.scala | 8 ++++---- .../blockchain/EthereumPostgresSource.scala | 8 ++++---- 37 files changed, 123 insertions(+), 108 deletions(-) create mode 100644 mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/GraphBuilder.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/GraphBuilder.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/GraphBuilder.scala new file mode 100644 index 0000000000..2b67ac9d00 --- /dev/null +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/GraphBuilder.scala @@ -0,0 +1,15 @@ +package com.raphtory.core.components.Router + +import com.raphtory.core.model.communication.GraphUpdate + +import scala.collection.mutable +import scala.collection.parallel.mutable.ParHashSet + +abstract class GraphBuilder[T] { + + var updates:mutable.HashSet[GraphUpdate] = mutable.HashSet[GraphUpdate]() + +// protected def parseTuple(tuple: T): + + +} diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala index 45f55db7a0..8f76a0a4bb 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala @@ -18,7 +18,7 @@ import scala.util.hashing.MurmurHash3 // TODO Add val name which sub classes that extend this trait must overwrite // e.g. BlockChainRouter val name = "Blockchain Router" // Log.debug that read 'Router' should then read 'Blockchain Router' -abstract class RouterWorker[In <: SpoutGoing](val routerId: Int, val workerID: Int, val initialManagerCount: Int,val initialRouterCount:Int) +abstract class RouterWorker[In <: Any](val routerId: Int, val workerID: Int, val initialManagerCount: Int,val initialRouterCount:Int) extends Actor with ActorLogging { implicit val executionContext: ExecutionContext = context.system.dispatcher diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/DataSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/DataSource.scala index 88f99c2457..bd57d4f28a 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/DataSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/DataSource.scala @@ -1,15 +1,12 @@ package com.raphtory.core.components.Spout -import com.raphtory.core.model.communication.SpoutGoing - - -trait DataSource{ +trait DataSource[+T]{ private var dataComplete = false def setupDataSource():Unit - def generateData():Option[SpoutGoing] + def generateData():Option[T] def closeDataSource():Unit diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/Spout.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/Spout.scala index 0a6972a97e..bbb48a643b 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/Spout.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/Spout.scala @@ -16,7 +16,7 @@ import scala.language.postfixOps -class Spout(datasource:DataSource) extends Actor with ActorLogging with Timers { +class Spout(datasource:DataSource[Any]) extends Actor with ActorLogging with Timers { // todo: wvv should assign the dispatcher when create the actor implicit val executionContext: ExecutionContext = context.system.dispatchers.lookup("spout-dispatcher") //implicit val executionContext: ExecutionContext = context.system.dispatcher @@ -58,6 +58,7 @@ class Spout(datasource:DataSource) extends Actor with ActorLogging with Timers { if(datasource.isComplete()) if(!datafinishedSent) { sender ! DataFinished + println(s"All data sent") datafinishedSent=true }else sender ! NoWork @@ -111,6 +112,7 @@ class Spout(datasource:DataSource) extends Actor with ActorLogging with Timers { }//mediator ! DistributedPubSubMediator.Send(lastRouter, message, localAffinity = false) if(datasource isComplete) { sender ! DataFinished + println(s"All data sent") datafinishedSent=true } } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/model/communication/raphtoryMessages.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/model/communication/raphtoryMessages.scala index 2f0af0844d..362199b45c 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/model/communication/raphtoryMessages.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/model/communication/raphtoryMessages.scala @@ -19,9 +19,7 @@ sealed trait TrackedGraphUpdate{ val spoutTime:Long } -trait SpoutGoing -final case class StringSpoutGoing(value: String) extends SpoutGoing -case class DataFinished() extends SpoutGoing +case class DataFinished() case class DataFinishedSync(time:Long) case class RouterUp(id: Int) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/BitcoinRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/BitcoinRouter.scala index 3fe72f9524..b28aec2f3c 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/BitcoinRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/BitcoinRouter.scala @@ -2,6 +2,7 @@ package com.raphtory.examples.blockchain.routers import com.raphtory.core.components.Router.RouterWorker import com.raphtory.core.model.communication._ +import com.raphtory.sources.blockchain.BitcoinTransaction import spray.json.JsArray import scala.collection.mutable.ListBuffer diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusRouter.scala index 3c44137d65..d87210ab98 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusRouter.scala @@ -11,10 +11,10 @@ import scala.collection.mutable.ListBuffer import scala.collection.parallel.mutable.ParHashSet class ChABClus2ClusRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) - extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount,initialRouterCount) { + extends RouterWorker[String](routerId,workerID, initialManagerCount,initialRouterCount) { - override protected def parseTuple(tuple: StringSpoutGoing): ParHashSet[GraphUpdate] = { - val dp = formatLine(tuple.value.split(",").map(_.trim)) + override protected def parseTuple(tuple: String): ParHashSet[GraphUpdate] = { + val dp = formatLine(tuple.split(",").map(_.trim)) val transactionTime = dp.time val srcClusterId = dp.srcCluster val dstClusterId = dp.dstCluster diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusRouterNoprop.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusRouterNoprop.scala index 0892c5950e..a34ca903d0 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusRouterNoprop.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusRouterNoprop.scala @@ -11,10 +11,10 @@ import scala.collection.mutable.ListBuffer import scala.collection.parallel.mutable.ParHashSet class ChABClus2ClusRouterNoprop(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) - extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount,initialRouterCount) { + extends RouterWorker[String](routerId,workerID, initialManagerCount,initialRouterCount) { - override protected def parseTuple(tuple: StringSpoutGoing): ParHashSet[GraphUpdate] = { - val dp = formatLine(tuple.value.split(",").map(_.trim)) + override protected def parseTuple(tuple: String): ParHashSet[GraphUpdate] = { + val dp = formatLine(tuple.split(",").map(_.trim)) val transactionTime = dp.time val srcClusterId = dp.srcCluster val dstClusterId = dp.dstCluster diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChainalysisABRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChainalysisABRouter.scala index fb9f673c21..0c7cdacc51 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChainalysisABRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChainalysisABRouter.scala @@ -10,10 +10,10 @@ import scala.collection.mutable.ListBuffer import scala.collection.parallel.mutable.ParHashSet class ChainalysisABRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) - extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount,initialRouterCount) { + extends RouterWorker[String](routerId,workerID, initialManagerCount,initialRouterCount) { - override protected def parseTuple(tuple: StringSpoutGoing): ParHashSet[GraphUpdate] = { - val dp = formatLine(tuple.value.split(",").map(_.trim)) + override protected def parseTuple(tuple: String): ParHashSet[GraphUpdate] = { + val dp = formatLine(tuple.split(",").map(_.trim)) val transactionTime = dp.time val srcClusterId = dp.srcCluster val dstClusterId = dp.dstCluster diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/DashcoinRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/DashcoinRouter.scala index 41a7cecc5f..5e1882c62e 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/DashcoinRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/DashcoinRouter.scala @@ -3,6 +3,7 @@ package com.raphtory.examples.blockchain.routers import com.raphtory.core.components.Router.RouterWorker import com.raphtory.core.model.communication._ import com.raphtory.examples.blockchain.routers +import com.raphtory.sources.blockchain.BitcoinTransaction import spray.json.{JsArray, JsValue} import scala.collection.mutable.ListBuffer diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumGethRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumGethRouter.scala index cac1981f39..bf41da22d9 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumGethRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumGethRouter.scala @@ -2,7 +2,7 @@ package com.raphtory.examples.blockchain.routers import akka.http.scaladsl.unmarshalling.Unmarshal import com.raphtory.core.components.Router.RouterWorker -import com.raphtory.core.model.communication.{EdgeAddWithProperties, GraphUpdate, ImmutableProperty, Properties, StringProperty, StringSpoutGoing, VertexAddWithProperties} +import com.raphtory.core.model.communication.{EdgeAddWithProperties, GraphUpdate, ImmutableProperty, Properties, StringProperty, VertexAddWithProperties} import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._ import akka.stream.ActorMaterializer import spray.json._ @@ -11,14 +11,14 @@ import scala.collection.mutable.ListBuffer import scala.collection.parallel.mutable.ParHashSet import scala.util.hashing.MurmurHash3 class EthereumGethRouter(override val routerId: Int,override val workerID:Int,override val initialManagerCount: Int, override val initialRouterCount: Int) - extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount, initialRouterCount) { + extends RouterWorker[String](routerId,workerID, initialManagerCount, initialRouterCount) { def hexToInt(hex: String) = Integer.parseInt(hex.drop(2), 16) print(routerId) - override protected def parseTuple(tuple: StringSpoutGoing): ParHashSet[GraphUpdate] = { + override protected def parseTuple(tuple: String): ParHashSet[GraphUpdate] = { print(tuple) - val transaction = tuple.value.split(",") + val transaction = tuple.split(",") val blockNumber = hexToInt(transaction(0)) val from = transaction(1).replaceAll("\"", "").toLowerCase diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumKafkaRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumKafkaRouter.scala index 324b4baeee..c8ea3563a3 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumKafkaRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumKafkaRouter.scala @@ -2,7 +2,7 @@ package com.raphtory.examples.blockchain.routers import akka.http.scaladsl.unmarshalling.Unmarshal import com.raphtory.core.components.Router.RouterWorker -import com.raphtory.core.model.communication.{EdgeAddWithProperties, GraphUpdate, ImmutableProperty, Properties, StringProperty, StringSpoutGoing, VertexAddWithProperties} +import com.raphtory.core.model.communication.{EdgeAddWithProperties, GraphUpdate, ImmutableProperty, Properties, StringProperty, VertexAddWithProperties} import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._ import akka.stream.ActorMaterializer import spray.json._ @@ -11,11 +11,11 @@ import scala.collection.mutable.ListBuffer import scala.collection.parallel.mutable.ParHashSet import scala.util.hashing.MurmurHash3 class EthereumKafkaRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) - extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount, initialRouterCount) { + extends RouterWorker[String](routerId,workerID, initialManagerCount, initialRouterCount) { def hexToInt(hex: String) = Integer.parseInt(hex.drop(2), 16) - override protected def parseTuple(tuple: StringSpoutGoing): ParHashSet[GraphUpdate] = { + override protected def parseTuple(tuple: String): ParHashSet[GraphUpdate] = { - val transaction = tuple.value.split(",") + val transaction = tuple.split(",") if(transaction(1).equals("block_number")) return ParHashSet() diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumTransactionRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumTransactionRouter.scala index 3b0fa3a405..5c9ff2c6e4 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumTransactionRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumTransactionRouter.scala @@ -1,17 +1,17 @@ package com.raphtory.examples.blockchain.routers import com.raphtory.core.components.Router.RouterWorker -import com.raphtory.core.model.communication.{EdgeAddWithProperties, GraphUpdate, Properties, StringProperty, StringSpoutGoing, VertexAddWithProperties} +import com.raphtory.core.model.communication.{EdgeAddWithProperties, GraphUpdate, Properties, StringProperty, VertexAddWithProperties} import scala.collection.mutable.ListBuffer import scala.collection.parallel.mutable.ParHashSet import scala.util.hashing.MurmurHash3 class EthereumTransactionRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) - extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount, initialRouterCount) { + extends RouterWorker[String](routerId,workerID, initialManagerCount, initialRouterCount) { - override protected def parseTuple(tuple: StringSpoutGoing): ParHashSet[GraphUpdate] = { - val components = tuple.value.drop(1).dropRight(1).split(",") + override protected def parseTuple(tuple: String): ParHashSet[GraphUpdate] = { + val components = tuple.drop(1).dropRight(1).split(",") val creationDate = components(3).toLong * 1000 //seconds to miliseconds val sourceNode = MurmurHash3.stringHash(components(0)) //hash the id to get a vertex ID val commands = new ParHashSet[GraphUpdate]() diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/FirehoseKafkaRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/FirehoseKafkaRouter.scala index 8419a8fde0..156bd6324d 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/FirehoseKafkaRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/FirehoseKafkaRouter.scala @@ -2,7 +2,7 @@ package com.raphtory.examples.blockchain.routers import akka.http.scaladsl.unmarshalling.Unmarshal import com.raphtory.core.components.Router.RouterWorker -import com.raphtory.core.model.communication.{DoubleProperty, EdgeAdd, EdgeAddWithProperties, EdgeDelete, GraphUpdate, ImmutableProperty, LongProperty, Properties, StringProperty, StringSpoutGoing, VertexAdd, VertexAddWithProperties, VertexDelete} +import com.raphtory.core.model.communication.{DoubleProperty, EdgeAdd, EdgeAddWithProperties, EdgeDelete, GraphUpdate, ImmutableProperty, LongProperty, Properties, StringProperty, VertexAdd, VertexAddWithProperties, VertexDelete} import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._ import akka.stream.ActorMaterializer import spray.json._ @@ -13,15 +13,15 @@ import scala.util.Random import scala.util.hashing.MurmurHash3 import scala.math.BigInt class FirehoseKafkaRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) - extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount, initialRouterCount) { + extends RouterWorker[String](routerId,workerID, initialManagerCount, initialRouterCount) { var DELETEPERCENT = System.getenv().getOrDefault("ETHER_DELETE_PERCENT", "0").trim.toDouble/100 var DELETESEED = System.getenv().getOrDefault("ETHER_DELETE_SEED", "123").trim.toInt val random = new Random(DELETESEED) def hexToInt(hex: String) = Integer.parseInt(hex.drop(2), 16) - override protected def parseTuple(tuple: StringSpoutGoing): ParHashSet[GraphUpdate] = { + override protected def parseTuple(tuple: String): ParHashSet[GraphUpdate] = { //if(value.toString.contains("0xa09871aeadf4994ca12f5c0b6056bbd1d343c029")) println(value.toString) - val transaction = tuple.value.split(",") + val transaction = tuple.split(",") if(transaction(1).equals("block_number")) return ParHashSet() val blockNumber = transaction(2).toInt val commands = new ParHashSet[GraphUpdate]() diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/LitecoinRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/LitecoinRouter.scala index 4dbdb1b118..66f20e5b3e 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/LitecoinRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/LitecoinRouter.scala @@ -2,12 +2,13 @@ package com.raphtory.examples.blockchain.routers import com.raphtory.core.components.Router.RouterWorker import com.raphtory.core.model.communication._ +import com.raphtory.sources.blockchain.BitcoinTransaction import spray.json.{JsArray, JsValue} import scala.collection.mutable.ListBuffer import scala.collection.parallel.mutable.ParHashSet import scala.util.hashing.MurmurHash3 -case class BitcoinTransaction(time: JsValue, block: Int, blockID: JsValue, transaction: JsValue) extends SpoutGoing + class LitecoinRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) extends RouterWorker[BitcoinTransaction](routerId,workerID,initialManagerCount, initialRouterCount) { diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/citationNetwork/CitationRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/citationNetwork/CitationRouter.scala index f5b3d9da09..c2d44019b2 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/citationNetwork/CitationRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/citationNetwork/CitationRouter.scala @@ -9,9 +9,9 @@ import scala.collection.mutable.ListBuffer import scala.collection.parallel.mutable.ParHashSet class CitationRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) - extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount, initialRouterCount) { - override protected def parseTuple(tuple: StringSpoutGoing): ParHashSet[GraphUpdate] = { - val fileLine = tuple.value.split(",").map(_.trim) //take the tuple and split on , as we are only interested in the first 4 fields + extends RouterWorker[String](routerId,workerID, initialManagerCount, initialRouterCount) { + override protected def parseTuple(tuple: String): ParHashSet[GraphUpdate] = { + val fileLine = tuple.split(",").map(_.trim) //take the tuple and split on , as we are only interested in the first 4 fields // title_paper,year,volume,title,pages,number,journal,author,ENTRYTYPE,ID val sourceTitle = fileLine(0) val destinationTitle = fileLine(3) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/datasources/GabMongoSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/datasources/GabMongoSource.scala index 545f5baab6..790293c88a 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/datasources/GabMongoSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/datasources/GabMongoSource.scala @@ -10,7 +10,7 @@ import org.slf4j.LoggerFactory import scala.collection.mutable import scala.language.postfixOps -final class GabMongoDataSource extends DataSource { +final class GabMongoDataSource extends DataSource[String] { //private val redis = new RedisClient("moe", 6379) //private val redisKey = "gab-posts" @@ -29,9 +29,9 @@ final class GabMongoDataSource extends DataSource { val root = LoggerFactory.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME).asInstanceOf[ch.qos.logback.classic.Logger] root.setLevel(Level.ERROR) - var queue = mutable.Queue[Option[StringSpoutGoing]]() + var queue = mutable.Queue[Option[String]]() - override def generateData(): Option[SpoutGoing] = { + override def generateData(): Option[String] = { if(queue.isEmpty) getNextPosts() queue.dequeue() @@ -44,7 +44,7 @@ final class GabMongoDataSource extends DataSource { for (x <- mongoColl.find("_id" $lt postMax $gt postMin)) try { val data = x.get("data").toString.drop(2).dropRight(1).replaceAll("""\\"""", "").replaceAll("""\\""", "") - queue += Some(StringSpoutGoing(data)) + queue += Some(data) } catch { case e: Throwable => println("Cannot parse record") diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabPostGraphRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabPostGraphRouter.scala index 0286c3285e..97e948db8f 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabPostGraphRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabPostGraphRouter.scala @@ -16,10 +16,10 @@ import scala.collection.parallel.mutable.ParHashSet // Then either the vertex or the edge are created accordingly. class GabPostGraphRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) - extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount, initialRouterCount) { + extends RouterWorker[String](routerId,workerID, initialManagerCount, initialRouterCount) { - override protected def parseTuple(tuple: StringSpoutGoing): ParHashSet[GraphUpdate] = { - val fileLine = tuple.value.split(";").map(_.trim) + override protected def parseTuple(tuple: String): ParHashSet[GraphUpdate] = { + val fileLine = tuple.split(";").map(_.trim) //user wise // val sourceNode=fileLine(2).toInt // val targetNode=fileLine(5).toInt diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabRawRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabRawRouter.scala index fbe78cf8eb..264c35ea26 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabRawRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabRawRouter.scala @@ -3,7 +3,7 @@ package com.raphtory.examples.gab.routers import java.time.OffsetDateTime import com.raphtory.core.components.Router.RouterWorker -import com.raphtory.core.model.communication.{EdgeAddWithProperties, GraphUpdate, Properties, StringProperty, StringSpoutGoing, VertexAddWithProperties} +import com.raphtory.core.model.communication.{EdgeAddWithProperties, GraphUpdate, Properties, StringProperty, VertexAddWithProperties} import com.raphtory.examples.gab.rawgraphmodel.GabPost import spray.json._ @@ -21,16 +21,16 @@ import scala.collection.parallel.mutable.ParHashSet * which will then pass it to the graph partition dealing with the associated vertex */ final class GabRawRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) - extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount, initialRouterCount) { + extends RouterWorker[String](routerId,workerID, initialManagerCount, initialRouterCount) { import com.raphtory.examples.gab.rawgraphmodel.GabJsonProtocol._ private val nullStr = "null" - override protected def parseTuple(tuple: StringSpoutGoing): ParHashSet[GraphUpdate] = { + override protected def parseTuple(tuple: String): ParHashSet[GraphUpdate] = { val commands = new ParHashSet[GraphUpdate]() try { - val command = tuple.value + val command = tuple val post = command.parseJson.convertTo[GabPost] sendPostToPartitions(post) } catch { diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabUserGraphRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabUserGraphRouter.scala index 66e5c1b579..7b7a66dae4 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabUserGraphRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabUserGraphRouter.scala @@ -18,10 +18,10 @@ import scala.util.Random // Then either the vertex or the edge are created accordingly. class GabUserGraphRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) - extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount, initialRouterCount) { + extends RouterWorker[String](routerId,workerID, initialManagerCount, initialRouterCount) { - override protected def parseTuple(tuple: StringSpoutGoing): ParHashSet[GraphUpdate] = { - val fileLine = tuple.value.split(";").map(_.trim) + override protected def parseTuple(tuple: String): ParHashSet[GraphUpdate] = { + val fileLine = tuple.split(";").map(_.trim) //user wise val sourceNode = fileLine(2).toInt val targetNode = fileLine(5).toInt diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCOldRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCOldRouter.scala index 3053409d5d..777f6b5879 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCOldRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCOldRouter.scala @@ -4,14 +4,14 @@ import java.text.SimpleDateFormat import java.util.Date import com.raphtory.core.components.Router.RouterWorker -import com.raphtory.core.model.communication.{EdgeAdd, EdgeDelete, GraphUpdate, StringSpoutGoing, Type, VertexAdd, VertexDelete} +import com.raphtory.core.model.communication.{EdgeAdd, EdgeDelete, GraphUpdate, Type, VertexAdd, VertexDelete} import scala.collection.mutable.ListBuffer import scala.collection.parallel.mutable.ParHashSet class LDBCOldRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) - extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount, initialRouterCount) { - override protected def parseTuple(tuple: StringSpoutGoing): ParHashSet[GraphUpdate] = { + extends RouterWorker[String](routerId,workerID, initialManagerCount, initialRouterCount) { + override protected def parseTuple(tuple: String): ParHashSet[GraphUpdate] = { val fileLine = tuple.asInstanceOf[String].split("\\|") val commands = new ParHashSet[GraphUpdate]() diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCRouter.scala index 94e72e94d6..ad3acfbda6 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCRouter.scala @@ -4,14 +4,14 @@ import java.text.SimpleDateFormat import java.util.Date import com.raphtory.core.components.Router.RouterWorker -import com.raphtory.core.model.communication.{EdgeAdd, EdgeDelete, GraphUpdate, StringSpoutGoing, Type, VertexAdd, VertexDelete} +import com.raphtory.core.model.communication.{EdgeAdd, EdgeDelete, GraphUpdate, Type, VertexAdd, VertexDelete} import scala.collection.mutable.ListBuffer import scala.collection.parallel.mutable.ParHashSet class LDBCRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) - extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount, initialRouterCount) { - override protected def parseTuple(tuple: StringSpoutGoing): ParHashSet[GraphUpdate] = { + extends RouterWorker[String](routerId,workerID, initialManagerCount, initialRouterCount) { + override protected def parseTuple(tuple: String): ParHashSet[GraphUpdate] = { val commands = new ParHashSet[GraphUpdate]() val fileLine = tuple.asInstanceOf[String].split("\\|") val date = fileLine(1).substring(0, 10) + fileLine(1).substring(11, 23); //extract the day of the event diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRDataSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRDataSource.scala index 27e8178955..350c9e0ff0 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRDataSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRDataSource.scala @@ -6,7 +6,7 @@ import com.raphtory.core.model.communication.{SpoutGoing, StringSpoutGoing} import scala.collection.mutable -class LOTRDataSource extends DataSource { +class LOTRDataSource extends DataSource[String] { val directory = System.getenv().getOrDefault("LOTR_DIRECTORY", "com/raphtory/example/lotr").trim val file_name = System.getenv().getOrDefault("LOTR_FILE_NAME", "lotr.csv").trim @@ -14,13 +14,13 @@ class LOTRDataSource extends DataSource { override def setupDataSource(): Unit = {}//no setup - override def generateData(): Option[SpoutGoing] = { + override def generateData(): Option[String] = { if(fileQueue isEmpty){ dataSourceComplete() None } else - Some(StringSpoutGoing(fileQueue.dequeue())) + Some(fileQueue.dequeue()) } override def closeDataSource(): Unit = {}//no file closure already done diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRRouter.scala index f0f9f2a821..08fe715373 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRRouter.scala @@ -8,11 +8,11 @@ import scala.collection.parallel.mutable.ParHashSet import scala.util.Random class LOTRRouter(override val routerId: Int, override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) - extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount, initialRouterCount) { + extends RouterWorker[String](routerId,workerID, initialManagerCount, initialRouterCount) { - override protected def parseTuple(tuple: StringSpoutGoing): ParHashSet[GraphUpdate] = { + override protected def parseTuple(tuple: String): ParHashSet[GraphUpdate] = { - val fileLine = tuple.value.split(",").map(_.trim) + val fileLine = tuple.split(",").map(_.trim) val commands = new ParHashSet[GraphUpdate]() val sourceNode = fileLine(0) val srcID = assignID(sourceNode) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomDataSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomDataSource.scala index 0c2fd37cc6..1acf9ced21 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomDataSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomDataSource.scala @@ -7,7 +7,7 @@ import com.raphtory.core.utils.Utils import scala.language.postfixOps import scala.util.Random -class RandomDataSource extends DataSource { +class RandomDataSource extends DataSource[String] { var totalCount = 100 var freq = 1000 @@ -16,7 +16,7 @@ class RandomDataSource extends DataSource { override def setupDataSource(): Unit = {} - override def generateData(): Option[SpoutGoing] = Some(StringSpoutGoing(distribution())) + override def generateData(): Option[String] = Some(distribution()) override def closeDataSource(): Unit = {} diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomRouter.scala index 1e259e61ab..baec54d5c9 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomRouter.scala @@ -17,11 +17,11 @@ import scala.collection.parallel.mutable.ParHashSet * which will then pass it to the graph partition dealing with the associated vertex */ class RandomRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) - extends RouterWorker[StringSpoutGoing](routerId,workerID,initialManagerCount, initialRouterCount) { + extends RouterWorker[String](routerId,workerID,initialManagerCount, initialRouterCount) { //************* MESSAGE HANDLING BLOCK - override protected def parseTuple(tuple:StringSpoutGoing): ParHashSet[GraphUpdate] = { + override protected def parseTuple(tuple:String): ParHashSet[GraphUpdate] = { val command = tuple.asInstanceOf[String] val parsedOBJ = command.parseJson.asJsObject //get the json object val commandKey = parsedOBJ.fields //get the command type diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleDataSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleDataSource.scala index 93b9321593..72b1784179 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleDataSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleDataSource.scala @@ -5,13 +5,13 @@ import com.raphtory.core.model.communication.{SpoutGoing, StringSpoutGoing} import scala.collection.mutable.Queue -class TriangleDataSource extends DataSource { +class TriangleDataSource extends DataSource[String] { - val edges = Queue[StringSpoutGoing](StringSpoutGoing("3,3,1"), StringSpoutGoing("4,3,4"), StringSpoutGoing("2,2,3"), StringSpoutGoing("5,4,1"), StringSpoutGoing("6,1,3"), StringSpoutGoing("1,1,2")) + val edges = Queue[String]("3,3,1", "4,3,4", "2,2,3", "5,4,1", "6,1,3", "1,1,2") override def setupDataSource(): Unit = {} - override def generateData(): Option[SpoutGoing] = { + override def generateData(): Option[String] = { if(edges.isEmpty) { dataSourceComplete() None diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleTestRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleTestRouter.scala index 26249dc062..9fb8835607 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleTestRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleTestRouter.scala @@ -2,15 +2,15 @@ package com.raphtory.examples.test.actors import com.raphtory.core.components.Router.RouterWorker import com.raphtory.core.components.Spout.Spout -import com.raphtory.core.model.communication.{EdgeAdd, GraphUpdate, StringSpoutGoing, VertexAddWithProperties} +import com.raphtory.core.model.communication.{EdgeAdd, GraphUpdate, VertexAddWithProperties} import scala.collection.mutable.ListBuffer import scala.collection.parallel.mutable.ParHashSet class TriangleTestRouter (override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) - extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount, initialRouterCount) { - override protected def parseTuple(tuple: StringSpoutGoing): ParHashSet[GraphUpdate] = { - val command = tuple.value.split(",") + extends RouterWorker[String](routerId,workerID, initialManagerCount, initialRouterCount) { + override protected def parseTuple(tuple: String): ParHashSet[GraphUpdate] = { + val command = tuple.split(",") val msgTime = command(0).toLong val sourceID = command(1).toLong val destID = command(2).toLong diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/trackAndTrace/routers/TrackAndTraceRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/trackAndTrace/routers/TrackAndTraceRouter.scala index 733e026a7e..a537d5cc38 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/trackAndTrace/routers/TrackAndTraceRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/trackAndTrace/routers/TrackAndTraceRouter.scala @@ -11,13 +11,13 @@ import scala.collection.parallel.mutable.ParHashSet import scala.util.control.Breaks._ class TrackAndTraceRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) - extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount, initialRouterCount) { + extends RouterWorker[String](routerId,workerID, initialManagerCount, initialRouterCount) { val EARTH_EQU = 6378137.0 //m val EARTH_POL = 6356752.3142 //m val STEPSIZE = System.getenv().getOrDefault("MAP_GRID_SIZE", "100").trim.toDouble //m - override protected def parseTuple(tuple: StringSpoutGoing): ParHashSet[GraphUpdate] = { - val datapoint = lineToDatapoint(tuple.value.split(",").map(_.trim)) + override protected def parseTuple(tuple: String): ParHashSet[GraphUpdate] = { + val datapoint = lineToDatapoint(tuple.split(",").map(_.trim)) val eventTime = datapoint.time val userID = datapoint.userId val latitude = datapoint.latitude diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/TSVRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/TSVRouter.scala index 3a75601c64..4f898bc19d 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/TSVRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/TSVRouter.scala @@ -1,17 +1,17 @@ package com.raphtory.examples.tsvnet import com.raphtory.core.components.Router.RouterWorker -import com.raphtory.core.model.communication.{EdgeAdd, GraphUpdate, StringSpoutGoing, Type, VertexAdd} +import com.raphtory.core.model.communication.{EdgeAdd, GraphUpdate, Type, VertexAdd} import scala.collection.mutable.ListBuffer import scala.collection.parallel.mutable.ParHashSet /** Spout for network datasets of the form SRC_NODE_ID DEST_NODE_ID TIMESTAMP */ class TSVRouter(override val routerId: Int, override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) - extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount, initialRouterCount) { + extends RouterWorker[String](routerId,workerID, initialManagerCount, initialRouterCount) { val commands = new ParHashSet[GraphUpdate]() - override protected def parseTuple(tuple: StringSpoutGoing): ParHashSet[GraphUpdate] = { - val fileLine = tuple.value.split(" ").map(_.trim) + override protected def parseTuple(tuple: String): ParHashSet[GraphUpdate] = { + val fileLine = tuple.split(" ").map(_.trim) //user wise val sourceNode = fileLine(0).toInt val targetNode = fileLine(1).toInt diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixRouter.scala index f83a114c5a..b79d01ff83 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixRouter.scala @@ -7,11 +7,11 @@ import scala.collection.mutable.ListBuffer import scala.collection.parallel.mutable.ParHashSet class CooccurrenceMatrixRouter(override val routerId: Int, override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) - extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount, initialRouterCount) { + extends RouterWorker[String](routerId,workerID, initialManagerCount, initialRouterCount) { - override protected def parseTuple(tuple: StringSpoutGoing): ParHashSet[GraphUpdate] = { + override protected def parseTuple(tuple: String): ParHashSet[GraphUpdate] = { //println(record) - var dp = tuple.value.split(" ").map(_.trim) + var dp = tuple.split(" ").map(_.trim) val commands = new ParHashSet[GraphUpdate]() val occurenceTime = dp.head.toLong//DateFormatting(dp.head) //.slice(4, dp.head.length) try { @@ -34,7 +34,7 @@ class CooccurrenceMatrixRouter(override val routerId: Int, override val workerID } }catch { - case e: Exception => println(e, dp.length, tuple.value) + case e: Exception => println(e, dp.length, tuple) } commands } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixRouterFiltered.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixRouterFiltered.scala index cc9d3795b7..e42bd33474 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixRouterFiltered.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixRouterFiltered.scala @@ -7,12 +7,12 @@ import scala.collection.mutable.ListBuffer import scala.collection.parallel.mutable.ParHashSet class CooccurrenceMatrixRouterFiltered(override val routerId: Int, override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) - extends RouterWorker[StringSpoutGoing](routerId,workerID, initialManagerCount, initialRouterCount) { + extends RouterWorker[String](routerId,workerID, initialManagerCount, initialRouterCount) { val THR = System.getenv().getOrDefault("COOC_FREQ_THRESHOLD ", "0.05").trim.toDouble - override protected def parseTuple(tuple: StringSpoutGoing): ParHashSet[GraphUpdate] = { + override protected def parseTuple(tuple: String): ParHashSet[GraphUpdate] = { //println(record) - var dp =tuple.value.split(" ").map(_.trim) + var dp =tuple.split(" ").map(_.trim) val occurenceTime = dp.head.toLong//DateFormatting(dp.head) //.slice(4, dp.head.length) val scale = dp(1).toDouble val commands = new ParHashSet[GraphUpdate]() diff --git a/mainproject/cluster/src/main/scala/com/raphtory/sources/FileSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/sources/FileSource.scala index 00d11e9624..cfe35a1bb2 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/sources/FileSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/sources/FileSource.scala @@ -11,7 +11,7 @@ import com.typesafe.scalalogging.LazyLogging import scala.annotation.tailrec import scala.concurrent.duration._ -class FileSource extends DataSource { +class FileSource extends DataSource[String] { //TODO work out loggging here //log.info("initialise FileSpout") private val directory = System.getenv().getOrDefault("FILE_SPOUT_DIRECTORY", "/app").trim @@ -23,7 +23,7 @@ class FileSource extends DataSource { private var fileManager = FileManager(directory, fileName, dropHeader) - override def generateData(): Option[StringSpoutGoing] = { + override def generateData(): Option[String] = { if (fileManager.allCompleted) { dataSourceComplete() None @@ -31,7 +31,7 @@ class FileSource extends DataSource { else { val (newFileManager, line) = fileManager.nextLine() fileManager = newFileManager - Some(StringSpoutGoing(line)) + Some(line) } } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/sources/KafkaSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/sources/KafkaSource.scala index 7d98a445a7..f91e27a1db 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/sources/KafkaSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/sources/KafkaSource.scala @@ -12,7 +12,7 @@ import scala.collection.JavaConverters._ import scala.collection.mutable import scala.util.Random -final case class KafkaSpout() extends DataSource { +final case class KafkaSpout() extends DataSource[String] { private val kafkaServer = System.getenv().getOrDefault("KAFKA_ADDRESS", "127.0.0.1").trim private val kafkaIp = System.getenv().getOrDefault("KAFKA_PORT", "9092").trim private val offset = System.getenv().getOrDefault("KAFKA_OFFSET", "earliest").trim @@ -22,14 +22,14 @@ final case class KafkaSpout() extends DataSource { private val startingSpeed = System.getenv().getOrDefault("STARTING_SPEED", "1000").trim.toInt private var kafkaManager = KafkaManager(kafkaServer, kafkaIp, groupId, topic, offset) - val messageQueue = mutable.Queue[StringSpoutGoing]() + val messageQueue = mutable.Queue[String]() override def setupDataSource(): Unit = {} - override def generateData(): Option[SpoutGoing] = { + override def generateData(): Option[String] = { if(messageQueue isEmpty) { val (newManager, block) = kafkaManager.nextNLine(startingSpeed / 100) kafkaManager = newManager - block.foreach(str => messageQueue += StringSpoutGoing(str)) + block.foreach(str => messageQueue += str) if(messageQueue isEmpty) //still empty None } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/BitcoinNodeSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/BitcoinNodeSource.scala index 41b73425a4..f7a9be96ab 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/BitcoinNodeSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/BitcoinNodeSource.scala @@ -13,9 +13,9 @@ import scala.collection.mutable import scala.language.postfixOps import scala.sys.process._ // -case class BitcoinTransaction(time: JsValue, block: Int, blockID: JsValue, transaction: JsValue) extends SpoutGoing +case class BitcoinTransaction(time: JsValue, block: Int, blockID: JsValue, transaction: JsValue) -class BitcoinSpout extends DataSource{ +class BitcoinSpout extends DataSource[BitcoinTransaction]{ var blockcount = 1 val rpcuser = System.getenv().getOrDefault("BITCOIN_USERNAME", "").trim @@ -29,7 +29,7 @@ class BitcoinSpout extends DataSource{ val queue = mutable.Queue[Option[BitcoinTransaction]]() - override def generateData(): Option[SpoutGoing] = { + override def generateData(): Option[BitcoinTransaction] = { if(queue.isEmpty) getTransactions() queue.dequeue() diff --git a/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumGethSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumGethSource.scala index 323b43d258..cbcbecb661 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumGethSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumGethSource.scala @@ -21,7 +21,7 @@ import scala.collection.mutable -class EthereumGethSpout extends DataSource { +class EthereumGethSpout extends DataSource[String] { var currentBlock = System.getenv().getOrDefault("SPOUT_ETHEREUM_START_BLOCK_INDEX", "9014194").trim.toInt var highestBlock = System.getenv().getOrDefault("SPOUT_ETHEREUM_MAXIMUM_BLOCK_INDEX", "10026447").trim.toInt @@ -31,13 +31,13 @@ class EthereumGethSpout extends DataSource { print(currentBlock) - val queue = mutable.Queue[Option[StringSpoutGoing]]() + val queue = mutable.Queue[Option[String]]() val baseRequest = requestBuilder() override def setupDataSource(): Unit = {} override def closeDataSource(): Unit = {} - override def generateData(): Option[SpoutGoing] = { + override def generateData(): Option[String] = { if(queue.isEmpty) pullNextBlock() queue.dequeue() @@ -68,7 +68,7 @@ class EthereumGethSpout extends DataSource { val trasnactionBlock = executeBatchRequest(transactions.dropRight(1)+"]") val transList = trasnactionBlock.parseJson.convertTo[List[EthTransaction]] transList.foreach(t => { //try needed to ignore contracts //todo include them - try{queue +=(Some(StringSpoutGoing(s"${t.result.blockNumber.get},${t.result.from.get},${t.result.to.get},${t.result.value.get}")))} + try{queue +=(Some(s"${t.result.blockNumber.get},${t.result.from.get},${t.result.to.get},${t.result.value.get}"))} catch {case e:NoSuchElementException =>} }) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumPostgresSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumPostgresSource.scala index bda843675c..2c22b59778 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumPostgresSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumPostgresSource.scala @@ -9,7 +9,7 @@ import doobie.util.transactor.Transactor import scala.collection.mutable -class EthereumPostgresSource extends DataSource{ +class EthereumPostgresSource extends DataSource[String]{ var startBlock = System.getenv().getOrDefault("STARTING_BLOCK", "46147").trim.toInt //first block to have a transaction by default val batchSize = System.getenv().getOrDefault("BLOCK_BATCH_SIZE", "100").trim.toInt //number of blocks to pull each query val maxblock = System.getenv().getOrDefault("MAX_BLOCK", "8828337").trim.toInt //Maximum block in database to stop querying once this is reached @@ -27,9 +27,9 @@ class EthereumPostgresSource extends DataSource{ dbPASSWORD, Blocker.liftExecutionContext(ExecutionContexts.synchronous) ) - val queue = mutable.Queue[Option[StringSpoutGoing]]() + val queue = mutable.Queue[Option[String]]() - override def generateData(): Option[SpoutGoing] = { + override def generateData(): Option[String] = { if(queue isEmpty) pullBlocks() queue.dequeue() @@ -43,7 +43,7 @@ class EthereumPostgresSource extends DataSource{ .to[List] // ConnectionIO[List[String]] .transact(dbconnector) // IO[List[String]] .unsafeRunSync // List[String] - .foreach(x => queue+=(Some(StringSpoutGoing(x.toString())))) //send each transaction to the routers + .foreach(x => queue+=(Some(x.toString()))) //send each transaction to the routers startBlock += batchSize//increment batch for the next query if (startBlock <= maxblock) From 5824cc6d5d9f6c5d4960f607bb6685a6c35e129e Mon Sep 17 00:00:00 2001 From: Miratepuffin Date: Thu, 26 Nov 2020 23:18:19 +0000 Subject: [PATCH 34/58] Created the new GraphBuilder API for the Router --- .../src/main/scala/com/raphtory/Server.scala | 4 +- .../core/clustersetup/RouterNode.scala | 10 +++-- .../singlenode/SingleNodeSetup.scala | 7 +++- .../RaphtoryReplicator.scala | 12 +++--- .../core/components/Router/GraphBuilder.scala | 16 ++++++-- .../components/Router/RouterManager.scala | 4 +- .../core/components/Router/RouterWorker.scala | 13 +++---- ...Router.scala => BitcoinGraphBuilder.scala} | 32 +++++++--------- ....scala => ChABClus2ClusGraphBuilder.scala} | 15 +++----- ... => ChABClus2ClusGraphBuilderNoprop.scala} | 15 +++----- ....scala => ChainalysisABGraphBuilder.scala} | 19 ++++------ ...outer.scala => DashcoinGraphBuilder.scala} | 21 +++++----- ...r.scala => EthereumGethGraphBuilder.scala} | 19 ++++------ ....scala => EthereumKafkaGraphBuilder.scala} | 19 ++++------ ... => EthereumTransactionGraphBuilder.scala} | 19 ++++------ ....scala => FirehoseKafkaGraphBuilder.scala} | 17 ++++----- ...outer.scala => LitecoinGraphBuilder.scala} | 21 +++++----- ...outer.scala => CitationGraphBuilder.scala} | 16 +++----- .../gab/datasources/GabMongoSource.scala | 1 - ...Router.scala => GabPostGraphBuilder.scala} | 15 +++----- ...wRouter.scala => GabRawGraphBuilder.scala} | 38 +++++++++---------- ...Router.scala => GabUserGraphBuilder.scala} | 15 +++----- ...DBCRouter.scala => LDBCGraphBuilder.scala} | 17 ++++----- ...Router.scala => LDBCOldGraphBuilder.scala} | 14 +++---- .../examples/lotr/LOTRDataSource.scala | 2 - ...OTRRouter.scala => LOTRGraphBuilder.scala} | 15 +++----- .../test/actors/RandomDataSource.scala | 2 - ...mRouter.scala => RandomGraphBuilder.scala} | 16 ++++---- .../test/actors/TriangleDataSource.scala | 1 - .../actors/TriangleTestGraphBuilder.scala | 18 +++++++++ .../test/actors/TriangleTestRouter.scala | 19 ---------- ....scala => TrackAndTraceGraphBuilder.scala} | 16 ++++---- ...{TSVRouter.scala => TSVGraphBuilder.scala} | 15 +++----- ...a => CooccurrenceMatrixGraphBuilder.scala} | 11 ++---- ...ccurrenceMatrixGraphBuilderFiltered.scala} | 11 ++---- .../com/raphtory/sources/FileSource.scala | 1 - .../com/raphtory/sources/KafkaSource.scala | 1 - .../blockchain/BitcoinNodeSource.scala | 1 - .../blockchain/EthereumGethSource.scala | 1 - .../blockchain/EthereumPostgresSource.scala | 1 - .../com/raphtory/tests/CitationExample.scala | 5 ++- .../scala/com/raphtory/tests/NaomiTest.scala | 5 ++- .../com/raphtory/tests/SingleNodeTest.scala | 6 ++- .../tests/TemporalTriangleCountExample.scala | 8 +++- 44 files changed, 240 insertions(+), 294 deletions(-) rename mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/{BitcoinRouter.scala => BitcoinGraphBuilder.scala} (81%) rename mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/{ChABClus2ClusRouter.scala => ChABClus2ClusGraphBuilder.scala} (76%) rename mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/{ChABClus2ClusRouterNoprop.scala => ChABClus2ClusGraphBuilderNoprop.scala} (73%) rename mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/{ChainalysisABRouter.scala => ChainalysisABGraphBuilder.scala} (77%) rename mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/{DashcoinRouter.scala => DashcoinGraphBuilder.scala} (87%) rename mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/{EthereumGethRouter.scala => EthereumGethGraphBuilder.scala} (80%) rename mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/{EthereumKafkaRouter.scala => EthereumKafkaGraphBuilder.scala} (79%) rename mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/{EthereumTransactionRouter.scala => EthereumTransactionGraphBuilder.scala} (75%) rename mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/{FirehoseKafkaRouter.scala => FirehoseKafkaGraphBuilder.scala} (80%) rename mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/{LitecoinRouter.scala => LitecoinGraphBuilder.scala} (88%) rename mainproject/cluster/src/main/scala/com/raphtory/examples/citationNetwork/{CitationRouter.scala => CitationGraphBuilder.scala} (69%) rename mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/{GabPostGraphRouter.scala => GabPostGraphBuilder.scala} (72%) rename mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/{GabRawRouter.scala => GabRawGraphBuilder.scala} (86%) rename mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/{GabUserGraphRouter.scala => GabUserGraphBuilder.scala} (78%) rename mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/{LDBCRouter.scala => LDBCGraphBuilder.scala} (73%) rename mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/{LDBCOldRouter.scala => LDBCOldGraphBuilder.scala} (78%) rename mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/{LOTRRouter.scala => LOTRGraphBuilder.scala} (52%) rename mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/{RandomRouter.scala => RandomGraphBuilder.scala} (87%) create mode 100644 mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleTestGraphBuilder.scala delete mode 100644 mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleTestRouter.scala rename mainproject/cluster/src/main/scala/com/raphtory/examples/trackAndTrace/routers/{TrackAndTraceRouter.scala => TrackAndTraceGraphBuilder.scala} (89%) rename mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/{TSVRouter.scala => TSVGraphBuilder.scala} (52%) rename mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/{CooccurrenceMatrixRouter.scala => CooccurrenceMatrixGraphBuilder.scala} (69%) rename mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/{CooccurrenceMatrixRouterFiltered.scala => CooccurrenceMatrixGraphBuilderFiltered.scala} (70%) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/Server.scala b/mainproject/cluster/src/main/scala/com/raphtory/Server.scala index cbd08acafd..ab877ba19e 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/Server.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/Server.scala @@ -7,7 +7,7 @@ import ch.qos.logback.classic.Level import com.raphtory.core.analysis.Tasks.AnalysisTask import com.raphtory.core.clustersetup._ import com.raphtory.core.clustersetup.singlenode.SingleNodeSetup -import com.raphtory.examples.test.actors.RandomRouter +import com.raphtory.examples.test.actors.RandomGraphBuilder import com.typesafe.config.Config import com.typesafe.config.ConfigFactory import kamon.metric.PeriodSnapshot @@ -28,7 +28,7 @@ object Go extends App { root.setLevel(Level.ERROR) // debug should give timing //root.setLevel(Level.DEBUG) - val routerName = s"${sys.env.getOrElse("ROUTERCLASS", classOf[RandomRouter].getClass.getName)}" + val routerName = s"${sys.env.getOrElse("ROUTERCLASS", classOf[RandomGraphBuilder].getClass.getName)}" val updaterName = s"${sys.env.getOrElse("SPOUTCLASS", "")}" val docker = System.getenv().getOrDefault("DOCKER", "false").trim.toBoolean diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/RouterNode.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/RouterNode.scala index 3f44b76934..d1099d6c83 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/RouterNode.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/RouterNode.scala @@ -6,13 +6,15 @@ package com.raphtory.core.clustersetup import akka.actor.ActorSystem import akka.actor.Props import com.raphtory.core.components.ClusterManagement.RaphtoryReplicator +import com.raphtory.core.components.Router.GraphBuilder case class RouterNode(seedLoc: String, partitionCount: Int, routerCount:Int, className: String) extends DocSvr { implicit val system: ActorSystem = initialiseActorSystem(seeds = List(seedLoc)) final val actorName: String = "Routers" - system.actorOf( - Props(RaphtoryReplicator(actorType = "Router", initialManagerCount = partitionCount, initialRouterCount = routerCount, routerName = className)), - actorName - ) + + val graphBuilder = Class.forName(className).getConstructor().newInstance().asInstanceOf[GraphBuilder[Any]] + val routerReplicator = RaphtoryReplicator.apply("Router", partitionCount, routerCount,graphBuilder) + system.actorOf(Props(routerReplicator), actorName) + } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/singlenode/SingleNodeSetup.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/singlenode/SingleNodeSetup.scala index 9d556174f5..73b5b47ede 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/singlenode/SingleNodeSetup.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/singlenode/SingleNodeSetup.scala @@ -14,6 +14,7 @@ import com.raphtory.core.analysis.Tasks.ViewTasks.ViewAnalysisTask import com.raphtory.core.analysis.Tasks.ViewTasks.WindowedViewAnalysisTask import com.raphtory.core.clustersetup.DocSvr import com.raphtory.core.components.ClusterManagement.{RaphtoryReplicator, SeedActor, WatchDog, WatermarkManager} +import com.raphtory.core.components.Router.GraphBuilder import com.typesafe.config.Config import com.typesafe.config.ConfigFactory @@ -31,7 +32,11 @@ case class SingleNodeSetup( system.actorOf(Props(new SeedActor(this)), "cluster") system.actorOf(Props(new WatchDog(1, 1)), "WatchDog") system.actorOf(Props(new WatermarkManager(managerCount = 1)),"WatermarkManager") - system.actorOf(Props(RaphtoryReplicator("Router", 1, 1,routerClassName)), s"Routers") + + val graphBuilder = Class.forName(routerClassName).getConstructor().newInstance().asInstanceOf[GraphBuilder[Any]] + val routerReplicator = RaphtoryReplicator.apply("Router", 1, 1,graphBuilder) + system.actorOf(Props(routerReplicator), s"Routers") + system.actorOf(Props(RaphtoryReplicator("Partition Manager", 1,1)), s"PartitionManager") system.actorOf(Props[AnalysisManager], s"AnalysisManager") diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/ClusterManagement/RaphtoryReplicator.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/ClusterManagement/RaphtoryReplicator.scala index 60cab5024e..b9c1b06f18 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/ClusterManagement/RaphtoryReplicator.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/ClusterManagement/RaphtoryReplicator.scala @@ -7,7 +7,7 @@ import akka.pattern.ask import akka.util.Timeout import com.raphtory.core.components.PartitionManager.Workers.IngestionWorker import com.raphtory.core.components.PartitionManager.{Archivist, Reader, Writer} -import com.raphtory.core.components.Router.RouterManager +import com.raphtory.core.components.Router.{GraphBuilder, RouterManager} import com.raphtory.core.model.communication._ import com.raphtory.core.storage.EntityStorage import com.raphtory.core.utils.{SchedulerUtil, Utils} @@ -19,14 +19,14 @@ import scala.concurrent.duration._ import scala.language.postfixOps object RaphtoryReplicator { - def apply(actorType: String, initialManagerCount: Int,initialRouterCount:Int, routerName: String): RaphtoryReplicator = - new RaphtoryReplicator(actorType, initialManagerCount,initialRouterCount, routerName) + def apply[T](actorType: String, initialManagerCount: Int,initialRouterCount:Int, graphBuilder: GraphBuilder[T]): RaphtoryReplicator[T] = + new RaphtoryReplicator(actorType, initialManagerCount,initialRouterCount, graphBuilder) - def apply(actorType: String, initialManagerCount: Int,initialRouterCount:Int): RaphtoryReplicator = + def apply[T](actorType: String, initialManagerCount: Int,initialRouterCount:Int): RaphtoryReplicator[T] = new RaphtoryReplicator(actorType, initialManagerCount,initialRouterCount, null) } -class RaphtoryReplicator(actorType: String, initialManagerCount: Int, initialRouterCount:Int, routerName: String) +class RaphtoryReplicator[T](actorType: String, initialManagerCount: Int, initialRouterCount:Int, graphBuilder: GraphBuilder[T]) extends Actor with ActorLogging { @@ -151,7 +151,7 @@ class RaphtoryReplicator(actorType: String, initialManagerCount: Int, initialRou log.info(s"Router $assignedId has come online.") actorRef = context.system.actorOf( - Props(new RouterManager(myId, currentCount, initialRouterCount, routerName)).withDispatcher("misc-dispatcher"), + Props(new RouterManager(myId, currentCount, initialRouterCount, graphBuilder)).withDispatcher("misc-dispatcher"), "router" ) } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/GraphBuilder.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/GraphBuilder.scala index 2b67ac9d00..23b860bb93 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/GraphBuilder.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/GraphBuilder.scala @@ -3,13 +3,23 @@ package com.raphtory.core.components.Router import com.raphtory.core.model.communication.GraphUpdate import scala.collection.mutable -import scala.collection.parallel.mutable.ParHashSet +import scala.util.hashing.MurmurHash3 -abstract class GraphBuilder[T] { +trait GraphBuilder[T]{ var updates:mutable.HashSet[GraphUpdate] = mutable.HashSet[GraphUpdate]() -// protected def parseTuple(tuple: T): + def getUpdates() = { //TODO hide from users + val toReturn = updates + updates = mutable.HashSet[GraphUpdate]() + toReturn + } + def sendUpdate(update:GraphUpdate):Unit ={ + updates += update + } + protected def assignID(uniqueChars: String): Long = MurmurHash3.stringHash(uniqueChars) + + def parseTuple(tuple: T):Unit } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterManager.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterManager.scala index eb48e31c15..ec97a687ae 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterManager.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterManager.scala @@ -18,7 +18,7 @@ import scala.language.postfixOps // TODO break object RouterManager { getProps = { routerManager(arg1, arg2...): Props }} // todo slave type should be Props -final case class RouterManager(routerId: Int, initialManagerCount: Int, initialRouterCount:Int, slaveType: String) +final case class RouterManager[T](routerId: Int, initialManagerCount: Int, initialRouterCount:Int, graphBuilder: GraphBuilder[T]) extends Actor with ActorLogging { implicit val executionContext: ExecutionContext = context.system.dispatcher @@ -26,7 +26,7 @@ final case class RouterManager(routerId: Int, initialManagerCount: Int, initialR private val childrenNumber = 10 private val children = (0 until childrenNumber).map { i => context.actorOf( - Props(Class.forName(slaveType), routerId, i, initialManagerCount,initialRouterCount).withDispatcher("router-dispatcher"), + Props(new RouterWorker(graphBuilder, routerId, i, initialManagerCount,initialRouterCount)).withDispatcher("router-dispatcher"), s"router_${routerId}_Worker_$i" ) }.toList diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala index 8f76a0a4bb..5f5c5cb0ba 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala @@ -18,7 +18,7 @@ import scala.util.hashing.MurmurHash3 // TODO Add val name which sub classes that extend this trait must overwrite // e.g. BlockChainRouter val name = "Blockchain Router" // Log.debug that read 'Router' should then read 'Blockchain Router' -abstract class RouterWorker[In <: Any](val routerId: Int, val workerID: Int, val initialManagerCount: Int,val initialRouterCount:Int) +class RouterWorker[T](val graphBuilder: GraphBuilder[T],val routerId: Int, val workerID: Int, val initialManagerCount: Int,val initialRouterCount:Int) extends Actor with ActorLogging { implicit val executionContext: ExecutionContext = context.system.dispatcher @@ -30,7 +30,6 @@ abstract class RouterWorker[In <: Any](val routerId: Int, val workerID: Int, val var update = 0 // todo: wvv let people know parseTuple will create a list of update message // and this trait will handle logic to send to graph - protected def parseTuple(tuple: In): ParHashSet[GraphUpdate] final protected val mediator = DistributedPubSub(context.system).mediator mediator ! DistributedPubSubMediator.Put(self) @@ -52,7 +51,7 @@ abstract class RouterWorker[In <: Any](val routerId: Int, val workerID: Int, val log.debug(s"RouterWorker [$routerId] received [$msg] request.") if (managerCount < msg.newValue) context.become(work(msg.newValue, trackedTime, newestTime)) - case AllocateTuple(record: In) => //todo: wvv AllocateTuple should hold type of record instead of using Any + case AllocateTuple(record: T) => //todo: wvv AllocateTuple should hold type of record instead of using Any log.debug(s"RouterWorker [$routerId] received AllocateTuple[$record] request.") parseTupleAndSendGraph(record, managerCount, false, trackedTime).foreach(newNewestTime => if(newNewestTime>newestTime) @@ -62,7 +61,7 @@ abstract class RouterWorker[In <: Any](val routerId: Int, val workerID: Int, val case msg @ AllocateTrackedTuple( wallClock, - record: In + record: T ) => //todo: wvv AllocateTrackedTuple should hold type of record instead of using Any log.debug(s"RouterWorker [$routerId] received [$msg] request.") val newNewestTime = parseTupleAndSendGraph(record, managerCount, true, wallClock).getOrElse(newestTime) @@ -116,15 +115,15 @@ abstract class RouterWorker[In <: Any](val routerId: Int, val workerID: Int, val } - protected def assignID(uniqueChars: String): Long = MurmurHash3.stringHash(uniqueChars) private def parseTupleAndSendGraph( - record: In, + record: T, managerCount: Int, trackedMessage: Boolean, trackedTime: Long ): Option[Long] =try{ - parseTuple(record).map(update => sendGraphUpdate(update, managerCount, trackedMessage, trackedTime)).lastOption + graphBuilder.parseTuple(record) + graphBuilder.getUpdates().map(update => sendGraphUpdate(update, managerCount, trackedMessage, trackedTime)).lastOption }catch {case e:Exception => None} private def sendGraphUpdate( diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/BitcoinRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/BitcoinGraphBuilder.scala similarity index 81% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/BitcoinRouter.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/BitcoinGraphBuilder.scala index b28aec2f3c..ddd3ab8051 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/BitcoinRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/BitcoinGraphBuilder.scala @@ -1,6 +1,6 @@ package com.raphtory.examples.blockchain.routers -import com.raphtory.core.components.Router.RouterWorker +import com.raphtory.core.components.Router.{GraphBuilder, RouterWorker} import com.raphtory.core.model.communication._ import com.raphtory.sources.blockchain.BitcoinTransaction import spray.json.JsArray @@ -8,10 +8,9 @@ import spray.json.JsArray import scala.collection.mutable.ListBuffer import scala.collection.parallel.mutable.{ParArray, ParHashSet} -class BitcoinRouter(override val routerId: Int, override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) - extends RouterWorker[BitcoinTransaction](routerId,workerID,initialManagerCount, initialRouterCount) { +class BitcoinGraphBuilder extends GraphBuilder[BitcoinTransaction]{ - override protected def parseTuple(tuple:BitcoinTransaction): ParHashSet[GraphUpdate] = { + override def parseTuple(tuple:BitcoinTransaction) = { val transaction = tuple.transaction val time = tuple.time @@ -25,7 +24,6 @@ class BitcoinRouter(override val routerId: Int, override val workerID:Int, overr val vouts = transaction.asJsObject.fields("vout") var total: Double = 0 - var commands = new ParHashSet[GraphUpdate]() for (vout <- vouts.asInstanceOf[JsArray].elements) { val voutOBJ = vout.asJsObject() var value = voutOBJ.fields("value").toString @@ -41,20 +39,20 @@ class BitcoinRouter(override val routerId: Int, override val workerID:Int, overr //println(s"Edge $timeAsLong, ${txid.hashCode}, ${address.hashCode}, $n, $value") //creates vertex for the receiving wallet - commands += VertexAddWithProperties( + sendUpdate(VertexAddWithProperties( msgTime = timeAsLong, srcID = address.hashCode, properties = Properties(StringProperty("type", "address"), - StringProperty("address", address))) + StringProperty("address", address)))) //creates edge between the transaction and the wallet - commands += EdgeAddWithProperties( + sendUpdate(EdgeAddWithProperties( msgTime = timeAsLong, srcID = txid.hashCode, dstID = address.hashCode, - properties = Properties(StringProperty("n", n), StringProperty("value", value))) + properties = Properties(StringProperty("n", n), StringProperty("value", value)))) } - commands += VertexAddWithProperties( + sendUpdate(VertexAddWithProperties( msgTime = timeAsLong, srcID = txid.hashCode, properties = Properties( @@ -64,19 +62,18 @@ class BitcoinRouter(override val routerId: Int, override val workerID:Int, overr StringProperty("total", total.toString), StringProperty("blockhash", blockID.toString), StringProperty("block", block.toString) - )) + ))) if (vins.toString().contains("coinbase")) { //creates the coingen node //TODO change so only added once - commands += VertexAddWithProperties( + sendUpdate(VertexAddWithProperties( msgTime = timeAsLong, srcID = "coingen".hashCode, - properties = Properties(StringProperty("type", "coingen"))) + properties = Properties(StringProperty("type", "coingen")))) //creates edge between coingen and the transaction - commands += EdgeAdd(msgTime = timeAsLong, srcID = "coingen".hashCode, dstID = txid.hashCode) - commands + sendUpdate(EdgeAdd(msgTime = timeAsLong, srcID = "coingen".hashCode, dstID = txid.hashCode)) } else for (vin <- vins.asInstanceOf[JsArray].elements) { val vinOBJ = vin.asJsObject() @@ -84,14 +81,13 @@ class BitcoinRouter(override val routerId: Int, override val workerID:Int, overr val prevtxid = vinOBJ.fields("txid").toString //no need to create node for prevtxid as should already exist //creates edge between the prev transaction and current transaction - commands += EdgeAddWithProperties( + sendUpdate(EdgeAddWithProperties( msgTime = timeAsLong, srcID = prevtxid.hashCode, dstID = txid.hashCode, - properties = Properties(StringProperty("vout", prevVout))) + properties = Properties(StringProperty("vout", prevVout)))) } - commands } } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusGraphBuilder.scala similarity index 76% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusRouter.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusGraphBuilder.scala index d87210ab98..bb4d233a49 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusGraphBuilder.scala @@ -3,17 +3,16 @@ package com.raphtory.examples.blockchain.routers import java.text.SimpleDateFormat import java.util.Calendar -import com.raphtory.core.components.Router.RouterWorker +import com.raphtory.core.components.Router.{GraphBuilder, RouterWorker} import com.raphtory.core.model.communication.Type import com.raphtory.core.model.communication._ import scala.collection.mutable.ListBuffer import scala.collection.parallel.mutable.ParHashSet -class ChABClus2ClusRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) - extends RouterWorker[String](routerId,workerID, initialManagerCount,initialRouterCount) { +class ChABClus2ClusGraphBuilder extends GraphBuilder[String]{ - override protected def parseTuple(tuple: String): ParHashSet[GraphUpdate] = { + override def parseTuple(tuple: String) = { val dp = formatLine(tuple.split(",").map(_.trim)) val transactionTime = dp.time val srcClusterId = dp.srcCluster @@ -22,12 +21,11 @@ class ChABClus2ClusRouter(override val routerId: Int,override val workerID:Int, val btcAmount = dp.amount val usdAmount = dp.usd - val commands = new ParHashSet[GraphUpdate]() - commands+= VertexAdd(msgTime = transactionTime, srcID = srcClusterId, Type("Cluster")) - commands+=(VertexAdd(msgTime = transactionTime, srcID = dstClusterId, Type("Cluster"))) + sendUpdate(VertexAdd(msgTime = transactionTime, srcID = srcClusterId, Type("Cluster"))) + sendUpdate((VertexAdd(msgTime = transactionTime, srcID = dstClusterId, Type("Cluster")))) - commands+=( + sendUpdate( EdgeAddWithProperties(msgTime = transactionTime, srcID = srcClusterId, dstID = dstClusterId, @@ -37,7 +35,6 @@ class ChABClus2ClusRouter(override val routerId: Int,override val workerID:Int, Type("Transfer") ) ) - commands } //converts the line into a case class which has all of the data via the correct name and type diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusRouterNoprop.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusGraphBuilderNoprop.scala similarity index 73% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusRouterNoprop.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusGraphBuilderNoprop.scala index a34ca903d0..7c33020eb4 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusRouterNoprop.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusGraphBuilderNoprop.scala @@ -3,26 +3,23 @@ package com.raphtory.examples.blockchain.routers import java.text.SimpleDateFormat import java.util.Calendar -import com.raphtory.core.components.Router.RouterWorker +import com.raphtory.core.components.Router.{GraphBuilder, RouterWorker} import com.raphtory.core.model.communication.Type import com.raphtory.core.model.communication._ import scala.collection.mutable.ListBuffer import scala.collection.parallel.mutable.ParHashSet -class ChABClus2ClusRouterNoprop(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) - extends RouterWorker[String](routerId,workerID, initialManagerCount,initialRouterCount) { +class ChABClus2ClusGraphBuilderNoprop extends GraphBuilder[String] { - override protected def parseTuple(tuple: String): ParHashSet[GraphUpdate] = { + override def parseTuple(tuple: String) = { val dp = formatLine(tuple.split(",").map(_.trim)) val transactionTime = dp.time val srcClusterId = dp.srcCluster val dstClusterId = dp.dstCluster - val commands = new ParHashSet[GraphUpdate]() - commands+=(VertexAdd(msgTime = transactionTime, srcID = srcClusterId, Type("Cluster"))) - commands+=(VertexAdd(msgTime = transactionTime, srcID = dstClusterId, Type("Cluster"))) - commands+=(EdgeAdd(msgTime = transactionTime, srcID = srcClusterId, dstID = dstClusterId, Type("Transfer"))) - commands + sendUpdate(VertexAdd(msgTime = transactionTime, srcID = srcClusterId, Type("Cluster"))) + sendUpdate(VertexAdd(msgTime = transactionTime, srcID = dstClusterId, Type("Cluster"))) + sendUpdate(EdgeAdd(msgTime = transactionTime, srcID = srcClusterId, dstID = dstClusterId, Type("Transfer"))) } //converts the line into a case class which has all of the data via the correct name and type diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChainalysisABRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChainalysisABGraphBuilder.scala similarity index 77% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChainalysisABRouter.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChainalysisABGraphBuilder.scala index 0c7cdacc51..143d52d1cf 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChainalysisABRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChainalysisABGraphBuilder.scala @@ -2,17 +2,16 @@ package com.raphtory.examples.blockchain.routers import java.text.SimpleDateFormat -import com.raphtory.core.components.Router.RouterWorker +import com.raphtory.core.components.Router.{GraphBuilder, RouterWorker} import com.raphtory.core.model.communication.Type import com.raphtory.core.model.communication._ import scala.collection.mutable.ListBuffer import scala.collection.parallel.mutable.ParHashSet -class ChainalysisABRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) - extends RouterWorker[String](routerId,workerID, initialManagerCount,initialRouterCount) { +class ChainalysisABGraphBuilder extends GraphBuilder[String] { - override protected def parseTuple(tuple: String): ParHashSet[GraphUpdate] = { + override def parseTuple(tuple: String) = { val dp = formatLine(tuple.split(",").map(_.trim)) val transactionTime = dp.time val srcClusterId = dp.srcCluster @@ -20,13 +19,12 @@ class ChainalysisABRouter(override val routerId: Int,override val workerID:Int, val transactionId = dp.txid val btcAmount = dp.amount val usdAmount = dp.usd - val commands = new ParHashSet[GraphUpdate]() - commands+=(VertexAdd(msgTime = transactionTime, srcID = srcClusterId, Type("Cluster"))) - commands+=(VertexAdd(msgTime = transactionTime, srcID = dstClusterId, Type("Cluster"))) - commands+=(VertexAdd(msgTime = transactionTime, srcID = transactionId, Type("Transaction"))) + sendUpdate(VertexAdd(msgTime = transactionTime, srcID = srcClusterId, Type("Cluster"))) + sendUpdate(VertexAdd(msgTime = transactionTime, srcID = dstClusterId, Type("Cluster"))) + sendUpdate(VertexAdd(msgTime = transactionTime, srcID = transactionId, Type("Transaction"))) - commands+=( + sendUpdate( EdgeAddWithProperties(msgTime = transactionTime, srcID = srcClusterId, dstID = transactionId, @@ -34,7 +32,7 @@ class ChainalysisABRouter(override val routerId: Int,override val workerID:Int, Type("Incoming Payment") ) ) - commands+=( + sendUpdate( EdgeAddWithProperties(msgTime = transactionTime, srcID = transactionId, dstID = dstClusterId, @@ -42,7 +40,6 @@ class ChainalysisABRouter(override val routerId: Int,override val workerID:Int, Type("Outgoing Payment") ) ) - commands } //converts the line into a case class which has all of the data via the correct name and type diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/DashcoinRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/DashcoinGraphBuilder.scala similarity index 87% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/DashcoinRouter.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/DashcoinGraphBuilder.scala index 5e1882c62e..492bed79d5 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/DashcoinRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/DashcoinGraphBuilder.scala @@ -1,6 +1,6 @@ package com.raphtory.examples.blockchain.routers -import com.raphtory.core.components.Router.RouterWorker +import com.raphtory.core.components.Router.{GraphBuilder, RouterWorker} import com.raphtory.core.model.communication._ import com.raphtory.examples.blockchain.routers import com.raphtory.sources.blockchain.BitcoinTransaction @@ -10,10 +10,9 @@ import scala.collection.mutable.ListBuffer import scala.collection.parallel.mutable.ParHashSet import scala.util.hashing.MurmurHash3 -class DashcoinRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) - extends RouterWorker[BitcoinTransaction](routerId,workerID, initialManagerCount,initialRouterCount) { +class DashcoinGraphBuilder extends GraphBuilder[BitcoinTransaction] { - override protected def parseTuple(tuple: BitcoinTransaction): ParHashSet[GraphUpdate] = { + override def parseTuple(tuple: BitcoinTransaction) = { val transaction = tuple.transaction val time = tuple.time val blockID = tuple.blockID @@ -27,7 +26,6 @@ class DashcoinRouter(override val routerId: Int,override val workerID:Int, overr val locktime = transaction.asJsObject.fields("locktime") val version = transaction.asJsObject.fields("version") var total: Double = 0 - val commands = new ParHashSet[GraphUpdate]() for (vout <- vouts.asInstanceOf[JsArray].elements) { val voutOBJ = vout.asJsObject() var value = voutOBJ.fields("value").toString @@ -43,7 +41,7 @@ class DashcoinRouter(override val routerId: Int,override val workerID:Int, overr else value = "0" //TODO deal with people burning money //creates vertex for the receiving wallet - commands+=( + sendUpdate( VertexAddWithProperties( msgTime = timeAsLong, srcID = MurmurHash3.stringHash(address), @@ -55,7 +53,7 @@ class DashcoinRouter(override val routerId: Int,override val workerID:Int, overr ) ) //creates edge between the transaction and the wallet - commands+=( + sendUpdate( EdgeAddWithProperties( msgTime = timeAsLong, srcID = MurmurHash3.stringHash(txid), @@ -65,7 +63,7 @@ class DashcoinRouter(override val routerId: Int,override val workerID:Int, overr ) } - commands+=( + sendUpdate( VertexAddWithProperties( msgTime = timeAsLong, srcID = MurmurHash3.stringHash(txid), @@ -84,10 +82,10 @@ class DashcoinRouter(override val routerId: Int,override val workerID:Int, overr if (vins.toString().contains("coinbase")) { //creates the coingen node //toPartitionManager(VertexAddWithProperties(msgTime = timeAsLong, srcID = MurmurHash3.stringHash("coingen"), properties = Map[String,String](("type","coingen")))) - commands+=(VertexAdd(msgTime = timeAsLong, srcID = MurmurHash3.stringHash("coingen"))) + sendUpdate(VertexAdd(msgTime = timeAsLong, srcID = MurmurHash3.stringHash("coingen"))) //creates edge between coingen and the transaction - commands+=( + sendUpdate( EdgeAdd( msgTime = timeAsLong, srcID = MurmurHash3.stringHash("coingen"), @@ -102,7 +100,7 @@ class DashcoinRouter(override val routerId: Int,override val workerID:Int, overr val sequence = vinOBJ.fields("sequence").toString //no need to create node for prevtxid as should already exist //creates edge between the prev transaction and current transaction - commands+=( + sendUpdate( EdgeAddWithProperties( msgTime = timeAsLong, srcID = MurmurHash3.stringHash(prevtxid), @@ -112,7 +110,6 @@ class DashcoinRouter(override val routerId: Int,override val workerID:Int, overr ) } - commands } } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumGethRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumGethGraphBuilder.scala similarity index 80% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumGethRouter.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumGethGraphBuilder.scala index bf41da22d9..a83a2d452d 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumGethRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumGethGraphBuilder.scala @@ -1,7 +1,7 @@ package com.raphtory.examples.blockchain.routers import akka.http.scaladsl.unmarshalling.Unmarshal -import com.raphtory.core.components.Router.RouterWorker +import com.raphtory.core.components.Router.{GraphBuilder, RouterWorker} import com.raphtory.core.model.communication.{EdgeAddWithProperties, GraphUpdate, ImmutableProperty, Properties, StringProperty, VertexAddWithProperties} import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._ import akka.stream.ActorMaterializer @@ -10,13 +10,12 @@ import spray.json._ import scala.collection.mutable.ListBuffer import scala.collection.parallel.mutable.ParHashSet import scala.util.hashing.MurmurHash3 -class EthereumGethRouter(override val routerId: Int,override val workerID:Int,override val initialManagerCount: Int, override val initialRouterCount: Int) - extends RouterWorker[String](routerId,workerID, initialManagerCount, initialRouterCount) { +class EthereumGethGraphBuilder extends GraphBuilder[String] { def hexToInt(hex: String) = Integer.parseInt(hex.drop(2), 16) - print(routerId) - override protected def parseTuple(tuple: String): ParHashSet[GraphUpdate] = { + + override def parseTuple(tuple: String) = { print(tuple) val transaction = tuple.split(",") val blockNumber = hexToInt(transaction(0)) @@ -27,17 +26,14 @@ class EthereumGethRouter(override val routerId: Int,override val workerID:Int,ov val sourceNode = assignID(from) //hash the id to get a vertex ID val destinationNode = assignID(to) //hash the id to get a vertex ID - print(from) - print(to) - val commands = new ParHashSet[GraphUpdate]() - commands+=( + sendUpdate( VertexAddWithProperties(blockNumber, sourceNode, properties = Properties(ImmutableProperty("id", from))) ) - commands+=( + sendUpdate( VertexAddWithProperties(blockNumber, destinationNode, properties = Properties(ImmutableProperty("id", to))) ) - commands+=( + sendUpdate( EdgeAddWithProperties( blockNumber, sourceNode, @@ -45,7 +41,6 @@ class EthereumGethRouter(override val routerId: Int,override val workerID:Int,ov properties = Properties(StringProperty("value", sent)) ) ) - commands } } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumKafkaRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumKafkaGraphBuilder.scala similarity index 79% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumKafkaRouter.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumKafkaGraphBuilder.scala index c8ea3563a3..11de2df4e0 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumKafkaRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumKafkaGraphBuilder.scala @@ -1,7 +1,7 @@ package com.raphtory.examples.blockchain.routers import akka.http.scaladsl.unmarshalling.Unmarshal -import com.raphtory.core.components.Router.RouterWorker +import com.raphtory.core.components.Router.{GraphBuilder, RouterWorker} import com.raphtory.core.model.communication.{EdgeAddWithProperties, GraphUpdate, ImmutableProperty, Properties, StringProperty, VertexAddWithProperties} import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._ import akka.stream.ActorMaterializer @@ -10,14 +10,13 @@ import spray.json._ import scala.collection.mutable.ListBuffer import scala.collection.parallel.mutable.ParHashSet import scala.util.hashing.MurmurHash3 -class EthereumKafkaRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) - extends RouterWorker[String](routerId,workerID, initialManagerCount, initialRouterCount) { +class EthereumKafkaGraphBuilder extends GraphBuilder[String] { def hexToInt(hex: String) = Integer.parseInt(hex.drop(2), 16) - override protected def parseTuple(tuple: String): ParHashSet[GraphUpdate] = { + override def parseTuple(tuple: String):Unit = { val transaction = tuple.split(",") - if(transaction(1).equals("block_number")) return ParHashSet() + if(transaction(1).equals("block_number")) return print(transaction) @@ -28,7 +27,7 @@ class EthereumKafkaRouter(override val routerId: Int,override val workerID:Int, // val to = transaction(2).replaceAll("\"", "").toLowerCase // val sent = transaction(5).replaceAll("\"", "") - if(transaction(2).equals("block_number")) return ParHashSet() + if(transaction(2).equals("block_number")) return val blockNumber = transaction(2).toInt val from = transaction(4).replaceAll("\"", "").toLowerCase @@ -37,15 +36,14 @@ class EthereumKafkaRouter(override val routerId: Int,override val workerID:Int, val sourceNode = assignID(from) //hash the id to get a vertex ID val destinationNode = assignID(to) //hash the id to get a vertex ID - val commands = new ParHashSet[GraphUpdate]() - commands+=( + sendUpdate( VertexAddWithProperties(blockNumber, sourceNode, properties = Properties(ImmutableProperty("id", from))) ) - commands+=( + sendUpdate( VertexAddWithProperties(blockNumber, destinationNode, properties = Properties(ImmutableProperty("id", to))) ) - commands+=( + sendUpdate( EdgeAddWithProperties( blockNumber, sourceNode, @@ -53,7 +51,6 @@ class EthereumKafkaRouter(override val routerId: Int,override val workerID:Int, properties = Properties(StringProperty("value", sent)) ) ) - commands } } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumTransactionRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumTransactionGraphBuilder.scala similarity index 75% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumTransactionRouter.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumTransactionGraphBuilder.scala index 5c9ff2c6e4..7b227ce79c 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumTransactionRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumTransactionGraphBuilder.scala @@ -1,29 +1,27 @@ package com.raphtory.examples.blockchain.routers -import com.raphtory.core.components.Router.RouterWorker +import com.raphtory.core.components.Router.{GraphBuilder, RouterWorker} import com.raphtory.core.model.communication.{EdgeAddWithProperties, GraphUpdate, Properties, StringProperty, VertexAddWithProperties} import scala.collection.mutable.ListBuffer import scala.collection.parallel.mutable.ParHashSet import scala.util.hashing.MurmurHash3 -class EthereumTransactionRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) - extends RouterWorker[String](routerId,workerID, initialManagerCount, initialRouterCount) { +class EthereumTransactionGraphBuilder extends GraphBuilder[String]{ - override protected def parseTuple(tuple: String): ParHashSet[GraphUpdate] = { + override def parseTuple(tuple: String):Unit = { val components = tuple.drop(1).dropRight(1).split(",") val creationDate = components(3).toLong * 1000 //seconds to miliseconds val sourceNode = MurmurHash3.stringHash(components(0)) //hash the id to get a vertex ID - val commands = new ParHashSet[GraphUpdate]() - commands+=( + sendUpdate( VertexAddWithProperties(creationDate, sourceNode, Properties(StringProperty("id", components(0)))) ) //create the source node and add the wallet ID as a property if (components(1).nonEmpty) { //money being sent to an actual user val targetNode = MurmurHash3.stringHash(components(1)) //hash the id of the to wallet to get a vertex ID - commands+=( + sendUpdate( VertexAddWithProperties(creationDate, targetNode, Properties(StringProperty("id", components(1)))) ) //create the destination vertex - commands+=( + sendUpdate( EdgeAddWithProperties( creationDate, sourceNode, @@ -33,8 +31,8 @@ class EthereumTransactionRouter(override val routerId: Int,override val workerID ) //create the edge between them adding the value as a property } else { //burnt cash val targetNode = MurmurHash3.stringHash("null") - commands+=(VertexAddWithProperties(creationDate, targetNode, Properties(StringProperty("id", "null")))) - commands+=( + sendUpdate(VertexAddWithProperties(creationDate, targetNode, Properties(StringProperty("id", "null")))) + sendUpdate( EdgeAddWithProperties( creationDate, sourceNode, @@ -43,6 +41,5 @@ class EthereumTransactionRouter(override val routerId: Int,override val workerID ) ) } - commands } } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/FirehoseKafkaRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/FirehoseKafkaGraphBuilder.scala similarity index 80% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/FirehoseKafkaRouter.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/FirehoseKafkaGraphBuilder.scala index 156bd6324d..d69716d85f 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/FirehoseKafkaRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/FirehoseKafkaGraphBuilder.scala @@ -1,7 +1,7 @@ package com.raphtory.examples.blockchain.routers import akka.http.scaladsl.unmarshalling.Unmarshal -import com.raphtory.core.components.Router.RouterWorker +import com.raphtory.core.components.Router.{GraphBuilder, RouterWorker} import com.raphtory.core.model.communication.{DoubleProperty, EdgeAdd, EdgeAddWithProperties, EdgeDelete, GraphUpdate, ImmutableProperty, LongProperty, Properties, StringProperty, VertexAdd, VertexAddWithProperties, VertexDelete} import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._ import akka.stream.ActorMaterializer @@ -12,19 +12,17 @@ import scala.collection.parallel.mutable.ParHashSet import scala.util.Random import scala.util.hashing.MurmurHash3 import scala.math.BigInt -class FirehoseKafkaRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) - extends RouterWorker[String](routerId,workerID, initialManagerCount, initialRouterCount) { +class FirehoseKafkaGraphBuilder extends GraphBuilder[String] { var DELETEPERCENT = System.getenv().getOrDefault("ETHER_DELETE_PERCENT", "0").trim.toDouble/100 var DELETESEED = System.getenv().getOrDefault("ETHER_DELETE_SEED", "123").trim.toInt val random = new Random(DELETESEED) def hexToInt(hex: String) = Integer.parseInt(hex.drop(2), 16) - override protected def parseTuple(tuple: String): ParHashSet[GraphUpdate] = { + override def parseTuple(tuple: String): Unit = { //if(value.toString.contains("0xa09871aeadf4994ca12f5c0b6056bbd1d343c029")) println(value.toString) val transaction = tuple.split(",") - if(transaction(1).equals("block_number")) return ParHashSet() + if(transaction(1).equals("block_number")) return val blockNumber = transaction(2).toInt - val commands = new ParHashSet[GraphUpdate]() val from = transaction(4).replaceAll("\"", "").toLowerCase val to = transaction(5).replaceAll("\"", "").toLowerCase @@ -33,18 +31,17 @@ class FirehoseKafkaRouter(override val routerId: Int,override val workerID:Int, val destinationNode = assignID(to) //hash the id to get a vertex ID //if(from.contains("0xa09871aeadf4994ca12f5c0b6056bbd1d343c029".toLowerCase())) println(from) //if(to.contains("0xa09871aeadf4994ca12f5c0b6056bbd1d343c029".toLowerCase())) println(to) - commands+=(VertexAddWithProperties(blockNumber, sourceNode, properties = Properties(ImmutableProperty("id", from)))) + sendUpdate(VertexAddWithProperties(blockNumber, sourceNode, properties = Properties(ImmutableProperty("id", from)))) // if(random.nextDouble()<=DELETEPERCENT) // sendGraphUpdate(VertexDelete(blockNumber+1,sourceNode)) - commands+=(VertexAddWithProperties(blockNumber, destinationNode, properties = Properties(ImmutableProperty("id", to)))) + sendUpdate(VertexAddWithProperties(blockNumber, destinationNode, properties = Properties(ImmutableProperty("id", to)))) // if(random.nextDouble()<=DELETEPERCENT) // sendGraphUpdate(VertexDelete(blockNumber+1,destinationNode)) - commands+=(EdgeAddWithProperties(blockNumber, sourceNode, destinationNode,properties = Properties(DoubleProperty("value", sent)))) + sendUpdate(EdgeAddWithProperties(blockNumber, sourceNode, destinationNode,properties = Properties(DoubleProperty("value", sent)))) // if(random.nextDouble()<=DELETEPERCENT) // sendGraphUpdate(EdgeDelete(blockNumber+1,sourceNode,destinationNode)) - commands } } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/LitecoinRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/LitecoinGraphBuilder.scala similarity index 88% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/LitecoinRouter.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/LitecoinGraphBuilder.scala index 66f20e5b3e..576b09511d 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/LitecoinRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/LitecoinGraphBuilder.scala @@ -1,6 +1,6 @@ package com.raphtory.examples.blockchain.routers -import com.raphtory.core.components.Router.RouterWorker +import com.raphtory.core.components.Router.{GraphBuilder, RouterWorker} import com.raphtory.core.model.communication._ import com.raphtory.sources.blockchain.BitcoinTransaction import spray.json.{JsArray, JsValue} @@ -10,10 +10,9 @@ import scala.collection.parallel.mutable.ParHashSet import scala.util.hashing.MurmurHash3 -class LitecoinRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) - extends RouterWorker[BitcoinTransaction](routerId,workerID,initialManagerCount, initialRouterCount) { +class LitecoinGraphBuilder extends GraphBuilder[BitcoinTransaction]{ - override protected def parseTuple(tuple:BitcoinTransaction): ParHashSet[GraphUpdate] = { + override def parseTuple(tuple:BitcoinTransaction) = { val transaction = tuple.transaction val time = tuple.time val blockID = tuple.blockID @@ -27,7 +26,6 @@ class LitecoinRouter(override val routerId: Int,override val workerID:Int, overr val locktime = transaction.asJsObject.fields("locktime") val version = transaction.asJsObject.fields("version") var total: Double = 0 - val commands = new ParHashSet[GraphUpdate]() for (vout <- vouts.asInstanceOf[JsArray].elements) { val voutOBJ = vout.asJsObject() var value = voutOBJ.fields("value").toString @@ -43,7 +41,7 @@ class LitecoinRouter(override val routerId: Int,override val workerID:Int, overr else value = "0" //TODO deal with people burning money //creates vertex for the receiving wallet - commands+=( + sendUpdate( VertexAddWithProperties( msgTime = timeAsLong, srcID = MurmurHash3.stringHash(address), @@ -55,7 +53,7 @@ class LitecoinRouter(override val routerId: Int,override val workerID:Int, overr ) ) //creates edge between the transaction and the wallet - commands+=( + sendUpdate( EdgeAddWithProperties( msgTime = timeAsLong, srcID = MurmurHash3.stringHash(txid), @@ -65,7 +63,7 @@ class LitecoinRouter(override val routerId: Int,override val workerID:Int, overr ) } - commands+=( + sendUpdate( VertexAddWithProperties( msgTime = timeAsLong, srcID = MurmurHash3.stringHash(txid), @@ -84,7 +82,7 @@ class LitecoinRouter(override val routerId: Int,override val workerID:Int, overr if (vins.toString().contains("coinbase")) { //creates the coingen node - commands+=( + sendUpdate( VertexAddWithProperties( msgTime = timeAsLong, srcID = MurmurHash3.stringHash("coingen"), @@ -93,7 +91,7 @@ class LitecoinRouter(override val routerId: Int,override val workerID:Int, overr ) //creates edge between coingen and the transaction - commands+=( + sendUpdate( EdgeAdd( msgTime = timeAsLong, srcID = MurmurHash3.stringHash("coingen"), @@ -108,7 +106,7 @@ class LitecoinRouter(override val routerId: Int,override val workerID:Int, overr val sequence = vinOBJ.fields("sequence").toString //no need to create node for prevtxid as should already exist //creates edge between the prev transaction and current transaction - commands+=( + sendUpdate( EdgeAddWithProperties( msgTime = timeAsLong, srcID = MurmurHash3.stringHash(prevtxid), @@ -119,7 +117,6 @@ class LitecoinRouter(override val routerId: Int,override val workerID:Int, overr } } - commands } } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/citationNetwork/CitationRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/citationNetwork/CitationGraphBuilder.scala similarity index 69% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/citationNetwork/CitationRouter.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/citationNetwork/CitationGraphBuilder.scala index c2d44019b2..ea57716485 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/citationNetwork/CitationRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/citationNetwork/CitationGraphBuilder.scala @@ -2,15 +2,14 @@ package com.raphtory.examples.citationNetwork import java.text.SimpleDateFormat -import com.raphtory.core.components.Router.RouterWorker +import com.raphtory.core.components.Router.{GraphBuilder, RouterWorker} import com.raphtory.core.model.communication._ import scala.collection.mutable.ListBuffer import scala.collection.parallel.mutable.ParHashSet -class CitationRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) - extends RouterWorker[String](routerId,workerID, initialManagerCount, initialRouterCount) { - override protected def parseTuple(tuple: String): ParHashSet[GraphUpdate] = { +class CitationGraphBuilder extends GraphBuilder[String] { + override def parseTuple(tuple: String) = { val fileLine = tuple.split(",").map(_.trim) //take the tuple and split on , as we are only interested in the first 4 fields // title_paper,year,volume,title,pages,number,journal,author,ENTRYTYPE,ID val sourceTitle = fileLine(0) @@ -20,16 +19,15 @@ class CitationRouter(override val routerId: Int,override val workerID:Int, overr val sourceID = assignID(sourceTitle) val destinationID = assignID(destinationTitle) - val commands = new ParHashSet[GraphUpdate]() //create sourceNode - commands+=(VertexAddWithProperties( + sendUpdate(VertexAddWithProperties( sourceYear, //when it happened ?? sourceID, // the id of the node Properties(ImmutableProperty("title", sourceTitle)), //properties for the node Type("Publication")) //node type ) //create destinationNode - commands+=(VertexAddWithProperties( + sendUpdate(VertexAddWithProperties( destinationYear, destinationID, Properties(ImmutableProperty("title", destinationTitle)), @@ -37,13 +35,11 @@ class CitationRouter(override val routerId: Int,override val workerID:Int, overr ) //create edge - commands+=(EdgeAdd( + sendUpdate(EdgeAdd( sourceYear, //time of edge ?? sourceID, //source of edge destinationID, //destination of edge Type("Cited") // edge type )) - - commands } } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/datasources/GabMongoSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/datasources/GabMongoSource.scala index 790293c88a..9551f95cbf 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/datasources/GabMongoSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/datasources/GabMongoSource.scala @@ -4,7 +4,6 @@ import akka.actor.Cancellable import ch.qos.logback.classic.Level import com.mongodb.casbah.Imports.{MongoConnection, _} import com.raphtory.core.components.Spout.DataSource -import com.raphtory.core.model.communication.{SpoutGoing, StringSpoutGoing} import org.slf4j.LoggerFactory import scala.collection.mutable diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabPostGraphRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabPostGraphBuilder.scala similarity index 72% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabPostGraphRouter.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabPostGraphBuilder.scala index 97e948db8f..8a260a32a7 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabPostGraphRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabPostGraphBuilder.scala @@ -2,7 +2,7 @@ package com.raphtory.examples.gab.routers import java.text.SimpleDateFormat -import com.raphtory.core.components.Router.RouterWorker +import com.raphtory.core.components.Router.{GraphBuilder, RouterWorker} import com.raphtory.core.model.communication._ import scala.collection.mutable.ListBuffer @@ -15,26 +15,23 @@ import scala.collection.parallel.mutable.ParHashSet // is equal to -1. Columns 1 and 4 correspond to the postId and parentPostid in the file. // Then either the vertex or the edge are created accordingly. -class GabPostGraphRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) - extends RouterWorker[String](routerId,workerID, initialManagerCount, initialRouterCount) { +class GabPostGraphBuilder extends GraphBuilder[String] { - override protected def parseTuple(tuple: String): ParHashSet[GraphUpdate] = { + override def parseTuple(tuple: String) = { val fileLine = tuple.split(";").map(_.trim) //user wise // val sourceNode=fileLine(2).toInt // val targetNode=fileLine(5).toInt - val commands = new ParHashSet[GraphUpdate]() //comment wise val sourceNode = fileLine(1).toInt val targetNode = fileLine(4).toInt if (targetNode > 0) { val creationDate = dateToUnixTime(timestamp = fileLine(0).slice(0, 19)) - commands+=(VertexAdd(creationDate, sourceNode)) - commands+=(VertexAdd(creationDate, targetNode)) - commands+=(EdgeAdd(creationDate, sourceNode, targetNode)) + sendUpdate(VertexAdd(creationDate, sourceNode)) + sendUpdate(VertexAdd(creationDate, targetNode)) + sendUpdate(EdgeAdd(creationDate, sourceNode, targetNode)) } - commands } def dateToUnixTime(timestamp: => String): Long = { diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabRawRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabRawGraphBuilder.scala similarity index 86% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabRawRouter.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabRawGraphBuilder.scala index 264c35ea26..dae6418363 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabRawRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabRawGraphBuilder.scala @@ -2,7 +2,7 @@ package com.raphtory.examples.gab.routers import java.time.OffsetDateTime -import com.raphtory.core.components.Router.RouterWorker +import com.raphtory.core.components.Router.{GraphBuilder, RouterWorker} import com.raphtory.core.model.communication.{EdgeAddWithProperties, GraphUpdate, Properties, StringProperty, VertexAddWithProperties} import com.raphtory.examples.gab.rawgraphmodel.GabPost import spray.json._ @@ -20,15 +20,13 @@ import scala.collection.parallel.mutable.ParHashSet * the correct case Class which can then be passed to the graph manager * which will then pass it to the graph partition dealing with the associated vertex */ -final class GabRawRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) - extends RouterWorker[String](routerId,workerID, initialManagerCount, initialRouterCount) { +final class GabRawGraphBuilder extends GraphBuilder[String] { import com.raphtory.examples.gab.rawgraphmodel.GabJsonProtocol._ private val nullStr = "null" - override protected def parseTuple(tuple: String): ParHashSet[GraphUpdate] = { - val commands = new ParHashSet[GraphUpdate]() + override def parseTuple(tuple: String) = { try { val command = tuple val post = command.parseJson.convertTo[GabPost] @@ -44,7 +42,7 @@ final class GabRawRouter(override val routerId: Int,override val workerID:Int, o def sendPostToPartitions(post: GabPost, recursiveCall: Boolean = false, parent: Int = 0): Unit = { val postUUID = post.id.get.toInt val timestamp = OffsetDateTime.parse(post.created_at.get).toEpochSecond - commands += ( + sendUpdate( VertexAddWithProperties( timestamp, postUUID, @@ -68,12 +66,12 @@ final class GabRawRouter(override val routerId: Int,override val workerID:Int, o StringProperty("type", "post") ) ) - ) + ) post.user match { case Some(user) => val userUUID: Int = "user".hashCode() + user.id //TODO improve in case of clashes - commands += ( + sendUpdate( VertexAddWithProperties( timestamp, userUUID, @@ -85,21 +83,21 @@ final class GabRawRouter(override val routerId: Int,override val workerID:Int, o StringProperty("verified", user.verified.toString) ) ) - ) + ) - commands += ( + sendUpdate( EdgeAddWithProperties(timestamp, userUUID, postUUID, Properties((StringProperty("type", "userToPost")))) - ) - commands += ( + ) + sendUpdate( EdgeAddWithProperties(timestamp, postUUID, userUUID, Properties(StringProperty("type", "postToUser"))) - ) + ) case None => } post.topic match { case Some(topic) => val topicUUID: Int = Math.pow(2, 24).toInt + (topic.id.hashCode()) - commands += ( + sendUpdate( VertexAddWithProperties( timestamp, topicUUID, @@ -111,19 +109,19 @@ final class GabRawRouter(override val routerId: Int,override val workerID:Int, o StringProperty("id", topic.id) ) ) - ) + ) - commands += ( + sendUpdate( EdgeAddWithProperties(timestamp, postUUID, topicUUID, Properties(StringProperty("type", "postToTopic"))) - ) + ) case None => } // Edge from child to parent post if (recursiveCall && parent != 0) - commands += ( + sendUpdate( EdgeAddWithProperties(timestamp, postUUID, parent, Properties(StringProperty("type", "childToParent"))) - ) + ) post.parent match { case Some(p) => if (!recursiveCall) // Allow only one recursion per post @@ -133,8 +131,6 @@ final class GabRawRouter(override val routerId: Int,override val workerID:Int, o } } - - commands } } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabUserGraphRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabUserGraphBuilder.scala similarity index 78% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabUserGraphRouter.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabUserGraphBuilder.scala index 7b7a66dae4..5012b87528 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabUserGraphRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabUserGraphBuilder.scala @@ -2,7 +2,7 @@ package com.raphtory.examples.gab.routers import java.text.SimpleDateFormat -import com.raphtory.core.components.Router.RouterWorker +import com.raphtory.core.components.Router.{GraphBuilder, RouterWorker} import com.raphtory.core.model.communication.Type import com.raphtory.core.model.communication._ @@ -17,28 +17,25 @@ import scala.util.Random // is equal to -1. Columns 2 and 5 correspond to the userid and parentUserid in the file. // Then either the vertex or the edge are created accordingly. -class GabUserGraphRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) - extends RouterWorker[String](routerId,workerID, initialManagerCount, initialRouterCount) { +class GabUserGraphBuilder extends GraphBuilder[String] { - override protected def parseTuple(tuple: String): ParHashSet[GraphUpdate] = { + override def parseTuple(tuple: String) = { val fileLine = tuple.split(";").map(_.trim) //user wise val sourceNode = fileLine(2).toInt val targetNode = fileLine(5).toInt - val commands = new ParHashSet[GraphUpdate]() //comment wise // val sourceNode=fileLine(1).toInt //val targetNode=fileLine(4).toInt if (targetNode > 0 && targetNode != sourceNode) { val creationDate = dateToUnixTime(timestamp = fileLine(0).slice(0, 19)) - commands+=(VertexAdd(creationDate, sourceNode, Type("User"))) - commands+=(VertexAdd(creationDate, targetNode, Type("User"))) - commands+=(EdgeAdd(creationDate, sourceNode, targetNode, Type("User to User"))) + sendUpdate(VertexAdd(creationDate, sourceNode, Type("User"))) + sendUpdate(VertexAdd(creationDate, targetNode, Type("User"))) + sendUpdate(EdgeAdd(creationDate, sourceNode, targetNode, Type("User to User"))) // sendGraphUpdate(VertexAddWithProperties(creationDate, sourceNode, Properties(StringProperty("test1","value1"),StringProperty("test2","Value2")),Type("User"))) // sendGraphUpdate(VertexAddWithProperties(creationDate, targetNode, Properties(StringProperty("test1","value1"),StringProperty("test2","Value2")),Type("User"))) // sendGraphUpdate(EdgeAddWithProperties(creationDate, sourceNode, targetNode, Properties(StringProperty("test1","value1"),StringProperty("test2","Value2")),Type("User To User"))) } - commands } def dateToUnixTime(timestamp: => String): Long = { diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCGraphBuilder.scala similarity index 73% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCRouter.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCGraphBuilder.scala index ad3acfbda6..3b08c5eb63 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCGraphBuilder.scala @@ -3,16 +3,14 @@ package com.raphtory.examples.ldbc.routers import java.text.SimpleDateFormat import java.util.Date -import com.raphtory.core.components.Router.RouterWorker +import com.raphtory.core.components.Router.{GraphBuilder, RouterWorker} import com.raphtory.core.model.communication.{EdgeAdd, EdgeDelete, GraphUpdate, Type, VertexAdd, VertexDelete} import scala.collection.mutable.ListBuffer import scala.collection.parallel.mutable.ParHashSet -class LDBCRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) - extends RouterWorker[String](routerId,workerID, initialManagerCount, initialRouterCount) { - override protected def parseTuple(tuple: String): ParHashSet[GraphUpdate] = { - val commands = new ParHashSet[GraphUpdate]() +class LDBCGraphBuilder extends GraphBuilder[String] { + override def parseTuple(tuple: String) = { val fileLine = tuple.asInstanceOf[String].split("\\|") val date = fileLine(1).substring(0, 10) + fileLine(1).substring(11, 23); //extract the day of the event val date2 = fileLine(2).substring(0, 10) + fileLine(1).substring(11, 23); //extract the day of the event @@ -22,13 +20,13 @@ class LDBCRouter(override val routerId: Int,override val workerID:Int, override val edgeDeletion = sys.env.getOrElse("LDBC_EDGE_DELETION", "false").trim.toBoolean fileLine(0) match { case "person" => - commands+=(VertexAdd(creationDate, assignID("person" + fileLine(3)), Type("person"))) + sendUpdate(VertexAdd(creationDate, assignID("person" + fileLine(3)), Type("person"))) //sendGraphUpdate(VertexAdd(creationDate, fileLine(3).toLong,Type("person"))) if(vertexDeletion) - commands+=(VertexDelete(deletionDate, assignID("person" + fileLine(3)))) + sendUpdate(VertexDelete(deletionDate, assignID("person" + fileLine(3)))) case "person_knows_person" => //sendGraphUpdate(EdgeAdd(creationDate, fileLine(3).toLong,fileLine(4).toLong,Type("person_knows_person"))) - commands+=( + sendUpdate( EdgeAdd( creationDate, assignID("person" + fileLine(3)), @@ -37,9 +35,8 @@ class LDBCRouter(override val routerId: Int,override val workerID:Int, override ) ) if(edgeDeletion) - commands+=(EdgeDelete(deletionDate, assignID("person"+fileLine(3)),assignID("person"+fileLine(4)))) + sendUpdate(EdgeDelete(deletionDate, assignID("person"+fileLine(3)),assignID("person"+fileLine(4)))) } - commands } } //2012-11-01T09:28:01.185+00:00|2019-07-22T11:24:24.362+00:00|35184372093644|Jose|Garcia|female|1988-05-20|111.68.47.44|Firefox diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCOldRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCOldGraphBuilder.scala similarity index 78% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCOldRouter.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCOldGraphBuilder.scala index 777f6b5879..b6f4197208 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCOldRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCOldGraphBuilder.scala @@ -3,26 +3,23 @@ package com.raphtory.examples.ldbc.routers import java.text.SimpleDateFormat import java.util.Date -import com.raphtory.core.components.Router.RouterWorker +import com.raphtory.core.components.Router.{GraphBuilder, RouterWorker} import com.raphtory.core.model.communication.{EdgeAdd, EdgeDelete, GraphUpdate, Type, VertexAdd, VertexDelete} import scala.collection.mutable.ListBuffer import scala.collection.parallel.mutable.ParHashSet -class LDBCOldRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) - extends RouterWorker[String](routerId,workerID, initialManagerCount, initialRouterCount) { - override protected def parseTuple(tuple: String): ParHashSet[GraphUpdate] = { +class LDBCOldGraphBuilder extends GraphBuilder[String] { + override def parseTuple(tuple: String) = { val fileLine = tuple.asInstanceOf[String].split("\\|") - val commands = new ParHashSet[GraphUpdate]() //val deletionDate:Long = new SimpleDateFormat("yyyy-MM-ddHH:mm:ss.SSS").parse(date2).getTime() fileLine(0) match { case "person" => val date = fileLine(6).substring(0, 10) + fileLine(5).substring(11, 23); //extract the day of the event //val date2 = fileLine(2).substring(0, 10) + fileLine(1).substring(11, 23); //extract the day of the event val creationDate: Long = new SimpleDateFormat("yyyy-MM-ddHH:mm:ss.SSS").parse(date).getTime() - commands+=(VertexAdd(creationDate, assignID("person" + fileLine(1)), Type("person"))) - commands + sendUpdate(VertexAdd(creationDate, assignID("person" + fileLine(1)), Type("person"))) //sendGraphUpdate(VertexAdd(creationDate, fileLine(3).toLong,Type("person"))) // sendGraphUpdate(VertexDelete(deletionDate, assignID("person"+fileLine(3)))) case "person_knows_person" => @@ -30,7 +27,7 @@ class LDBCOldRouter(override val routerId: Int,override val workerID:Int, overri //val date2 = fileLine(2).substring(0, 10) + fileLine(1).substring(11, 23); //extract the day of the event val creationDate: Long = new SimpleDateFormat("yyyy-MM-ddHH:mm:ss.SSS").parse(date).getTime() //sendGraphUpdate(EdgeAdd(creationDate, fileLine(3).toLong,fileLine(4).toLong,Type("person_knows_person"))) - commands+=( + sendUpdate( EdgeAdd( creationDate, assignID("person" + fileLine(1)), @@ -38,7 +35,6 @@ class LDBCOldRouter(override val routerId: Int,override val workerID:Int, overri Type("person_knows_person") ) ) - commands //sendGraphUpdate(EdgeDelete(deletionDate, assignID("person"+fileLine(3)),assignID("person"+fileLine(4)))) } } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRDataSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRDataSource.scala index 350c9e0ff0..5bad0d114a 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRDataSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRDataSource.scala @@ -1,8 +1,6 @@ package com.raphtory.examples.lotr import com.raphtory.core.components.Spout.{DataSource} -import com.raphtory.core.model.communication.{SpoutGoing, StringSpoutGoing} - import scala.collection.mutable diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRGraphBuilder.scala similarity index 52% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRRouter.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRGraphBuilder.scala index 08fe715373..6c55f13e05 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRGraphBuilder.scala @@ -1,19 +1,17 @@ package com.raphtory.examples.lotr -import com.raphtory.core.components.Router.RouterWorker +import com.raphtory.core.components.Router.{GraphBuilder, RouterWorker} import com.raphtory.core.model.communication.{EdgeAdd, Type, VertexAdd, _} import scala.collection.mutable.ListBuffer import scala.collection.parallel.mutable.ParHashSet import scala.util.Random -class LOTRRouter(override val routerId: Int, override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) - extends RouterWorker[String](routerId,workerID, initialManagerCount, initialRouterCount) { +class LOTRGraphBuilder extends GraphBuilder[String]{ - override protected def parseTuple(tuple: String): ParHashSet[GraphUpdate] = { + override def parseTuple(tuple: String) = { val fileLine = tuple.split(",").map(_.trim) - val commands = new ParHashSet[GraphUpdate]() val sourceNode = fileLine(0) val srcID = assignID(sourceNode) @@ -22,9 +20,8 @@ class LOTRRouter(override val routerId: Int, override val workerID:Int, override val timeStamp = fileLine(2).toLong - commands+=(VertexAddWithProperties(timeStamp, srcID, Properties(LongProperty("test",Random.nextLong()),ImmutableProperty("name",sourceNode)),Type("Character"))) - commands+=(VertexAddWithProperties(timeStamp, tarID, Properties(LongProperty("test2",Random.nextLong()),ImmutableProperty("name",targetNode)),Type("Character"))) - commands+=(EdgeAdd(timeStamp,srcID,tarID, Type("Character Co-occurence"))) - commands + sendUpdate(VertexAddWithProperties(timeStamp, srcID, Properties(LongProperty("test",Random.nextLong()),ImmutableProperty("name",sourceNode)),Type("Character"))) + sendUpdate(VertexAddWithProperties(timeStamp, tarID, Properties(LongProperty("test2",Random.nextLong()),ImmutableProperty("name",targetNode)),Type("Character"))) + sendUpdate(EdgeAdd(timeStamp,srcID,tarID, Type("Character Co-occurence"))) } } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomDataSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomDataSource.scala index 1acf9ced21..49848a0571 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomDataSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomDataSource.scala @@ -1,8 +1,6 @@ package com.raphtory.examples.test.actors import com.raphtory.core.components.Spout.DataSource -import com.raphtory.core.model.communication.{SpoutGoing, StringSpoutGoing} -import com.raphtory.core.utils.Utils import scala.language.postfixOps import scala.util.Random diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomGraphBuilder.scala similarity index 87% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomRouter.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomGraphBuilder.scala index baec54d5c9..316302d3f9 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomGraphBuilder.scala @@ -1,6 +1,6 @@ package com.raphtory.examples.test.actors -import com.raphtory.core.components.Router.RouterWorker +import com.raphtory.core.components.Router.{GraphBuilder, RouterWorker} import com.raphtory.core.model.communication.{GraphUpdate, _} import spray.json._ @@ -16,26 +16,24 @@ import scala.collection.parallel.mutable.ParHashSet * the correct case Class which can then be passed to the graph manager * which will then pass it to the graph partition dealing with the associated vertex */ -class RandomRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) - extends RouterWorker[String](routerId,workerID,initialManagerCount, initialRouterCount) { +class RandomGraphBuilder extends GraphBuilder[String]{ //************* MESSAGE HANDLING BLOCK - override protected def parseTuple(tuple:String): ParHashSet[GraphUpdate] = { + override def parseTuple(tuple:String) = { val command = tuple.asInstanceOf[String] val parsedOBJ = command.parseJson.asJsObject //get the json object val commandKey = parsedOBJ.fields //get the command type if (commandKey.contains("VertexAdd")) - ParHashSet(vertexAdd(parsedOBJ.getFields("VertexAdd").head.asJsObject)) + sendUpdate(vertexAdd(parsedOBJ.getFields("VertexAdd").head.asJsObject)) //else if(commandKey.contains("VertexUpdateProperties")) vertexUpdateProperties(parsedOBJ.getFields("VertexUpdateProperties").head.asJsObject) else if (commandKey.contains("VertexRemoval")) - ParHashSet(vertexRemoval(parsedOBJ.getFields("VertexRemoval").head.asJsObject)) + sendUpdate(vertexRemoval(parsedOBJ.getFields("VertexRemoval").head.asJsObject)) else if (commandKey.contains("EdgeAdd")) - ParHashSet(edgeAdd(parsedOBJ.getFields("EdgeAdd").head.asJsObject)) //if addVertex, parse to handling function + sendUpdate(edgeAdd(parsedOBJ.getFields("EdgeAdd").head.asJsObject)) //if addVertex, parse to handling function // else if(commandKey.contains("EdgeUpdateProperties")) edgeUpdateProperties(parsedOBJ.getFields("EdgeUpdateProperties").head.asJsObject) else if (commandKey.contains("EdgeRemoval")) - ParHashSet(edgeRemoval(parsedOBJ.getFields("EdgeRemoval").head.asJsObject)) - else ParHashSet() + sendUpdate(edgeRemoval(parsedOBJ.getFields("EdgeRemoval").head.asJsObject)) } def vertexAdd(command: JsObject) = { diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleDataSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleDataSource.scala index 72b1784179..40e1153b54 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleDataSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleDataSource.scala @@ -1,7 +1,6 @@ package com.raphtory.examples.test.actors import com.raphtory.core.components.Spout.{DataSource} -import com.raphtory.core.model.communication.{SpoutGoing, StringSpoutGoing} import scala.collection.mutable.Queue diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleTestGraphBuilder.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleTestGraphBuilder.scala new file mode 100644 index 0000000000..1b248f73a8 --- /dev/null +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleTestGraphBuilder.scala @@ -0,0 +1,18 @@ +package com.raphtory.examples.test.actors + +import com.raphtory.core.components.Router.{GraphBuilder, RouterWorker} +import com.raphtory.core.components.Spout.Spout +import com.raphtory.core.model.communication.{EdgeAdd, GraphUpdate, VertexAddWithProperties} + +import scala.collection.mutable.ListBuffer +import scala.collection.parallel.mutable.ParHashSet + +class TriangleTestGraphBuilder extends GraphBuilder[String] { + override def parseTuple(tuple: String)= { + val command = tuple.split(",") + val msgTime = command(0).toLong + val sourceID = command(1).toLong + val destID = command(2).toLong + sendUpdate(EdgeAdd(msgTime, sourceID, destID)) + } +} diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleTestRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleTestRouter.scala deleted file mode 100644 index 9fb8835607..0000000000 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleTestRouter.scala +++ /dev/null @@ -1,19 +0,0 @@ -package com.raphtory.examples.test.actors - -import com.raphtory.core.components.Router.RouterWorker -import com.raphtory.core.components.Spout.Spout -import com.raphtory.core.model.communication.{EdgeAdd, GraphUpdate, VertexAddWithProperties} - -import scala.collection.mutable.ListBuffer -import scala.collection.parallel.mutable.ParHashSet - -class TriangleTestRouter (override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) - extends RouterWorker[String](routerId,workerID, initialManagerCount, initialRouterCount) { - override protected def parseTuple(tuple: String): ParHashSet[GraphUpdate] = { - val command = tuple.split(",") - val msgTime = command(0).toLong - val sourceID = command(1).toLong - val destID = command(2).toLong - ParHashSet((EdgeAdd(msgTime, sourceID, destID))) - } -} diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/trackAndTrace/routers/TrackAndTraceRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/trackAndTrace/routers/TrackAndTraceGraphBuilder.scala similarity index 89% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/trackAndTrace/routers/TrackAndTraceRouter.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/trackAndTrace/routers/TrackAndTraceGraphBuilder.scala index a537d5cc38..0b97848bb7 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/trackAndTrace/routers/TrackAndTraceRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/trackAndTrace/routers/TrackAndTraceGraphBuilder.scala @@ -2,7 +2,7 @@ package com.raphtory.examples.trackAndTrace.routers import java.text.SimpleDateFormat -import com.raphtory.core.components.Router.RouterWorker +import com.raphtory.core.components.Router.{GraphBuilder, RouterWorker} import com.raphtory.core.model.communication.Type import com.raphtory.core.model.communication._ @@ -10,23 +10,22 @@ import scala.collection.mutable.ListBuffer import scala.collection.parallel.mutable.ParHashSet import scala.util.control.Breaks._ -class TrackAndTraceRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) - extends RouterWorker[String](routerId,workerID, initialManagerCount, initialRouterCount) { +class TrackAndTraceGraphBuilder extends GraphBuilder[String] { val EARTH_EQU = 6378137.0 //m val EARTH_POL = 6356752.3142 //m val STEPSIZE = System.getenv().getOrDefault("MAP_GRID_SIZE", "100").trim.toDouble //m - override protected def parseTuple(tuple: String): ParHashSet[GraphUpdate] = { + override def parseTuple(tuple: String) = { val datapoint = lineToDatapoint(tuple.split(",").map(_.trim)) val eventTime = datapoint.time val userID = datapoint.userId val latitude = datapoint.latitude val longitude = datapoint.longitude val locationID = locationIDGenerator(latitude, longitude) - val commands = new ParHashSet[GraphUpdate]() - commands+=(VertexAdd(eventTime, userID, Type("User"))) - commands+=( + sendUpdate(VertexAdd(eventTime, userID, Type("User"))) + + sendUpdate( VertexAddWithProperties( eventTime, locationID, @@ -35,8 +34,7 @@ class TrackAndTraceRouter(override val routerId: Int,override val workerID:Int, ) ) - commands+=(EdgeAdd(eventTime, userID, locationID, Type("User Visted Location"))) - commands + sendUpdate(EdgeAdd(eventTime, userID, locationID, Type("User Visted Location"))) } //converts the line into a case class which has all of the data via the correct name and type diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/TSVRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/TSVGraphBuilder.scala similarity index 52% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/TSVRouter.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/TSVGraphBuilder.scala index 4f898bc19d..8914a4447b 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/TSVRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/TSVGraphBuilder.scala @@ -1,16 +1,14 @@ package com.raphtory.examples.tsvnet -import com.raphtory.core.components.Router.RouterWorker +import com.raphtory.core.components.Router.{GraphBuilder, RouterWorker} import com.raphtory.core.model.communication.{EdgeAdd, GraphUpdate, Type, VertexAdd} import scala.collection.mutable.ListBuffer import scala.collection.parallel.mutable.ParHashSet /** Spout for network datasets of the form SRC_NODE_ID DEST_NODE_ID TIMESTAMP */ -class TSVRouter(override val routerId: Int, override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) - extends RouterWorker[String](routerId,workerID, initialManagerCount, initialRouterCount) { - val commands = new ParHashSet[GraphUpdate]() - override protected def parseTuple(tuple: String): ParHashSet[GraphUpdate] = { +class TSVGraphBuilder extends GraphBuilder[String] { + override def parseTuple(tuple: String) = { val fileLine = tuple.split(" ").map(_.trim) //user wise val sourceNode = fileLine(0).toInt @@ -24,11 +22,10 @@ class TSVRouter(override val routerId: Int, override val workerID:Int, override //val targetNode=fileLine(4).toInt if (targetNode > 0) { val creationDate = fileLine(2).toLong - commands+=(VertexAdd(creationDate, sourceNode, Type("User"))) - commands+=(VertexAdd(creationDate, targetNode, Type("User"))) - commands+=(EdgeAdd(creationDate, sourceNode, targetNode, Type("User to User"))) + sendUpdate(VertexAdd(creationDate, sourceNode, Type("User"))) + sendUpdate(VertexAdd(creationDate, targetNode, Type("User"))) + sendUpdate(EdgeAdd(creationDate, sourceNode, targetNode, Type("User to User"))) } - commands } } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixRouter.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixGraphBuilder.scala similarity index 69% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixRouter.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixGraphBuilder.scala index b79d01ff83..eb7afe49af 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixRouter.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixGraphBuilder.scala @@ -1,18 +1,16 @@ package com.raphtory.examples.wordSemantic.routers -import com.raphtory.core.components.Router.RouterWorker +import com.raphtory.core.components.Router.{GraphBuilder, RouterWorker} import com.raphtory.core.model.communication.{Type, _} import scala.collection.mutable.ListBuffer import scala.collection.parallel.mutable.ParHashSet -class CooccurrenceMatrixRouter(override val routerId: Int, override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) - extends RouterWorker[String](routerId,workerID, initialManagerCount, initialRouterCount) { +class CooccurrenceMatrixGraphBuilder extends GraphBuilder[String] { - override protected def parseTuple(tuple: String): ParHashSet[GraphUpdate] = { + override def parseTuple(tuple: String) = { //println(record) var dp = tuple.split(" ").map(_.trim) - val commands = new ParHashSet[GraphUpdate]() val occurenceTime = dp.head.toLong//DateFormatting(dp.head) //.slice(4, dp.head.length) try { dp = dp.last.split("\t") @@ -23,7 +21,7 @@ class CooccurrenceMatrixRouter(override val routerId: Int, override val workerID val dstClusterId = dp(i).toLong//assignID(dp(i)) val coocWeight = dp(i + 1).toLong - commands+=( + sendUpdate( EdgeAddWithProperties( msgTime = occurenceTime, srcID = srcClusterId, @@ -36,7 +34,6 @@ class CooccurrenceMatrixRouter(override val routerId: Int, override val workerID }catch { case e: Exception => println(e, dp.length, tuple) } - commands } def DateFormatting(date: String): Long = { diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixRouterFiltered.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixGraphBuilderFiltered.scala similarity index 70% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixRouterFiltered.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixGraphBuilderFiltered.scala index e42bd33474..1d53281d12 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixRouterFiltered.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixGraphBuilderFiltered.scala @@ -1,21 +1,19 @@ package com.raphtory.examples.wordSemantic.routers -import com.raphtory.core.components.Router.RouterWorker +import com.raphtory.core.components.Router.{GraphBuilder, RouterWorker} import com.raphtory.core.model.communication._ import scala.collection.mutable.ListBuffer import scala.collection.parallel.mutable.ParHashSet -class CooccurrenceMatrixRouterFiltered(override val routerId: Int, override val workerID:Int, override val initialManagerCount: Int, override val initialRouterCount: Int) - extends RouterWorker[String](routerId,workerID, initialManagerCount, initialRouterCount) { +class CooccurrenceMatrixGraphBuilderFiltered extends GraphBuilder[String] { val THR = System.getenv().getOrDefault("COOC_FREQ_THRESHOLD ", "0.05").trim.toDouble - override protected def parseTuple(tuple: String): ParHashSet[GraphUpdate] = { + override def parseTuple(tuple: String) = { //println(record) var dp =tuple.split(" ").map(_.trim) val occurenceTime = dp.head.toLong//DateFormatting(dp.head) //.slice(4, dp.head.length) val scale = dp(1).toDouble - val commands = new ParHashSet[GraphUpdate]() try { dp = dp.last.split("\t") val srcClusterId = assignID(dp.head) @@ -25,7 +23,7 @@ class CooccurrenceMatrixRouterFiltered(override val routerId: Int, override val val dstClusterId = assignID(dp(i)) val coocWeight = dp(i + 1).toLong - commands+= ( + sendUpdate( EdgeAddWithProperties( msgTime = occurenceTime, srcID = srcClusterId, @@ -39,6 +37,5 @@ class CooccurrenceMatrixRouterFiltered(override val routerId: Int, override val }catch { case e: Exception => println(e, dp.length, tuple.asInstanceOf[String]) } - commands } } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/sources/FileSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/sources/FileSource.scala index cfe35a1bb2..a1962ba887 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/sources/FileSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/sources/FileSource.scala @@ -5,7 +5,6 @@ import java.nio.charset.StandardCharsets import java.util.zip.GZIPInputStream import com.raphtory.core.components.Spout.{DataSource} -import com.raphtory.core.model.communication.StringSpoutGoing import com.typesafe.scalalogging.LazyLogging import scala.annotation.tailrec diff --git a/mainproject/cluster/src/main/scala/com/raphtory/sources/KafkaSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/sources/KafkaSource.scala index f91e27a1db..71753b4708 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/sources/KafkaSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/sources/KafkaSource.scala @@ -4,7 +4,6 @@ import java.util import java.util.Properties import com.raphtory.core.components.Spout.{DataSource} -import com.raphtory.core.model.communication.{SpoutGoing, StringSpoutGoing} import org.apache.kafka.clients.consumer.KafkaConsumer import scala.annotation.tailrec diff --git a/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/BitcoinNodeSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/BitcoinNodeSource.scala index f7a9be96ab..14239366b6 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/BitcoinNodeSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/BitcoinNodeSource.scala @@ -4,7 +4,6 @@ import java.io.File import java.io.PrintWriter import com.raphtory.core.components.Spout.DataSource -import com.raphtory.core.model.communication.SpoutGoing import scalaj.http.Http import scalaj.http.HttpRequest import spray.json._ diff --git a/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumGethSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumGethSource.scala index cbcbecb661..0103f78201 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumGethSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumGethSource.scala @@ -14,7 +14,6 @@ import scala.sys.process._ import scalaj.http.Http import scalaj.http.HttpRequest import spray.json._ -import com.raphtory.core.model.communication.{SpoutGoing, StringSpoutGoing} import spray.json.DefaultJsonProtocol._ import scala.collection.mutable diff --git a/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumPostgresSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumPostgresSource.scala index 2c22b59778..ec9b113e77 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumPostgresSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumPostgresSource.scala @@ -2,7 +2,6 @@ package com.raphtory.sources.blockchain import cats.effect.{Blocker, IO} import com.raphtory.core.components.Spout.{DataSource} -import com.raphtory.core.model.communication.{SpoutGoing, StringSpoutGoing} import doobie.implicits._ import doobie.util.ExecutionContexts import doobie.util.transactor.Transactor diff --git a/mainproject/cluster/src/main/scala/com/raphtory/tests/CitationExample.scala b/mainproject/cluster/src/main/scala/com/raphtory/tests/CitationExample.scala index d9f163484e..5b0909e464 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/tests/CitationExample.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/tests/CitationExample.scala @@ -4,6 +4,7 @@ import akka.actor.{ActorSystem, Props} import ch.qos.logback.classic.Level import com.raphtory.core.analysis.{AnalysisManager, AnalysisRestApi} import com.raphtory.core.components.ClusterManagement.{RaphtoryReplicator, WatchDog, WatermarkManager} +import com.raphtory.core.components.Router.GraphBuilder import kamon.Kamon import org.slf4j.LoggerFactory @@ -26,7 +27,9 @@ object CitationExample extends App{ var routerClassName = "com.raphtory.examples.gab.actors.GabUserGraphRouter" //var routerClassName = "com.raphtory.examples.citationNetwork.CitationRouter" - system.actorOf(Props(RaphtoryReplicator("Router", 1, 1,routerClassName)), s"Routers") + val graphBuilder = Class.forName(routerClassName).getConstructor().newInstance().asInstanceOf[GraphBuilder[Any]] + val routerReplicator = RaphtoryReplicator.apply("Router", 1, 1,graphBuilder) + system.actorOf(Props(routerReplicator), s"Routers") system.actorOf(Props(RaphtoryReplicator("Partition Manager", 1,1)), s"PartitionManager") diff --git a/mainproject/cluster/src/main/scala/com/raphtory/tests/NaomiTest.scala b/mainproject/cluster/src/main/scala/com/raphtory/tests/NaomiTest.scala index 4926daac98..622df806f6 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/tests/NaomiTest.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/tests/NaomiTest.scala @@ -4,6 +4,7 @@ import akka.actor.{ActorSystem, Props} import ch.qos.logback.classic.Level import com.raphtory.core.analysis.{AnalysisManager, AnalysisRestApi} import com.raphtory.core.components.ClusterManagement.{RaphtoryReplicator, WatchDog, WatermarkManager} +import com.raphtory.core.components.Router.GraphBuilder import kamon.Kamon import org.slf4j.LoggerFactory @@ -31,7 +32,9 @@ object NaomiTest extends App{ system.actorOf(Props(new WatermarkManager(managerCount = 1)),"WatermarkManager") system.actorOf(Props(new WatchDog(partitionNumber, minimumRouters)), "WatchDog") - system.actorOf(Props(RaphtoryReplicator("Router", 1, 1,routerClassName)), s"Routers") + val graphBuilder = Class.forName(routerClassName).getConstructor().newInstance().asInstanceOf[GraphBuilder[Any]] + val routerReplicator = RaphtoryReplicator.apply("Router", 1, 1,graphBuilder) + system.actorOf(Props(routerReplicator), s"Routers") system.actorOf(Props(RaphtoryReplicator("Partition Manager", 1,1)), s"PartitionManager") system.actorOf(Props(Class.forName(SpoutName)), "Spout") val analysisManager = system.actorOf(Props[AnalysisManager], s"AnalysisManager") diff --git a/mainproject/cluster/src/main/scala/com/raphtory/tests/SingleNodeTest.scala b/mainproject/cluster/src/main/scala/com/raphtory/tests/SingleNodeTest.scala index deefd8149a..be3a6895ee 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/tests/SingleNodeTest.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/tests/SingleNodeTest.scala @@ -6,12 +6,12 @@ import ch.qos.logback.classic.Level import com.raphtory.core.analysis.API.Analyser import com.raphtory.core.analysis.{AnalysisManager, AnalysisRestApi} import com.raphtory.core.components.ClusterManagement.{RaphtoryReplicator, WatchDog, WatermarkManager} +import com.raphtory.core.components.Router.GraphBuilder import com.raphtory.core.model.communication.{LiveAnalysisRequest, RangeAnalysisRequest, ViewAnalysisRequest} import com.raphtory.examples.blockchain.analysers.TaintTrackExchangeStop import kamon.Kamon import org.slf4j.LoggerFactory - import scala.language.postfixOps object SingleNodeTest extends App { @@ -73,7 +73,9 @@ object SingleNodeTest extends App { system.actorOf(Props(new WatermarkManager(managerCount = 1)),"WatermarkManager") system.actorOf(Props(new WatchDog(partitionNumber, minimumRouters)), "WatchDog") - system.actorOf(Props(RaphtoryReplicator("Router", 1,1, routerClassName)), s"Routers") + val graphBuilder = Class.forName(routerClassName).getConstructor().newInstance().asInstanceOf[GraphBuilder[Any]] + val routerReplicator = RaphtoryReplicator.apply("Router", 1, 1,graphBuilder) + system.actorOf(Props(routerReplicator), s"Routers") system.actorOf(Props(RaphtoryReplicator("Partition Manager", 1,1)), s"PartitionManager") system.actorOf(Props(Class.forName(SpoutName)), "Spout") val analysisManager = system.actorOf(Props[AnalysisManager], s"AnalysisManager") diff --git a/mainproject/cluster/src/main/scala/com/raphtory/tests/TemporalTriangleCountExample.scala b/mainproject/cluster/src/main/scala/com/raphtory/tests/TemporalTriangleCountExample.scala index c120644ad2..9621ba2239 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/tests/TemporalTriangleCountExample.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/tests/TemporalTriangleCountExample.scala @@ -4,7 +4,9 @@ import akka.actor.{ActorSystem, Props} import ch.qos.logback.classic.Level import com.raphtory.core.analysis.{AnalysisManager, AnalysisRestApi} import com.raphtory.core.components.ClusterManagement.{RaphtoryReplicator, WatchDog, WatermarkManager} +import com.raphtory.core.components.Router.GraphBuilder import com.raphtory.core.components.Spout.Spout +import com.raphtory.examples.lotr.LOTRGraphBuilder import com.raphtory.sources.FileSource import kamon.Kamon import org.slf4j.LoggerFactory @@ -34,8 +36,10 @@ object TemporalTriangleCountExample extends App{ system.actorOf(Props(new Spout(new FileSource())), "Spout") //var routerClassName = "com.raphtory.examples.test.actors.TriangleTestRouter" - var routerClassName = "com.raphtory.examples.lotr.LOTRRouter" - system.actorOf(Props(RaphtoryReplicator("Router", 1, 1,routerClassName)), s"Routers") + var graphBuilderClassName = "com.raphtory.examples.lotr.LOTRGraphBuilder" + + + system.actorOf(Props(RaphtoryReplicator.apply("Router", 1, 1,new LOTRGraphBuilder())), s"Routers") system.actorOf(Props(RaphtoryReplicator("Partition Manager", 1,1)), s"PartitionManager") From 2612ade5182fd9aa2068c15b87edcc4dab131706 Mon Sep 17 00:00:00 2001 From: Miratepuffin Date: Thu, 26 Nov 2020 23:21:10 +0000 Subject: [PATCH 35/58] Deleted old tests as swapping to new API --- .../blockchain/EthereumGethSource.scala | 6 +- .../com/raphtory/tests/CitationExample.scala | 38 -------- .../com/raphtory/tests/EtherAPITest.scala | 33 ------- .../scala/com/raphtory/tests/NaomiTest.scala | 44 --------- .../com/raphtory/tests/SingleNodeTest.scala | 91 ------------------- .../scala/com/raphtory/tests/compileTest.sc | 48 ---------- .../main/scala/com/raphtory/tests/curltest.sc | 51 ----------- 7 files changed, 3 insertions(+), 308 deletions(-) delete mode 100644 mainproject/cluster/src/main/scala/com/raphtory/tests/CitationExample.scala delete mode 100644 mainproject/cluster/src/main/scala/com/raphtory/tests/EtherAPITest.scala delete mode 100644 mainproject/cluster/src/main/scala/com/raphtory/tests/NaomiTest.scala delete mode 100644 mainproject/cluster/src/main/scala/com/raphtory/tests/SingleNodeTest.scala delete mode 100644 mainproject/cluster/src/main/scala/com/raphtory/tests/compileTest.sc delete mode 100644 mainproject/cluster/src/main/scala/com/raphtory/tests/curltest.sc diff --git a/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumGethSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumGethSource.scala index 0103f78201..b9039e9c00 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumGethSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumGethSource.scala @@ -5,9 +5,9 @@ import java.util.NoSuchElementException import com.raphtory.core.components.Spout.{DataSource, Spout} import com.raphtory.core.utils.Utils -import com.raphtory.tests.EtherAPITest.baseRequest -import com.raphtory.tests.EtherAPITest.currentBlock -import com.raphtory.tests.EtherAPITest.request +import .baseRequest +import .currentBlock +import .request import scala.language.postfixOps import scala.sys.process._ diff --git a/mainproject/cluster/src/main/scala/com/raphtory/tests/CitationExample.scala b/mainproject/cluster/src/main/scala/com/raphtory/tests/CitationExample.scala deleted file mode 100644 index 5b0909e464..0000000000 --- a/mainproject/cluster/src/main/scala/com/raphtory/tests/CitationExample.scala +++ /dev/null @@ -1,38 +0,0 @@ -package com.raphtory.tests - -import akka.actor.{ActorSystem, Props} -import ch.qos.logback.classic.Level -import com.raphtory.core.analysis.{AnalysisManager, AnalysisRestApi} -import com.raphtory.core.components.ClusterManagement.{RaphtoryReplicator, WatchDog, WatermarkManager} -import com.raphtory.core.components.Router.GraphBuilder -import kamon.Kamon -import org.slf4j.LoggerFactory - -object CitationExample extends App{ - Kamon.init() //start tool logging - - val root = LoggerFactory.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME).asInstanceOf[ch.qos.logback.classic.Logger] - root.setLevel(Level.ERROR) - val system = ActorSystem("Citation-system") - - val partitionNumber = 1 - val minimumRouters = 1 - system.actorOf(Props(new WatermarkManager(managerCount = 1)),"WatermarkManager") - system.actorOf(Props(new WatchDog(partitionNumber, minimumRouters)), "WatchDog") - - - //var SpoutName = "com.raphtory.spouts.FileSpout" - var SpoutName = "com.raphtory.examples.gab.actors.GabExampleSpout" - system.actorOf(Props(Class.forName(SpoutName)), "Spout") - - var routerClassName = "com.raphtory.examples.gab.actors.GabUserGraphRouter" - //var routerClassName = "com.raphtory.examples.citationNetwork.CitationRouter" - val graphBuilder = Class.forName(routerClassName).getConstructor().newInstance().asInstanceOf[GraphBuilder[Any]] - val routerReplicator = RaphtoryReplicator.apply("Router", 1, 1,graphBuilder) - system.actorOf(Props(routerReplicator), s"Routers") - - system.actorOf(Props(RaphtoryReplicator("Partition Manager", 1,1)), s"PartitionManager") - - system.actorOf(Props[AnalysisManager], s"AnalysisManager") - AnalysisRestApi(system) -} diff --git a/mainproject/cluster/src/main/scala/com/raphtory/tests/EtherAPITest.scala b/mainproject/cluster/src/main/scala/com/raphtory/tests/EtherAPITest.scala deleted file mode 100644 index c3e8356805..0000000000 --- a/mainproject/cluster/src/main/scala/com/raphtory/tests/EtherAPITest.scala +++ /dev/null @@ -1,33 +0,0 @@ -package com.raphtory.tests - -import scalaj.http.Http -import scalaj.http.HttpRequest -import spray.json._ -object EtherAPITest extends App { - val nodeIP = System.getenv().getOrDefault("ETHEREUM_IP_ADDRESS", "http://127.0.0.1").trim - val nodePort = System.getenv().getOrDefault("ETHEREUM_PORT", "8545").trim - val baseRequest = Http(nodeIP + ":" + nodePort).header("content-type", "application/json") - var currentBlock = System.getenv().getOrDefault("ETHEREUM_START_BLOCK_INDEX", "9000000").trim.toInt - var highestBlock = System.getenv().getOrDefault("ETHEREUM_MAXIMUM_BLOCK_INDEX", "999999999").trim.toInt - def request(command: String, params: String = ""): HttpRequest = - baseRequest.postData(s"""{"jsonrpc": "2.0", "id":"100", "method": "$command", "params": [$params]}""") - var total = 0L - for (i <- 9000000 to 9700000) { - val transactionCountHex = request("eth_getBlockTransactionCountByNumber", "\"0x" + currentBlock.toHexString + "\"") - .execute() - .body - .toString - .parseJson - .asJsObject - val transactionCount = Integer.parseInt(transactionCountHex.fields("result").toString().drop(3).dropRight(1), 16) - total += transactionCount - if (i % 1000 == 0) - println(total) - } - println(total) -// for(i <- 0 until transactionCount) { -// println(s"At index $i") -// println(s""""0x${currentBlock.toHexString}","0x${i.toHexString}"""") -// println(request("eth_getTransactionByBlockNumberAndIndex",s""""0x${currentBlock.toHexString}","0x${i.toHexString}"""").execute().body.toString.parseJson.asJsObject) -// } -} diff --git a/mainproject/cluster/src/main/scala/com/raphtory/tests/NaomiTest.scala b/mainproject/cluster/src/main/scala/com/raphtory/tests/NaomiTest.scala deleted file mode 100644 index 622df806f6..0000000000 --- a/mainproject/cluster/src/main/scala/com/raphtory/tests/NaomiTest.scala +++ /dev/null @@ -1,44 +0,0 @@ -package com.raphtory.tests - -import akka.actor.{ActorSystem, Props} -import ch.qos.logback.classic.Level -import com.raphtory.core.analysis.{AnalysisManager, AnalysisRestApi} -import com.raphtory.core.components.ClusterManagement.{RaphtoryReplicator, WatchDog, WatermarkManager} -import com.raphtory.core.components.Router.GraphBuilder -import kamon.Kamon -import org.slf4j.LoggerFactory - -object NaomiTest extends App{ - - Kamon.init() - - val root = LoggerFactory.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME).asInstanceOf[ch.qos.logback.classic.Logger] - root.setLevel(Level.ERROR) - - val partitionNumber = 1 - val minimumRouters = 1 - - var Analyser = "com.raphtory.core.analysis.Algorithms.ConnectedComponents" - Analyser = "com.raphtory.core.analysis.Algorithms.PageRank" - - val start = 1470837600000L - val end = 31525368897000L - - val jump = 3600000 - var SpoutName = "com.raphtory.examples.lotr.LOTRSpout" - var routerClassName = "com.raphtory.examples.lotr.LOTRRouter" - - val system = ActorSystem("Single-Node-test") - - system.actorOf(Props(new WatermarkManager(managerCount = 1)),"WatermarkManager") - system.actorOf(Props(new WatchDog(partitionNumber, minimumRouters)), "WatchDog") - val graphBuilder = Class.forName(routerClassName).getConstructor().newInstance().asInstanceOf[GraphBuilder[Any]] - val routerReplicator = RaphtoryReplicator.apply("Router", 1, 1,graphBuilder) - system.actorOf(Props(routerReplicator), s"Routers") - system.actorOf(Props(RaphtoryReplicator("Partition Manager", 1,1)), s"PartitionManager") - system.actorOf(Props(Class.forName(SpoutName)), "Spout") - val analysisManager = system.actorOf(Props[AnalysisManager], s"AnalysisManager") - AnalysisRestApi(system) - - -} \ No newline at end of file diff --git a/mainproject/cluster/src/main/scala/com/raphtory/tests/SingleNodeTest.scala b/mainproject/cluster/src/main/scala/com/raphtory/tests/SingleNodeTest.scala deleted file mode 100644 index be3a6895ee..0000000000 --- a/mainproject/cluster/src/main/scala/com/raphtory/tests/SingleNodeTest.scala +++ /dev/null @@ -1,91 +0,0 @@ -package com.raphtory.tests - -import akka.actor.ActorSystem -import akka.actor.Props -import ch.qos.logback.classic.Level -import com.raphtory.core.analysis.API.Analyser -import com.raphtory.core.analysis.{AnalysisManager, AnalysisRestApi} -import com.raphtory.core.components.ClusterManagement.{RaphtoryReplicator, WatchDog, WatermarkManager} -import com.raphtory.core.components.Router.GraphBuilder -import com.raphtory.core.model.communication.{LiveAnalysisRequest, RangeAnalysisRequest, ViewAnalysisRequest} -import com.raphtory.examples.blockchain.analysers.TaintTrackExchangeStop -import kamon.Kamon -import org.slf4j.LoggerFactory - -import scala.language.postfixOps - -object SingleNodeTest extends App { - Kamon.init() //start tool logging - - val root = LoggerFactory.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME).asInstanceOf[ch.qos.logback.classic.Logger] - root.setLevel(Level.ERROR) - - val partitionNumber = 1 - val minimumRouters = 1 - -// var Analyser = "com.raphtory.core.analysis.Algorithms.ConnectedComponents" -// Analyser = "com.raphtory.core.analysis.Algorithms.DegreeBasic" - var Analyser = "com.raphtory.examples.blockchain.analysers.TaintTrackExchangeStop" - - //var UpdaterName = "com.raphtory.examples.ldbc.spouts.LDBCSpout" - //var routerClassName = "com.raphtory.examples.ldbc.routers.LDBCRouter" - //val start = 1262394061000L - //val end = 1357002061000L - //val jump = 86400000 - //curl -X POST -H "Content-Type: application/json" --data '{"jsonrpc":"2.0", "jobID":"ldbcdegreeTest","analyserName":"com.raphtory.core.analysis.Algorithms.DegreeBasic","start":1262394061000,"end":1357002061000,"jump":86400000}' 127.0.0.1:8081/RangeAnalysisRequest - - //ether test -// val start = 4000000L -// val end = 6000000L -// val jump = 10 - //var SpoutName = "com.raphtory.spouts.KafkaSpout" - //var routerClassName = "com.raphtory.examples.blockchain.routers.EthereumKafkaRouter" -// Analyser = "com.raphtory.examples.blockchain.analysers.EthereumTaintTracking" - - //Gab test - val start = 1470837600000L - val end = 31525368897000L - - val jump = 3600000 - - // var SpoutName = "com.raphtory.examples.gab.actors.GabExampleSpout" - //var SpoutName = "com.raphtory.examples.random.actors.RandomSpout" - // var routerClassName = "com.raphtory.examples.gab.actors.GabUserGraphRouter" - //var routerClassName = "com.raphtory.examples.random.actors.RandomRouter" - - - //track and trace test - //var UpdaterName = "com.raphtory.examples.trackAndTrace.spouts.TrackAndTraceSpout" - //var routerClassName = "com.raphtory.examples.trackAndTrace.routers.TrackAndTraceRouter" - // Analyser = "com.raphtory.examples.blockchain.analysers.EthereumTaintTracking" - -//chainalysisAB - - var SpoutName = "com.raphtory.examples.blockchain.spouts.ChainalysisABSpout" - // var SpoutName = "com.raphtory.spouts.KafkaSpout" - var routerClassName = "com.raphtory.examples.blockchain.routers.ChABClus2ClusRouter" -// var routerClassName = "com.raphtory.examples.blockchain.routers.EthereumKafkaRouter" - - -// var SpoutName = "com.raphtory.examples.twitterRumour.networkxSpout" -// var routerClassName = "com.raphtory.examples.twitterRumour.networkxRouter" - val system = ActorSystem("Single-Node-test") - - system.actorOf(Props(new WatermarkManager(managerCount = 1)),"WatermarkManager") - system.actorOf(Props(new WatchDog(partitionNumber, minimumRouters)), "WatchDog") - val graphBuilder = Class.forName(routerClassName).getConstructor().newInstance().asInstanceOf[GraphBuilder[Any]] - val routerReplicator = RaphtoryReplicator.apply("Router", 1, 1,graphBuilder) - system.actorOf(Props(routerReplicator), s"Routers") - system.actorOf(Props(RaphtoryReplicator("Partition Manager", 1,1)), s"PartitionManager") - system.actorOf(Props(Class.forName(SpoutName)), "Spout") - val analysisManager = system.actorOf(Props[AnalysisManager], s"AnalysisManager") - //AnalysisRestApi(system) - - //analysisManager ! ViewAnalysisRequest("jobID","com.raphtory.core.analysis.Algorithms.TriangleCounting", 1L) - - //curl -X POST -H "Content-Type: application/json" --data '{"jsonrpc":"2.0", "jobID":"connectedComponentsTest","analyserName":"com.raphtory.core.analysis.Algorithms.ConnectedComponents"}' 127.0.0.1:8080/LiveAnalysisRequest - //curl -X POST -H "Content-Type: application/json" --data '{"jsonrpc":"2.0", "jobID":"connectedComponentsViewTest","analyserName":"com.raphtory.core.analysis.Algorithms.ConnectedComponents","timestamp":1476113856000}' 127.0.0.1:8080/ViewAnalysisRequest - //curl -X POST -H "Content-Type: application/json" --data '{"jsonrpc":"2.0", "jobID":"connectedComponentsRangeWindowTest","analyserName":"com.raphtory.core.analysis.Algorithms.ConnectedComponents","start":1475113856000,"end":1475113856000,"jump":3600000,"windowType":"batched","windowSet":[3600000,86000000]}' 127.0.0.1:8080/RangeAnalysisRequest -//curl -X POST -H "Content-Type: application/json" --data '{"jsonrpc":"2.0", "jobID":"connectedComponentsRangeWindowTest","analyserName":"com.raphtory.examples.blockchain.analysers.EthereumTaintTracking","start":4000000,"end":6000000,"jump":10,"args":["0xa09871aeadf4994ca12f5c0b6056bbd1d343c029","4000000"]}' 127.0.0.1:8081/RangeAnalysisRequest -} - diff --git a/mainproject/cluster/src/main/scala/com/raphtory/tests/compileTest.sc b/mainproject/cluster/src/main/scala/com/raphtory/tests/compileTest.sc deleted file mode 100644 index c61b52f39a..0000000000 --- a/mainproject/cluster/src/main/scala/com/raphtory/tests/compileTest.sc +++ /dev/null @@ -1,48 +0,0 @@ - -import com.raphtory.core.analysis.API.{Analyser, BlankAnalyser} -import scala.language.postfixOps -import scala.reflect.runtime.currentMirror -import scala.tools.reflect.ToolBox - -//val newText = "import scala.collection.mutable.ArrayBuffer\r\nimport com.raphtory.core.analysis.API.Analyser\r\nclass Blank(args:Array[String]) extends Analyser(args) {\r\n override def analyse(): Unit = {}\r\n override def setup(): Unit = {}\r\n override def returnResults(): Any = {}\r\n override def defineMaxSteps(): Int = 1\r\n override def processResults(results: ArrayBuffer[Any], timeStamp: Long, viewCompleteTime: Long): Unit = {println(\"howdy!\")}\r\n}" - -object Files { - case class Person(a: String, b: Int) - - val analyser = """ - | import scala.collection.mutable.ArrayBuffer - | import com.raphtory.core.analysis.API.Analyser - | class Blanklyser(args:Array[String]) extends Analyser(args) { - | val myUniqString = 1 - | override def analyse(): Unit = {println(args(0))} - | override def setup(): Unit = {} - | override def returnResults(): Any = {} - | override def defineMaxSteps(): Int = 1 - | override def processResults(results: ArrayBuffer[Any], timeStamp: Long, viewCompleteTime: Long): Unit = {println("howdy!")} - | } - | scala.reflect.classTag[Blanklyser].runtimeClass - |""".stripMargin - - val test = - """ - | class Person(a: String, b: Int) - |""".stripMargin -} - -case class LoadExternalAnalyser(rawFile: String,args:Array[String]) { - val toolbox = currentMirror.mkToolBox() - val tree = toolbox.parse(rawFile) - val compiledCode = toolbox.compile(tree).apply().asInstanceOf[Class[Analyser]] - def newAnalyser = compiledCode.getConstructor(classOf[Array[String]]).newInstance(args).asInstanceOf[Analyser] - -} - - try{ - val analyser = LoadExternalAnalyser(Files.analyser,Array("null")).newAnalyser.analyse() - println(analyser) - } - - - - - diff --git a/mainproject/cluster/src/main/scala/com/raphtory/tests/curltest.sc b/mainproject/cluster/src/main/scala/com/raphtory/tests/curltest.sc deleted file mode 100644 index c46ee59cef..0000000000 --- a/mainproject/cluster/src/main/scala/com/raphtory/tests/curltest.sc +++ /dev/null @@ -1,51 +0,0 @@ -val nodeIP = System.getenv().getOrDefault("SPOUT_ETHEREUM_IP_ADDRESS", "54.184.149.221").trim -val nodePort = System.getenv().getOrDefault("SPOUT_ETHEREUM_PORT", "8545").trim -import com.raphtory.core.components.Spout.Spout -import com.raphtory.core.utils.Utils -import com.raphtory.tests.EtherAPITest.baseRequest -import com.raphtory.tests.EtherAPITest.currentBlock -import com.raphtory.tests.EtherAPITest.request - -import scala.concurrent.duration.Duration -import scala.concurrent.duration.MILLISECONDS -import scala.concurrent.duration.NANOSECONDS -import scala.concurrent.duration.SECONDS -import scala.language.postfixOps -import scala.sys.process._ -import scalaj.http.Http -import scalaj.http.HttpRequest -import spray.json._ -import java.net.InetAddress -import scala.collection.mutable.ArrayBuffer -val currentBlock = 9000000 - -def hostname2Ip(hostname: String): String = InetAddress.getByName(hostname).getHostAddress() - -def requestBuilder() = - if (nodeIP.matches(Utils.IPRegex)) - Http("http://" + nodeIP + ":" + nodePort).header("content-type", "application/json") - else - Http("http://" + hostname2Ip(nodeIP) + ":" + nodePort).header("content-type", "application/json") - -val baseRequest = requestBuilder() - -def requestBatch(data: String): HttpRequest = baseRequest.postData(data) - -def executeBatchRequest(data: String) = requestBatch(data).execute().body.toString.parseJson - -def request(command: String, params: String = ""): HttpRequest = - baseRequest.postData(s"""{"jsonrpc": "2.0", "id":"100", "method": "$command", "params": [$params]}""") -def executeRequest(command: String, params: String = "") = - request(command, params).execute().body.toString.parseJson.asJsObject - -def batchRequestBuilder(command:String,params:String):String = s"""{"jsonrpc": "2.0", "id":"100", "method": "$command", "params": [$params]}""" - -def pullNextBlock(): Unit = { - val transactionCountHex = executeRequest("eth_getBlockTransactionCountByNumber", "\"0x" + currentBlock.toHexString + "\"") - val transactionCount = Integer.parseInt(transactionCountHex.fields("result").toString().drop(3).dropRight(1), 16) - var transactions = "[" - for (i <- 0 until transactionCount) - transactions = transactions + batchRequestBuilder("eth_getTransactionByBlockNumberAndIndex",s""""0x${currentBlock.toHexString}","0x${i.toHexString}"""")+"," - println(executeBatchRequest(transactions.dropRight(1)+"]")) -} -pullNextBlock() \ No newline at end of file From daad2dfc5b3734e011fc93ea92d4d2729fad3a52 Mon Sep 17 00:00:00 2001 From: Miratepuffin Date: Thu, 26 Nov 2020 23:37:04 +0000 Subject: [PATCH 36/58] Created Raphtory single node main class which will be the entry point for library --- ...angleCountExample.scala => Raphtory.scala} | 33 ++++++------------- .../examples/lotr/LOTRDataSource.scala | 5 +-- .../examples/lotr/LOTRGraphBuilder.scala | 6 ++-- ...ilder.scala => TriangleGraphBuilder.scala} | 2 +- .../blockchain/EthereumGethSource.scala | 4 --- .../com/raphtory/tests/LOTRDeployment.scala | 8 +++++ 6 files changed, 22 insertions(+), 36 deletions(-) rename mainproject/cluster/src/main/scala/com/raphtory/{tests/TemporalTriangleCountExample.scala => Raphtory.scala} (50%) rename mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/{TriangleTestGraphBuilder.scala => TriangleGraphBuilder.scala} (90%) create mode 100644 mainproject/cluster/src/main/scala/com/raphtory/tests/LOTRDeployment.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/tests/TemporalTriangleCountExample.scala b/mainproject/cluster/src/main/scala/com/raphtory/Raphtory.scala similarity index 50% rename from mainproject/cluster/src/main/scala/com/raphtory/tests/TemporalTriangleCountExample.scala rename to mainproject/cluster/src/main/scala/com/raphtory/Raphtory.scala index 9621ba2239..8d132f9776 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/tests/TemporalTriangleCountExample.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/Raphtory.scala @@ -1,24 +1,21 @@ -import java.lang.management.ManagementFactory +package com.raphtory import akka.actor.{ActorSystem, Props} import ch.qos.logback.classic.Level import com.raphtory.core.analysis.{AnalysisManager, AnalysisRestApi} import com.raphtory.core.components.ClusterManagement.{RaphtoryReplicator, WatchDog, WatermarkManager} import com.raphtory.core.components.Router.GraphBuilder -import com.raphtory.core.components.Spout.Spout -import com.raphtory.examples.lotr.LOTRGraphBuilder -import com.raphtory.sources.FileSource +import com.raphtory.core.components.Spout.{DataSource, Spout} import kamon.Kamon import org.slf4j.LoggerFactory -object TemporalTriangleCountExample extends App{ - Kamon.init() //start tool logging +object Raphtory { + def apply[T](dataSource: DataSource[T],graphBuilder: GraphBuilder[T]) : Raphtory[T] = + new Raphtory(dataSource, graphBuilder) +} - val runtimeMxBean = ManagementFactory.getRuntimeMXBean - val arguments = runtimeMxBean.getInputArguments - import java.nio.file.Paths - println(Paths.get(".").toAbsolutePath) - println(s"Current java options: $arguments") +class Raphtory[T](dataSource: DataSource[T],graphBuilder: GraphBuilder[T]) { + Kamon.init() //start tool logging val root = LoggerFactory.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME).asInstanceOf[ch.qos.logback.classic.Logger] root.setLevel(Level.ERROR) @@ -29,18 +26,8 @@ object TemporalTriangleCountExample extends App{ system.actorOf(Props(new WatermarkManager(managerCount = 1)),"WatermarkManager") system.actorOf(Props(new WatchDog(partitionNumber, minimumRouters)), "WatchDog") - - //var SpoutName ="com.raphtory.examples.test.actors.TriangleTestSpout" - //var SpoutName = "com.raphtory.examples.gab.actors.GabExampleSpout" - //var SpoutName = "com.raphtory.spouts.FileSpout" - system.actorOf(Props(new Spout(new FileSource())), "Spout") - - //var routerClassName = "com.raphtory.examples.test.actors.TriangleTestRouter" - var graphBuilderClassName = "com.raphtory.examples.lotr.LOTRGraphBuilder" - - - system.actorOf(Props(RaphtoryReplicator.apply("Router", 1, 1,new LOTRGraphBuilder())), s"Routers") - + system.actorOf(Props(new Spout(dataSource)), "Spout") + system.actorOf(Props(RaphtoryReplicator.apply("Router", 1, 1,graphBuilder)), s"Routers") system.actorOf(Props(RaphtoryReplicator("Partition Manager", 1,1)), s"PartitionManager") system.actorOf(Props[AnalysisManager], s"AnalysisManager") diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRDataSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRDataSource.scala index 5bad0d114a..7901fef758 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRDataSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRDataSource.scala @@ -6,9 +6,7 @@ import scala.collection.mutable class LOTRDataSource extends DataSource[String] { - val directory = System.getenv().getOrDefault("LOTR_DIRECTORY", "com/raphtory/example/lotr").trim - val file_name = System.getenv().getOrDefault("LOTR_FILE_NAME", "lotr.csv").trim - val fileQueue = mutable.Queue[String]() ++= scala.io.Source.fromFile(directory + "/" + file_name).getLines + val fileQueue = mutable.Queue[String]() ++= scala.io.Source.fromFile("cluster/src/main/scala/com/raphtory/examples/lotr/lotr.csv").getLines override def setupDataSource(): Unit = {}//no setup @@ -20,6 +18,5 @@ class LOTRDataSource extends DataSource[String] { else Some(fileQueue.dequeue()) } - override def closeDataSource(): Unit = {}//no file closure already done } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRGraphBuilder.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRGraphBuilder.scala index 6c55f13e05..d76b0d98ef 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRGraphBuilder.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRGraphBuilder.scala @@ -1,10 +1,8 @@ package com.raphtory.examples.lotr -import com.raphtory.core.components.Router.{GraphBuilder, RouterWorker} -import com.raphtory.core.model.communication.{EdgeAdd, Type, VertexAdd, _} +import com.raphtory.core.components.Router.GraphBuilder +import com.raphtory.core.model.communication._ -import scala.collection.mutable.ListBuffer -import scala.collection.parallel.mutable.ParHashSet import scala.util.Random class LOTRGraphBuilder extends GraphBuilder[String]{ diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleTestGraphBuilder.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleGraphBuilder.scala similarity index 90% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleTestGraphBuilder.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleGraphBuilder.scala index 1b248f73a8..e55c8dbe39 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleTestGraphBuilder.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleGraphBuilder.scala @@ -7,7 +7,7 @@ import com.raphtory.core.model.communication.{EdgeAdd, GraphUpdate, VertexAddWit import scala.collection.mutable.ListBuffer import scala.collection.parallel.mutable.ParHashSet -class TriangleTestGraphBuilder extends GraphBuilder[String] { +class TriangleGraphBuilder extends GraphBuilder[String] { override def parseTuple(tuple: String)= { val command = tuple.split(",") val msgTime = command(0).toLong diff --git a/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumGethSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumGethSource.scala index b9039e9c00..4638bb45e3 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumGethSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumGethSource.scala @@ -5,10 +5,6 @@ import java.util.NoSuchElementException import com.raphtory.core.components.Spout.{DataSource, Spout} import com.raphtory.core.utils.Utils -import .baseRequest -import .currentBlock -import .request - import scala.language.postfixOps import scala.sys.process._ import scalaj.http.Http diff --git a/mainproject/cluster/src/main/scala/com/raphtory/tests/LOTRDeployment.scala b/mainproject/cluster/src/main/scala/com/raphtory/tests/LOTRDeployment.scala new file mode 100644 index 0000000000..f64aa49b76 --- /dev/null +++ b/mainproject/cluster/src/main/scala/com/raphtory/tests/LOTRDeployment.scala @@ -0,0 +1,8 @@ +import com.raphtory.Raphtory +import com.raphtory.examples.lotr.{LOTRDataSource, LOTRGraphBuilder} + +object LOTRDeployment extends App{ + val source = new LOTRDataSource() + val builder = new LOTRGraphBuilder() + Raphtory[String](source,builder) +} From 38a34a2d90c7563a1a0743a6c71115dd6f168bcc Mon Sep 17 00:00:00 2001 From: Miratepuffin Date: Thu, 26 Nov 2020 23:45:17 +0000 Subject: [PATCH 37/58] Removed tests completly as can now be in example folders --- .../com/raphtory/examples/lotr/LOTRDataSource.scala | 4 +++- .../{tests => examples/lotr}/LOTRDeployment.scala | 3 ++- .../com/raphtory/examples/lotr/LOTRGraphBuilder.scala | 9 +++++---- 3 files changed, 10 insertions(+), 6 deletions(-) rename mainproject/cluster/src/main/scala/com/raphtory/{tests => examples/lotr}/LOTRDeployment.scala (72%) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRDataSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRDataSource.scala index 7901fef758..bba81cc245 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRDataSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRDataSource.scala @@ -6,7 +6,9 @@ import scala.collection.mutable class LOTRDataSource extends DataSource[String] { - val fileQueue = mutable.Queue[String]() ++= scala.io.Source.fromFile("cluster/src/main/scala/com/raphtory/examples/lotr/lotr.csv").getLines + val fileQueue = mutable.Queue[String]() ++= + scala.io.Source.fromFile("cluster/src/main/scala/com/raphtory/examples/lotr/lotr.csv") + .getLines override def setupDataSource(): Unit = {}//no setup diff --git a/mainproject/cluster/src/main/scala/com/raphtory/tests/LOTRDeployment.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRDeployment.scala similarity index 72% rename from mainproject/cluster/src/main/scala/com/raphtory/tests/LOTRDeployment.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRDeployment.scala index f64aa49b76..fc8bc50e94 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/tests/LOTRDeployment.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRDeployment.scala @@ -1,5 +1,6 @@ +package com.raphtory.examples.lotr + import com.raphtory.Raphtory -import com.raphtory.examples.lotr.{LOTRDataSource, LOTRGraphBuilder} object LOTRDeployment extends App{ val source = new LOTRDataSource() diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRGraphBuilder.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRGraphBuilder.scala index d76b0d98ef..ca57d39535 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRGraphBuilder.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRGraphBuilder.scala @@ -3,8 +3,6 @@ package com.raphtory.examples.lotr import com.raphtory.core.components.Router.GraphBuilder import com.raphtory.core.model.communication._ -import scala.util.Random - class LOTRGraphBuilder extends GraphBuilder[String]{ override def parseTuple(tuple: String) = { @@ -18,8 +16,11 @@ class LOTRGraphBuilder extends GraphBuilder[String]{ val timeStamp = fileLine(2).toLong - sendUpdate(VertexAddWithProperties(timeStamp, srcID, Properties(LongProperty("test",Random.nextLong()),ImmutableProperty("name",sourceNode)),Type("Character"))) - sendUpdate(VertexAddWithProperties(timeStamp, tarID, Properties(LongProperty("test2",Random.nextLong()),ImmutableProperty("name",targetNode)),Type("Character"))) + sendUpdate(VertexAddWithProperties(timeStamp, srcID, + Properties(ImmutableProperty("name",sourceNode)),Type("Character"))) + sendUpdate(VertexAddWithProperties(timeStamp, tarID, + Properties(ImmutableProperty("name",targetNode)),Type("Character"))) + sendUpdate(EdgeAdd(timeStamp,srcID,tarID, Type("Character Co-occurence"))) } } From a8a33a739a759fdfa7fcbc0c4247c622459c0379 Mon Sep 17 00:00:00 2001 From: Miratepuffin Date: Sat, 28 Nov 2020 23:27:57 +0000 Subject: [PATCH 38/58] Optimised imports --- mainproject/build.sbt | 8 ++++- .../src/main/scala/com/raphtory/Server.scala | 6 ---- .../raphtory/core/analysis/API/Analyser.scala | 1 - .../analysis/API/GraphLenses/GraphLens.scala | 7 ++-- .../analysis/API/GraphLenses/ViewLens.scala | 1 - .../API/entityVisitors/EntityVisitor.scala | 1 - .../API/entityVisitors/VertexVisitor.scala | 6 ++-- .../analysis/Algorithms/MultiLayerLPA.scala | 5 --- .../Algorithms/TemporalTriangleCount.scala | 2 -- .../analysis/Algorithms/WattsCascade.scala | 2 -- .../analysis/Algorithms/lotrExample.scala | 1 + .../core/analysis/AnalysisManager.scala | 15 +++----- .../core/analysis/AnalysisRestApi.scala | 18 ++++------ .../core/analysis/Tasks/AnalysisTask.scala | 18 ++++------ .../LiveTasks/BWindowedLiveAnalysisTask.scala | 1 - .../Tasks/LiveTasks/LiveAnalysisTask.scala | 2 +- .../LiveTasks/WindowedLiveAnalysisTask.scala | 1 - .../Tasks/RangeTasks/RangeAnalysisTask.scala | 5 +-- .../Tasks/ViewTasks/ViewAnalysisTask.scala | 2 -- .../analysis/serialisers/S3Serialiser.scala | 8 ++--- .../raphtory/core/clustersetup/DocSvr.scala | 9 ++--- .../core/clustersetup/ManagerNode.scala | 3 +- .../core/clustersetup/RouterNode.scala | 3 +- .../raphtory/core/clustersetup/SeedNode.scala | 6 ++-- .../core/clustersetup/UpdateNode.scala | 3 +- .../core/clustersetup/WatchDogNode.scala | 3 +- .../singlenode/SingleNodeSetup.scala | 15 ++------ .../core/clustersetup/util/ConfigUtils.scala | 3 +- .../RaphtoryReplicator.scala | 4 +-- .../ClusterManagement/SeedActor.scala | 3 +- .../ClusterManagement/WatchDog.scala | 11 ++---- .../PartitionManager/Archivist.scala | 4 +-- .../components/PartitionManager/Reader.scala | 11 ++---- .../Workers/ArchivistWorker.scala | 3 +- .../Workers/IngestionWorker.scala | 5 ++- .../Workers/ReaderWorker.scala | 17 ++++----- .../components/PartitionManager/Writer.scala | 11 ++---- .../components/Router/RouterManager.scala | 8 ++--- .../core/components/Router/RouterWorker.scala | 8 ++--- .../core/components/Spout/Spout.scala | 5 +-- .../communication/raphtoryMessages.scala | 2 -- .../core/model/graphentities/Entity.scala | 1 - .../model/graphentities/MutableProperty.scala | 1 - .../core/model/graphentities/SplitEdge.scala | 2 -- .../raphtory/core/storage/EntityStorage.scala | 11 ++---- .../raphtory/core/utils/SchedulerUtil.scala | 5 +-- .../scala/com/raphtory/core/utils/Utils.scala | 2 -- .../analysers/EthereumTaintTracking.scala | 2 -- .../analysers/TaintTrackExchangeStop.scala | 5 --- .../routers/BitcoinGraphBuilder.scala | 5 +-- .../routers/ChABClus2ClusGraphBuilder.scala | 11 ++---- .../ChABClus2ClusGraphBuilderNoprop.scala | 11 ++---- .../routers/ChainalysisABGraphBuilder.scala | 10 ++---- .../routers/DashcoinGraphBuilder.scala | 7 ++-- .../routers/EthereumGethGraphBuilder.scala | 12 ++----- .../routers/EthereumKafkaGraphBuilder.scala | 12 ++----- .../EthereumTransactionGraphBuilder.scala | 6 ++-- .../routers/FirehoseKafkaGraphBuilder.scala | 12 ++----- .../routers/LitecoinGraphBuilder.scala | 6 ++-- .../CitationGraphBuilder.scala | 7 +--- .../examples/concirrus/BoatCoOcurrance.scala | 35 +++++++++++++++++++ .../gab/routers/GabPostGraphBuilder.scala | 5 +-- .../gab/routers/GabRawGraphBuilder.scala | 7 ++-- .../gab/routers/GabUserGraphBuilder.scala | 9 ++--- .../ldbc/routers/LDBCGraphBuilder.scala | 8 ++--- .../ldbc/routers/LDBCOldGraphBuilder.scala | 8 ++--- .../examples/lotr/LOTRDataSource.scala | 13 ++++--- .../test/actors/RandomGraphBuilder.scala | 4 +-- .../test/actors/TriangleDataSource.scala | 2 +- .../test/actors/TriangleGraphBuilder.scala | 8 ++--- .../routers/TrackAndTraceGraphBuilder.scala | 11 ++---- .../examples/tsvnet/TSVGraphBuilder.scala | 7 ++-- .../CooccurrenceMatrixGraphBuilder.scala | 7 ++-- ...occurrenceMatrixGraphBuilderFiltered.scala | 5 +-- .../com/raphtory/sources/FileSource.scala | 8 ++--- .../com/raphtory/sources/KafkaSource.scala | 2 +- .../blockchain/BitcoinNodeSource.scala | 6 ++-- .../blockchain/EthereumGethSource.scala | 10 +++--- .../blockchain/EthereumPostgresSource.scala | 2 +- 79 files changed, 168 insertions(+), 364 deletions(-) create mode 100644 mainproject/cluster/src/main/scala/com/raphtory/examples/concirrus/BoatCoOcurrance.scala diff --git a/mainproject/build.sbt b/mainproject/build.sbt index 7160ab72e7..5d778916ea 100644 --- a/mainproject/build.sbt +++ b/mainproject/build.sbt @@ -65,9 +65,12 @@ val lift = "net.liftweb" %% "lift-json" % "3.3.0" val bitcoin = "org.scalaj" %% "scalaj-http" % "2.3.0" val twitter_eval = "com.twitter" %% "util-eval" % "6.43.0" +val hadoop = "org.apache.hadoop" % "hadoop-client" % "3.3.0" // https://mvnrepository.com/artifact/com.amazonaws/aws-java-sdk val aws = "com.amazonaws" % "aws-java-sdk" % "1.11.897" +val parquet = "com.github.mjakubowski84" %% "parquet4s-core" % "1.6.0" +val h3 = "com.uber" % "h3" % "3.6.4" val IP = java.net.InetAddress.getLocalHost.getHostAddress @@ -155,7 +158,10 @@ lazy val cluster = project joda, casbah, mongo, - aws + aws, + parquet, + hadoop, + h3 ) ) .settings( diff --git a/mainproject/cluster/src/main/scala/com/raphtory/Server.scala b/mainproject/cluster/src/main/scala/com/raphtory/Server.scala index ab877ba19e..25aa6f4aec 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/Server.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/Server.scala @@ -4,16 +4,10 @@ import java.lang.management.ManagementFactory import java.net.InetAddress import ch.qos.logback.classic.Level -import com.raphtory.core.analysis.Tasks.AnalysisTask import com.raphtory.core.clustersetup._ import com.raphtory.core.clustersetup.singlenode.SingleNodeSetup import com.raphtory.examples.test.actors.RandomGraphBuilder -import com.typesafe.config.Config import com.typesafe.config.ConfigFactory -import kamon.metric.PeriodSnapshot -import kamon.prometheus.PrometheusReporter -import kamon.Kamon -import kamon.module.MetricReporter import org.slf4j.LoggerFactory import scala.language.postfixOps diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/Analyser.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/Analyser.scala index c6aee5e997..9844da7e88 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/Analyser.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/Analyser.scala @@ -5,7 +5,6 @@ import com.raphtory.core.analysis.API.GraphLenses.GraphLens import scala.collection.mutable import scala.collection.mutable.ArrayBuffer -import scala.io.Source import scala.reflect.runtime.currentMirror import scala.tools.reflect.ToolBox case class ManagerCount(count: Int) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/GraphLenses/GraphLens.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/GraphLenses/GraphLens.scala index 09357c418b..0c96260cbc 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/GraphLenses/GraphLens.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/GraphLenses/GraphLens.scala @@ -1,16 +1,15 @@ package com.raphtory.core.analysis.API.GraphLenses +import java.util.concurrent.atomic.AtomicInteger + import akka.actor.ActorContext import com.raphtory.core.analysis.API.ManagerCount import com.raphtory.core.analysis.API.entityVisitors.VertexVisitor +import com.raphtory.core.components.PartitionManager.Workers.ViewJob import com.raphtory.core.model.graphentities.Vertex import com.raphtory.core.storage.EntityStorage -import java.util.concurrent.atomic.AtomicInteger - -import com.raphtory.core.components.PartitionManager.Workers.ViewJob import scala.collection.parallel.ParIterable -import scala.collection.parallel.mutable.ParTrieMap abstract class GraphLens(jobID: ViewJob, superstep: Int, storage: EntityStorage, managerCount: ManagerCount) { private val messages = new AtomicInteger(0) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/GraphLenses/ViewLens.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/GraphLenses/ViewLens.scala index a181a29e98..ef6898b249 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/GraphLenses/ViewLens.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/GraphLenses/ViewLens.scala @@ -9,7 +9,6 @@ import com.raphtory.core.storage.EntityStorage import kamon.Kamon import scala.collection.parallel.ParIterable -import scala.collection.parallel.mutable.ParTrieMap class ViewLens( jobID: ViewJob, diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/EntityVisitor.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/EntityVisitor.scala index 1e18548075..35f555febc 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/EntityVisitor.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/EntityVisitor.scala @@ -4,7 +4,6 @@ import com.raphtory.core.components.PartitionManager.Workers.ViewJob import com.raphtory.core.model.graphentities.Entity import scala.collection.mutable -import scala.collection.parallel.ParSet import scala.collection.parallel.mutable.ParTrieMap abstract class EntityVisitor(entity:Entity,viewJob:ViewJob) { diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/VertexVisitor.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/VertexVisitor.scala index 93fa85e9d5..2045cc9a08 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/VertexVisitor.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/VertexVisitor.scala @@ -6,12 +6,10 @@ import com.raphtory.core.analysis.API.GraphLenses.GraphLens import com.raphtory.core.analysis.API.ManagerCount import com.raphtory.core.components.PartitionManager.Workers.ViewJob import com.raphtory.core.model.communication._ -import com.raphtory.core.model.graphentities.{Edge, MutableProperty, Vertex} +import com.raphtory.core.model.graphentities.{Edge, Vertex} import com.raphtory.core.utils.Utils -import scala.collection.mutable -import scala.collection.parallel.ParSet -import scala.collection.parallel.mutable.{ParIterable, ParTrieMap} +import scala.collection.parallel.mutable.ParIterable import scala.reflect.ClassTag object VertexVisitor { def apply(v: Vertex, jobID: ViewJob, superStep: Int, proxy: GraphLens)(implicit context: ActorContext, managerCount: ManagerCount) = diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/MultiLayerLPA.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/MultiLayerLPA.scala index 186c431bfe..37d7a1b296 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/MultiLayerLPA.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/MultiLayerLPA.scala @@ -1,11 +1,6 @@ package com.raphtory.core.analysis.Algorithms -import com.raphtory.core.analysis.API.Analyser -import com.raphtory.core.analysis.API.entityVisitors.VertexVisitor - import scala.collection.mutable.ArrayBuffer -import scala.collection.parallel.immutable -import scala.collection.parallel.mutable.{ParArray, ParTrieMap} class MultiLayerLPA(args:Array[String]) extends LPA(args) { val snapshotSize:Long = args(2).toLong diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/TemporalTriangleCount.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/TemporalTriangleCount.scala index 0c92cd5c45..38cfec8d13 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/TemporalTriangleCount.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/TemporalTriangleCount.scala @@ -1,10 +1,8 @@ package com.raphtory.core.analysis.Algorithms import com.raphtory.core.analysis.API.Analyser -import com.raphtory.core.utils.Utils import scala.collection.mutable.ArrayBuffer -import scala.collection.parallel.immutable import scala.collection.parallel.mutable.ParArray class TemporalTriangleCount(args:Array[String]) extends Analyser(args) { diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/WattsCascade.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/WattsCascade.scala index d68de7dad1..f15907cda4 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/WattsCascade.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/WattsCascade.scala @@ -3,8 +3,6 @@ package com.raphtory.core.analysis.Algorithms import com.raphtory.core.analysis.API.Analyser import scala.collection.mutable.ArrayBuffer -import scala.collection.mutable -import scala.collection.parallel.immutable import scala.util.Random diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/lotrExample.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/lotrExample.scala index e2472ecd79..712c676d37 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/lotrExample.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/lotrExample.scala @@ -1,6 +1,7 @@ package com.raphtory.core.analysis.Algorithms import com.raphtory.core.analysis.API.Analyser + import scala.collection.mutable.ArrayBuffer import scala.collection.parallel.immutable diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/AnalysisManager.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/AnalysisManager.scala index 0fdb4eb581..161c168610 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/AnalysisManager.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/AnalysisManager.scala @@ -1,26 +1,19 @@ package com.raphtory.core.analysis -import com.raphtory.core.model.communication.{ClusterStatusRequest, ClusterStatusResponse} -import akka.actor.{Actor, ActorRef, ActorSystem, Cancellable, InvalidActorNameException, PoisonPill, Props} -import akka.cluster.pubsub.DistributedPubSub -import akka.cluster.pubsub.DistributedPubSubMediator -import akka.http.scaladsl.model.HttpResponse +import akka.actor.{Actor, ActorRef, InvalidActorNameException, PoisonPill, Props} +import akka.cluster.pubsub.{DistributedPubSub, DistributedPubSubMediator} import akka.pattern.ask import akka.util.Timeout import com.raphtory.core.analysis.API.{Analyser, BlankAnalyser, LoadExternalAnalyser} import com.raphtory.core.analysis.Tasks.LiveTasks.{BWindowedLiveAnalysisTask, LiveAnalysisTask, WindowedLiveAnalysisTask} import com.raphtory.core.analysis.Tasks.RangeTasks.{BWindowedRangeAnalysisTask, RangeAnalysisTask, WindowedRangeAnalysisTask} import com.raphtory.core.analysis.Tasks.ViewTasks.{BWindowedViewAnalysisTask, ViewAnalysisTask, WindowedViewAnalysisTask} -import com.raphtory.core.model.communication._ -import com.twitter.util.Eval +import com.raphtory.core.model.communication.{ClusterStatusRequest, ClusterStatusResponse, _} -import scala.collection.mutable import scala.collection.parallel.mutable.ParTrieMap import scala.concurrent.Await -import scala.concurrent.duration.Duration -import scala.concurrent.duration._ +import scala.concurrent.duration.{Duration, _} import scala.language.postfixOps -import scala.util.Try case class StartAnalysis() class AnalysisManager() extends Actor{ implicit val executionContext = context.system.dispatchers.lookup("misc-dispatcher") diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/AnalysisRestApi.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/AnalysisRestApi.scala index d0784f1966..398413e645 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/AnalysisRestApi.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/AnalysisRestApi.scala @@ -1,26 +1,22 @@ package com.raphtory.core.analysis -import akka.actor.ActorSystem -import akka.actor._ +import akka.actor.{ActorSystem, _} +import akka.cluster.pubsub._ import akka.http.scaladsl.Http +import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._ import akka.http.scaladsl.model.HttpMethods._ import akka.http.scaladsl.model.{HttpRequest, HttpResponse, _} +import akka.http.scaladsl.unmarshalling.Unmarshal import akka.pattern.ask import akka.stream.ActorMaterializer import akka.stream.scaladsl.{Sink, Source} import akka.util.Timeout +import com.raphtory.core.model.communication._ import com.typesafe.config.ConfigFactory - -import scala.concurrent.duration._ -import scala.concurrent.{Await, Future} -import akka.cluster.pubsub._ -import akka.http.scaladsl.unmarshalling.Unmarshal -import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._ -import com.raphtory.core.clustersetup.util.ConfigUtils.SystemConfig -import com.raphtory.core.model.communication.{AnalysisRequest, JobDoesntExist, JobKilled, KillTask, LiveAnalysisPOST, LiveAnalysisRequest, RangeAnalysisPOST, RangeAnalysisRequest, RequestResults, ResultsForApiPI, ViewAnalysisPOST, ViewAnalysisRequest} import spray.json.DefaultJsonProtocol._ -import scala.concurrent.Future +import scala.concurrent.{Await, Future} +import scala.concurrent.duration._ case class AnalysisRestApi(system:ActorSystem){ implicit val system2 = system diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/AnalysisTask.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/AnalysisTask.scala index f6b02d06bb..8d59f6cdb2 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/AnalysisTask.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/AnalysisTask.scala @@ -1,28 +1,22 @@ package com.raphtory.core.analysis.Tasks -import java.io.FileNotFoundException import java.net.InetAddress -import java.util.Date -import java.net.InetAddress -import akka.actor.{Actor, Cancellable, PoisonPill} -import akka.cluster.pubsub.DistributedPubSub -import akka.cluster.pubsub.DistributedPubSubMediator + +import akka.actor.{Actor, PoisonPill} +import akka.cluster.pubsub.{DistributedPubSub, DistributedPubSubMediator} +import com.mongodb.DBObject +import com.mongodb.casbah.{MongoClient, MongoClientURI} +import com.mongodb.util.JSON import com.raphtory.core.analysis.API.Analyser import com.raphtory.core.analysis.StartAnalysis import com.raphtory.core.components.PartitionManager.Workers.ViewJob import com.raphtory.core.model.communication._ import com.raphtory.core.utils.Utils import kamon.Kamon -import com.mongodb.DBObject -import com.mongodb.casbah.MongoClient -import com.mongodb.casbah.MongoClientURI -import com.mongodb.util.JSON import scala.collection.mutable import scala.collection.mutable.ArrayBuffer import scala.concurrent.duration._ -import scala.io.Source -import scala.sys.process._ abstract class AnalysisTask(jobID: String, args:Array[String], analyser: Analyser, managerCount:Int,newAnalyser: Boolean,rawFile:String) extends Actor { protected var currentSuperStep = 0 //SuperStep the algorithm is currently on diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/LiveTasks/BWindowedLiveAnalysisTask.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/LiveTasks/BWindowedLiveAnalysisTask.scala index 34e0cf5579..3cd3947ad3 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/LiveTasks/BWindowedLiveAnalysisTask.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/LiveTasks/BWindowedLiveAnalysisTask.scala @@ -1,7 +1,6 @@ package com.raphtory.core.analysis.Tasks.LiveTasks import com.raphtory.core.analysis.API.Analyser -import com.raphtory.core.analysis.Tasks.AnalysisTask import com.raphtory.core.model.communication.AnalysisType import scala.collection.mutable.ArrayBuffer diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/LiveTasks/LiveAnalysisTask.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/LiveTasks/LiveAnalysisTask.scala index 42d8d2e693..070d122328 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/LiveTasks/LiveAnalysisTask.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/LiveTasks/LiveAnalysisTask.scala @@ -3,7 +3,7 @@ package com.raphtory.core.analysis.Tasks.LiveTasks import akka.cluster.pubsub.DistributedPubSubMediator import com.raphtory.core.analysis.API.Analyser import com.raphtory.core.analysis.Tasks.AnalysisTask -import com.raphtory.core.model.communication.{AnalyserPresentCheck, AnalysisType, Finish, Setup, TimeCheck} +import com.raphtory.core.model.communication.{AnalysisType, Finish, Setup, TimeCheck} import com.raphtory.core.utils.Utils import scala.collection.mutable diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/LiveTasks/WindowedLiveAnalysisTask.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/LiveTasks/WindowedLiveAnalysisTask.scala index 186834a1a8..32f7811c82 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/LiveTasks/WindowedLiveAnalysisTask.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/LiveTasks/WindowedLiveAnalysisTask.scala @@ -1,7 +1,6 @@ package com.raphtory.core.analysis.Tasks.LiveTasks import com.raphtory.core.analysis.API.Analyser -import com.raphtory.core.analysis.Tasks.AnalysisTask import com.raphtory.core.model.communication.AnalysisType class WindowedLiveAnalysisTask(managerCount:Int, jobID: String,args:Array[String], analyser: Analyser,repeatTime:Long,eventTime:Boolean, windowSize:Long,newAnalyser:Boolean,rawFile:String) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/RangeTasks/RangeAnalysisTask.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/RangeTasks/RangeAnalysisTask.scala index 16836ea4f7..6a26805dac 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/RangeTasks/RangeAnalysisTask.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/RangeTasks/RangeAnalysisTask.scala @@ -1,12 +1,9 @@ package com.raphtory.core.analysis.Tasks.RangeTasks -import java.util.Date - import akka.cluster.pubsub.DistributedPubSubMediator import com.raphtory.core.analysis.API.Analyser import com.raphtory.core.analysis.Tasks.AnalysisTask -import com.raphtory.core.model.communication.AnalyserPresentCheck -import com.raphtory.core.model.communication.AnalysisType +import com.raphtory.core.model.communication.{AnalyserPresentCheck, AnalysisType} import com.raphtory.core.utils.Utils class RangeAnalysisTask(managerCount:Int, jobID: String, args:Array[String],analyser: Analyser, start: Long, end: Long, jump: Long,newAnalyser:Boolean,rawFile:String) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/ViewTasks/ViewAnalysisTask.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/ViewTasks/ViewAnalysisTask.scala index b6eac68a60..086ef9c8b1 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/ViewTasks/ViewAnalysisTask.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/ViewTasks/ViewAnalysisTask.scala @@ -1,7 +1,5 @@ package com.raphtory.core.analysis.Tasks.ViewTasks -import java.util.Date - import com.raphtory.core.analysis.API.Analyser import com.raphtory.core.analysis.Tasks.AnalysisTask import com.raphtory.core.model.communication.AnalysisType diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/serialisers/S3Serialiser.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/serialisers/S3Serialiser.scala index 7b49e117f1..3056ba26c3 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/serialisers/S3Serialiser.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/serialisers/S3Serialiser.scala @@ -1,10 +1,8 @@ package com.raphtory.core.analysis.serialisers -import com.amazonaws.auth.BasicAWSCredentials import java.io.{BufferedWriter, File, FileWriter} -import com.amazonaws.auth.AWSStaticCredentialsProvider -import com.amazonaws.services.s3.AmazonS3 -import com.amazonaws.services.s3.AmazonS3ClientBuilder -import com.amazonaws.regions.Regions + +import com.amazonaws.auth.{AWSStaticCredentialsProvider, BasicAWSCredentials} +import com.amazonaws.services.s3.{AmazonS3, AmazonS3ClientBuilder} abstract class S3Serialiser extends Serialiser { val AWS_ACCESS_KEY = s"${sys.env.getOrElse("AWS_ACCESS_KEY", "")}" diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/DocSvr.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/DocSvr.scala index ad1d93fd38..ddf1550a20 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/DocSvr.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/DocSvr.scala @@ -1,18 +1,13 @@ package com.raphtory.core.clustersetup -import akka.actor.ActorSystem -import akka.actor.Address -import akka.actor.ExtendedActorSystem -import akka.cluster.Cluster +import akka.actor.{ActorSystem, Address, ExtendedActorSystem} import akka.cluster.Member import akka.event.LoggingAdapter import akka.management.cluster.bootstrap.ClusterBootstrap import akka.management.javadsl.AkkaManagement import com.raphtory.core.clustersetup.util.ConfigUtils._ import com.raphtory.core.utils.Utils -import com.typesafe.config.Config -import com.typesafe.config.ConfigFactory -import com.typesafe.config.ConfigValueFactory +import com.typesafe.config.{Config, ConfigFactory, ConfigValueFactory} import scala.collection.JavaConversions import scala.collection.JavaConversions._ diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/ManagerNode.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/ManagerNode.scala index 0cff34b686..ac0525d48f 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/ManagerNode.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/ManagerNode.scala @@ -1,7 +1,6 @@ package com.raphtory.core.clustersetup -import akka.actor.ActorSystem -import akka.actor.Props +import akka.actor.{ActorSystem, Props} import com.raphtory.core.components.ClusterManagement.RaphtoryReplicator import com.raphtory.core.utils.Utils diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/RouterNode.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/RouterNode.scala index d1099d6c83..5ab06f21ca 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/RouterNode.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/RouterNode.scala @@ -3,8 +3,7 @@ package com.raphtory.core.clustersetup /** * Created by Mirate on 31/05/2017. */ -import akka.actor.ActorSystem -import akka.actor.Props +import akka.actor.{ActorSystem, Props} import com.raphtory.core.components.ClusterManagement.RaphtoryReplicator import com.raphtory.core.components.Router.GraphBuilder diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/SeedNode.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/SeedNode.scala index b22d4b590f..b120df3fe0 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/SeedNode.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/SeedNode.scala @@ -1,10 +1,8 @@ package com.raphtory.core.clustersetup -import akka.actor.ActorSystem -import akka.actor.Props +import akka.actor.{ActorSystem, Props} import com.raphtory.core.components.ClusterManagement.SeedActor -import com.typesafe.config.Config -import com.typesafe.config.ConfigFactory +import com.typesafe.config.{Config, ConfigFactory} case class SeedNode(seedLoc: String) extends DocSvr { val conf: Config = ConfigFactory.load() diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/UpdateNode.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/UpdateNode.scala index d11689cfd9..bd972b037a 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/UpdateNode.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/UpdateNode.scala @@ -3,8 +3,7 @@ package com.raphtory.core.clustersetup /** * Created by Mirate on 13/06/2017. */ -import akka.actor.ActorSystem -import akka.actor.Props +import akka.actor.{ActorSystem, Props} import scala.language.postfixOps diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/WatchDogNode.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/WatchDogNode.scala index 9f523d4496..f24b85856b 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/WatchDogNode.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/WatchDogNode.scala @@ -3,8 +3,7 @@ package com.raphtory.core.clustersetup /** * Created by Mirate on 11/07/2017. */ -import akka.actor.ActorSystem -import akka.actor.Props +import akka.actor.{ActorSystem, Props} import com.raphtory.core.components.ClusterManagement.{WatchDog, WatermarkManager} case class WatchDogNode(seedLoc: String, partitionNumber: Int, minimumRouters: Int) extends DocSvr { diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/singlenode/SingleNodeSetup.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/singlenode/SingleNodeSetup.scala index 73b5b47ede..f7418d10e7 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/singlenode/SingleNodeSetup.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/singlenode/SingleNodeSetup.scala @@ -1,22 +1,11 @@ package com.raphtory.core.clustersetup.singlenode -import akka.actor.{ActorSystem, Props} -import com.raphtory.core.analysis.API.Analyser +import akka.actor.Props import com.raphtory.core.analysis.{AnalysisManager, AnalysisRestApi} -import com.raphtory.core.analysis.Tasks.LiveTasks.BWindowedLiveAnalysisTask -import com.raphtory.core.analysis.Tasks.LiveTasks.LiveAnalysisTask -import com.raphtory.core.analysis.Tasks.LiveTasks.WindowedLiveAnalysisTask -import com.raphtory.core.analysis.Tasks.RangeTasks.BWindowedRangeAnalysisTask -import com.raphtory.core.analysis.Tasks.RangeTasks.RangeAnalysisTask -import com.raphtory.core.analysis.Tasks.RangeTasks.WindowedRangeAnalysisTask -import com.raphtory.core.analysis.Tasks.ViewTasks.BWindowedViewAnalysisTask -import com.raphtory.core.analysis.Tasks.ViewTasks.ViewAnalysisTask -import com.raphtory.core.analysis.Tasks.ViewTasks.WindowedViewAnalysisTask import com.raphtory.core.clustersetup.DocSvr import com.raphtory.core.components.ClusterManagement.{RaphtoryReplicator, SeedActor, WatchDog, WatermarkManager} import com.raphtory.core.components.Router.GraphBuilder -import com.typesafe.config.Config -import com.typesafe.config.ConfigFactory +import com.typesafe.config.{Config, ConfigFactory} import scala.language.postfixOps diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/util/ConfigUtils.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/util/ConfigUtils.scala index 984357fbd3..a4a39af8a1 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/util/ConfigUtils.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/util/ConfigUtils.scala @@ -1,7 +1,6 @@ package com.raphtory.core.clustersetup.util -import com.typesafe.config.Config -import com.typesafe.config.ConfigValue +import com.typesafe.config.{Config, ConfigValue} import scala.collection.JavaConversions._ diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/ClusterManagement/RaphtoryReplicator.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/ClusterManagement/RaphtoryReplicator.scala index b9c1b06f18..01fc2fdb31 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/ClusterManagement/RaphtoryReplicator.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/ClusterManagement/RaphtoryReplicator.scala @@ -1,8 +1,8 @@ package com.raphtory.core.components.ClusterManagement import akka.actor.{Actor, ActorLogging, ActorRef, Cancellable, Props} -import akka.cluster.pubsub.{DistributedPubSub, DistributedPubSubMediator} import akka.cluster.pubsub.DistributedPubSubMediator.SubscribeAck +import akka.cluster.pubsub.{DistributedPubSub, DistributedPubSubMediator} import akka.pattern.ask import akka.util.Timeout import com.raphtory.core.components.PartitionManager.Workers.IngestionWorker @@ -14,8 +14,8 @@ import com.raphtory.core.utils.{SchedulerUtil, Utils} import scala.collection.mutable import scala.collection.parallel.mutable.ParTrieMap -import scala.concurrent.{Await, Future} import scala.concurrent.duration._ +import scala.concurrent.{Await, Future} import scala.language.postfixOps object RaphtoryReplicator { diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/ClusterManagement/SeedActor.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/ClusterManagement/SeedActor.scala index b07ec7e1d5..7f50929895 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/ClusterManagement/SeedActor.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/ClusterManagement/SeedActor.scala @@ -1,7 +1,6 @@ package com.raphtory.core.components.ClusterManagement -import akka.actor.Actor -import akka.actor.ActorLogging +import akka.actor.{Actor, ActorLogging} import akka.cluster.Cluster import akka.cluster.ClusterEvent._ import com.raphtory.core.clustersetup.DocSvr diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/ClusterManagement/WatchDog.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/ClusterManagement/WatchDog.scala index 9beadd0f84..995f11ba37 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/ClusterManagement/WatchDog.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/ClusterManagement/WatchDog.scala @@ -3,15 +3,10 @@ package com.raphtory.core.components.ClusterManagement /** * Created by Mirate on 11/07/2017. */ -import akka.actor.Actor -import akka.actor.ActorLogging -import akka.actor.ActorRef -import akka.actor.Cancellable -import akka.cluster.pubsub.DistributedPubSub -import akka.cluster.pubsub.DistributedPubSubMediator +import akka.actor.{Actor, ActorLogging, ActorRef, Cancellable} +import akka.cluster.pubsub.{DistributedPubSub, DistributedPubSubMediator} import com.raphtory.core.model.communication._ -import com.raphtory.core.utils.SchedulerUtil -import com.raphtory.core.utils.Utils +import com.raphtory.core.utils.{SchedulerUtil, Utils} import scala.collection.concurrent.TrieMap import scala.collection.mutable diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Archivist.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Archivist.scala index 60168b8322..5db634b43c 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Archivist.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Archivist.scala @@ -1,8 +1,6 @@ package com.raphtory.core.components.PartitionManager -import akka.actor.Actor -import akka.actor.ActorRef -import akka.actor.Props +import akka.actor.{Actor, ActorRef, Props} import ch.qos.logback.classic.Level import com.raphtory.core.components.PartitionManager.Workers.ArchivistWorker import com.raphtory.core.model.communication._ diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Reader.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Reader.scala index ee8e2ac81d..0915932534 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Reader.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Reader.scala @@ -1,19 +1,12 @@ package com.raphtory.core.components.PartitionManager -import akka.actor.Actor -import akka.actor.ActorLogging -import akka.actor.ActorRef -import akka.actor.Props -import akka.actor.Terminated +import akka.actor.{Actor, ActorLogging, ActorRef, Props, Terminated} +import akka.cluster.pubsub.{DistributedPubSub, DistributedPubSubMediator} import akka.cluster.pubsub.DistributedPubSubMediator.SubscribeAck -import akka.cluster.pubsub.DistributedPubSub -import akka.cluster.pubsub.DistributedPubSubMediator -import com.raphtory.core.analysis.API.Analyser import com.raphtory.core.components.PartitionManager.Workers.ReaderWorker import com.raphtory.core.model.communication._ import com.raphtory.core.storage.EntityStorage import com.raphtory.core.utils.Utils -import com.twitter.util.Eval import scala.collection.parallel.mutable.ParTrieMap import scala.util.Try diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/ArchivistWorker.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/ArchivistWorker.scala index 4c43ad6e86..0d1b6e695f 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/ArchivistWorker.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/ArchivistWorker.scala @@ -1,7 +1,6 @@ package com.raphtory.core.components.PartitionManager.Workers -import akka.actor.Actor -import akka.actor.ActorRef +import akka.actor.{Actor, ActorRef} import com.raphtory.core.model.communication._ import com.raphtory.core.storage.EntityStorage import com.raphtory.core.utils.Utils diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/IngestionWorker.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/IngestionWorker.scala index b942463476..38d2d8e989 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/IngestionWorker.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/IngestionWorker.scala @@ -3,16 +3,15 @@ package com.raphtory.core.components.PartitionManager.Workers import java.util.concurrent.atomic.AtomicInteger import akka.actor.{Actor, ActorLogging, ActorRef, Cancellable} -import akka.cluster.pubsub.DistributedPubSub -import akka.cluster.pubsub.DistributedPubSubMediator +import akka.cluster.pubsub.{DistributedPubSub, DistributedPubSubMediator} import com.raphtory.core.model.communication._ import com.raphtory.core.storage.EntityStorage import com.raphtory.core.utils.SchedulerUtil import kamon.Kamon -import scala.concurrent.duration._ import scala.collection.mutable import scala.collection.parallel.mutable.ParTrieMap +import scala.concurrent.duration._ case class queueItem(routerEpoch:Int,timestamp:Long)extends Ordered[queueItem] { def compare(that: queueItem): Int = that.routerEpoch-this.routerEpoch diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/ReaderWorker.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/ReaderWorker.scala index 9ba79aff8d..8346474c67 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/ReaderWorker.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/ReaderWorker.scala @@ -1,20 +1,15 @@ package com.raphtory.core.components.PartitionManager.Workers -import akka.actor.Actor -import akka.actor.ActorLogging -import akka.actor.ActorRef -import akka.cluster.pubsub.DistributedPubSub -import akka.cluster.pubsub.DistributedPubSubMediator -import com.raphtory.core.analysis.API.GraphLenses.GraphLens -import com.raphtory.core.analysis.API.GraphLenses.ViewLens -import com.raphtory.core.analysis.API.GraphLenses.WindowLens -import com.raphtory.core.analysis.API.Analyser -import com.raphtory.core.analysis.API._ +import java.util.concurrent.atomic.AtomicInteger + +import akka.actor.{Actor, ActorLogging, ActorRef} +import akka.cluster.pubsub.{DistributedPubSub, DistributedPubSubMediator} +import com.raphtory.core.analysis.API.GraphLenses.{GraphLens, ViewLens, WindowLens} +import com.raphtory.core.analysis.API.{Analyser, _} import com.raphtory.core.model.communication._ import com.raphtory.core.storage.EntityStorage import com.raphtory.core.utils.Utils import kamon.Kamon -import java.util.concurrent.atomic.AtomicInteger import scala.collection.concurrent.TrieMap import scala.collection.mutable diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Writer.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Writer.scala index 0d0755be3f..4ee5620edc 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Writer.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Writer.scala @@ -1,15 +1,8 @@ package com.raphtory.core.components.PartitionManager import akka.actor.SupervisorStrategy.Resume -import akka.actor.Actor -import akka.actor.ActorLogging -import akka.actor.ActorRef -import akka.actor.Cancellable -import akka.actor.OneForOneStrategy -import akka.actor.Props -import akka.actor.Terminated -import akka.cluster.pubsub.DistributedPubSub -import akka.cluster.pubsub.DistributedPubSubMediator +import akka.actor.{Actor, ActorLogging, ActorRef, Cancellable, OneForOneStrategy, Terminated} +import akka.cluster.pubsub.{DistributedPubSub, DistributedPubSubMediator} import com.raphtory.core.model.communication._ import com.raphtory.core.storage.EntityStorage import com.raphtory.core.utils.{SchedulerUtil, Utils} diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterManager.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterManager.scala index ec97a687ae..0e09aa82d5 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterManager.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterManager.scala @@ -1,11 +1,7 @@ package com.raphtory.core.components.Router -import akka.actor.Actor -import akka.actor.ActorLogging -import akka.actor.ActorRef -import akka.actor.Props -import akka.cluster.pubsub.DistributedPubSub -import akka.cluster.pubsub.DistributedPubSubMediator +import akka.actor.{Actor, ActorLogging, ActorRef, Props} +import akka.cluster.pubsub.{DistributedPubSub, DistributedPubSubMediator} import com.raphtory.core.components.Router.RouterManager.Message.KeepAlive import com.raphtory.core.model.communication._ diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala index 5f5c5cb0ba..8345cb93bc 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala @@ -1,8 +1,7 @@ package com.raphtory.core.components.Router -import akka.actor.{Actor, ActorLogging, Cancellable} -import akka.cluster.pubsub.DistributedPubSub -import akka.cluster.pubsub.DistributedPubSubMediator +import akka.actor.{Actor, ActorLogging} +import akka.cluster.pubsub.{DistributedPubSub, DistributedPubSubMediator} import com.raphtory.core.components.Router.RouterWorker.CommonMessage.TimeBroadcast import com.raphtory.core.components.Spout.Spout.CommonMessage.{NoWork, SpoutOnline, WorkPlease} import com.raphtory.core.model.communication._ @@ -10,10 +9,9 @@ import com.raphtory.core.utils.Utils import com.raphtory.core.utils.Utils.getManager import kamon.Kamon -import scala.collection.parallel.mutable.{ParArray, ParHashSet, ParTrieMap} +import scala.collection.parallel.mutable.ParTrieMap import scala.concurrent.ExecutionContext import scala.concurrent.duration._ -import scala.util.hashing.MurmurHash3 // TODO Add val name which sub classes that extend this trait must overwrite // e.g. BlockChainRouter val name = "Blockchain Router" diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/Spout.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/Spout.scala index bbb48a643b..3c90ed1286 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/Spout.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/Spout.scala @@ -1,16 +1,13 @@ package com.raphtory.core.components.Spout import akka.actor.{Actor, ActorLogging, ActorRef, Cancellable, Timers} -import akka.cluster.pubsub.DistributedPubSub -import akka.cluster.pubsub.DistributedPubSubMediator +import akka.cluster.pubsub.{DistributedPubSub, DistributedPubSubMediator} import com.raphtory.core.components.Spout.Spout.CommonMessage._ import com.raphtory.core.model.communication._ import com.raphtory.core.utils.Utils import kamon.Kamon import scala.concurrent.ExecutionContext - - import scala.concurrent.duration._ import scala.language.postfixOps diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/model/communication/raphtoryMessages.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/model/communication/raphtoryMessages.scala index 362199b45c..f7af56719e 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/model/communication/raphtoryMessages.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/model/communication/raphtoryMessages.scala @@ -1,7 +1,5 @@ package com.raphtory.core.model.communication -import java.util.concurrent.ConcurrentHashMap - import com.raphtory.core.analysis.API.Analyser import com.raphtory.core.components.PartitionManager.Workers.ViewJob import com.raphtory.core.model.graphentities.Edge diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/Entity.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/Entity.scala index 250b7401bd..c739ac14b7 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/Entity.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/Entity.scala @@ -1,6 +1,5 @@ package com.raphtory.core.model.graphentities -import com.raphtory.core.storage.EntityStorage import com.raphtory.core.utils.HistoryOrdering import scala.collection.mutable diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/MutableProperty.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/MutableProperty.scala index c1f7120097..a0fb01a7bd 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/MutableProperty.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/MutableProperty.scala @@ -1,6 +1,5 @@ package com.raphtory.core.model.graphentities -import com.raphtory.core.storage.EntityStorage import com.raphtory.core.utils.HistoryOrdering import scala.collection.mutable diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/SplitEdge.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/SplitEdge.scala index 975f8580e8..862e4acb97 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/SplitEdge.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/SplitEdge.scala @@ -1,7 +1,5 @@ package com.raphtory.core.model.graphentities -import com.raphtory.core.storage.EntityStorage - import scala.collection.mutable import scala.collection.parallel.mutable.ParTrieMap diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/storage/EntityStorage.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/storage/EntityStorage.scala index fd0adbfa4c..4c54c1cab2 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/storage/EntityStorage.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/storage/EntityStorage.scala @@ -3,15 +3,11 @@ package com.raphtory.core.storage import akka.actor.ActorRef import akka.cluster.pubsub.DistributedPubSubMediator import com.raphtory.core.model.communication._ -import com.raphtory.core.model.graphentities.Edge -import com.raphtory.core.model.graphentities.Entity -import com.raphtory.core.model.graphentities.SplitEdge -import com.raphtory.core.model.graphentities.Vertex +import com.raphtory.core.model.graphentities.{Edge, Entity, SplitEdge, Vertex} import com.raphtory.core.utils.Utils import kamon.Kamon import scala.collection.mutable -import scala.collection.mutable.ArrayBuffer import scala.collection.parallel.mutable.ParTrieMap /** @@ -21,10 +17,7 @@ import scala.collection.parallel.mutable.ParTrieMap //TODO What happens when an edge which has been archived gets readded class EntityStorage(partitionID:Int,workerID: Int) { - import com.raphtory.core.utils.Utils.checkDst - import com.raphtory.core.utils.Utils.checkWorker - import com.raphtory.core.utils.Utils.getManager - import com.raphtory.core.utils.Utils.getPartition + import com.raphtory.core.utils.Utils.{checkDst, checkWorker, getManager} val debug = System.getenv().getOrDefault("DEBUG", "false").trim.toBoolean /** diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/utils/SchedulerUtil.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/utils/SchedulerUtil.scala index ae10043b4e..0f2d93eb23 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/utils/SchedulerUtil.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/utils/SchedulerUtil.scala @@ -1,9 +1,6 @@ package com.raphtory.core.utils -import akka.actor.Actor -import akka.actor.ActorContext -import akka.actor.ActorRef -import akka.actor.Cancellable +import akka.actor.{Actor, ActorContext, ActorRef, Cancellable} import scala.concurrent.ExecutionContext import scala.concurrent.duration.FiniteDuration diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/utils/Utils.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/utils/Utils.scala index 424c85061c..3b3122620e 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/utils/Utils.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/utils/Utils.scala @@ -4,11 +4,9 @@ import java.io._ import java.text.SimpleDateFormat import akka.actor.ActorContext -import com.raphtory.core.analysis.API.Analyser import com.typesafe.config.ConfigFactory import org.apache.commons.lang.StringEscapeUtils -import scala.collection.concurrent.TrieMap import scala.collection.mutable object Utils { diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/analysers/EthereumTaintTracking.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/analysers/EthereumTaintTracking.scala index 4fc02ee636..6adb45cdd7 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/analysers/EthereumTaintTracking.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/analysers/EthereumTaintTracking.scala @@ -1,11 +1,9 @@ package com.raphtory.examples.blockchain.analysers import com.raphtory.core.analysis.API.Analyser -import com.raphtory.core.utils.Utils import scala.collection.mutable.ArrayBuffer import scala.collection.parallel.immutable -import scala.util.Random class EthereumTaintTracking(args:Array[String]) extends Analyser(args) { //val infectedNode =System.getenv().getOrDefault("ETHEREUM_TAINT_TRACKING_BAD_ACTOR", "0xa09871aeadf4994ca12f5c0b6056bbd1d343c029").trim diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/analysers/TaintTrackExchangeStop.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/analysers/TaintTrackExchangeStop.scala index 203c122b90..efcf35b489 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/analysers/TaintTrackExchangeStop.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/analysers/TaintTrackExchangeStop.scala @@ -1,15 +1,10 @@ package com.raphtory.examples.blockchain.analysers -import com.raphtory.core.analysis.API.Analyser - -import scala.collection.mutable.ArrayBuffer -import scala.collection.parallel.immutable import com.raphtory.core.analysis.API.Analyser import com.raphtory.core.utils.Utils import scala.collection.mutable.ArrayBuffer import scala.collection.parallel.immutable -import scala.util.Random class TaintTrackExchangeStop(args:Array[String]) extends Analyser(args) { //val infectedNode =System.getenv().getOrDefault("ETHEREUM_TAINT_TRACKING_BAD_ACTOR", "0xa09871aeadf4994ca12f5c0b6056bbd1d343c029").trim diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/BitcoinGraphBuilder.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/BitcoinGraphBuilder.scala index ddd3ab8051..6532250a13 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/BitcoinGraphBuilder.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/BitcoinGraphBuilder.scala @@ -1,13 +1,10 @@ package com.raphtory.examples.blockchain.routers -import com.raphtory.core.components.Router.{GraphBuilder, RouterWorker} +import com.raphtory.core.components.Router.GraphBuilder import com.raphtory.core.model.communication._ import com.raphtory.sources.blockchain.BitcoinTransaction import spray.json.JsArray -import scala.collection.mutable.ListBuffer -import scala.collection.parallel.mutable.{ParArray, ParHashSet} - class BitcoinGraphBuilder extends GraphBuilder[BitcoinTransaction]{ override def parseTuple(tuple:BitcoinTransaction) = { diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusGraphBuilder.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusGraphBuilder.scala index bb4d233a49..786f8444ad 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusGraphBuilder.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusGraphBuilder.scala @@ -1,14 +1,7 @@ package com.raphtory.examples.blockchain.routers -import java.text.SimpleDateFormat -import java.util.Calendar - -import com.raphtory.core.components.Router.{GraphBuilder, RouterWorker} -import com.raphtory.core.model.communication.Type -import com.raphtory.core.model.communication._ - -import scala.collection.mutable.ListBuffer -import scala.collection.parallel.mutable.ParHashSet +import com.raphtory.core.components.Router.GraphBuilder +import com.raphtory.core.model.communication.{Type, _} class ChABClus2ClusGraphBuilder extends GraphBuilder[String]{ diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusGraphBuilderNoprop.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusGraphBuilderNoprop.scala index 7c33020eb4..4255e4cca4 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusGraphBuilderNoprop.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusGraphBuilderNoprop.scala @@ -1,14 +1,7 @@ package com.raphtory.examples.blockchain.routers -import java.text.SimpleDateFormat -import java.util.Calendar - -import com.raphtory.core.components.Router.{GraphBuilder, RouterWorker} -import com.raphtory.core.model.communication.Type -import com.raphtory.core.model.communication._ - -import scala.collection.mutable.ListBuffer -import scala.collection.parallel.mutable.ParHashSet +import com.raphtory.core.components.Router.GraphBuilder +import com.raphtory.core.model.communication.{Type, _} class ChABClus2ClusGraphBuilderNoprop extends GraphBuilder[String] { diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChainalysisABGraphBuilder.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChainalysisABGraphBuilder.scala index 143d52d1cf..df0cd860c1 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChainalysisABGraphBuilder.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChainalysisABGraphBuilder.scala @@ -1,13 +1,7 @@ package com.raphtory.examples.blockchain.routers -import java.text.SimpleDateFormat - -import com.raphtory.core.components.Router.{GraphBuilder, RouterWorker} -import com.raphtory.core.model.communication.Type -import com.raphtory.core.model.communication._ - -import scala.collection.mutable.ListBuffer -import scala.collection.parallel.mutable.ParHashSet +import com.raphtory.core.components.Router.GraphBuilder +import com.raphtory.core.model.communication.{Type, _} class ChainalysisABGraphBuilder extends GraphBuilder[String] { diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/DashcoinGraphBuilder.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/DashcoinGraphBuilder.scala index 492bed79d5..fedf7a418c 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/DashcoinGraphBuilder.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/DashcoinGraphBuilder.scala @@ -1,13 +1,10 @@ package com.raphtory.examples.blockchain.routers -import com.raphtory.core.components.Router.{GraphBuilder, RouterWorker} +import com.raphtory.core.components.Router.GraphBuilder import com.raphtory.core.model.communication._ -import com.raphtory.examples.blockchain.routers import com.raphtory.sources.blockchain.BitcoinTransaction -import spray.json.{JsArray, JsValue} +import spray.json.JsArray -import scala.collection.mutable.ListBuffer -import scala.collection.parallel.mutable.ParHashSet import scala.util.hashing.MurmurHash3 class DashcoinGraphBuilder extends GraphBuilder[BitcoinTransaction] { diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumGethGraphBuilder.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumGethGraphBuilder.scala index a83a2d452d..1d0196a710 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumGethGraphBuilder.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumGethGraphBuilder.scala @@ -1,15 +1,7 @@ package com.raphtory.examples.blockchain.routers -import akka.http.scaladsl.unmarshalling.Unmarshal -import com.raphtory.core.components.Router.{GraphBuilder, RouterWorker} -import com.raphtory.core.model.communication.{EdgeAddWithProperties, GraphUpdate, ImmutableProperty, Properties, StringProperty, VertexAddWithProperties} -import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._ -import akka.stream.ActorMaterializer -import spray.json._ - -import scala.collection.mutable.ListBuffer -import scala.collection.parallel.mutable.ParHashSet -import scala.util.hashing.MurmurHash3 +import com.raphtory.core.components.Router.GraphBuilder +import com.raphtory.core.model.communication._ class EthereumGethGraphBuilder extends GraphBuilder[String] { def hexToInt(hex: String) = Integer.parseInt(hex.drop(2), 16) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumKafkaGraphBuilder.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumKafkaGraphBuilder.scala index 11de2df4e0..24af7c6b97 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumKafkaGraphBuilder.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumKafkaGraphBuilder.scala @@ -1,15 +1,7 @@ package com.raphtory.examples.blockchain.routers -import akka.http.scaladsl.unmarshalling.Unmarshal -import com.raphtory.core.components.Router.{GraphBuilder, RouterWorker} -import com.raphtory.core.model.communication.{EdgeAddWithProperties, GraphUpdate, ImmutableProperty, Properties, StringProperty, VertexAddWithProperties} -import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._ -import akka.stream.ActorMaterializer -import spray.json._ - -import scala.collection.mutable.ListBuffer -import scala.collection.parallel.mutable.ParHashSet -import scala.util.hashing.MurmurHash3 +import com.raphtory.core.components.Router.GraphBuilder +import com.raphtory.core.model.communication._ class EthereumKafkaGraphBuilder extends GraphBuilder[String] { def hexToInt(hex: String) = Integer.parseInt(hex.drop(2), 16) override def parseTuple(tuple: String):Unit = { diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumTransactionGraphBuilder.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumTransactionGraphBuilder.scala index 7b227ce79c..41cd409cec 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumTransactionGraphBuilder.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumTransactionGraphBuilder.scala @@ -1,10 +1,8 @@ package com.raphtory.examples.blockchain.routers -import com.raphtory.core.components.Router.{GraphBuilder, RouterWorker} -import com.raphtory.core.model.communication.{EdgeAddWithProperties, GraphUpdate, Properties, StringProperty, VertexAddWithProperties} +import com.raphtory.core.components.Router.GraphBuilder +import com.raphtory.core.model.communication.{EdgeAddWithProperties, Properties, StringProperty, VertexAddWithProperties} -import scala.collection.mutable.ListBuffer -import scala.collection.parallel.mutable.ParHashSet import scala.util.hashing.MurmurHash3 class EthereumTransactionGraphBuilder extends GraphBuilder[String]{ diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/FirehoseKafkaGraphBuilder.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/FirehoseKafkaGraphBuilder.scala index d69716d85f..72a87f0b7e 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/FirehoseKafkaGraphBuilder.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/FirehoseKafkaGraphBuilder.scala @@ -1,17 +1,9 @@ package com.raphtory.examples.blockchain.routers -import akka.http.scaladsl.unmarshalling.Unmarshal -import com.raphtory.core.components.Router.{GraphBuilder, RouterWorker} -import com.raphtory.core.model.communication.{DoubleProperty, EdgeAdd, EdgeAddWithProperties, EdgeDelete, GraphUpdate, ImmutableProperty, LongProperty, Properties, StringProperty, VertexAdd, VertexAddWithProperties, VertexDelete} -import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._ -import akka.stream.ActorMaterializer -import spray.json._ +import com.raphtory.core.components.Router.GraphBuilder +import com.raphtory.core.model.communication._ -import scala.collection.mutable.ListBuffer -import scala.collection.parallel.mutable.ParHashSet import scala.util.Random -import scala.util.hashing.MurmurHash3 -import scala.math.BigInt class FirehoseKafkaGraphBuilder extends GraphBuilder[String] { var DELETEPERCENT = System.getenv().getOrDefault("ETHER_DELETE_PERCENT", "0").trim.toDouble/100 var DELETESEED = System.getenv().getOrDefault("ETHER_DELETE_SEED", "123").trim.toInt diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/LitecoinGraphBuilder.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/LitecoinGraphBuilder.scala index 576b09511d..a334405348 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/LitecoinGraphBuilder.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/LitecoinGraphBuilder.scala @@ -1,12 +1,10 @@ package com.raphtory.examples.blockchain.routers -import com.raphtory.core.components.Router.{GraphBuilder, RouterWorker} +import com.raphtory.core.components.Router.GraphBuilder import com.raphtory.core.model.communication._ import com.raphtory.sources.blockchain.BitcoinTransaction -import spray.json.{JsArray, JsValue} +import spray.json.JsArray -import scala.collection.mutable.ListBuffer -import scala.collection.parallel.mutable.ParHashSet import scala.util.hashing.MurmurHash3 diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/citationNetwork/CitationGraphBuilder.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/citationNetwork/CitationGraphBuilder.scala index ea57716485..3a15e85d9d 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/citationNetwork/CitationGraphBuilder.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/citationNetwork/CitationGraphBuilder.scala @@ -1,13 +1,8 @@ package com.raphtory.examples.citationNetwork -import java.text.SimpleDateFormat - -import com.raphtory.core.components.Router.{GraphBuilder, RouterWorker} +import com.raphtory.core.components.Router.GraphBuilder import com.raphtory.core.model.communication._ -import scala.collection.mutable.ListBuffer -import scala.collection.parallel.mutable.ParHashSet - class CitationGraphBuilder extends GraphBuilder[String] { override def parseTuple(tuple: String) = { val fileLine = tuple.split(",").map(_.trim) //take the tuple and split on , as we are only interested in the first 4 fields diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/concirrus/BoatCoOcurrance.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/concirrus/BoatCoOcurrance.scala new file mode 100644 index 0000000000..12efef51e3 --- /dev/null +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/concirrus/BoatCoOcurrance.scala @@ -0,0 +1,35 @@ +package com.raphtory.examples.concirrus + +import com.raphtory.core.analysis.API.Analyser + +import scala.collection.mutable.ArrayBuffer + +class BoatCoOcurrance(args: Array[String]) extends Analyser(args) { + + + override def returnResults(): Any = { + view.getVertices().filter(v => v.Type() equals ("Location")).map(loc => { + loc.getIncEdges.map(e => { //We get all incoming edges + val toCompare = loc.getIncEdges.filter(edge => edge != e) //then filter incoming edges to remove itself + e.getHistory().flatMap(f => { + //for each element in the history of the edge + toCompare.filter(edge => //go through all other edges + edge.getHistory() + .exists(p => Math.abs(p._1 - f._1) <= 3600)) //and see if they contain a point +/- an hour + .map(edge => edge.src()) //and if so return their ID + }) + }) + }) + } + + override def processResults(results: ArrayBuffer[Any], timeStamp: Long, viewCompleteTime: Long): Unit = ??? + + + override def setup(): Unit = {} + + override def analyse(): Unit = {} + + + override def defineMaxSteps(): Int = 1 + +} diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabPostGraphBuilder.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabPostGraphBuilder.scala index 8a260a32a7..1b77fb3fe6 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabPostGraphBuilder.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabPostGraphBuilder.scala @@ -2,12 +2,9 @@ package com.raphtory.examples.gab.routers import java.text.SimpleDateFormat -import com.raphtory.core.components.Router.{GraphBuilder, RouterWorker} +import com.raphtory.core.components.Router.GraphBuilder import com.raphtory.core.model.communication._ -import scala.collection.mutable.ListBuffer -import scala.collection.parallel.mutable.ParHashSet - // The lines sent by the Gab mining spout are read and processed accordingly. //In this router we needed to transform the data that was sent by the spout by turning it into a epoch value (long value) // in order to be accepted by Raphtory to create the corresponding entity. diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabRawGraphBuilder.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabRawGraphBuilder.scala index dae6418363..c4cf81fc1f 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabRawGraphBuilder.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabRawGraphBuilder.scala @@ -2,14 +2,11 @@ package com.raphtory.examples.gab.routers import java.time.OffsetDateTime -import com.raphtory.core.components.Router.{GraphBuilder, RouterWorker} -import com.raphtory.core.model.communication.{EdgeAddWithProperties, GraphUpdate, Properties, StringProperty, VertexAddWithProperties} +import com.raphtory.core.components.Router.GraphBuilder +import com.raphtory.core.model.communication.{EdgeAddWithProperties, Properties, StringProperty, VertexAddWithProperties} import com.raphtory.examples.gab.rawgraphmodel.GabPost import spray.json._ -import scala.collection.mutable.ListBuffer -import scala.collection.parallel.mutable.ParHashSet - /** * The Graph Manager is the top level actor in this system (under the stream) * which tracks all the graph partitions - passing commands processed by the 'command processor' actors diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabUserGraphBuilder.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabUserGraphBuilder.scala index 5012b87528..e5c86db133 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabUserGraphBuilder.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabUserGraphBuilder.scala @@ -2,13 +2,8 @@ package com.raphtory.examples.gab.routers import java.text.SimpleDateFormat -import com.raphtory.core.components.Router.{GraphBuilder, RouterWorker} -import com.raphtory.core.model.communication.Type -import com.raphtory.core.model.communication._ - -import scala.collection.mutable.ListBuffer -import scala.collection.parallel.mutable.ParHashSet -import scala.util.Random +import com.raphtory.core.components.Router.GraphBuilder +import com.raphtory.core.model.communication.{Type, _} // The lines sent by the Gab mining spout are read and processed accordingly. //In this router we needed to transform the data that was sent by the spout by turning it into a epoch value (long value) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCGraphBuilder.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCGraphBuilder.scala index 3b08c5eb63..8ab68deab4 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCGraphBuilder.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCGraphBuilder.scala @@ -1,13 +1,9 @@ package com.raphtory.examples.ldbc.routers import java.text.SimpleDateFormat -import java.util.Date -import com.raphtory.core.components.Router.{GraphBuilder, RouterWorker} -import com.raphtory.core.model.communication.{EdgeAdd, EdgeDelete, GraphUpdate, Type, VertexAdd, VertexDelete} - -import scala.collection.mutable.ListBuffer -import scala.collection.parallel.mutable.ParHashSet +import com.raphtory.core.components.Router.GraphBuilder +import com.raphtory.core.model.communication._ class LDBCGraphBuilder extends GraphBuilder[String] { override def parseTuple(tuple: String) = { diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCOldGraphBuilder.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCOldGraphBuilder.scala index b6f4197208..c82699bdf0 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCOldGraphBuilder.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCOldGraphBuilder.scala @@ -1,13 +1,9 @@ package com.raphtory.examples.ldbc.routers import java.text.SimpleDateFormat -import java.util.Date -import com.raphtory.core.components.Router.{GraphBuilder, RouterWorker} -import com.raphtory.core.model.communication.{EdgeAdd, EdgeDelete, GraphUpdate, Type, VertexAdd, VertexDelete} - -import scala.collection.mutable.ListBuffer -import scala.collection.parallel.mutable.ParHashSet +import com.raphtory.core.components.Router.GraphBuilder +import com.raphtory.core.model.communication.{EdgeAdd, Type, VertexAdd} class LDBCOldGraphBuilder extends GraphBuilder[String] { override def parseTuple(tuple: String) = { diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRDataSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRDataSource.scala index bba81cc245..76601f16fe 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRDataSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRDataSource.scala @@ -1,16 +1,19 @@ package com.raphtory.examples.lotr -import com.raphtory.core.components.Spout.{DataSource} +import com.raphtory.core.components.Spout.DataSource + import scala.collection.mutable class LOTRDataSource extends DataSource[String] { - val fileQueue = mutable.Queue[String]() ++= - scala.io.Source.fromFile("cluster/src/main/scala/com/raphtory/examples/lotr/lotr.csv") - .getLines + val fileQueue = mutable.Queue[String]() - override def setupDataSource(): Unit = {}//no setup + override def setupDataSource(): Unit = { + fileQueue++= + scala.io.Source.fromFile("cluster/src/main/scala/com/raphtory/examples/lotr/lotr.csv") + .getLines + }//no setup override def generateData(): Option[String] = { if(fileQueue isEmpty){ diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomGraphBuilder.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomGraphBuilder.scala index 316302d3f9..e6631d6290 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomGraphBuilder.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomGraphBuilder.scala @@ -1,11 +1,9 @@ package com.raphtory.examples.test.actors -import com.raphtory.core.components.Router.{GraphBuilder, RouterWorker} +import com.raphtory.core.components.Router.GraphBuilder import com.raphtory.core.model.communication.{GraphUpdate, _} import spray.json._ -import scala.collection.parallel.mutable.ParHashSet - /** * The Graph Manager is the top level actor in this system (under the stream) * which tracks all the graph partitions - passing commands processed by the 'command processor' actors diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleDataSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleDataSource.scala index 40e1153b54..f877c90c64 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleDataSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleDataSource.scala @@ -1,6 +1,6 @@ package com.raphtory.examples.test.actors -import com.raphtory.core.components.Spout.{DataSource} +import com.raphtory.core.components.Spout.DataSource import scala.collection.mutable.Queue diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleGraphBuilder.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleGraphBuilder.scala index e55c8dbe39..6e15beba69 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleGraphBuilder.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleGraphBuilder.scala @@ -1,11 +1,7 @@ package com.raphtory.examples.test.actors -import com.raphtory.core.components.Router.{GraphBuilder, RouterWorker} -import com.raphtory.core.components.Spout.Spout -import com.raphtory.core.model.communication.{EdgeAdd, GraphUpdate, VertexAddWithProperties} - -import scala.collection.mutable.ListBuffer -import scala.collection.parallel.mutable.ParHashSet +import com.raphtory.core.components.Router.GraphBuilder +import com.raphtory.core.model.communication.EdgeAdd class TriangleGraphBuilder extends GraphBuilder[String] { override def parseTuple(tuple: String)= { diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/trackAndTrace/routers/TrackAndTraceGraphBuilder.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/trackAndTrace/routers/TrackAndTraceGraphBuilder.scala index 0b97848bb7..4938204539 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/trackAndTrace/routers/TrackAndTraceGraphBuilder.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/trackAndTrace/routers/TrackAndTraceGraphBuilder.scala @@ -1,14 +1,7 @@ package com.raphtory.examples.trackAndTrace.routers -import java.text.SimpleDateFormat - -import com.raphtory.core.components.Router.{GraphBuilder, RouterWorker} -import com.raphtory.core.model.communication.Type -import com.raphtory.core.model.communication._ - -import scala.collection.mutable.ListBuffer -import scala.collection.parallel.mutable.ParHashSet -import scala.util.control.Breaks._ +import com.raphtory.core.components.Router.GraphBuilder +import com.raphtory.core.model.communication.{Type, _} class TrackAndTraceGraphBuilder extends GraphBuilder[String] { val EARTH_EQU = 6378137.0 //m diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/TSVGraphBuilder.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/TSVGraphBuilder.scala index 8914a4447b..a7bcb4f367 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/TSVGraphBuilder.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/TSVGraphBuilder.scala @@ -1,10 +1,7 @@ package com.raphtory.examples.tsvnet -import com.raphtory.core.components.Router.{GraphBuilder, RouterWorker} -import com.raphtory.core.model.communication.{EdgeAdd, GraphUpdate, Type, VertexAdd} - -import scala.collection.mutable.ListBuffer -import scala.collection.parallel.mutable.ParHashSet +import com.raphtory.core.components.Router.GraphBuilder +import com.raphtory.core.model.communication.{EdgeAdd, Type, VertexAdd} /** Spout for network datasets of the form SRC_NODE_ID DEST_NODE_ID TIMESTAMP */ class TSVGraphBuilder extends GraphBuilder[String] { diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixGraphBuilder.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixGraphBuilder.scala index eb7afe49af..7b5c4ce64f 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixGraphBuilder.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixGraphBuilder.scala @@ -1,10 +1,7 @@ package com.raphtory.examples.wordSemantic.routers -import com.raphtory.core.components.Router.{GraphBuilder, RouterWorker} -import com.raphtory.core.model.communication.{Type, _} - -import scala.collection.mutable.ListBuffer -import scala.collection.parallel.mutable.ParHashSet +import com.raphtory.core.components.Router.GraphBuilder +import com.raphtory.core.model.communication._ class CooccurrenceMatrixGraphBuilder extends GraphBuilder[String] { diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixGraphBuilderFiltered.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixGraphBuilderFiltered.scala index 1d53281d12..ebd9d105be 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixGraphBuilderFiltered.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixGraphBuilderFiltered.scala @@ -1,11 +1,8 @@ package com.raphtory.examples.wordSemantic.routers -import com.raphtory.core.components.Router.{GraphBuilder, RouterWorker} +import com.raphtory.core.components.Router.GraphBuilder import com.raphtory.core.model.communication._ -import scala.collection.mutable.ListBuffer -import scala.collection.parallel.mutable.ParHashSet - class CooccurrenceMatrixGraphBuilderFiltered extends GraphBuilder[String] { val THR = System.getenv().getOrDefault("COOC_FREQ_THRESHOLD ", "0.05").trim.toDouble diff --git a/mainproject/cluster/src/main/scala/com/raphtory/sources/FileSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/sources/FileSource.scala index a1962ba887..47d874c38c 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/sources/FileSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/sources/FileSource.scala @@ -1,15 +1,11 @@ package com.raphtory.sources -import java.io.{BufferedReader, File, FileInputStream, FileReader, InputStreamReader} -import java.nio.charset.StandardCharsets +import java.io._ import java.util.zip.GZIPInputStream -import com.raphtory.core.components.Spout.{DataSource} +import com.raphtory.core.components.Spout.DataSource import com.typesafe.scalalogging.LazyLogging -import scala.annotation.tailrec -import scala.concurrent.duration._ - class FileSource extends DataSource[String] { //TODO work out loggging here //log.info("initialise FileSpout") diff --git a/mainproject/cluster/src/main/scala/com/raphtory/sources/KafkaSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/sources/KafkaSource.scala index 71753b4708..3eaad0dd85 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/sources/KafkaSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/sources/KafkaSource.scala @@ -3,7 +3,7 @@ package com.raphtory.spouts import java.util import java.util.Properties -import com.raphtory.core.components.Spout.{DataSource} +import com.raphtory.core.components.Spout.DataSource import org.apache.kafka.clients.consumer.KafkaConsumer import scala.annotation.tailrec diff --git a/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/BitcoinNodeSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/BitcoinNodeSource.scala index 14239366b6..fb20b5842e 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/BitcoinNodeSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/BitcoinNodeSource.scala @@ -1,11 +1,9 @@ package com.raphtory.sources.blockchain // -import java.io.File -import java.io.PrintWriter +import java.io.{File, PrintWriter} import com.raphtory.core.components.Spout.DataSource -import scalaj.http.Http -import scalaj.http.HttpRequest +import scalaj.http.{Http, HttpRequest} import spray.json._ import scala.collection.mutable diff --git a/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumGethSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumGethSource.scala index 4638bb45e3..104051c0c8 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumGethSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumGethSource.scala @@ -3,16 +3,14 @@ package com.raphtory.sources.blockchain import java.net.InetAddress import java.util.NoSuchElementException -import com.raphtory.core.components.Spout.{DataSource, Spout} +import com.raphtory.core.components.Spout.DataSource import com.raphtory.core.utils.Utils -import scala.language.postfixOps -import scala.sys.process._ -import scalaj.http.Http -import scalaj.http.HttpRequest -import spray.json._ +import scalaj.http.{Http, HttpRequest} import spray.json.DefaultJsonProtocol._ +import spray.json._ import scala.collection.mutable +import scala.language.postfixOps diff --git a/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumPostgresSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumPostgresSource.scala index ec9b113e77..0c67b7bb0d 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumPostgresSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumPostgresSource.scala @@ -1,7 +1,7 @@ package com.raphtory.sources.blockchain import cats.effect.{Blocker, IO} -import com.raphtory.core.components.Spout.{DataSource} +import com.raphtory.core.components.Spout.DataSource import doobie.implicits._ import doobie.util.ExecutionContexts import doobie.util.transactor.Transactor From ed443441eab7c2c65e6a777f04e035550c14fe81 Mon Sep 17 00:00:00 2001 From: Miratepuffin Date: Sun, 29 Nov 2020 00:15:06 +0000 Subject: [PATCH 39/58] Changed DataSource back to spout as discussed --- .../{Raphtory.scala => RaphtoryGraph.scala} | 12 +- .../core/components/Router/RouterWorker.scala | 2 +- .../core/components/Spout/DataSource.scala | 15 -- .../core/components/Spout/Spout.scala | 130 +---------------- .../core/components/Spout/SpoutAgent.scala | 131 ++++++++++++++++++ .../BitcoinGraphBuilder.scala | 4 +- .../ChABClus2ClusGraphBuilder.scala | 2 +- .../ChABClus2ClusGraphBuilderNoprop.scala | 2 +- .../ChABClus2ClusRouter.scala.save | 0 .../ChainalysisABGraphBuilder.scala | 2 +- .../DashcoinGraphBuilder.scala | 4 +- .../EthereumGethGraphBuilder.scala | 2 +- .../EthereumKafkaGraphBuilder.scala | 2 +- .../EthereumTransactionGraphBuilder.scala | 2 +- .../FirehoseKafkaGraphBuilder.scala | 2 +- .../LitecoinGraphBuilder.scala | 4 +- .../examples/concirrus/BoatCoOcurrance.scala | 35 ----- .../gab/datasources/GabMongoSource.scala | 4 +- .../GabPostGraphBuilder.scala | 2 +- .../GabRawGraphBuilder.scala | 2 +- .../GabUserGraphBuilder.scala | 2 +- .../LDBCGraphBuilder.scala | 2 +- .../LDBCOldGraphBuilder.scala | 2 +- .../examples/lotr/LOTRDeployment.scala | 6 +- .../{LOTRDataSource.scala => LOTRSpout.scala} | 4 +- ...ndomDataSource.scala => RandomSpout.scala} | 4 +- ...leDataSource.scala => TriangleSpout.scala} | 4 +- .../TrackAndTraceGraphBuilder.scala | 2 +- .../CooccurrenceMatrixGraphBuilder.scala | 2 +- ...occurrenceMatrixGraphBuilderFiltered.scala | 2 +- .../FileSpout.scala} | 6 +- .../KafkaSpout.scala} | 4 +- .../blockchain/BitcoinNodeSpout.scala} | 6 +- .../blockchain/EthereumGethSpout.scala} | 6 +- .../blockchain/EthereumPostgresSpout.scala} | 6 +- 35 files changed, 191 insertions(+), 226 deletions(-) rename mainproject/cluster/src/main/scala/com/raphtory/{Raphtory.scala => RaphtoryGraph.scala} (75%) delete mode 100644 mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/DataSource.scala create mode 100644 mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/SpoutAgent.scala rename mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/{routers => graphbuilders}/BitcoinGraphBuilder.scala (97%) rename mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/{routers => graphbuilders}/ChABClus2ClusGraphBuilder.scala (97%) rename mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/{routers => graphbuilders}/ChABClus2ClusGraphBuilderNoprop.scala (97%) rename mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/{routers => graphbuilders}/ChABClus2ClusRouter.scala.save (100%) rename mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/{routers => graphbuilders}/ChainalysisABGraphBuilder.scala (97%) rename mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/{routers => graphbuilders}/DashcoinGraphBuilder.scala (97%) rename mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/{routers => graphbuilders}/EthereumGethGraphBuilder.scala (97%) rename mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/{routers => graphbuilders}/EthereumKafkaGraphBuilder.scala (97%) rename mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/{routers => graphbuilders}/EthereumTransactionGraphBuilder.scala (97%) rename mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/{routers => graphbuilders}/FirehoseKafkaGraphBuilder.scala (98%) rename mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/{routers => graphbuilders}/LitecoinGraphBuilder.scala (97%) delete mode 100644 mainproject/cluster/src/main/scala/com/raphtory/examples/concirrus/BoatCoOcurrance.scala rename mainproject/cluster/src/main/scala/com/raphtory/examples/gab/{routers => graphbuilders}/GabPostGraphBuilder.scala (97%) rename mainproject/cluster/src/main/scala/com/raphtory/examples/gab/{routers => graphbuilders}/GabRawGraphBuilder.scala (98%) rename mainproject/cluster/src/main/scala/com/raphtory/examples/gab/{routers => graphbuilders}/GabUserGraphBuilder.scala (98%) rename mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/{routers => graphbuilders}/LDBCGraphBuilder.scala (97%) rename mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/{routers => graphbuilders}/LDBCOldGraphBuilder.scala (97%) rename mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/{LOTRDataSource.scala => LOTRSpout.scala} (84%) rename mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/{RandomDataSource.scala => RandomSpout.scala} (96%) rename mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/{TriangleDataSource.scala => TriangleSpout.scala} (79%) rename mainproject/cluster/src/main/scala/com/raphtory/examples/trackAndTrace/{routers => }/TrackAndTraceGraphBuilder.scala (98%) rename mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/{routers => graphbuilders}/CooccurrenceMatrixGraphBuilder.scala (95%) rename mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/{routers => graphbuilders}/CooccurrenceMatrixGraphBuilderFiltered.scala (95%) rename mainproject/cluster/src/main/scala/com/raphtory/{sources/FileSource.scala => spouts/FileSpout.scala} (96%) rename mainproject/cluster/src/main/scala/com/raphtory/{sources/KafkaSource.scala => spouts/KafkaSpout.scala} (96%) rename mainproject/cluster/src/main/scala/com/raphtory/{sources/blockchain/BitcoinNodeSource.scala => spouts/blockchain/BitcoinNodeSpout.scala} (94%) rename mainproject/cluster/src/main/scala/com/raphtory/{sources/blockchain/EthereumGethSource.scala => spouts/blockchain/EthereumGethSpout.scala} (96%) rename mainproject/cluster/src/main/scala/com/raphtory/{sources/blockchain/EthereumPostgresSource.scala => spouts/blockchain/EthereumPostgresSpout.scala} (94%) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/Raphtory.scala b/mainproject/cluster/src/main/scala/com/raphtory/RaphtoryGraph.scala similarity index 75% rename from mainproject/cluster/src/main/scala/com/raphtory/Raphtory.scala rename to mainproject/cluster/src/main/scala/com/raphtory/RaphtoryGraph.scala index 8d132f9776..b1378a84ee 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/Raphtory.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/RaphtoryGraph.scala @@ -5,16 +5,16 @@ import ch.qos.logback.classic.Level import com.raphtory.core.analysis.{AnalysisManager, AnalysisRestApi} import com.raphtory.core.components.ClusterManagement.{RaphtoryReplicator, WatchDog, WatermarkManager} import com.raphtory.core.components.Router.GraphBuilder -import com.raphtory.core.components.Spout.{DataSource, Spout} +import com.raphtory.core.components.Spout.{Spout, SpoutAgent} import kamon.Kamon import org.slf4j.LoggerFactory -object Raphtory { - def apply[T](dataSource: DataSource[T],graphBuilder: GraphBuilder[T]) : Raphtory[T] = - new Raphtory(dataSource, graphBuilder) +object RaphtoryGraph { + def apply[T](dataSource: Spout[T], graphBuilder: GraphBuilder[T]) : RaphtoryGraph[T] = + new RaphtoryGraph(dataSource, graphBuilder) } -class Raphtory[T](dataSource: DataSource[T],graphBuilder: GraphBuilder[T]) { +class RaphtoryGraph[T](dataSource: Spout[T], graphBuilder: GraphBuilder[T]) { Kamon.init() //start tool logging val root = LoggerFactory.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME).asInstanceOf[ch.qos.logback.classic.Logger] @@ -26,7 +26,7 @@ class Raphtory[T](dataSource: DataSource[T],graphBuilder: GraphBuilder[T]) { system.actorOf(Props(new WatermarkManager(managerCount = 1)),"WatermarkManager") system.actorOf(Props(new WatchDog(partitionNumber, minimumRouters)), "WatchDog") - system.actorOf(Props(new Spout(dataSource)), "Spout") + system.actorOf(Props(new SpoutAgent(dataSource)), "Spout") system.actorOf(Props(RaphtoryReplicator.apply("Router", 1, 1,graphBuilder)), s"Routers") system.actorOf(Props(RaphtoryReplicator("Partition Manager", 1,1)), s"PartitionManager") diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala index 8345cb93bc..1d10c54add 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala @@ -3,7 +3,7 @@ package com.raphtory.core.components.Router import akka.actor.{Actor, ActorLogging} import akka.cluster.pubsub.{DistributedPubSub, DistributedPubSubMediator} import com.raphtory.core.components.Router.RouterWorker.CommonMessage.TimeBroadcast -import com.raphtory.core.components.Spout.Spout.CommonMessage.{NoWork, SpoutOnline, WorkPlease} +import com.raphtory.core.components.Spout.SpoutAgent.CommonMessage.{NoWork, SpoutOnline, WorkPlease} import com.raphtory.core.model.communication._ import com.raphtory.core.utils.Utils import com.raphtory.core.utils.Utils.getManager diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/DataSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/DataSource.scala deleted file mode 100644 index bd57d4f28a..0000000000 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/DataSource.scala +++ /dev/null @@ -1,15 +0,0 @@ -package com.raphtory.core.components.Spout - - -trait DataSource[+T]{ - private var dataComplete = false - - def setupDataSource():Unit - - def generateData():Option[T] - - def closeDataSource():Unit - - def dataSourceComplete():Unit = dataComplete=true - def isComplete():Boolean = dataComplete -} diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/Spout.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/Spout.scala index 3c90ed1286..fffa09dc3d 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/Spout.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/Spout.scala @@ -1,131 +1,15 @@ package com.raphtory.core.components.Spout -import akka.actor.{Actor, ActorLogging, ActorRef, Cancellable, Timers} -import akka.cluster.pubsub.{DistributedPubSub, DistributedPubSubMediator} -import com.raphtory.core.components.Spout.Spout.CommonMessage._ -import com.raphtory.core.model.communication._ -import com.raphtory.core.utils.Utils -import kamon.Kamon -import scala.concurrent.ExecutionContext -import scala.concurrent.duration._ -import scala.language.postfixOps +trait Spout[+T]{ + private var dataComplete = false + def setupDataSource():Unit + def generateData():Option[T] -class Spout(datasource:DataSource[Any]) extends Actor with ActorLogging with Timers { - // todo: wvv should assign the dispatcher when create the actor - implicit val executionContext: ExecutionContext = context.system.dispatchers.lookup("spout-dispatcher") - //implicit val executionContext: ExecutionContext = context.system.dispatcher + def closeDataSource():Unit - private val spoutTuples = Kamon.counter("Raphtory_Spout_Tuples").withTag("actor", self.path.name) - private var count = 0 - private var partitionManagers = 0 - private var routers = 0 - private var datafinishedSent = false - - private def recordUpdate(): Unit = { - spoutTuples.increment() - count += 1 - } - - private val mediator = DistributedPubSub(context.system).mediator - - override def preStart() { - log.debug("Spout is being started.") - mediator ! DistributedPubSubMediator.Put(self) - context.system.scheduler.scheduleOnce(7 seconds, self, StateCheck) - context.system.scheduler.scheduleOnce(1 seconds, self, IsSafe) - } - - final override def receive: Receive = work(false,1,1) - - - private def work(safe: Boolean, pmCounter:Int, rmCounter:Int): Receive = { - case StateCheck => processStateCheckMessage(safe) - case ClusterStatusResponse(clusterUp,pmCounter,rmCounter) => - context.become(work(clusterUp,pmCounter,rmCounter)) - context.system.scheduler.scheduleOnce(1 second, self, StateCheck) - case IsSafe => processIsSafeMessage(safe,pmCounter,rmCounter) - case WorkPlease => processWorkPlease() - case unhandled => log.error(s"Unable to handle message [$unhandled].") - } - - private def processWorkPlease():Unit = { - if(datasource.isComplete()) - if(!datafinishedSent) { - sender ! DataFinished - println(s"All data sent") - datafinishedSent=true - }else - sender ! NoWork - else - sendData(context.sender()) - } - - private def processStateCheckMessage(safe: Boolean): Unit = { - log.debug(s"Spout is handling [StateCheck] message.") - if (!safe) { - val sendMessage = ClusterStatusRequest() - val sendPath = "/user/WatchDog" - log.debug(s"Sending DPSM message [$sendMessage] to path [$sendPath].") - mediator ! DistributedPubSubMediator.Send(sendPath, sendMessage, localAffinity = false) - } - } - - private def processIsSafeMessage(safe: Boolean,pmCount:Int,roCount:Int): Unit = { - log.debug(s"Spout is handling [IsSafe] message.") - if (safe) { - datasource.setupDataSource() - partitionManagers=pmCount - routers=roCount - Utils.getAllRouterWorkers(roCount).foreach { workerPath => - mediator ! DistributedPubSubMediator.Send( - workerPath, - SpoutOnline, - false - ) - } - println(s"Number of routers: $routers") - println(s"Number of partitions: $partitionManagers") - - } else - context.system.scheduler.scheduleOnce(delay = 1 second, receiver = self, message = IsSafe) - } - - - def sendData(sender:ActorRef): Unit = { - - datasource.generateData() match { - case Some(work) => - val message = if (count % 10000 == 0) - AllocateTrackedTuple(System.currentTimeMillis(),work) - else - AllocateTuple(work) - sender ! message - recordUpdate() - if (count % 10000 == 0) println(s"Spout at Message $count") - case None => sender ! NoWork - }//mediator ! DistributedPubSubMediator.Send(lastRouter, message, localAffinity = false) - if(datasource isComplete) { - sender ! DataFinished - println(s"All data sent") - datafinishedSent=true - } - } - - def AllocateSpoutTask(duration: FiniteDuration, task: Any): Cancellable = { - val taskCancellable = context.system.scheduler.scheduleOnce(duration, self, task) - taskCancellable - } -} - -object Spout { - object CommonMessage { - case object StateCheck - case object IsSafe - case object WorkPlease - case object NoWork - case object SpoutOnline - } + def dataSourceComplete():Unit = dataComplete=true + def isComplete():Boolean = dataComplete } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/SpoutAgent.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/SpoutAgent.scala new file mode 100644 index 0000000000..b334a8e9b8 --- /dev/null +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/SpoutAgent.scala @@ -0,0 +1,131 @@ +package com.raphtory.core.components.Spout + +import akka.actor.{Actor, ActorLogging, ActorRef, Cancellable, Timers} +import akka.cluster.pubsub.{DistributedPubSub, DistributedPubSubMediator} +import com.raphtory.core.components.Spout.SpoutAgent.CommonMessage._ +import com.raphtory.core.model.communication._ +import com.raphtory.core.utils.Utils +import kamon.Kamon + +import scala.concurrent.ExecutionContext +import scala.concurrent.duration._ +import scala.language.postfixOps + + + +class SpoutAgent(datasource:Spout[Any]) extends Actor with ActorLogging with Timers { + // todo: wvv should assign the dispatcher when create the actor + implicit val executionContext: ExecutionContext = context.system.dispatchers.lookup("spout-dispatcher") + //implicit val executionContext: ExecutionContext = context.system.dispatcher + + private val spoutTuples = Kamon.counter("Raphtory_Spout_Tuples").withTag("actor", self.path.name) + private var count = 0 + private var partitionManagers = 0 + private var routers = 0 + private var datafinishedSent = false + + private def recordUpdate(): Unit = { + spoutTuples.increment() + count += 1 + } + + private val mediator = DistributedPubSub(context.system).mediator + + override def preStart() { + log.debug("Spout is being started.") + mediator ! DistributedPubSubMediator.Put(self) + context.system.scheduler.scheduleOnce(7 seconds, self, StateCheck) + context.system.scheduler.scheduleOnce(1 seconds, self, IsSafe) + } + + final override def receive: Receive = work(false,1,1) + + + private def work(safe: Boolean, pmCounter:Int, rmCounter:Int): Receive = { + case StateCheck => processStateCheckMessage(safe) + case ClusterStatusResponse(clusterUp,pmCounter,rmCounter) => + context.become(work(clusterUp,pmCounter,rmCounter)) + context.system.scheduler.scheduleOnce(1 second, self, StateCheck) + case IsSafe => processIsSafeMessage(safe,pmCounter,rmCounter) + case WorkPlease => processWorkPlease() + case unhandled => log.error(s"Unable to handle message [$unhandled].") + } + + private def processWorkPlease():Unit = { + if(datasource.isComplete()) + if(!datafinishedSent) { + sender ! DataFinished + println(s"All data sent") + datafinishedSent=true + }else + sender ! NoWork + else + sendData(context.sender()) + } + + private def processStateCheckMessage(safe: Boolean): Unit = { + log.debug(s"Spout is handling [StateCheck] message.") + if (!safe) { + val sendMessage = ClusterStatusRequest() + val sendPath = "/user/WatchDog" + log.debug(s"Sending DPSM message [$sendMessage] to path [$sendPath].") + mediator ! DistributedPubSubMediator.Send(sendPath, sendMessage, localAffinity = false) + } + } + + private def processIsSafeMessage(safe: Boolean,pmCount:Int,roCount:Int): Unit = { + log.debug(s"Spout is handling [IsSafe] message.") + if (safe) { + datasource.setupDataSource() + partitionManagers=pmCount + routers=roCount + Utils.getAllRouterWorkers(roCount).foreach { workerPath => + mediator ! DistributedPubSubMediator.Send( + workerPath, + SpoutOnline, + false + ) + } + println(s"Number of routers: $routers") + println(s"Number of partitions: $partitionManagers") + + } else + context.system.scheduler.scheduleOnce(delay = 1 second, receiver = self, message = IsSafe) + } + + + def sendData(sender:ActorRef): Unit = { + + datasource.generateData() match { + case Some(work) => + val message = if (count % 10000 == 0) + AllocateTrackedTuple(System.currentTimeMillis(),work) + else + AllocateTuple(work) + sender ! message + recordUpdate() + if (count % 10000 == 0) println(s"Spout at Message $count") + case None => sender ! NoWork + }//mediator ! DistributedPubSubMediator.Send(lastRouter, message, localAffinity = false) + if(datasource isComplete) { + sender ! DataFinished + println(s"All data sent") + datafinishedSent=true + } + } + + def AllocateSpoutTask(duration: FiniteDuration, task: Any): Cancellable = { + val taskCancellable = context.system.scheduler.scheduleOnce(duration, self, task) + taskCancellable + } +} + +object SpoutAgent { + object CommonMessage { + case object StateCheck + case object IsSafe + case object WorkPlease + case object NoWork + case object SpoutOnline + } +} diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/BitcoinGraphBuilder.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/BitcoinGraphBuilder.scala similarity index 97% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/BitcoinGraphBuilder.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/BitcoinGraphBuilder.scala index 6532250a13..e820a064d3 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/BitcoinGraphBuilder.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/BitcoinGraphBuilder.scala @@ -1,8 +1,8 @@ -package com.raphtory.examples.blockchain.routers +package com.raphtory.examples.blockchain.graphbuilders import com.raphtory.core.components.Router.GraphBuilder import com.raphtory.core.model.communication._ -import com.raphtory.sources.blockchain.BitcoinTransaction +import com.raphtory.spouts.blockchain.BitcoinTransaction import spray.json.JsArray class BitcoinGraphBuilder extends GraphBuilder[BitcoinTransaction]{ diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusGraphBuilder.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChABClus2ClusGraphBuilder.scala similarity index 97% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusGraphBuilder.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChABClus2ClusGraphBuilder.scala index 786f8444ad..13af351514 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusGraphBuilder.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChABClus2ClusGraphBuilder.scala @@ -1,4 +1,4 @@ -package com.raphtory.examples.blockchain.routers +package com.raphtory.examples.blockchain.graphbuilders import com.raphtory.core.components.Router.GraphBuilder import com.raphtory.core.model.communication.{Type, _} diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusGraphBuilderNoprop.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChABClus2ClusGraphBuilderNoprop.scala similarity index 97% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusGraphBuilderNoprop.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChABClus2ClusGraphBuilderNoprop.scala index 4255e4cca4..759d0401f2 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusGraphBuilderNoprop.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChABClus2ClusGraphBuilderNoprop.scala @@ -1,4 +1,4 @@ -package com.raphtory.examples.blockchain.routers +package com.raphtory.examples.blockchain.graphbuilders import com.raphtory.core.components.Router.GraphBuilder import com.raphtory.core.model.communication.{Type, _} diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusRouter.scala.save b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChABClus2ClusRouter.scala.save similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChABClus2ClusRouter.scala.save rename to mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChABClus2ClusRouter.scala.save diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChainalysisABGraphBuilder.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChainalysisABGraphBuilder.scala similarity index 97% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChainalysisABGraphBuilder.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChainalysisABGraphBuilder.scala index df0cd860c1..6b6d74351c 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/ChainalysisABGraphBuilder.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChainalysisABGraphBuilder.scala @@ -1,4 +1,4 @@ -package com.raphtory.examples.blockchain.routers +package com.raphtory.examples.blockchain.graphbuilders import com.raphtory.core.components.Router.GraphBuilder import com.raphtory.core.model.communication.{Type, _} diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/DashcoinGraphBuilder.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/DashcoinGraphBuilder.scala similarity index 97% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/DashcoinGraphBuilder.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/DashcoinGraphBuilder.scala index fedf7a418c..7ad4cb7306 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/DashcoinGraphBuilder.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/DashcoinGraphBuilder.scala @@ -1,8 +1,8 @@ -package com.raphtory.examples.blockchain.routers +package com.raphtory.examples.blockchain.graphbuilders import com.raphtory.core.components.Router.GraphBuilder import com.raphtory.core.model.communication._ -import com.raphtory.sources.blockchain.BitcoinTransaction +import com.raphtory.spouts.blockchain.BitcoinTransaction import spray.json.JsArray import scala.util.hashing.MurmurHash3 diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumGethGraphBuilder.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/EthereumGethGraphBuilder.scala similarity index 97% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumGethGraphBuilder.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/EthereumGethGraphBuilder.scala index 1d0196a710..22a6bfb349 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumGethGraphBuilder.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/EthereumGethGraphBuilder.scala @@ -1,4 +1,4 @@ -package com.raphtory.examples.blockchain.routers +package com.raphtory.examples.blockchain.graphbuilders import com.raphtory.core.components.Router.GraphBuilder import com.raphtory.core.model.communication._ diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumKafkaGraphBuilder.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/EthereumKafkaGraphBuilder.scala similarity index 97% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumKafkaGraphBuilder.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/EthereumKafkaGraphBuilder.scala index 24af7c6b97..3ad0387929 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumKafkaGraphBuilder.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/EthereumKafkaGraphBuilder.scala @@ -1,4 +1,4 @@ -package com.raphtory.examples.blockchain.routers +package com.raphtory.examples.blockchain.graphbuilders import com.raphtory.core.components.Router.GraphBuilder import com.raphtory.core.model.communication._ diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumTransactionGraphBuilder.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/EthereumTransactionGraphBuilder.scala similarity index 97% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumTransactionGraphBuilder.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/EthereumTransactionGraphBuilder.scala index 41cd409cec..673e8b6ec1 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/EthereumTransactionGraphBuilder.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/EthereumTransactionGraphBuilder.scala @@ -1,4 +1,4 @@ -package com.raphtory.examples.blockchain.routers +package com.raphtory.examples.blockchain.graphbuilders import com.raphtory.core.components.Router.GraphBuilder import com.raphtory.core.model.communication.{EdgeAddWithProperties, Properties, StringProperty, VertexAddWithProperties} diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/FirehoseKafkaGraphBuilder.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/FirehoseKafkaGraphBuilder.scala similarity index 98% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/FirehoseKafkaGraphBuilder.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/FirehoseKafkaGraphBuilder.scala index 72a87f0b7e..8f94d5863f 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/FirehoseKafkaGraphBuilder.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/FirehoseKafkaGraphBuilder.scala @@ -1,4 +1,4 @@ -package com.raphtory.examples.blockchain.routers +package com.raphtory.examples.blockchain.graphbuilders import com.raphtory.core.components.Router.GraphBuilder import com.raphtory.core.model.communication._ diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/LitecoinGraphBuilder.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/LitecoinGraphBuilder.scala similarity index 97% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/LitecoinGraphBuilder.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/LitecoinGraphBuilder.scala index a334405348..e09ea5827a 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/routers/LitecoinGraphBuilder.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/LitecoinGraphBuilder.scala @@ -1,8 +1,8 @@ -package com.raphtory.examples.blockchain.routers +package com.raphtory.examples.blockchain.graphbuilders import com.raphtory.core.components.Router.GraphBuilder import com.raphtory.core.model.communication._ -import com.raphtory.sources.blockchain.BitcoinTransaction +import com.raphtory.spouts.blockchain.BitcoinTransaction import spray.json.JsArray import scala.util.hashing.MurmurHash3 diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/concirrus/BoatCoOcurrance.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/concirrus/BoatCoOcurrance.scala deleted file mode 100644 index 12efef51e3..0000000000 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/concirrus/BoatCoOcurrance.scala +++ /dev/null @@ -1,35 +0,0 @@ -package com.raphtory.examples.concirrus - -import com.raphtory.core.analysis.API.Analyser - -import scala.collection.mutable.ArrayBuffer - -class BoatCoOcurrance(args: Array[String]) extends Analyser(args) { - - - override def returnResults(): Any = { - view.getVertices().filter(v => v.Type() equals ("Location")).map(loc => { - loc.getIncEdges.map(e => { //We get all incoming edges - val toCompare = loc.getIncEdges.filter(edge => edge != e) //then filter incoming edges to remove itself - e.getHistory().flatMap(f => { - //for each element in the history of the edge - toCompare.filter(edge => //go through all other edges - edge.getHistory() - .exists(p => Math.abs(p._1 - f._1) <= 3600)) //and see if they contain a point +/- an hour - .map(edge => edge.src()) //and if so return their ID - }) - }) - }) - } - - override def processResults(results: ArrayBuffer[Any], timeStamp: Long, viewCompleteTime: Long): Unit = ??? - - - override def setup(): Unit = {} - - override def analyse(): Unit = {} - - - override def defineMaxSteps(): Int = 1 - -} diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/datasources/GabMongoSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/datasources/GabMongoSource.scala index 9551f95cbf..84de1acff8 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/datasources/GabMongoSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/datasources/GabMongoSource.scala @@ -3,13 +3,13 @@ package com.raphtory.examples.gab.datasources import akka.actor.Cancellable import ch.qos.logback.classic.Level import com.mongodb.casbah.Imports.{MongoConnection, _} -import com.raphtory.core.components.Spout.DataSource +import com.raphtory.core.components.Spout.Spout import org.slf4j.LoggerFactory import scala.collection.mutable import scala.language.postfixOps -final class GabMongoDataSource extends DataSource[String] { +final class GabMongoSpout extends Spout[String] { //private val redis = new RedisClient("moe", 6379) //private val redisKey = "gab-posts" diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabPostGraphBuilder.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/graphbuilders/GabPostGraphBuilder.scala similarity index 97% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabPostGraphBuilder.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/gab/graphbuilders/GabPostGraphBuilder.scala index 1b77fb3fe6..bf81b5cf86 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabPostGraphBuilder.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/graphbuilders/GabPostGraphBuilder.scala @@ -1,4 +1,4 @@ -package com.raphtory.examples.gab.routers +package com.raphtory.examples.gab.graphbuilders import java.text.SimpleDateFormat diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabRawGraphBuilder.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/graphbuilders/GabRawGraphBuilder.scala similarity index 98% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabRawGraphBuilder.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/gab/graphbuilders/GabRawGraphBuilder.scala index c4cf81fc1f..c001f69561 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabRawGraphBuilder.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/graphbuilders/GabRawGraphBuilder.scala @@ -1,4 +1,4 @@ -package com.raphtory.examples.gab.routers +package com.raphtory.examples.gab.graphbuilders import java.time.OffsetDateTime diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabUserGraphBuilder.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/graphbuilders/GabUserGraphBuilder.scala similarity index 98% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabUserGraphBuilder.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/gab/graphbuilders/GabUserGraphBuilder.scala index e5c86db133..d85da967e8 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/routers/GabUserGraphBuilder.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/graphbuilders/GabUserGraphBuilder.scala @@ -1,4 +1,4 @@ -package com.raphtory.examples.gab.routers +package com.raphtory.examples.gab.graphbuilders import java.text.SimpleDateFormat diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCGraphBuilder.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/graphbuilders/LDBCGraphBuilder.scala similarity index 97% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCGraphBuilder.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/graphbuilders/LDBCGraphBuilder.scala index 8ab68deab4..1ed0993e2f 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCGraphBuilder.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/graphbuilders/LDBCGraphBuilder.scala @@ -1,4 +1,4 @@ -package com.raphtory.examples.ldbc.routers +package com.raphtory.examples.ldbc.graphbuilders import java.text.SimpleDateFormat diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCOldGraphBuilder.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/graphbuilders/LDBCOldGraphBuilder.scala similarity index 97% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCOldGraphBuilder.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/graphbuilders/LDBCOldGraphBuilder.scala index c82699bdf0..9a026ddec1 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/routers/LDBCOldGraphBuilder.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/graphbuilders/LDBCOldGraphBuilder.scala @@ -1,4 +1,4 @@ -package com.raphtory.examples.ldbc.routers +package com.raphtory.examples.ldbc.graphbuilders import java.text.SimpleDateFormat diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRDeployment.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRDeployment.scala index fc8bc50e94..3f363d516d 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRDeployment.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRDeployment.scala @@ -1,9 +1,9 @@ package com.raphtory.examples.lotr -import com.raphtory.Raphtory +import com.raphtory.RaphtoryGraph object LOTRDeployment extends App{ - val source = new LOTRDataSource() + val source = new LOTRSpout() val builder = new LOTRGraphBuilder() - Raphtory[String](source,builder) + RaphtoryGraph[String](source,builder) } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRDataSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRSpout.scala similarity index 84% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRDataSource.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRSpout.scala index 76601f16fe..98affff7ac 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRDataSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRSpout.scala @@ -1,11 +1,11 @@ package com.raphtory.examples.lotr -import com.raphtory.core.components.Spout.DataSource +import com.raphtory.core.components.Spout.Spout import scala.collection.mutable -class LOTRDataSource extends DataSource[String] { +class LOTRSpout extends Spout[String] { val fileQueue = mutable.Queue[String]() diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomDataSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomSpout.scala similarity index 96% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomDataSource.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomSpout.scala index 49848a0571..b7d160027f 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomDataSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomSpout.scala @@ -1,11 +1,11 @@ package com.raphtory.examples.test.actors -import com.raphtory.core.components.Spout.DataSource +import com.raphtory.core.components.Spout.Spout import scala.language.postfixOps import scala.util.Random -class RandomDataSource extends DataSource[String] { +class RandomSpout extends Spout[String] { var totalCount = 100 var freq = 1000 diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleDataSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleSpout.scala similarity index 79% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleDataSource.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleSpout.scala index f877c90c64..5e5e2171d2 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleDataSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleSpout.scala @@ -1,10 +1,10 @@ package com.raphtory.examples.test.actors -import com.raphtory.core.components.Spout.DataSource +import com.raphtory.core.components.Spout.Spout import scala.collection.mutable.Queue -class TriangleDataSource extends DataSource[String] { +class TriangleSpout extends Spout[String] { val edges = Queue[String]("3,3,1", "4,3,4", "2,2,3", "5,4,1", "6,1,3", "1,1,2") diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/trackAndTrace/routers/TrackAndTraceGraphBuilder.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/trackAndTrace/TrackAndTraceGraphBuilder.scala similarity index 98% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/trackAndTrace/routers/TrackAndTraceGraphBuilder.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/trackAndTrace/TrackAndTraceGraphBuilder.scala index 4938204539..47b5a27bec 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/trackAndTrace/routers/TrackAndTraceGraphBuilder.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/trackAndTrace/TrackAndTraceGraphBuilder.scala @@ -1,4 +1,4 @@ -package com.raphtory.examples.trackAndTrace.routers +package com.raphtory.examples.trackAndTrace import com.raphtory.core.components.Router.GraphBuilder import com.raphtory.core.model.communication.{Type, _} diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixGraphBuilder.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/graphbuilders/CooccurrenceMatrixGraphBuilder.scala similarity index 95% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixGraphBuilder.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/graphbuilders/CooccurrenceMatrixGraphBuilder.scala index 7b5c4ce64f..15ddd258f2 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixGraphBuilder.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/graphbuilders/CooccurrenceMatrixGraphBuilder.scala @@ -1,4 +1,4 @@ -package com.raphtory.examples.wordSemantic.routers +package com.raphtory.examples.wordSemantic.graphbuilders import com.raphtory.core.components.Router.GraphBuilder import com.raphtory.core.model.communication._ diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixGraphBuilderFiltered.scala b/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/graphbuilders/CooccurrenceMatrixGraphBuilderFiltered.scala similarity index 95% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixGraphBuilderFiltered.scala rename to mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/graphbuilders/CooccurrenceMatrixGraphBuilderFiltered.scala index ebd9d105be..3a424840ae 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/routers/CooccurrenceMatrixGraphBuilderFiltered.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/graphbuilders/CooccurrenceMatrixGraphBuilderFiltered.scala @@ -1,4 +1,4 @@ -package com.raphtory.examples.wordSemantic.routers +package com.raphtory.examples.wordSemantic.graphbuilders import com.raphtory.core.components.Router.GraphBuilder import com.raphtory.core.model.communication._ diff --git a/mainproject/cluster/src/main/scala/com/raphtory/sources/FileSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/spouts/FileSpout.scala similarity index 96% rename from mainproject/cluster/src/main/scala/com/raphtory/sources/FileSource.scala rename to mainproject/cluster/src/main/scala/com/raphtory/spouts/FileSpout.scala index 47d874c38c..2db287a58c 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/sources/FileSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/spouts/FileSpout.scala @@ -1,12 +1,12 @@ -package com.raphtory.sources +package com.raphtory.spouts import java.io._ import java.util.zip.GZIPInputStream -import com.raphtory.core.components.Spout.DataSource +import com.raphtory.core.components.Spout.Spout import com.typesafe.scalalogging.LazyLogging -class FileSource extends DataSource[String] { +class FileSpout extends Spout[String] { //TODO work out loggging here //log.info("initialise FileSpout") private val directory = System.getenv().getOrDefault("FILE_SPOUT_DIRECTORY", "/app").trim diff --git a/mainproject/cluster/src/main/scala/com/raphtory/sources/KafkaSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/spouts/KafkaSpout.scala similarity index 96% rename from mainproject/cluster/src/main/scala/com/raphtory/sources/KafkaSource.scala rename to mainproject/cluster/src/main/scala/com/raphtory/spouts/KafkaSpout.scala index 3eaad0dd85..4361676ff6 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/sources/KafkaSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/spouts/KafkaSpout.scala @@ -3,7 +3,7 @@ package com.raphtory.spouts import java.util import java.util.Properties -import com.raphtory.core.components.Spout.DataSource +import com.raphtory.core.components.Spout.Spout import org.apache.kafka.clients.consumer.KafkaConsumer import scala.annotation.tailrec @@ -11,7 +11,7 @@ import scala.collection.JavaConverters._ import scala.collection.mutable import scala.util.Random -final case class KafkaSpout() extends DataSource[String] { +final case class KafkaSpout() extends Spout[String] { private val kafkaServer = System.getenv().getOrDefault("KAFKA_ADDRESS", "127.0.0.1").trim private val kafkaIp = System.getenv().getOrDefault("KAFKA_PORT", "9092").trim private val offset = System.getenv().getOrDefault("KAFKA_OFFSET", "earliest").trim diff --git a/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/BitcoinNodeSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/spouts/blockchain/BitcoinNodeSpout.scala similarity index 94% rename from mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/BitcoinNodeSource.scala rename to mainproject/cluster/src/main/scala/com/raphtory/spouts/blockchain/BitcoinNodeSpout.scala index fb20b5842e..78693514ba 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/BitcoinNodeSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/spouts/blockchain/BitcoinNodeSpout.scala @@ -1,8 +1,8 @@ -package com.raphtory.sources.blockchain +package com.raphtory.spouts.blockchain // import java.io.{File, PrintWriter} -import com.raphtory.core.components.Spout.DataSource +import com.raphtory.core.components.Spout.Spout import scalaj.http.{Http, HttpRequest} import spray.json._ @@ -12,7 +12,7 @@ import scala.sys.process._ // case class BitcoinTransaction(time: JsValue, block: Int, blockID: JsValue, transaction: JsValue) -class BitcoinSpout extends DataSource[BitcoinTransaction]{ +class BitcoinNodeSpout extends Spout[BitcoinTransaction]{ var blockcount = 1 val rpcuser = System.getenv().getOrDefault("BITCOIN_USERNAME", "").trim diff --git a/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumGethSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/spouts/blockchain/EthereumGethSpout.scala similarity index 96% rename from mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumGethSource.scala rename to mainproject/cluster/src/main/scala/com/raphtory/spouts/blockchain/EthereumGethSpout.scala index 104051c0c8..398f306ccb 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumGethSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/spouts/blockchain/EthereumGethSpout.scala @@ -1,9 +1,9 @@ -package com.raphtory.sources.blockchain +package com.raphtory.spouts.blockchain import java.net.InetAddress import java.util.NoSuchElementException -import com.raphtory.core.components.Spout.DataSource +import com.raphtory.core.components.Spout.Spout import com.raphtory.core.utils.Utils import scalaj.http.{Http, HttpRequest} import spray.json.DefaultJsonProtocol._ @@ -14,7 +14,7 @@ import scala.language.postfixOps -class EthereumGethSpout extends DataSource[String] { +class EthereumGethSpout extends Spout[String] { var currentBlock = System.getenv().getOrDefault("SPOUT_ETHEREUM_START_BLOCK_INDEX", "9014194").trim.toInt var highestBlock = System.getenv().getOrDefault("SPOUT_ETHEREUM_MAXIMUM_BLOCK_INDEX", "10026447").trim.toInt diff --git a/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumPostgresSource.scala b/mainproject/cluster/src/main/scala/com/raphtory/spouts/blockchain/EthereumPostgresSpout.scala similarity index 94% rename from mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumPostgresSource.scala rename to mainproject/cluster/src/main/scala/com/raphtory/spouts/blockchain/EthereumPostgresSpout.scala index 0c67b7bb0d..d091fb6be1 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/sources/blockchain/EthereumPostgresSource.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/spouts/blockchain/EthereumPostgresSpout.scala @@ -1,14 +1,14 @@ -package com.raphtory.sources.blockchain +package com.raphtory.spouts.blockchain import cats.effect.{Blocker, IO} -import com.raphtory.core.components.Spout.DataSource +import com.raphtory.core.components.Spout.Spout import doobie.implicits._ import doobie.util.ExecutionContexts import doobie.util.transactor.Transactor import scala.collection.mutable -class EthereumPostgresSource extends DataSource[String]{ +class EthereumPostgresSpout extends Spout[String]{ var startBlock = System.getenv().getOrDefault("STARTING_BLOCK", "46147").trim.toInt //first block to have a transaction by default val batchSize = System.getenv().getOrDefault("BLOCK_BATCH_SIZE", "100").trim.toInt //number of blocks to pull each query val maxblock = System.getenv().getOrDefault("MAX_BLOCK", "8828337").trim.toInt //Maximum block in database to stop querying once this is reached From a4c8bffd0b6470abb93b5e642874af8ecf0e1f80 Mon Sep 17 00:00:00 2001 From: Miratepuffin Date: Sun, 29 Nov 2020 01:07:34 +0000 Subject: [PATCH 40/58] Removed the need to specify a job ID --- mainproject/build.sbt | 4 ++-- .../core/analysis/AnalysisManager.scala | 13 +++++++------ .../core/analysis/AnalysisRestApi.scala | 18 +++++++++--------- .../model/communication/raphtoryMessages.scala | 9 +++------ 4 files changed, 21 insertions(+), 23 deletions(-) diff --git a/mainproject/build.sbt b/mainproject/build.sbt index 3b7774c369..a9835b2dad 100644 --- a/mainproject/build.sbt +++ b/mainproject/build.sbt @@ -124,9 +124,9 @@ lazy val mergeStrategy: String => MergeStrategy = { case "spring.schemas" :: Nil | "spring.handlers" :: Nil | "io.netty.versions.properties" :: Nil => MergeStrategy.filterDistinctLines - case _ => MergeStrategy.deduplicate + case _ => MergeStrategy.discard } - case _ => MergeStrategy.deduplicate + case _ => MergeStrategy.discard } lazy val root = Project(id = "raphtory", base = file(".")) aggregate (cluster) diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/AnalysisManager.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/AnalysisManager.scala index 161c168610..f330ce4894 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/AnalysisManager.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/AnalysisManager.scala @@ -63,9 +63,9 @@ class AnalysisManager() extends Actor{ } def spawnLiveAnalysisManager(request: LiveAnalysisRequest): Unit = { - println(s"Live Analysis Task ${request.jobID} received, running ${request.analyserName}") try { - val jobID = request.jobID + val jobID = request.analyserName+"_"+System.currentTimeMillis().toString + println(s"Live Analysis Task received, your job ID is ${jobID}") val args = request.args val repeatTime = request.repeatTime val eventTime = request.eventTime @@ -90,9 +90,10 @@ class AnalysisManager() extends Actor{ } } def spawnViewAnalysisManager(request: ViewAnalysisRequest): Unit = { - println(s"View Analysis Task ${request.jobID} received, running ${request.analyserName} at time ${request.timestamp}") + try{ - val jobID = request.jobID + val jobID = request.analyserName+"_"+System.currentTimeMillis().toString + println(s"View Analysis Task received, your job ID is ${jobID}") val timestamp = request.timestamp val args = request.args val buildAnalyser = getAnalyser(request.analyserName,args,request.rawFile) @@ -124,9 +125,9 @@ class AnalysisManager() extends Actor{ } def spawnRangeAnalysisManager(request: RangeAnalysisRequest): Unit = { - println(s"Range Analysis Task ${request.jobID} received, running ${request.analyserName}, between ${request.start} and ${request.end} jumping ${request.jump} at a time.") try{ - val jobID = request.jobID + val jobID = request.analyserName+"_"+System.currentTimeMillis().toString + println(s"Range Analysis Task received, your job ID is ${jobID}, running ${request.analyserName}, between ${request.start} and ${request.end} jumping ${request.jump} at a time.") val start = request.start val end = request.end val jump = request.jump diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/AnalysisRestApi.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/AnalysisRestApi.scala index 398413e645..2863251616 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/AnalysisRestApi.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/AnalysisRestApi.scala @@ -31,32 +31,32 @@ case class AnalysisRestApi(system:ActorSystem){ //Submit Analysis case HttpRequest(POST,Uri.Path("/LiveAnalysisRequest"),_,entity,_) => { try{ - implicit val LiveAnalysisFormat = jsonFormat9(LiveAnalysisPOST) + implicit val LiveAnalysisFormat = jsonFormat8(LiveAnalysisPOST) val in:LiveAnalysisPOST = Await.result(Unmarshal(entity).to[LiveAnalysisPOST], 10.second) - val response = LiveAnalysisRequest(in.jobID,in.analyserName,in.repeatTime.getOrElse(0),in.eventTime.getOrElse(false),in.windowType.getOrElse("false"),in.windowSize.getOrElse(0),in.windowSet.getOrElse(Array()),in.args.getOrElse(Array()),in.rawFile.getOrElse("")) + val response = LiveAnalysisRequest(in.analyserName,in.repeatTime.getOrElse(0),in.eventTime.getOrElse(false),in.windowType.getOrElse("false"),in.windowSize.getOrElse(0),in.windowSet.getOrElse(Array()),in.args.getOrElse(Array()),in.rawFile.getOrElse("")) mediator ! DistributedPubSubMediator.Send("/user/AnalysisManager", response, false) - HttpResponse(entity = s"""Your Task ${in.jobID} Has been successfully submitted as a Live Analysis Task!""") + HttpResponse(entity = s"""Your Task Has been successfully submitted as a Live Analysis Task!""") } catch { case e:Exception => e.printStackTrace();HttpResponse(entity = "Your Task Appeared to have some issue, please check your JSON and resubmit")} } case HttpRequest(POST,Uri.Path("/ViewAnalysisRequest"),_,entity,_) => { try{ - implicit val viewAnalysisPOST = jsonFormat8(ViewAnalysisPOST) + implicit val viewAnalysisPOST = jsonFormat7(ViewAnalysisPOST) val in:ViewAnalysisPOST = Await.result(Unmarshal(entity).to[ViewAnalysisPOST], 100 .second) - val response = ViewAnalysisRequest(in.jobID,in.analyserName,in.timestamp,in.windowType.getOrElse("false"),in.windowSize.getOrElse(0),in.windowSet.getOrElse(Array()),in.args.getOrElse(Array()),in.rawFile.getOrElse("")) + val response = ViewAnalysisRequest(in.analyserName,in.timestamp,in.windowType.getOrElse("false"),in.windowSize.getOrElse(0),in.windowSet.getOrElse(Array()),in.args.getOrElse(Array()),in.rawFile.getOrElse("")) mediator ! DistributedPubSubMediator.Send("/user/AnalysisManager", response, false) - HttpResponse(entity = s"""Your Task ${in.jobID} Has been successfully submitted as a View Analysis Task!""") + HttpResponse(entity = s"""Your Task Has been successfully submitted as a View Analysis Task!""") } catch {case e:Exception => e.printStackTrace();HttpResponse(entity = "Your Task Appeared to have some issue, please check your JSON and resubmit")} } case HttpRequest(POST,Uri.Path("/RangeAnalysisRequest"),_,entity,_) => { try{ - implicit val rangeAnalysisPOST = jsonFormat10(RangeAnalysisPOST) + implicit val rangeAnalysisPOST = jsonFormat9(RangeAnalysisPOST) val in:RangeAnalysisPOST = Await.result(Unmarshal(entity).to[RangeAnalysisPOST], 10.second) - val response = RangeAnalysisRequest(in.jobID,in.analyserName,in.start,in.end,in.jump,in.windowType.getOrElse("false"),in.windowSize.getOrElse(0),in.windowSet.getOrElse(Array()),in.args.getOrElse(Array()),in.rawFile.getOrElse("")) + val response = RangeAnalysisRequest(in.analyserName,in.start,in.end,in.jump,in.windowType.getOrElse("false"),in.windowSize.getOrElse(0),in.windowSet.getOrElse(Array()),in.args.getOrElse(Array()),in.rawFile.getOrElse("")) mediator ! DistributedPubSubMediator.Send("/user/AnalysisManager", response, false) - HttpResponse(entity = s"""Your Task ${in.jobID} Has been successfully submitted as a Range Analysis Task!""") + HttpResponse(entity = s"""Your Task Has been successfully submitted as a Range Analysis Task!""") } catch {case e:Exception => e.printStackTrace();HttpResponse(entity = "Your Task Appeared to have some issue, please check your JSON and resubmit")} } diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/model/communication/raphtoryMessages.scala b/mainproject/cluster/src/main/scala/com/raphtory/core/model/communication/raphtoryMessages.scala index f7af56719e..483dbffb13 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/model/communication/raphtoryMessages.scala +++ b/mainproject/cluster/src/main/scala/com/raphtory/core/model/communication/raphtoryMessages.scala @@ -115,12 +115,11 @@ case class ReaderWorkersACK() extends RaphReadClasses -case class LiveAnalysisPOST(jobID:String, analyserName:String, windowType:Option[String], windowSize:Option[Long], windowSet:Option[Array[Long]],repeatTime:Option[Long],eventTime:Option[Boolean],args:Option[Array[String]],rawFile:Option[String]) -case class ViewAnalysisPOST(jobID:String,analyserName:String,timestamp:Long,windowType:Option[String],windowSize:Option[Long],windowSet:Option[Array[Long]],args:Option[Array[String]],rawFile:Option[String]) -case class RangeAnalysisPOST(jobID:String,analyserName:String,start:Long,end:Long,jump:Long,windowType:Option[String],windowSize:Option[Long],windowSet:Option[Array[Long]],args:Option[Array[String]],rawFile:Option[String]) +case class LiveAnalysisPOST(analyserName:String, windowType:Option[String], windowSize:Option[Long], windowSet:Option[Array[Long]],repeatTime:Option[Long],eventTime:Option[Boolean],args:Option[Array[String]],rawFile:Option[String]) +case class ViewAnalysisPOST(analyserName:String,timestamp:Long,windowType:Option[String],windowSize:Option[Long],windowSet:Option[Array[Long]],args:Option[Array[String]],rawFile:Option[String]) +case class RangeAnalysisPOST(analyserName:String,start:Long,end:Long,jump:Long,windowType:Option[String],windowSize:Option[Long],windowSet:Option[Array[Long]],args:Option[Array[String]],rawFile:Option[String]) trait AnalysisRequest case class LiveAnalysisRequest( - jobID: String, analyserName: String, repeatTime:Long =0L, eventTime:Boolean=false, @@ -131,7 +130,6 @@ case class LiveAnalysisRequest( rawFile:String="" ) extends AnalysisRequest case class ViewAnalysisRequest( - jobID: String, analyserName: String, timestamp: Long, windowType: String = "false", @@ -141,7 +139,6 @@ case class ViewAnalysisRequest( rawFile:String="" ) extends AnalysisRequest case class RangeAnalysisRequest( - jobID: String, analyserName: String, start: Long, end: Long, From 4a1f7e01ca963e2880f581a73445b574a08f10fb Mon Sep 17 00:00:00 2001 From: Miratepuffin Date: Sun, 29 Nov 2020 19:05:15 +0000 Subject: [PATCH 41/58] Moved src to top level --- mainproject/.jvmopts | 5 +++++ mainproject/build.sbt | 8 ++++---- mainproject/{cluster => }/src/main/main.iml | 0 .../{cluster => }/src/main/resources/application.conf | 0 .../src/main/scala/com/raphtory/RaphtoryGraph.scala | 4 ++-- .../src/main/scala/com/raphtory/Server.scala | 0 .../scala/com/raphtory/core/analysis/API/Analyser.scala | 0 .../core/analysis/API/GraphLenses/GraphLens.scala | 0 .../raphtory/core/analysis/API/GraphLenses/ViewLens.scala | 0 .../core/analysis/API/GraphLenses/WindowLens.scala | 0 .../core/analysis/API/entityVisitors/EdgeVisitor.scala | 0 .../core/analysis/API/entityVisitors/EntityVisitor.scala | 0 .../core/analysis/API/entityVisitors/VertexVisitor.scala | 0 .../core/analysis/Algorithms/BinaryDefusion.scala | 0 .../analysis/Algorithms/CommunityOutlierDetection.scala | 0 .../core/analysis/Algorithms/ConnectedComponents.scala | 0 .../raphtory/core/analysis/Algorithms/DegreeBasic.scala | 0 .../core/analysis/Algorithms/DegreeDistribution.scala | 0 .../raphtory/core/analysis/Algorithms/DegreeRanking.scala | 0 .../raphtory/core/analysis/Algorithms/Distributions.scala | 0 .../core/analysis/Algorithms/GabWeightedPageRank.scala | 0 .../scala/com/raphtory/core/analysis/Algorithms/LPA.scala | 0 .../raphtory/core/analysis/Algorithms/MultiLayerLPA.scala | 0 .../com/raphtory/core/analysis/Algorithms/PageRank.scala | 0 .../core/analysis/Algorithms/TemporalTriangleCount.scala | 0 .../raphtory/core/analysis/Algorithms/TriangleCount.scala | 0 .../raphtory/core/analysis/Algorithms/WattsCascade.scala | 0 .../core/analysis/Algorithms/WeightedPageRank.scala | 0 .../raphtory/core/analysis/Algorithms/lotrExample.scala | 0 .../com/raphtory/core/analysis/AnalysisManager.scala | 0 .../com/raphtory/core/analysis/AnalysisRestApi.scala | 0 .../com/raphtory/core/analysis/Tasks/AnalysisTask.scala | 0 .../Tasks/LiveTasks/BWindowedLiveAnalysisTask.scala | 0 .../core/analysis/Tasks/LiveTasks/LiveAnalysisTask.scala | 0 .../Tasks/LiveTasks/WindowedLiveAnalysisTask.scala | 0 .../Tasks/RangeTasks/BWindowedRangeAnalysisTask.scala | 0 .../analysis/Tasks/RangeTasks/RangeAnalysisTask.scala | 0 .../Tasks/RangeTasks/WindowedRangeAnalysisTask.scala | 0 .../Tasks/ViewTasks/BWindowedViewAnalysisTask.scala | 0 .../core/analysis/Tasks/ViewTasks/ViewAnalysisTask.scala | 0 .../Tasks/ViewTasks/WindowedViewAnalysisTask.scala | 0 .../core/analysis/serialisers/JSONSerialiser.scala | 0 .../core/analysis/serialisers/NetworkXSerialiser.scala | 0 .../raphtory/core/analysis/serialisers/S3Serialiser.scala | 0 .../raphtory/core/analysis/serialisers/Serialiser.scala | 0 .../scala/com/raphtory/core/clustersetup/DocSvr.scala | 0 .../com/raphtory/core/clustersetup/LiveAnalysisNode.scala | 1 - .../com/raphtory/core/clustersetup/ManagerNode.scala | 0 .../scala/com/raphtory/core/clustersetup/RouterNode.scala | 0 .../scala/com/raphtory/core/clustersetup/SeedNode.scala | 0 .../scala/com/raphtory/core/clustersetup/UpdateNode.scala | 0 .../com/raphtory/core/clustersetup/WatchDogNode.scala | 0 .../core/clustersetup/singlenode/SingleNodeSetup.scala | 0 .../com/raphtory/core/clustersetup/util/ConfigUtils.scala | 0 .../components/ClusterManagement/RaphtoryReplicator.scala | 0 .../core/components/ClusterManagement/SeedActor.scala | 0 .../core/components/ClusterManagement/WatchDog.scala | 0 .../components/ClusterManagement/WatermarkManager.scala | 0 .../core/components/PartitionManager/Archivist.scala | 0 .../core/components/PartitionManager/Reader.scala | 0 .../PartitionManager/Workers/ArchivistWorker.scala | 0 .../PartitionManager/Workers/IngestionWorker.scala | 0 .../PartitionManager/Workers/ReaderWorker.scala | 0 .../core/components/PartitionManager/Writer.scala | 0 .../raphtory/core/components/Router/GraphBuilder.scala | 0 .../raphtory/core/components/Router/RouterManager.scala | 0 .../raphtory/core/components/Router/RouterWorker.scala | 0 .../scala/com/raphtory/core/components/Spout/Spout.scala | 0 .../com/raphtory/core/components/Spout/SpoutAgent.scala | 0 .../raphtory/core/model/communication/AnalysisType.scala | 0 .../core/model/communication/VertexMutliQueue.scala | 0 .../core/model/communication/raphtoryMessages.scala | 0 .../com/raphtory/core/model/graphentities/Edge.scala | 0 .../com/raphtory/core/model/graphentities/Entity.scala | 0 .../core/model/graphentities/ImmutableProperty.scala | 0 .../core/model/graphentities/MutableProperty.scala | 0 .../com/raphtory/core/model/graphentities/Property.scala | 0 .../com/raphtory/core/model/graphentities/SplitEdge.scala | 0 .../com/raphtory/core/model/graphentities/Vertex.scala | 0 .../scala/com/raphtory/core/storage/EntityStorage.scala | 0 .../scala/com/raphtory/core/utils/HistoryOrdering.scala | 0 .../scala/com/raphtory/core/utils/SchedulerUtil.scala | 0 .../src/main/scala/com/raphtory/core/utils/Utils.scala | 0 .../blockchain/analysers/EthereumDegreeRanking.scala | 0 .../blockchain/analysers/EthereumTaintTracking.scala | 0 .../blockchain/analysers/TaintTrackExchangeStop.scala | 0 .../blockchain/graphbuilders/BitcoinGraphBuilder.scala | 0 .../graphbuilders/ChABClus2ClusGraphBuilder.scala | 0 .../graphbuilders/ChABClus2ClusGraphBuilderNoprop.scala | 0 .../graphbuilders/ChABClus2ClusRouter.scala.save | 0 .../graphbuilders/ChainalysisABGraphBuilder.scala | 0 .../blockchain/graphbuilders/DashcoinGraphBuilder.scala | 0 .../graphbuilders/EthereumGethGraphBuilder.scala | 0 .../graphbuilders/EthereumKafkaGraphBuilder.scala | 0 .../graphbuilders/EthereumTransactionGraphBuilder.scala | 0 .../graphbuilders/FirehoseKafkaGraphBuilder.scala | 0 .../blockchain/graphbuilders/LitecoinGraphBuilder.scala | 0 .../examples/citationNetwork/CitationGraphBuilder.scala | 0 .../examples/gab/analysis/GabMostUsedTopics.scala | 0 .../examples/gab/datasources/GabMongoSource.scala | 0 .../examples/gab/graphbuilders/GabPostGraphBuilder.scala | 0 .../examples/gab/graphbuilders/GabRawGraphBuilder.scala | 0 .../examples/gab/graphbuilders/GabUserGraphBuilder.scala | 0 .../examples/gab/rawgraphmodel/GabEntityType.scala | 0 .../examples/gab/rawgraphmodel/GabJsonProtocol.scala | 0 .../com/raphtory/examples/gab/rawgraphmodel/GabPost.scala | 0 .../examples/ldbc/graphbuilders/LDBCGraphBuilder.scala | 0 .../examples/ldbc/graphbuilders/LDBCOldGraphBuilder.scala | 0 .../scala/com/raphtory/examples/lotr/LOTRDeployment.scala | 0 .../com/raphtory/examples/lotr/LOTRGraphBuilder.scala | 0 .../main/scala/com/raphtory/examples/lotr/LOTRSpout.scala | 0 .../src/main/scala/com/raphtory/examples/lotr/lotr.csv | 0 .../examples/test/actors/RandomGraphBuilder.scala | 0 .../com/raphtory/examples/test/actors/RandomSpout.scala | 0 .../examples/test/actors/TriangleGraphBuilder.scala | 0 .../com/raphtory/examples/test/actors/TriangleSpout.scala | 0 .../trackAndTrace/TrackAndTraceGraphBuilder.scala | 0 .../scala/com/raphtory/examples/tsvnet/SamplerSpout.scala | 0 .../com/raphtory/examples/tsvnet/TSVGraphBuilder.scala | 0 .../graphbuilders/CooccurrenceMatrixGraphBuilder.scala | 0 .../CooccurrenceMatrixGraphBuilderFiltered.scala | 0 .../wordSemantic/spouts/CooccurrenceMatrixSpout.scala | 0 .../spouts/CooccurrenceMatrixSpoutFiltered.scala | 0 .../src/main/scala/com/raphtory/spouts/FileSpout.scala | 0 .../src/main/scala/com/raphtory/spouts/KafkaSpout.scala | 0 .../com/raphtory/spouts/blockchain/BitcoinNodeSpout.scala | 0 .../raphtory/spouts/blockchain/EthereumGethSpout.scala | 0 .../spouts/blockchain/EthereumPostgresSpout.scala | 0 128 files changed, 11 insertions(+), 7 deletions(-) create mode 100644 mainproject/.jvmopts rename mainproject/{cluster => }/src/main/main.iml (100%) rename mainproject/{cluster => }/src/main/resources/application.conf (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/RaphtoryGraph.scala (89%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/Server.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/analysis/API/Analyser.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/analysis/API/GraphLenses/GraphLens.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/analysis/API/GraphLenses/ViewLens.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/analysis/API/GraphLenses/WindowLens.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/EdgeVisitor.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/EntityVisitor.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/VertexVisitor.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/analysis/Algorithms/BinaryDefusion.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/analysis/Algorithms/CommunityOutlierDetection.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/analysis/Algorithms/ConnectedComponents.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/analysis/Algorithms/DegreeBasic.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/analysis/Algorithms/DegreeDistribution.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/analysis/Algorithms/DegreeRanking.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/analysis/Algorithms/Distributions.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/analysis/Algorithms/GabWeightedPageRank.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/analysis/Algorithms/LPA.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/analysis/Algorithms/MultiLayerLPA.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/analysis/Algorithms/PageRank.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/analysis/Algorithms/TemporalTriangleCount.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/analysis/Algorithms/TriangleCount.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/analysis/Algorithms/WattsCascade.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/analysis/Algorithms/WeightedPageRank.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/analysis/Algorithms/lotrExample.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/analysis/AnalysisManager.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/analysis/AnalysisRestApi.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/analysis/Tasks/AnalysisTask.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/analysis/Tasks/LiveTasks/BWindowedLiveAnalysisTask.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/analysis/Tasks/LiveTasks/LiveAnalysisTask.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/analysis/Tasks/LiveTasks/WindowedLiveAnalysisTask.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/analysis/Tasks/RangeTasks/BWindowedRangeAnalysisTask.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/analysis/Tasks/RangeTasks/RangeAnalysisTask.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/analysis/Tasks/RangeTasks/WindowedRangeAnalysisTask.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/analysis/Tasks/ViewTasks/BWindowedViewAnalysisTask.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/analysis/Tasks/ViewTasks/ViewAnalysisTask.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/analysis/Tasks/ViewTasks/WindowedViewAnalysisTask.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/analysis/serialisers/JSONSerialiser.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/analysis/serialisers/NetworkXSerialiser.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/analysis/serialisers/S3Serialiser.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/analysis/serialisers/Serialiser.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/clustersetup/DocSvr.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/clustersetup/LiveAnalysisNode.scala (99%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/clustersetup/ManagerNode.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/clustersetup/RouterNode.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/clustersetup/SeedNode.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/clustersetup/UpdateNode.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/clustersetup/WatchDogNode.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/clustersetup/singlenode/SingleNodeSetup.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/clustersetup/util/ConfigUtils.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/components/ClusterManagement/RaphtoryReplicator.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/components/ClusterManagement/SeedActor.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/components/ClusterManagement/WatchDog.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/components/ClusterManagement/WatermarkManager.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/components/PartitionManager/Archivist.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/components/PartitionManager/Reader.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/ArchivistWorker.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/IngestionWorker.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/ReaderWorker.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/components/PartitionManager/Writer.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/components/Router/GraphBuilder.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/components/Router/RouterManager.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/components/Spout/Spout.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/components/Spout/SpoutAgent.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/model/communication/AnalysisType.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/model/communication/VertexMutliQueue.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/model/communication/raphtoryMessages.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/model/graphentities/Edge.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/model/graphentities/Entity.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/model/graphentities/ImmutableProperty.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/model/graphentities/MutableProperty.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/model/graphentities/Property.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/model/graphentities/SplitEdge.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/model/graphentities/Vertex.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/storage/EntityStorage.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/utils/HistoryOrdering.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/utils/SchedulerUtil.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/core/utils/Utils.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/examples/blockchain/analysers/EthereumDegreeRanking.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/examples/blockchain/analysers/EthereumTaintTracking.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/examples/blockchain/analysers/TaintTrackExchangeStop.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/BitcoinGraphBuilder.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChABClus2ClusGraphBuilder.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChABClus2ClusGraphBuilderNoprop.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChABClus2ClusRouter.scala.save (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChainalysisABGraphBuilder.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/DashcoinGraphBuilder.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/EthereumGethGraphBuilder.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/EthereumKafkaGraphBuilder.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/EthereumTransactionGraphBuilder.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/FirehoseKafkaGraphBuilder.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/LitecoinGraphBuilder.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/examples/citationNetwork/CitationGraphBuilder.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/examples/gab/analysis/GabMostUsedTopics.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/examples/gab/datasources/GabMongoSource.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/examples/gab/graphbuilders/GabPostGraphBuilder.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/examples/gab/graphbuilders/GabRawGraphBuilder.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/examples/gab/graphbuilders/GabUserGraphBuilder.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/examples/gab/rawgraphmodel/GabEntityType.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/examples/gab/rawgraphmodel/GabJsonProtocol.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/examples/gab/rawgraphmodel/GabPost.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/examples/ldbc/graphbuilders/LDBCGraphBuilder.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/examples/ldbc/graphbuilders/LDBCOldGraphBuilder.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/examples/lotr/LOTRDeployment.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/examples/lotr/LOTRGraphBuilder.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/examples/lotr/LOTRSpout.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/examples/lotr/lotr.csv (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/examples/test/actors/RandomGraphBuilder.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/examples/test/actors/RandomSpout.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/examples/test/actors/TriangleGraphBuilder.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/examples/test/actors/TriangleSpout.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/examples/trackAndTrace/TrackAndTraceGraphBuilder.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/examples/tsvnet/SamplerSpout.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/examples/tsvnet/TSVGraphBuilder.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/examples/wordSemantic/graphbuilders/CooccurrenceMatrixGraphBuilder.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/examples/wordSemantic/graphbuilders/CooccurrenceMatrixGraphBuilderFiltered.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/examples/wordSemantic/spouts/CooccurrenceMatrixSpout.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/examples/wordSemantic/spouts/CooccurrenceMatrixSpoutFiltered.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/spouts/FileSpout.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/spouts/KafkaSpout.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/spouts/blockchain/BitcoinNodeSpout.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/spouts/blockchain/EthereumGethSpout.scala (100%) rename mainproject/{cluster => }/src/main/scala/com/raphtory/spouts/blockchain/EthereumPostgresSpout.scala (100%) diff --git a/mainproject/.jvmopts b/mainproject/.jvmopts new file mode 100644 index 0000000000..90051c88c2 --- /dev/null +++ b/mainproject/.jvmopts @@ -0,0 +1,5 @@ + +-Xms512M +-Xmx4096M +-Xss2M +-XX:MaxMetaspaceSize=1024M diff --git a/mainproject/build.sbt b/mainproject/build.sbt index a9835b2dad..4475ef105d 100644 --- a/mainproject/build.sbt +++ b/mainproject/build.sbt @@ -99,7 +99,7 @@ lazy val basicSettings = Seq( lazy val dockerStuff = Seq( maintainer := "Imane Hafnaoui ", dockerBaseImage := "miratepuffin/raphtory-redis:latest", - dockerRepository := Some("niallroche"), + dockerRepository := Some("miratepuffin"), dockerExposedPorts := Seq(2551, 8080, 2552, 1600, 11600,8081,46339,9100), ) @@ -124,15 +124,15 @@ lazy val mergeStrategy: String => MergeStrategy = { case "spring.schemas" :: Nil | "spring.handlers" :: Nil | "io.netty.versions.properties" :: Nil => MergeStrategy.filterDistinctLines - case _ => MergeStrategy.discard + case _ => MergeStrategy.first } - case _ => MergeStrategy.discard + case _ => MergeStrategy.first } lazy val root = Project(id = "raphtory", base = file(".")) aggregate (cluster) lazy val cluster = project - .in(file("cluster")) + .in(file(".")) .enablePlugins(JavaAppPackaging) .enablePlugins(AshScriptPlugin) .enablePlugins(JavaAgent) diff --git a/mainproject/cluster/src/main/main.iml b/mainproject/src/main/main.iml similarity index 100% rename from mainproject/cluster/src/main/main.iml rename to mainproject/src/main/main.iml diff --git a/mainproject/cluster/src/main/resources/application.conf b/mainproject/src/main/resources/application.conf similarity index 100% rename from mainproject/cluster/src/main/resources/application.conf rename to mainproject/src/main/resources/application.conf diff --git a/mainproject/cluster/src/main/scala/com/raphtory/RaphtoryGraph.scala b/mainproject/src/main/scala/com/raphtory/RaphtoryGraph.scala similarity index 89% rename from mainproject/cluster/src/main/scala/com/raphtory/RaphtoryGraph.scala rename to mainproject/src/main/scala/com/raphtory/RaphtoryGraph.scala index b1378a84ee..0ebc10a189 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/RaphtoryGraph.scala +++ b/mainproject/src/main/scala/com/raphtory/RaphtoryGraph.scala @@ -17,8 +17,8 @@ object RaphtoryGraph { class RaphtoryGraph[T](dataSource: Spout[T], graphBuilder: GraphBuilder[T]) { Kamon.init() //start tool logging - val root = LoggerFactory.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME).asInstanceOf[ch.qos.logback.classic.Logger] - root.setLevel(Level.ERROR) +// val root = LoggerFactory.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME).asInstanceOf[ch.qos.logback.classic.Logger] + //root.setLevel(Level.ERROR) val system = ActorSystem("Citation-system") val partitionNumber = 1 diff --git a/mainproject/cluster/src/main/scala/com/raphtory/Server.scala b/mainproject/src/main/scala/com/raphtory/Server.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/Server.scala rename to mainproject/src/main/scala/com/raphtory/Server.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/Analyser.scala b/mainproject/src/main/scala/com/raphtory/core/analysis/API/Analyser.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/Analyser.scala rename to mainproject/src/main/scala/com/raphtory/core/analysis/API/Analyser.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/GraphLenses/GraphLens.scala b/mainproject/src/main/scala/com/raphtory/core/analysis/API/GraphLenses/GraphLens.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/GraphLenses/GraphLens.scala rename to mainproject/src/main/scala/com/raphtory/core/analysis/API/GraphLenses/GraphLens.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/GraphLenses/ViewLens.scala b/mainproject/src/main/scala/com/raphtory/core/analysis/API/GraphLenses/ViewLens.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/GraphLenses/ViewLens.scala rename to mainproject/src/main/scala/com/raphtory/core/analysis/API/GraphLenses/ViewLens.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/GraphLenses/WindowLens.scala b/mainproject/src/main/scala/com/raphtory/core/analysis/API/GraphLenses/WindowLens.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/GraphLenses/WindowLens.scala rename to mainproject/src/main/scala/com/raphtory/core/analysis/API/GraphLenses/WindowLens.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/EdgeVisitor.scala b/mainproject/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/EdgeVisitor.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/EdgeVisitor.scala rename to mainproject/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/EdgeVisitor.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/EntityVisitor.scala b/mainproject/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/EntityVisitor.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/EntityVisitor.scala rename to mainproject/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/EntityVisitor.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/VertexVisitor.scala b/mainproject/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/VertexVisitor.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/VertexVisitor.scala rename to mainproject/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/VertexVisitor.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/BinaryDefusion.scala b/mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/BinaryDefusion.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/BinaryDefusion.scala rename to mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/BinaryDefusion.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/CommunityOutlierDetection.scala b/mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/CommunityOutlierDetection.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/CommunityOutlierDetection.scala rename to mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/CommunityOutlierDetection.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/ConnectedComponents.scala b/mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/ConnectedComponents.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/ConnectedComponents.scala rename to mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/ConnectedComponents.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/DegreeBasic.scala b/mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/DegreeBasic.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/DegreeBasic.scala rename to mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/DegreeBasic.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/DegreeDistribution.scala b/mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/DegreeDistribution.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/DegreeDistribution.scala rename to mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/DegreeDistribution.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/DegreeRanking.scala b/mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/DegreeRanking.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/DegreeRanking.scala rename to mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/DegreeRanking.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/Distributions.scala b/mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/Distributions.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/Distributions.scala rename to mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/Distributions.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/GabWeightedPageRank.scala b/mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/GabWeightedPageRank.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/GabWeightedPageRank.scala rename to mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/GabWeightedPageRank.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/LPA.scala b/mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/LPA.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/LPA.scala rename to mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/LPA.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/MultiLayerLPA.scala b/mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/MultiLayerLPA.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/MultiLayerLPA.scala rename to mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/MultiLayerLPA.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/PageRank.scala b/mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/PageRank.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/PageRank.scala rename to mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/PageRank.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/TemporalTriangleCount.scala b/mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/TemporalTriangleCount.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/TemporalTriangleCount.scala rename to mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/TemporalTriangleCount.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/TriangleCount.scala b/mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/TriangleCount.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/TriangleCount.scala rename to mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/TriangleCount.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/WattsCascade.scala b/mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/WattsCascade.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/WattsCascade.scala rename to mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/WattsCascade.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/WeightedPageRank.scala b/mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/WeightedPageRank.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/WeightedPageRank.scala rename to mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/WeightedPageRank.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/lotrExample.scala b/mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/lotrExample.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Algorithms/lotrExample.scala rename to mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/lotrExample.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/AnalysisManager.scala b/mainproject/src/main/scala/com/raphtory/core/analysis/AnalysisManager.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/analysis/AnalysisManager.scala rename to mainproject/src/main/scala/com/raphtory/core/analysis/AnalysisManager.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/AnalysisRestApi.scala b/mainproject/src/main/scala/com/raphtory/core/analysis/AnalysisRestApi.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/analysis/AnalysisRestApi.scala rename to mainproject/src/main/scala/com/raphtory/core/analysis/AnalysisRestApi.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/AnalysisTask.scala b/mainproject/src/main/scala/com/raphtory/core/analysis/Tasks/AnalysisTask.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/AnalysisTask.scala rename to mainproject/src/main/scala/com/raphtory/core/analysis/Tasks/AnalysisTask.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/LiveTasks/BWindowedLiveAnalysisTask.scala b/mainproject/src/main/scala/com/raphtory/core/analysis/Tasks/LiveTasks/BWindowedLiveAnalysisTask.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/LiveTasks/BWindowedLiveAnalysisTask.scala rename to mainproject/src/main/scala/com/raphtory/core/analysis/Tasks/LiveTasks/BWindowedLiveAnalysisTask.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/LiveTasks/LiveAnalysisTask.scala b/mainproject/src/main/scala/com/raphtory/core/analysis/Tasks/LiveTasks/LiveAnalysisTask.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/LiveTasks/LiveAnalysisTask.scala rename to mainproject/src/main/scala/com/raphtory/core/analysis/Tasks/LiveTasks/LiveAnalysisTask.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/LiveTasks/WindowedLiveAnalysisTask.scala b/mainproject/src/main/scala/com/raphtory/core/analysis/Tasks/LiveTasks/WindowedLiveAnalysisTask.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/LiveTasks/WindowedLiveAnalysisTask.scala rename to mainproject/src/main/scala/com/raphtory/core/analysis/Tasks/LiveTasks/WindowedLiveAnalysisTask.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/RangeTasks/BWindowedRangeAnalysisTask.scala b/mainproject/src/main/scala/com/raphtory/core/analysis/Tasks/RangeTasks/BWindowedRangeAnalysisTask.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/RangeTasks/BWindowedRangeAnalysisTask.scala rename to mainproject/src/main/scala/com/raphtory/core/analysis/Tasks/RangeTasks/BWindowedRangeAnalysisTask.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/RangeTasks/RangeAnalysisTask.scala b/mainproject/src/main/scala/com/raphtory/core/analysis/Tasks/RangeTasks/RangeAnalysisTask.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/RangeTasks/RangeAnalysisTask.scala rename to mainproject/src/main/scala/com/raphtory/core/analysis/Tasks/RangeTasks/RangeAnalysisTask.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/RangeTasks/WindowedRangeAnalysisTask.scala b/mainproject/src/main/scala/com/raphtory/core/analysis/Tasks/RangeTasks/WindowedRangeAnalysisTask.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/RangeTasks/WindowedRangeAnalysisTask.scala rename to mainproject/src/main/scala/com/raphtory/core/analysis/Tasks/RangeTasks/WindowedRangeAnalysisTask.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/ViewTasks/BWindowedViewAnalysisTask.scala b/mainproject/src/main/scala/com/raphtory/core/analysis/Tasks/ViewTasks/BWindowedViewAnalysisTask.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/ViewTasks/BWindowedViewAnalysisTask.scala rename to mainproject/src/main/scala/com/raphtory/core/analysis/Tasks/ViewTasks/BWindowedViewAnalysisTask.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/ViewTasks/ViewAnalysisTask.scala b/mainproject/src/main/scala/com/raphtory/core/analysis/Tasks/ViewTasks/ViewAnalysisTask.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/ViewTasks/ViewAnalysisTask.scala rename to mainproject/src/main/scala/com/raphtory/core/analysis/Tasks/ViewTasks/ViewAnalysisTask.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/ViewTasks/WindowedViewAnalysisTask.scala b/mainproject/src/main/scala/com/raphtory/core/analysis/Tasks/ViewTasks/WindowedViewAnalysisTask.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/analysis/Tasks/ViewTasks/WindowedViewAnalysisTask.scala rename to mainproject/src/main/scala/com/raphtory/core/analysis/Tasks/ViewTasks/WindowedViewAnalysisTask.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/serialisers/JSONSerialiser.scala b/mainproject/src/main/scala/com/raphtory/core/analysis/serialisers/JSONSerialiser.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/analysis/serialisers/JSONSerialiser.scala rename to mainproject/src/main/scala/com/raphtory/core/analysis/serialisers/JSONSerialiser.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/serialisers/NetworkXSerialiser.scala b/mainproject/src/main/scala/com/raphtory/core/analysis/serialisers/NetworkXSerialiser.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/analysis/serialisers/NetworkXSerialiser.scala rename to mainproject/src/main/scala/com/raphtory/core/analysis/serialisers/NetworkXSerialiser.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/serialisers/S3Serialiser.scala b/mainproject/src/main/scala/com/raphtory/core/analysis/serialisers/S3Serialiser.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/analysis/serialisers/S3Serialiser.scala rename to mainproject/src/main/scala/com/raphtory/core/analysis/serialisers/S3Serialiser.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/analysis/serialisers/Serialiser.scala b/mainproject/src/main/scala/com/raphtory/core/analysis/serialisers/Serialiser.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/analysis/serialisers/Serialiser.scala rename to mainproject/src/main/scala/com/raphtory/core/analysis/serialisers/Serialiser.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/DocSvr.scala b/mainproject/src/main/scala/com/raphtory/core/clustersetup/DocSvr.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/DocSvr.scala rename to mainproject/src/main/scala/com/raphtory/core/clustersetup/DocSvr.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/LiveAnalysisNode.scala b/mainproject/src/main/scala/com/raphtory/core/clustersetup/LiveAnalysisNode.scala similarity index 99% rename from mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/LiveAnalysisNode.scala rename to mainproject/src/main/scala/com/raphtory/core/clustersetup/LiveAnalysisNode.scala index 5a6e8fbea1..d3369fa27f 100644 --- a/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/LiveAnalysisNode.scala +++ b/mainproject/src/main/scala/com/raphtory/core/clustersetup/LiveAnalysisNode.scala @@ -14,4 +14,3 @@ case class LiveAnalysisNode(seedLoc: String) extends DocSvr { - diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/ManagerNode.scala b/mainproject/src/main/scala/com/raphtory/core/clustersetup/ManagerNode.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/ManagerNode.scala rename to mainproject/src/main/scala/com/raphtory/core/clustersetup/ManagerNode.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/RouterNode.scala b/mainproject/src/main/scala/com/raphtory/core/clustersetup/RouterNode.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/RouterNode.scala rename to mainproject/src/main/scala/com/raphtory/core/clustersetup/RouterNode.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/SeedNode.scala b/mainproject/src/main/scala/com/raphtory/core/clustersetup/SeedNode.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/SeedNode.scala rename to mainproject/src/main/scala/com/raphtory/core/clustersetup/SeedNode.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/UpdateNode.scala b/mainproject/src/main/scala/com/raphtory/core/clustersetup/UpdateNode.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/UpdateNode.scala rename to mainproject/src/main/scala/com/raphtory/core/clustersetup/UpdateNode.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/WatchDogNode.scala b/mainproject/src/main/scala/com/raphtory/core/clustersetup/WatchDogNode.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/WatchDogNode.scala rename to mainproject/src/main/scala/com/raphtory/core/clustersetup/WatchDogNode.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/singlenode/SingleNodeSetup.scala b/mainproject/src/main/scala/com/raphtory/core/clustersetup/singlenode/SingleNodeSetup.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/singlenode/SingleNodeSetup.scala rename to mainproject/src/main/scala/com/raphtory/core/clustersetup/singlenode/SingleNodeSetup.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/util/ConfigUtils.scala b/mainproject/src/main/scala/com/raphtory/core/clustersetup/util/ConfigUtils.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/clustersetup/util/ConfigUtils.scala rename to mainproject/src/main/scala/com/raphtory/core/clustersetup/util/ConfigUtils.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/ClusterManagement/RaphtoryReplicator.scala b/mainproject/src/main/scala/com/raphtory/core/components/ClusterManagement/RaphtoryReplicator.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/components/ClusterManagement/RaphtoryReplicator.scala rename to mainproject/src/main/scala/com/raphtory/core/components/ClusterManagement/RaphtoryReplicator.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/ClusterManagement/SeedActor.scala b/mainproject/src/main/scala/com/raphtory/core/components/ClusterManagement/SeedActor.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/components/ClusterManagement/SeedActor.scala rename to mainproject/src/main/scala/com/raphtory/core/components/ClusterManagement/SeedActor.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/ClusterManagement/WatchDog.scala b/mainproject/src/main/scala/com/raphtory/core/components/ClusterManagement/WatchDog.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/components/ClusterManagement/WatchDog.scala rename to mainproject/src/main/scala/com/raphtory/core/components/ClusterManagement/WatchDog.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/ClusterManagement/WatermarkManager.scala b/mainproject/src/main/scala/com/raphtory/core/components/ClusterManagement/WatermarkManager.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/components/ClusterManagement/WatermarkManager.scala rename to mainproject/src/main/scala/com/raphtory/core/components/ClusterManagement/WatermarkManager.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Archivist.scala b/mainproject/src/main/scala/com/raphtory/core/components/PartitionManager/Archivist.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Archivist.scala rename to mainproject/src/main/scala/com/raphtory/core/components/PartitionManager/Archivist.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Reader.scala b/mainproject/src/main/scala/com/raphtory/core/components/PartitionManager/Reader.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Reader.scala rename to mainproject/src/main/scala/com/raphtory/core/components/PartitionManager/Reader.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/ArchivistWorker.scala b/mainproject/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/ArchivistWorker.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/ArchivistWorker.scala rename to mainproject/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/ArchivistWorker.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/IngestionWorker.scala b/mainproject/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/IngestionWorker.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/IngestionWorker.scala rename to mainproject/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/IngestionWorker.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/ReaderWorker.scala b/mainproject/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/ReaderWorker.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/ReaderWorker.scala rename to mainproject/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/ReaderWorker.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Writer.scala b/mainproject/src/main/scala/com/raphtory/core/components/PartitionManager/Writer.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/components/PartitionManager/Writer.scala rename to mainproject/src/main/scala/com/raphtory/core/components/PartitionManager/Writer.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/GraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/core/components/Router/GraphBuilder.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/GraphBuilder.scala rename to mainproject/src/main/scala/com/raphtory/core/components/Router/GraphBuilder.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterManager.scala b/mainproject/src/main/scala/com/raphtory/core/components/Router/RouterManager.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterManager.scala rename to mainproject/src/main/scala/com/raphtory/core/components/Router/RouterManager.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala b/mainproject/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala rename to mainproject/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/Spout.scala b/mainproject/src/main/scala/com/raphtory/core/components/Spout/Spout.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/Spout.scala rename to mainproject/src/main/scala/com/raphtory/core/components/Spout/Spout.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/SpoutAgent.scala b/mainproject/src/main/scala/com/raphtory/core/components/Spout/SpoutAgent.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/components/Spout/SpoutAgent.scala rename to mainproject/src/main/scala/com/raphtory/core/components/Spout/SpoutAgent.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/model/communication/AnalysisType.scala b/mainproject/src/main/scala/com/raphtory/core/model/communication/AnalysisType.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/model/communication/AnalysisType.scala rename to mainproject/src/main/scala/com/raphtory/core/model/communication/AnalysisType.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/model/communication/VertexMutliQueue.scala b/mainproject/src/main/scala/com/raphtory/core/model/communication/VertexMutliQueue.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/model/communication/VertexMutliQueue.scala rename to mainproject/src/main/scala/com/raphtory/core/model/communication/VertexMutliQueue.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/model/communication/raphtoryMessages.scala b/mainproject/src/main/scala/com/raphtory/core/model/communication/raphtoryMessages.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/model/communication/raphtoryMessages.scala rename to mainproject/src/main/scala/com/raphtory/core/model/communication/raphtoryMessages.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/Edge.scala b/mainproject/src/main/scala/com/raphtory/core/model/graphentities/Edge.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/Edge.scala rename to mainproject/src/main/scala/com/raphtory/core/model/graphentities/Edge.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/Entity.scala b/mainproject/src/main/scala/com/raphtory/core/model/graphentities/Entity.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/Entity.scala rename to mainproject/src/main/scala/com/raphtory/core/model/graphentities/Entity.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/ImmutableProperty.scala b/mainproject/src/main/scala/com/raphtory/core/model/graphentities/ImmutableProperty.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/ImmutableProperty.scala rename to mainproject/src/main/scala/com/raphtory/core/model/graphentities/ImmutableProperty.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/MutableProperty.scala b/mainproject/src/main/scala/com/raphtory/core/model/graphentities/MutableProperty.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/MutableProperty.scala rename to mainproject/src/main/scala/com/raphtory/core/model/graphentities/MutableProperty.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/Property.scala b/mainproject/src/main/scala/com/raphtory/core/model/graphentities/Property.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/Property.scala rename to mainproject/src/main/scala/com/raphtory/core/model/graphentities/Property.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/SplitEdge.scala b/mainproject/src/main/scala/com/raphtory/core/model/graphentities/SplitEdge.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/SplitEdge.scala rename to mainproject/src/main/scala/com/raphtory/core/model/graphentities/SplitEdge.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/Vertex.scala b/mainproject/src/main/scala/com/raphtory/core/model/graphentities/Vertex.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/model/graphentities/Vertex.scala rename to mainproject/src/main/scala/com/raphtory/core/model/graphentities/Vertex.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/storage/EntityStorage.scala b/mainproject/src/main/scala/com/raphtory/core/storage/EntityStorage.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/storage/EntityStorage.scala rename to mainproject/src/main/scala/com/raphtory/core/storage/EntityStorage.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/utils/HistoryOrdering.scala b/mainproject/src/main/scala/com/raphtory/core/utils/HistoryOrdering.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/utils/HistoryOrdering.scala rename to mainproject/src/main/scala/com/raphtory/core/utils/HistoryOrdering.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/utils/SchedulerUtil.scala b/mainproject/src/main/scala/com/raphtory/core/utils/SchedulerUtil.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/utils/SchedulerUtil.scala rename to mainproject/src/main/scala/com/raphtory/core/utils/SchedulerUtil.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/core/utils/Utils.scala b/mainproject/src/main/scala/com/raphtory/core/utils/Utils.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/core/utils/Utils.scala rename to mainproject/src/main/scala/com/raphtory/core/utils/Utils.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/analysers/EthereumDegreeRanking.scala b/mainproject/src/main/scala/com/raphtory/examples/blockchain/analysers/EthereumDegreeRanking.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/analysers/EthereumDegreeRanking.scala rename to mainproject/src/main/scala/com/raphtory/examples/blockchain/analysers/EthereumDegreeRanking.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/analysers/EthereumTaintTracking.scala b/mainproject/src/main/scala/com/raphtory/examples/blockchain/analysers/EthereumTaintTracking.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/analysers/EthereumTaintTracking.scala rename to mainproject/src/main/scala/com/raphtory/examples/blockchain/analysers/EthereumTaintTracking.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/analysers/TaintTrackExchangeStop.scala b/mainproject/src/main/scala/com/raphtory/examples/blockchain/analysers/TaintTrackExchangeStop.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/analysers/TaintTrackExchangeStop.scala rename to mainproject/src/main/scala/com/raphtory/examples/blockchain/analysers/TaintTrackExchangeStop.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/BitcoinGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/BitcoinGraphBuilder.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/BitcoinGraphBuilder.scala rename to mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/BitcoinGraphBuilder.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChABClus2ClusGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChABClus2ClusGraphBuilder.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChABClus2ClusGraphBuilder.scala rename to mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChABClus2ClusGraphBuilder.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChABClus2ClusGraphBuilderNoprop.scala b/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChABClus2ClusGraphBuilderNoprop.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChABClus2ClusGraphBuilderNoprop.scala rename to mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChABClus2ClusGraphBuilderNoprop.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChABClus2ClusRouter.scala.save b/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChABClus2ClusRouter.scala.save similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChABClus2ClusRouter.scala.save rename to mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChABClus2ClusRouter.scala.save diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChainalysisABGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChainalysisABGraphBuilder.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChainalysisABGraphBuilder.scala rename to mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChainalysisABGraphBuilder.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/DashcoinGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/DashcoinGraphBuilder.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/DashcoinGraphBuilder.scala rename to mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/DashcoinGraphBuilder.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/EthereumGethGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/EthereumGethGraphBuilder.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/EthereumGethGraphBuilder.scala rename to mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/EthereumGethGraphBuilder.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/EthereumKafkaGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/EthereumKafkaGraphBuilder.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/EthereumKafkaGraphBuilder.scala rename to mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/EthereumKafkaGraphBuilder.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/EthereumTransactionGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/EthereumTransactionGraphBuilder.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/EthereumTransactionGraphBuilder.scala rename to mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/EthereumTransactionGraphBuilder.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/FirehoseKafkaGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/FirehoseKafkaGraphBuilder.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/FirehoseKafkaGraphBuilder.scala rename to mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/FirehoseKafkaGraphBuilder.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/LitecoinGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/LitecoinGraphBuilder.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/LitecoinGraphBuilder.scala rename to mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/LitecoinGraphBuilder.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/citationNetwork/CitationGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/citationNetwork/CitationGraphBuilder.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/citationNetwork/CitationGraphBuilder.scala rename to mainproject/src/main/scala/com/raphtory/examples/citationNetwork/CitationGraphBuilder.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/analysis/GabMostUsedTopics.scala b/mainproject/src/main/scala/com/raphtory/examples/gab/analysis/GabMostUsedTopics.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/gab/analysis/GabMostUsedTopics.scala rename to mainproject/src/main/scala/com/raphtory/examples/gab/analysis/GabMostUsedTopics.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/datasources/GabMongoSource.scala b/mainproject/src/main/scala/com/raphtory/examples/gab/datasources/GabMongoSource.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/gab/datasources/GabMongoSource.scala rename to mainproject/src/main/scala/com/raphtory/examples/gab/datasources/GabMongoSource.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/graphbuilders/GabPostGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/gab/graphbuilders/GabPostGraphBuilder.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/gab/graphbuilders/GabPostGraphBuilder.scala rename to mainproject/src/main/scala/com/raphtory/examples/gab/graphbuilders/GabPostGraphBuilder.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/graphbuilders/GabRawGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/gab/graphbuilders/GabRawGraphBuilder.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/gab/graphbuilders/GabRawGraphBuilder.scala rename to mainproject/src/main/scala/com/raphtory/examples/gab/graphbuilders/GabRawGraphBuilder.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/graphbuilders/GabUserGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/gab/graphbuilders/GabUserGraphBuilder.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/gab/graphbuilders/GabUserGraphBuilder.scala rename to mainproject/src/main/scala/com/raphtory/examples/gab/graphbuilders/GabUserGraphBuilder.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/rawgraphmodel/GabEntityType.scala b/mainproject/src/main/scala/com/raphtory/examples/gab/rawgraphmodel/GabEntityType.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/gab/rawgraphmodel/GabEntityType.scala rename to mainproject/src/main/scala/com/raphtory/examples/gab/rawgraphmodel/GabEntityType.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/rawgraphmodel/GabJsonProtocol.scala b/mainproject/src/main/scala/com/raphtory/examples/gab/rawgraphmodel/GabJsonProtocol.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/gab/rawgraphmodel/GabJsonProtocol.scala rename to mainproject/src/main/scala/com/raphtory/examples/gab/rawgraphmodel/GabJsonProtocol.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/gab/rawgraphmodel/GabPost.scala b/mainproject/src/main/scala/com/raphtory/examples/gab/rawgraphmodel/GabPost.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/gab/rawgraphmodel/GabPost.scala rename to mainproject/src/main/scala/com/raphtory/examples/gab/rawgraphmodel/GabPost.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/graphbuilders/LDBCGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/ldbc/graphbuilders/LDBCGraphBuilder.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/graphbuilders/LDBCGraphBuilder.scala rename to mainproject/src/main/scala/com/raphtory/examples/ldbc/graphbuilders/LDBCGraphBuilder.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/graphbuilders/LDBCOldGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/ldbc/graphbuilders/LDBCOldGraphBuilder.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/ldbc/graphbuilders/LDBCOldGraphBuilder.scala rename to mainproject/src/main/scala/com/raphtory/examples/ldbc/graphbuilders/LDBCOldGraphBuilder.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRDeployment.scala b/mainproject/src/main/scala/com/raphtory/examples/lotr/LOTRDeployment.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRDeployment.scala rename to mainproject/src/main/scala/com/raphtory/examples/lotr/LOTRDeployment.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/lotr/LOTRGraphBuilder.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRGraphBuilder.scala rename to mainproject/src/main/scala/com/raphtory/examples/lotr/LOTRGraphBuilder.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRSpout.scala b/mainproject/src/main/scala/com/raphtory/examples/lotr/LOTRSpout.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/LOTRSpout.scala rename to mainproject/src/main/scala/com/raphtory/examples/lotr/LOTRSpout.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/lotr.csv b/mainproject/src/main/scala/com/raphtory/examples/lotr/lotr.csv similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/lotr/lotr.csv rename to mainproject/src/main/scala/com/raphtory/examples/lotr/lotr.csv diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/test/actors/RandomGraphBuilder.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomGraphBuilder.scala rename to mainproject/src/main/scala/com/raphtory/examples/test/actors/RandomGraphBuilder.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomSpout.scala b/mainproject/src/main/scala/com/raphtory/examples/test/actors/RandomSpout.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/RandomSpout.scala rename to mainproject/src/main/scala/com/raphtory/examples/test/actors/RandomSpout.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/test/actors/TriangleGraphBuilder.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleGraphBuilder.scala rename to mainproject/src/main/scala/com/raphtory/examples/test/actors/TriangleGraphBuilder.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleSpout.scala b/mainproject/src/main/scala/com/raphtory/examples/test/actors/TriangleSpout.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/test/actors/TriangleSpout.scala rename to mainproject/src/main/scala/com/raphtory/examples/test/actors/TriangleSpout.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/trackAndTrace/TrackAndTraceGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/trackAndTrace/TrackAndTraceGraphBuilder.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/trackAndTrace/TrackAndTraceGraphBuilder.scala rename to mainproject/src/main/scala/com/raphtory/examples/trackAndTrace/TrackAndTraceGraphBuilder.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/SamplerSpout.scala b/mainproject/src/main/scala/com/raphtory/examples/tsvnet/SamplerSpout.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/SamplerSpout.scala rename to mainproject/src/main/scala/com/raphtory/examples/tsvnet/SamplerSpout.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/TSVGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/tsvnet/TSVGraphBuilder.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/tsvnet/TSVGraphBuilder.scala rename to mainproject/src/main/scala/com/raphtory/examples/tsvnet/TSVGraphBuilder.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/graphbuilders/CooccurrenceMatrixGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/wordSemantic/graphbuilders/CooccurrenceMatrixGraphBuilder.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/graphbuilders/CooccurrenceMatrixGraphBuilder.scala rename to mainproject/src/main/scala/com/raphtory/examples/wordSemantic/graphbuilders/CooccurrenceMatrixGraphBuilder.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/graphbuilders/CooccurrenceMatrixGraphBuilderFiltered.scala b/mainproject/src/main/scala/com/raphtory/examples/wordSemantic/graphbuilders/CooccurrenceMatrixGraphBuilderFiltered.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/graphbuilders/CooccurrenceMatrixGraphBuilderFiltered.scala rename to mainproject/src/main/scala/com/raphtory/examples/wordSemantic/graphbuilders/CooccurrenceMatrixGraphBuilderFiltered.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/spouts/CooccurrenceMatrixSpout.scala b/mainproject/src/main/scala/com/raphtory/examples/wordSemantic/spouts/CooccurrenceMatrixSpout.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/spouts/CooccurrenceMatrixSpout.scala rename to mainproject/src/main/scala/com/raphtory/examples/wordSemantic/spouts/CooccurrenceMatrixSpout.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/spouts/CooccurrenceMatrixSpoutFiltered.scala b/mainproject/src/main/scala/com/raphtory/examples/wordSemantic/spouts/CooccurrenceMatrixSpoutFiltered.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/examples/wordSemantic/spouts/CooccurrenceMatrixSpoutFiltered.scala rename to mainproject/src/main/scala/com/raphtory/examples/wordSemantic/spouts/CooccurrenceMatrixSpoutFiltered.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/spouts/FileSpout.scala b/mainproject/src/main/scala/com/raphtory/spouts/FileSpout.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/spouts/FileSpout.scala rename to mainproject/src/main/scala/com/raphtory/spouts/FileSpout.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/spouts/KafkaSpout.scala b/mainproject/src/main/scala/com/raphtory/spouts/KafkaSpout.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/spouts/KafkaSpout.scala rename to mainproject/src/main/scala/com/raphtory/spouts/KafkaSpout.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/spouts/blockchain/BitcoinNodeSpout.scala b/mainproject/src/main/scala/com/raphtory/spouts/blockchain/BitcoinNodeSpout.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/spouts/blockchain/BitcoinNodeSpout.scala rename to mainproject/src/main/scala/com/raphtory/spouts/blockchain/BitcoinNodeSpout.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/spouts/blockchain/EthereumGethSpout.scala b/mainproject/src/main/scala/com/raphtory/spouts/blockchain/EthereumGethSpout.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/spouts/blockchain/EthereumGethSpout.scala rename to mainproject/src/main/scala/com/raphtory/spouts/blockchain/EthereumGethSpout.scala diff --git a/mainproject/cluster/src/main/scala/com/raphtory/spouts/blockchain/EthereumPostgresSpout.scala b/mainproject/src/main/scala/com/raphtory/spouts/blockchain/EthereumPostgresSpout.scala similarity index 100% rename from mainproject/cluster/src/main/scala/com/raphtory/spouts/blockchain/EthereumPostgresSpout.scala rename to mainproject/src/main/scala/com/raphtory/spouts/blockchain/EthereumPostgresSpout.scala From 4d2b932edc271f4542ef26e48d13379a6e4f0fbc Mon Sep 17 00:00:00 2001 From: Miratepuffin Date: Sun, 29 Nov 2020 19:05:51 +0000 Subject: [PATCH 42/58] Archived Archivist --- .../RaphtoryReplicator.scala | 2 +- .../PartitionManager/Archivist.scala | 165 ------------------ .../Workers/ArchivistWorker.scala | 64 ------- 3 files changed, 1 insertion(+), 230 deletions(-) delete mode 100644 mainproject/src/main/scala/com/raphtory/core/components/PartitionManager/Archivist.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/ArchivistWorker.scala diff --git a/mainproject/src/main/scala/com/raphtory/core/components/ClusterManagement/RaphtoryReplicator.scala b/mainproject/src/main/scala/com/raphtory/core/components/ClusterManagement/RaphtoryReplicator.scala index 01fc2fdb31..4da850cf59 100644 --- a/mainproject/src/main/scala/com/raphtory/core/components/ClusterManagement/RaphtoryReplicator.scala +++ b/mainproject/src/main/scala/com/raphtory/core/components/ClusterManagement/RaphtoryReplicator.scala @@ -6,7 +6,7 @@ import akka.cluster.pubsub.{DistributedPubSub, DistributedPubSubMediator} import akka.pattern.ask import akka.util.Timeout import com.raphtory.core.components.PartitionManager.Workers.IngestionWorker -import com.raphtory.core.components.PartitionManager.{Archivist, Reader, Writer} +import com.raphtory.core.components.PartitionManager.{Reader, Writer} import com.raphtory.core.components.Router.{GraphBuilder, RouterManager} import com.raphtory.core.model.communication._ import com.raphtory.core.storage.EntityStorage diff --git a/mainproject/src/main/scala/com/raphtory/core/components/PartitionManager/Archivist.scala b/mainproject/src/main/scala/com/raphtory/core/components/PartitionManager/Archivist.scala deleted file mode 100644 index 5db634b43c..0000000000 --- a/mainproject/src/main/scala/com/raphtory/core/components/PartitionManager/Archivist.scala +++ /dev/null @@ -1,165 +0,0 @@ -package com.raphtory.core.components.PartitionManager - -import akka.actor.{Actor, ActorRef, Props} -import ch.qos.logback.classic.Level -import com.raphtory.core.components.PartitionManager.Workers.ArchivistWorker -import com.raphtory.core.model.communication._ -import com.raphtory.core.storage.EntityStorage -import com.raphtory.core.utils.Utils -import kamon.Kamon -import org.slf4j.LoggerFactory - -import scala.collection.parallel.mutable.ParTrieMap -import scala.concurrent.duration._ - -// TODO fix edges -// TODO Will be revamped -class Archivist(maximumMem: Double, workers: ParTrieMap[Int, ActorRef], storages: ParTrieMap[Int, EntityStorage]) - extends Actor { - val compressing: Boolean = Utils.compressing - val saving: Boolean = Utils.persistenceEnabled - val archiving: Boolean = Utils.archiving - val debug = System.getenv().getOrDefault("DEBUG", "false").trim.toBoolean - if (debug) println(s"Archivist compressing = $compressing, Saving = $saving, Archiving = $archiving") - implicit val executionContext = context.system.dispatchers.lookup("worker-dispatcher") - //Turn logging off - val root = LoggerFactory.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME).asInstanceOf[ch.qos.logback.classic.Logger] - root.setLevel(Level.ERROR) - - //get the runtime for memory usage - val runtime = Runtime.getRuntime - //times to track how long compression and archiving takes - var totalCompressionTime: Long = 0L - var totalArchiveTime: Long = 0L - // percent of history to be compressed/archived - var compressionPercent = 90f - var archivePercentage = 10f - // vars for the latest point the graph is saved to - var lastSaved = 0L - var newLastSaved = 0L - //var for the new oldest point after archiving - var removePointGlobal: Long = 0L - var removalPoint: Long = 0L - - var currentWorker = 0 - var currentState: Boolean = false - - // children for distribution of compresssion and archiving - val vertexManager = context - .actorOf(Props(new ArchivistWorker(workers, storages)), "vertexcompressor"); - - val archGauge = Kamon.gauge("raphtory_archivist") - - override def preStart() { - context.system.scheduler.scheduleOnce(60.seconds, self, "archive") //start the compression process in 20 seconds - } - - override def receive: Receive = { - case "archive" => archive() - case FinishedVertexCompression(key) => compressEnder("vertex") - case FinishedVertexArchiving(key) => archiveEnder("vertex") - } - - def archive(): Unit = - if (archiving) { - println(s"Currently archiving worker $currentWorker") - if (spaceForExtraHistory) //check if we need to archive - if (compressing) { - if (currentWorker == 0) - newLastSaved = cutOff(true) //get the cut off boundry for 90% of history in meme - vertexManager ! CompressVertices(newLastSaved, workerID = currentWorker) - } else - context.system.scheduler.scheduleOnce(60.second, self, "archive") - else { - if (currentWorker == 0) { - removalPoint = cutOff(false) // get the cut off for 10% of the compressed history - removePointGlobal = removalPoint - newLastSaved = cutOff(true) - } - vertexManager ! ArchiveVertices(newLastSaved, removalPoint, workerID = currentWorker) - } - } - - def compressEnder(name: String): Unit = { - if (debug) - println( - s"finished total compression in ${(System.currentTimeMillis() - totalCompressionTime) / 1000} seconds" - ) //report this to the user -// archGauge -// .refine("actor" -> "Archivist", "name" -> "totalCompressionTime") -// .set((System.currentTimeMillis() - totalCompressionTime) / 1000) - if (currentWorker == 9) - lastSaved = newLastSaved - //EntityStorage.lastCompressedAt = lastSaved //update the saved vals so we know where we are compressed up to ?????????????????????????????? - nextWorker() - } - - def archiveEnder(name: String): Unit = { - // archGauge.refine("actor" -> "Archivist", "name" -> "edgeHistoryRemoved").set(storage.edgeHistoryDeletionCount.sum) - // archGauge.refine("actor" -> "Archivist", "name" -> "edgePropertyRemoved").set(EntityStorage.edgePropertyDeletionCount.sum) - // archGauge.refine("actor" -> "Archivist", "name" -> "edgeEdgesRemoved").set(EntityStorage.edgeDeletionCount.sum) - // archGauge.refine("actor" -> "Archivist", "name" -> "vertexHistoryRemoved").set(EntityStorage.vertexHistoryDeletionCount.sum) - // archGauge.refine("actor" -> "Archivist", "name" -> "vertexPropertyRemoved").set(EntityStorage.vertexPropertyDeletionCount.sum) - // archGauge.refine("actor" -> "Archivist", "name" -> "vertexVerticesRemoved").set(EntityStorage.vertexDeletionCount.sum) - // archGauge.refine("actor" -> "Archivist", "name" -> "totalArchiveTime").set((System.currentTimeMillis()-totalArchiveTime)/1000) - if (currentWorker == 9) - // EntityStorage.oldestTime = removePointGlobal - System.gc() //suggest a good time to garbage collect - nextWorker() - if (debug) println(s"finished total archiving in ${(System.currentTimeMillis() - totalArchiveTime) / 1000} seconds") - } - - def nextWorker() = - if (currentWorker == 9) { - currentWorker = 0 - context.system.scheduler.scheduleOnce(60.second, self, "archive") //start the archiving process - } else { - currentWorker += 1 - context.system.scheduler.scheduleOnce(1.millisecond, self, "archive") //start the archiving process - } - - def spaceForExtraHistory: Boolean = { - if (currentWorker == 0) { - val totalMemory = runtime.maxMemory - val freeMemory = runtime.freeMemory - val usedMemory = (totalMemory - freeMemory) - val total = usedMemory / (totalMemory).asInstanceOf[Float] - //println(s"max ${runtime.maxMemory()} total ${runtime.totalMemory()} diff ${runtime.maxMemory()-runtime.totalMemory()} ") - println(s"Memory usage at ${total * 100}% of ${totalMemory / (1024 * 1024)}MB") - //archGauge.refine("actor" -> "Archivist", "name" -> "memoryPercentage").set((total * 100).asInstanceOf[Long]) - if (total < (1 - maximumMem)) currentState = true else currentState = false - } - currentState - } //check if used memory less than set maximum - - def toCompress(newestPoint: Long, oldestPoint: Long): Long = - (((newestPoint - oldestPoint) / 100f) * compressionPercent).asInstanceOf[Long] - def toArchive(newestPoint: Long, oldestPoint: Long): Long = - (((newestPoint - oldestPoint) / 100f) * archivePercentage).asInstanceOf[Long] - - def cutOff(compress: Boolean) = { - val oldestPoint = storages.map(s => s._2.oldestTime).max - val newestPoint = storages.map(s => s._2.newestTime).max - setActionTime(compress) - if (compress) - println( - s" Difference between oldest $oldestPoint to newest point $newestPoint --- ${((newestPoint - oldestPoint) / 1000)}, ${(toCompress(newestPoint, oldestPoint)) / 1000} seconds compressed" - ) - if (oldestPoint != Long.MaxValue) - if (compress) - oldestPoint + toCompress( - newestPoint, - oldestPoint - ) //oldestpoint + halfway to the newest point == always keep half of in memory stuff compressed - else oldestPoint + toArchive(newestPoint, oldestPoint) - else newestPoint - } - - def setActionTime(CorA: Boolean) = - if (CorA) totalCompressionTime = System.currentTimeMillis() - else totalArchiveTime = System.currentTimeMillis() - -} - -//export JAVA_HOME=/Library/Java/JavaVirtualMachines/jdk1.8.0_162.jdk/Contents/Home/ -//JAVA_OPTS=-XX:+UseConcMarkSweepGC -XX:+DisableExplicitGC -XX:+UseParNewGC -Xms10g -Xmx10g -XX:NewRatio=3 diff --git a/mainproject/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/ArchivistWorker.scala b/mainproject/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/ArchivistWorker.scala deleted file mode 100644 index 0d1b6e695f..0000000000 --- a/mainproject/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/ArchivistWorker.scala +++ /dev/null @@ -1,64 +0,0 @@ -package com.raphtory.core.components.PartitionManager.Workers - -import akka.actor.{Actor, ActorRef} -import com.raphtory.core.model.communication._ -import com.raphtory.core.storage.EntityStorage -import com.raphtory.core.utils.Utils - -import scala.collection.parallel.mutable.ParTrieMap - -// TODO Will be revamped -class ArchivistWorker(workers: ParTrieMap[Int, ActorRef], storages: ParTrieMap[Int, EntityStorage]) extends Actor { - - //timestamps to make sure all entities are compressed to exactly the same point - val compressing: Boolean = Utils.compressing - val saving: Boolean = Utils.persistenceEnabled - - var startedCompressions = 0 - var finishedCompressions = 0 - var startedArchiving = 0 - var finishedArchiving = 0 - - override def receive: Receive = { - case CompressVertices(compressTime, workerID) => compressVertices(compressTime, workerID) - case ArchiveVertices(compressTime, archiveTime, workerID) => - archiveVertices(compressTime, archiveTime, workerID) - case FinishedVertexCompression(key) => finishedVertexCompression(key) - case FinishedVertexArchiving(key) => finishedVertexArchiving(key) - } - - def compressVertices(compressTime: Long, workerID: Int): Unit = { - val worker = workers(workerID) - storages(workerID).vertices foreach (pair => { - worker ! CompressVertex(pair._1, compressTime) - startedCompressions += 1 - }) - } - - def archiveVertices(compressTime: Long, archiveTime: Long, workerID: Int): Unit = { - val worker = workers(workerID) - storages(workerID).vertices foreach (key => { - if (compressing) worker ! ArchiveVertex(key._1, compressTime, archiveTime) - else worker ! ArchiveOnlyVertex(key._1, archiveTime) - startedArchiving += 1 - }) - } - - def finishedVertexCompression(key: Long): Unit = { - finishedCompressions += 1 - if (startedCompressions == finishedCompressions) { - context.parent ! FinishedVertexCompression(startedCompressions) - startedCompressions = 0 - finishedCompressions = 0 - } - } - - def finishedVertexArchiving(ID: Long): Unit = { - finishedArchiving += 1 - if (startedArchiving == finishedArchiving) { - context.parent ! FinishedVertexArchiving(startedArchiving) - startedArchiving = 0 - finishedArchiving = 0 - } - } -} From e7bf1f488716d4c40bd3b81d48702c2471e64ed7 Mon Sep 17 00:00:00 2001 From: Miratepuffin Date: Mon, 30 Nov 2020 00:37:50 +0000 Subject: [PATCH 43/58] Finished sbt assembly. Closes issue #78 --- .../core/components/ClusterManagement/RaphtoryReplicator.scala | 2 -- .../src/main/scala/com/raphtory/examples/lotr/LOTRSpout.scala | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/mainproject/src/main/scala/com/raphtory/core/components/ClusterManagement/RaphtoryReplicator.scala b/mainproject/src/main/scala/com/raphtory/core/components/ClusterManagement/RaphtoryReplicator.scala index 4da850cf59..71d39ca3b0 100644 --- a/mainproject/src/main/scala/com/raphtory/core/components/ClusterManagement/RaphtoryReplicator.scala +++ b/mainproject/src/main/scala/com/raphtory/core/components/ClusterManagement/RaphtoryReplicator.scala @@ -143,8 +143,6 @@ class RaphtoryReplicator[T](actorType: String, initialManagerCount: Int, initial actorRefReader = context.system.actorOf(Props(new Reader(myId, false, currentCount, storages)), s"ManagerReader_$myId") - context.system.actorOf(Props(new Archivist(0.3, workers, storages))) - } def createNewRouter(assignedId: Int): Unit = { diff --git a/mainproject/src/main/scala/com/raphtory/examples/lotr/LOTRSpout.scala b/mainproject/src/main/scala/com/raphtory/examples/lotr/LOTRSpout.scala index 98affff7ac..058396eea6 100644 --- a/mainproject/src/main/scala/com/raphtory/examples/lotr/LOTRSpout.scala +++ b/mainproject/src/main/scala/com/raphtory/examples/lotr/LOTRSpout.scala @@ -11,7 +11,7 @@ class LOTRSpout extends Spout[String] { override def setupDataSource(): Unit = { fileQueue++= - scala.io.Source.fromFile("cluster/src/main/scala/com/raphtory/examples/lotr/lotr.csv") + scala.io.Source.fromFile("src/main/scala/com/raphtory/examples/lotr/lotr.csv") .getLines }//no setup From e7ec23543d465448e3a4e3533795a5a41b340a3f Mon Sep 17 00:00:00 2001 From: Miratepuffin Date: Mon, 30 Nov 2020 02:44:36 +0000 Subject: [PATCH 44/58] Refactored the whole cluster setup file into RaphtoryServer class --- .../scala/com/raphtory/RaphtoryGraph.scala | 21 +- .../src/main/scala/com/raphtory/Server.scala | 245 ++++++++++++------ .../raphtory/core/clustersetup/DocSvr.scala | 86 ------ .../core/clustersetup/LiveAnalysisNode.scala | 16 -- .../core/clustersetup/ManagerNode.scala | 21 -- .../core/clustersetup/RouterNode.scala | 19 -- .../raphtory/core/clustersetup/SeedNode.scala | 15 -- .../core/clustersetup/UpdateNode.scala | 15 -- .../core/clustersetup/WatchDogNode.scala | 13 - .../singlenode/SingleNodeSetup.scala | 36 --- .../core/clustersetup/util/ConfigUtils.scala | 34 --- .../ClusterManagement/SeedActor.scala | 21 +- 12 files changed, 187 insertions(+), 355 deletions(-) delete mode 100644 mainproject/src/main/scala/com/raphtory/core/clustersetup/DocSvr.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/core/clustersetup/LiveAnalysisNode.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/core/clustersetup/ManagerNode.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/core/clustersetup/RouterNode.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/core/clustersetup/SeedNode.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/core/clustersetup/UpdateNode.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/core/clustersetup/WatchDogNode.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/core/clustersetup/singlenode/SingleNodeSetup.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/core/clustersetup/util/ConfigUtils.scala diff --git a/mainproject/src/main/scala/com/raphtory/RaphtoryGraph.scala b/mainproject/src/main/scala/com/raphtory/RaphtoryGraph.scala index 0ebc10a189..f3973bcb21 100644 --- a/mainproject/src/main/scala/com/raphtory/RaphtoryGraph.scala +++ b/mainproject/src/main/scala/com/raphtory/RaphtoryGraph.scala @@ -10,11 +10,18 @@ import kamon.Kamon import org.slf4j.LoggerFactory object RaphtoryGraph { - def apply[T](dataSource: Spout[T], graphBuilder: GraphBuilder[T]) : RaphtoryGraph[T] = - new RaphtoryGraph(dataSource, graphBuilder) + def apply[T](spout: Spout[T], graphBuilder: GraphBuilder[T]) : RaphtoryGraph[T] = + new RaphtoryGraph(spout, graphBuilder) + + def apply[T](spoutPath: String, graphBuilderPath: String) : RaphtoryGraph[T] ={ + val spout = Class.forName(spoutPath).getConstructor().newInstance().asInstanceOf[Spout[T]] + val graphBuilder = Class.forName(graphBuilderPath).getConstructor().newInstance().asInstanceOf[GraphBuilder[T]] + new RaphtoryGraph(spout, graphBuilder) + } + } -class RaphtoryGraph[T](dataSource: Spout[T], graphBuilder: GraphBuilder[T]) { +class RaphtoryGraph[T](spout: Spout[T], graphBuilder: GraphBuilder[T]) { Kamon.init() //start tool logging // val root = LoggerFactory.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME).asInstanceOf[ch.qos.logback.classic.Logger] @@ -23,12 +30,12 @@ class RaphtoryGraph[T](dataSource: Spout[T], graphBuilder: GraphBuilder[T]) { val partitionNumber = 1 val minimumRouters = 1 - system.actorOf(Props(new WatermarkManager(managerCount = 1)),"WatermarkManager") + system.actorOf(Props(new WatermarkManager(partitionNumber)),"WatermarkManager") system.actorOf(Props(new WatchDog(partitionNumber, minimumRouters)), "WatchDog") - system.actorOf(Props(new SpoutAgent(dataSource)), "Spout") - system.actorOf(Props(RaphtoryReplicator.apply("Router", 1, 1,graphBuilder)), s"Routers") - system.actorOf(Props(RaphtoryReplicator("Partition Manager", 1,1)), s"PartitionManager") + system.actorOf(Props(new SpoutAgent(spout)), "Spout") + system.actorOf(Props(RaphtoryReplicator.apply("Router", partitionNumber, minimumRouters,graphBuilder)), s"Routers") + system.actorOf(Props(RaphtoryReplicator("Partition Manager", partitionNumber,minimumRouters)), s"PartitionManager") system.actorOf(Props[AnalysisManager], s"AnalysisManager") AnalysisRestApi(system) diff --git a/mainproject/src/main/scala/com/raphtory/Server.scala b/mainproject/src/main/scala/com/raphtory/Server.scala index 25aa6f4aec..e9a69d3277 100644 --- a/mainproject/src/main/scala/com/raphtory/Server.scala +++ b/mainproject/src/main/scala/com/raphtory/Server.scala @@ -3,109 +3,190 @@ package com.raphtory import java.lang.management.ManagementFactory import java.net.InetAddress -import ch.qos.logback.classic.Level -import com.raphtory.core.clustersetup._ -import com.raphtory.core.clustersetup.singlenode.SingleNodeSetup -import com.raphtory.examples.test.actors.RandomGraphBuilder -import com.typesafe.config.ConfigFactory -import org.slf4j.LoggerFactory +import akka.actor.{ActorSystem, Address, ExtendedActorSystem, Props} +import akka.cluster.Member +import akka.event.LoggingAdapter +import akka.management.cluster.bootstrap.ClusterBootstrap +import akka.management.javadsl.AkkaManagement +import com.raphtory.core.analysis.{AnalysisManager, AnalysisRestApi} +import com.raphtory.core.components.ClusterManagement.{RaphtoryReplicator, SeedActor, WatchDog, WatermarkManager} +import com.raphtory.core.components.Router.GraphBuilder +import com.raphtory.core.components.Spout.{Spout, SpoutAgent} +import com.raphtory.core.utils.Utils +import com.typesafe.config.{Config, ConfigFactory, ConfigValue, ConfigValueFactory} +import scala.collection.JavaConversions +import scala.collection.JavaConversions._ import scala.language.postfixOps import scala.sys.process._ //main function object Go extends App { // Kamon.init() //start tool logging + + printJavaOptions() + + val conf = ConfigFactory.load() - val seedLoc = s"${sys.env("HOST_IP")}:${conf.getInt("settings.bport")}" - val root = LoggerFactory.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME).asInstanceOf[ch.qos.logback.classic.Logger] - root.setLevel(Level.ERROR) - // debug should give timing - //root.setLevel(Level.DEBUG) - val routerName = s"${sys.env.getOrElse("ROUTERCLASS", classOf[RandomGraphBuilder].getClass.getName)}" - val updaterName = s"${sys.env.getOrElse("SPOUTCLASS", "")}" - val docker = System.getenv().getOrDefault("DOCKER", "false").trim.toBoolean - - val runtimeMxBean = ManagementFactory.getRuntimeMXBean - val arguments = runtimeMxBean.getInputArguments - - println(s"Current java options: $arguments") + + + + val docker = System.getenv().getOrDefault("DOCKER", "false").trim.toBoolean + + val clusterSystemName: String = Utils.clusterSystemName + val ssn: String = java.util.UUID.randomUUID.toString + + val partitionCount = sys.env("PARTITION_MIN").toInt + val routerCount = sys.env("ROUTER_MIN").toInt + + /** Node lookup directory for registered members in the System */ + var nodes = Set.empty[Member] + args(0) match { - case "seedNode" => - println("Creating seed node") - setConf(seedLoc) - SeedNode(seedLoc) - case "router" => - println("Creating Router") - RouterNode(getConf(), sys.env("PARTITION_MIN").toInt, sys.env("ROUTER_MIN").toInt, routerName) - case "partitionManager" => - println(s"Creating Partition Manager...") - ManagerNode(getConf(), sys.env("PARTITION_MIN").toInt, sys.env("ROUTER_MIN").toInt) - - case "updater" => - println("Creating Update Generator") - UpdateNode(getConf(), updaterName) - - case "analysisManager" => - println("Creating Analysis Manager") - LiveAnalysisNode(getConf()) - case "clusterUp" => - println("Cluster Up, informing Partition Managers and Routers") - WatchDogNode(getConf(), sys.env("PARTITION_MIN").toInt, sys.env("ROUTER_MIN").toInt) - - case "singleNodeSetup" => - println("putting up cluster in one node") - SingleNodeSetup( - hostname2Ip(seedLoc), - routerName, - updaterName - ) - prometheusReporter() + case "seedNode" => seedNode() + case "router" => router() + case "partitionManager" => partition() + case "spout" => spout() + case "analysisManager" =>analysis() + case "clusterUp" => watchDog() + case "local" => local() + } + + def seedNode() = { + val seedLoc = s"${sys.env("HOST_IP")}:${conf.getInt("settings.bport")}" + println(s"Creating seed node at $seedLoc") + implicit val system: ActorSystem = initialiseActorSystem(seeds = List(seedLoc)) + system.actorOf(Props(new SeedActor()), "cluster") + } + + def router() = { + println("Creating Router") + implicit val system: ActorSystem = initialiseActorSystem(seeds = List(locateSeed())) + + val builderPath = s"${sys.env.getOrElse("GRAPHBUILDER", "")}" + val graphBuilder = Class.forName(builderPath).getConstructor().newInstance().asInstanceOf[GraphBuilder[Any]] + + val routerReplicator = RaphtoryReplicator.apply("Router", partitionCount, routerCount,graphBuilder) + system.actorOf(Props(routerReplicator), "Routers") + } + + def partition() = { + println(s"Creating Partition Manager...") + implicit val system: ActorSystem = initialiseActorSystem(seeds = List(locateSeed())) + system.actorOf(Props(RaphtoryReplicator(actorType = "Partition Manager", initialManagerCount = partitionCount,initialRouterCount = routerCount)), "PartitionManager") + } + + def spout() = { + println("Creating Update Generator") + implicit val system: ActorSystem = initialiseActorSystem(seeds = List(locateSeed())) + val spoutPath = s"${sys.env.getOrElse("SPOUT", "")}" + val spout = Class.forName(spoutPath).getConstructor().newInstance().asInstanceOf[Spout[Any]] + system.actorOf(Props(new SpoutAgent(spout)), "Spout") } - def setConf(seedLoc: String): Unit = { - println(s"I AM AT $seedLoc") - prometheusReporter() + def analysis() = { + println("Creating Analysis Manager") + implicit val system: ActorSystem = initialiseActorSystem(List(locateSeed())) + system.actorOf(Props[AnalysisManager], s"AnalysisManager") + AnalysisRestApi(system) } - def getConf(): String = + def watchDog() = { + println("Cluster Up, informing Partition Managers and Routers") + implicit val system: ActorSystem = initialiseActorSystem(seeds = List(locateSeed())) + system.actorOf(Props(new WatermarkManager(managerCount = partitionCount)),"WatermarkManager") + system.actorOf(Props(new WatchDog(managerCount = partitionCount, minimumRouters = routerCount)), "WatchDog") + } + + + def local() = { + println("putting up cluster in one node") + val spoutPath = s"${sys.env.getOrElse("SPOUT", "")}" + val builderPath = s"${sys.env.getOrElse("GRAPHBUILDER", "")}" + RaphtoryGraph[Any](spoutPath,builderPath) + } + + + /** Initialise a new ActorSystem with configured name and seed nods + * + * @param seeds the set of Seed nodes to be added to the System + * @return A new Akka ActorSystem object with the set config and seed nodes + * as determined by the ${seeds} parameter + */ + def initialiseActorSystem(seeds: List[String]): ActorSystem = { + var config = ConfigFactory.load() + val seedLoc = seeds.head + if (docker) + config = config.withValue( + "akka.cluster.seed-nodes", + ConfigValueFactory.fromIterable( + JavaConversions.asJavaIterable( + seeds.map(_ => s"akka.tcp://$clusterSystemName@$seedLoc") + ) + ) + ) + + val actorSystem = ActorSystem(clusterSystemName, config) + if (!docker) { + AkkaManagement.get(actorSystem).start() + ClusterBootstrap.get(actorSystem).start() + } + printConfigInfo(config, actorSystem) + actorSystem + } + + def locateSeed(): String = if (docker) { while (!("nc seedNode 1600" !).equals(0)) { println("Waiting for seednode to come online") Thread.sleep(3000) } - prometheusReporter() - hostname2Ip("seedNode:1600") + InetAddress.getByName("seedNode").getHostAddress() + ":1600" } else "127.0.0.1" - def prometheusReporter() = { -// try //SystemMetrics.startCollecting() -// catch { -// case e: Exception => println("Error in pro") -// } -// val prom = new PrometheusReporter() -// -// val testLogger = new MetricReporter { -// -// override def reportPeriodSnapshot(snapshot: PeriodSnapshot): Unit = -// try prom.reportPeriodSnapshot(snapshot) -// catch { -// case e: Exception => -// println(e) -// println("Hello I have broken and I cannot get up") -// } -// -// override def reconfigure(config: Config): Unit = prom.reconfigure(config) -// -// override def stop(): Unit = prom.stop() -// } -// Kamon.attachInstrumentation() -// //Kamon. -// //Kamon.addReporter(testLogger) + + case class SocketAddress(host: String, port: String) + case class SystemConfig(bindAddress: SocketAddress, tcpAddress: SocketAddress, seeds: List[ConfigValue], roles: List[ConfigValue]) + def parseConfig(config: Config): SystemConfig = { + val bindHost = config.getString("akka.remote.netty.tcp.bind-hostname") + val bindPort = config.getString("akka.remote.netty.tcp.bind-port") + val bindAddress = SocketAddress(bindHost, bindPort) + + val tcpHost = config.getString("akka.remote.netty.tcp.hostname") + val tcpPort = config.getString("akka.remote.netty.tcp.port") + val tcpAddress = SocketAddress(tcpHost, tcpPort) + + val seeds = config.getList("akka.cluster.seed-nodes").toList + val roles = config.getList("akka.cluster.roles").toList + + SystemConfig(bindAddress = bindAddress, tcpAddress = tcpAddress, seeds, roles) } - def hostname2Ip(seedLoc: String): String = { - val t = seedLoc.split(":") - InetAddress.getByName(t(0)).getHostAddress() + ":" + t(1) + /** Utility method to print the configuration which an ActorSystem has been created under + * + * @param config a TypeSafe config object detailing the Akka system configuration + * @param system an initialised ActorSystem object + */ + def printConfigInfo(config: Config, system: ActorSystem): Unit = { + val log: LoggingAdapter = system.log + + val systemConfig: SystemConfig = parseConfig(config) + val bindAddress: SocketAddress = systemConfig.bindAddress + val tcpAddress: SocketAddress = systemConfig.tcpAddress + + log.info(s"Created ActorSystem with ID: $ssn") + + log.info(s"Binding ActorSystem internally to address ${bindAddress.host}:${bindAddress.port}") + log.info(s"Binding ActorSystem externally to host ${tcpAddress.host}:${tcpAddress.port}") + + log.info(s"Registering the following seeds to ActorSystem: ${systemConfig.seeds}") + log.info(s"Registering the following roles to ActorSystem: ${systemConfig.roles}") + + // FIXME: This is bit unorthodox ... + val akkaSystemUrl: Address = system.asInstanceOf[ExtendedActorSystem].provider.getDefaultAddress + log.info(s"ActorSystem successfully initialised at the following Akka URL: $akkaSystemUrl") } + + def printJavaOptions(): Unit = println(s"Current java options: ${ManagementFactory.getRuntimeMXBean.getInputArguments}") + } diff --git a/mainproject/src/main/scala/com/raphtory/core/clustersetup/DocSvr.scala b/mainproject/src/main/scala/com/raphtory/core/clustersetup/DocSvr.scala deleted file mode 100644 index ddf1550a20..0000000000 --- a/mainproject/src/main/scala/com/raphtory/core/clustersetup/DocSvr.scala +++ /dev/null @@ -1,86 +0,0 @@ -package com.raphtory.core.clustersetup - -import akka.actor.{ActorSystem, Address, ExtendedActorSystem} -import akka.cluster.Member -import akka.event.LoggingAdapter -import akka.management.cluster.bootstrap.ClusterBootstrap -import akka.management.javadsl.AkkaManagement -import com.raphtory.core.clustersetup.util.ConfigUtils._ -import com.raphtory.core.utils.Utils -import com.typesafe.config.{Config, ConfigFactory, ConfigValueFactory} - -import scala.collection.JavaConversions -import scala.collection.JavaConversions._ - -trait DocSvr { - - def seedLoc: String - - implicit val system: ActorSystem - val docker = System.getenv().getOrDefault("DOCKER", "false").trim.toBoolean - - val clusterSystemName: String = Utils.clusterSystemName - val ssn: String = java.util.UUID.randomUUID.toString - - /** Node lookup directory for registered members in the System */ - var nodes = Set.empty[Member] - - /** Initialise a new ActorSystem with configured name and seed nods - * - * @param seeds the set of Seed nodes to be added to the System - * @return A new Akka ActorSystem object with the set config and seed nodes - * as determined by the ${seeds} parameter - */ - def initialiseActorSystem(seeds: List[String]): ActorSystem = { - var config = ConfigFactory.load() - if (docker) - config = config.withValue( - "akka.cluster.seed-nodes", - ConfigValueFactory.fromIterable( - JavaConversions.asJavaIterable( - seeds.map(_ => s"akka.tcp://$clusterSystemName@$seedLoc") - ) - ) - ) - - val actorSystem = ActorSystem(clusterSystemName, config) - if (!docker) { - AkkaManagement.get(actorSystem).start() - ClusterBootstrap.get(actorSystem).start() - } - printConfigInfo(config, actorSystem) - actorSystem - } - - /** Returns the set of Nodes in this ActorSystem - * - * @return A string object representing the address and roles of the members in this System - */ - def getNodes: String = - nodes.map(member => s"${member.address} ${member.getRoles.mkString}").mkString(",") - - /** Utility method to print the configuration which an ActorSystem has been created under - * - * @param config a TypeSafe config object detailing the Akka system configuration - * @param system an initialised ActorSystem object - */ - def printConfigInfo(config: Config, system: ActorSystem): Unit = { - val log: LoggingAdapter = system.log - - val systemConfig: SystemConfig = config.parse() - val bindAddress: SocketAddress = systemConfig.bindAddress - val tcpAddress: SocketAddress = systemConfig.tcpAddress - - log.info(s"Created ActorSystem with ID: $ssn") - - log.info(s"Binding ActorSystem internally to address ${bindAddress.host}:${bindAddress.port}") - log.info(s"Binding ActorSystem externally to host ${tcpAddress.host}:${tcpAddress.port}") - - log.info(s"Registering the following seeds to ActorSystem: ${systemConfig.seeds}") - log.info(s"Registering the following roles to ActorSystem: ${systemConfig.roles}") - - // FIXME: This is bit unorthodox ... - val akkaSystemUrl: Address = system.asInstanceOf[ExtendedActorSystem].provider.getDefaultAddress - log.info(s"ActorSystem successfully initialised at the following Akka URL: $akkaSystemUrl") - } -} diff --git a/mainproject/src/main/scala/com/raphtory/core/clustersetup/LiveAnalysisNode.scala b/mainproject/src/main/scala/com/raphtory/core/clustersetup/LiveAnalysisNode.scala deleted file mode 100644 index d3369fa27f..0000000000 --- a/mainproject/src/main/scala/com/raphtory/core/clustersetup/LiveAnalysisNode.scala +++ /dev/null @@ -1,16 +0,0 @@ -package com.raphtory.core.clustersetup - -/** - * Created by Mirate on 29/09/2017. - */ - -import akka.actor.Props -import com.raphtory.core.analysis.{AnalysisManager, AnalysisRestApi} -case class LiveAnalysisNode(seedLoc: String) extends DocSvr { - implicit val system = initialiseActorSystem(List(seedLoc)) - system.actorOf(Props[AnalysisManager], s"AnalysisManager") - AnalysisRestApi(system) -} - - - diff --git a/mainproject/src/main/scala/com/raphtory/core/clustersetup/ManagerNode.scala b/mainproject/src/main/scala/com/raphtory/core/clustersetup/ManagerNode.scala deleted file mode 100644 index ac0525d48f..0000000000 --- a/mainproject/src/main/scala/com/raphtory/core/clustersetup/ManagerNode.scala +++ /dev/null @@ -1,21 +0,0 @@ -package com.raphtory.core.clustersetup - -import akka.actor.{ActorSystem, Props} -import com.raphtory.core.components.ClusterManagement.RaphtoryReplicator -import com.raphtory.core.utils.Utils - -import scala.language.postfixOps - -case class ManagerNode(seedLoc: String, partitionCount: Int,routerCount:Int) extends DocSvr { - - implicit val system: ActorSystem = initialiseActorSystem(seeds = List(seedLoc)) - - final val persistenceEnabled: Boolean = Utils.persistenceEnabled - final val actorName: String = "PartitionManager" - - system.actorOf( - Props(RaphtoryReplicator(actorType = "Partition Manager", initialManagerCount = partitionCount,initialRouterCount = routerCount)), - actorName - ) - -} diff --git a/mainproject/src/main/scala/com/raphtory/core/clustersetup/RouterNode.scala b/mainproject/src/main/scala/com/raphtory/core/clustersetup/RouterNode.scala deleted file mode 100644 index 5ab06f21ca..0000000000 --- a/mainproject/src/main/scala/com/raphtory/core/clustersetup/RouterNode.scala +++ /dev/null @@ -1,19 +0,0 @@ -package com.raphtory.core.clustersetup - -/** - * Created by Mirate on 31/05/2017. - */ -import akka.actor.{ActorSystem, Props} -import com.raphtory.core.components.ClusterManagement.RaphtoryReplicator -import com.raphtory.core.components.Router.GraphBuilder - -case class RouterNode(seedLoc: String, partitionCount: Int, routerCount:Int, className: String) extends DocSvr { - implicit val system: ActorSystem = initialiseActorSystem(seeds = List(seedLoc)) - - final val actorName: String = "Routers" - - val graphBuilder = Class.forName(className).getConstructor().newInstance().asInstanceOf[GraphBuilder[Any]] - val routerReplicator = RaphtoryReplicator.apply("Router", partitionCount, routerCount,graphBuilder) - system.actorOf(Props(routerReplicator), actorName) - -} diff --git a/mainproject/src/main/scala/com/raphtory/core/clustersetup/SeedNode.scala b/mainproject/src/main/scala/com/raphtory/core/clustersetup/SeedNode.scala deleted file mode 100644 index b120df3fe0..0000000000 --- a/mainproject/src/main/scala/com/raphtory/core/clustersetup/SeedNode.scala +++ /dev/null @@ -1,15 +0,0 @@ -package com.raphtory.core.clustersetup - -import akka.actor.{ActorSystem, Props} -import com.raphtory.core.components.ClusterManagement.SeedActor -import com.typesafe.config.{Config, ConfigFactory} - -case class SeedNode(seedLoc: String) extends DocSvr { - val conf: Config = ConfigFactory.load() - implicit val system: ActorSystem = initialiseActorSystem(seeds = List(seedLoc)) - - system.log.info("Bleep Bloop I am a seed node.") - - val actorName: String = "cluster" - system.actorOf(Props(new SeedActor(svr = this)), actorName) -} diff --git a/mainproject/src/main/scala/com/raphtory/core/clustersetup/UpdateNode.scala b/mainproject/src/main/scala/com/raphtory/core/clustersetup/UpdateNode.scala deleted file mode 100644 index bd972b037a..0000000000 --- a/mainproject/src/main/scala/com/raphtory/core/clustersetup/UpdateNode.scala +++ /dev/null @@ -1,15 +0,0 @@ -package com.raphtory.core.clustersetup - -/** - * Created by Mirate on 13/06/2017. - */ -import akka.actor.{ActorSystem, Props} - -import scala.language.postfixOps - -case class UpdateNode(seedLoc: String, className: String) extends DocSvr { - implicit val system: ActorSystem = initialiseActorSystem(seeds = List(seedLoc)) - - val actorName: String = "UpdateGen" - system.actorOf(Props(Class.forName(className)), actorName) -} diff --git a/mainproject/src/main/scala/com/raphtory/core/clustersetup/WatchDogNode.scala b/mainproject/src/main/scala/com/raphtory/core/clustersetup/WatchDogNode.scala deleted file mode 100644 index f24b85856b..0000000000 --- a/mainproject/src/main/scala/com/raphtory/core/clustersetup/WatchDogNode.scala +++ /dev/null @@ -1,13 +0,0 @@ -package com.raphtory.core.clustersetup - -/** - * Created by Mirate on 11/07/2017. - */ -import akka.actor.{ActorSystem, Props} -import com.raphtory.core.components.ClusterManagement.{WatchDog, WatermarkManager} - -case class WatchDogNode(seedLoc: String, partitionNumber: Int, minimumRouters: Int) extends DocSvr { - implicit val system: ActorSystem = initialiseActorSystem(seeds = List(seedLoc)) - system.actorOf(Props(new WatermarkManager(managerCount = 1)),"WatermarkManager") - system.actorOf(Props(new WatchDog(managerCount = partitionNumber, minimumRouters = minimumRouters)), "WatchDog") -} diff --git a/mainproject/src/main/scala/com/raphtory/core/clustersetup/singlenode/SingleNodeSetup.scala b/mainproject/src/main/scala/com/raphtory/core/clustersetup/singlenode/SingleNodeSetup.scala deleted file mode 100644 index f7418d10e7..0000000000 --- a/mainproject/src/main/scala/com/raphtory/core/clustersetup/singlenode/SingleNodeSetup.scala +++ /dev/null @@ -1,36 +0,0 @@ -package com.raphtory.core.clustersetup.singlenode - -import akka.actor.Props -import com.raphtory.core.analysis.{AnalysisManager, AnalysisRestApi} -import com.raphtory.core.clustersetup.DocSvr -import com.raphtory.core.components.ClusterManagement.{RaphtoryReplicator, SeedActor, WatchDog, WatermarkManager} -import com.raphtory.core.components.Router.GraphBuilder -import com.typesafe.config.{Config, ConfigFactory} - -import scala.language.postfixOps - -// TODO migrate to object props -case class SingleNodeSetup( - seedLoc: String, - routerClassName: String, - UpdaterName: String -) extends DocSvr { - val conf: Config = ConfigFactory.load() - implicit val system = initialiseActorSystem(List(seedLoc)) - AnalysisRestApi(system) - system.actorOf(Props(new SeedActor(this)), "cluster") - system.actorOf(Props(new WatchDog(1, 1)), "WatchDog") - system.actorOf(Props(new WatermarkManager(managerCount = 1)),"WatermarkManager") - - val graphBuilder = Class.forName(routerClassName).getConstructor().newInstance().asInstanceOf[GraphBuilder[Any]] - val routerReplicator = RaphtoryReplicator.apply("Router", 1, 1,graphBuilder) - system.actorOf(Props(routerReplicator), s"Routers") - - system.actorOf(Props(RaphtoryReplicator("Partition Manager", 1,1)), s"PartitionManager") - system.actorOf(Props[AnalysisManager], s"AnalysisManager") - - system.actorOf(Props(Class.forName(UpdaterName)), "Spout") - - - -} diff --git a/mainproject/src/main/scala/com/raphtory/core/clustersetup/util/ConfigUtils.scala b/mainproject/src/main/scala/com/raphtory/core/clustersetup/util/ConfigUtils.scala deleted file mode 100644 index a4a39af8a1..0000000000 --- a/mainproject/src/main/scala/com/raphtory/core/clustersetup/util/ConfigUtils.scala +++ /dev/null @@ -1,34 +0,0 @@ -package com.raphtory.core.clustersetup.util - -import com.typesafe.config.{Config, ConfigValue} - -import scala.collection.JavaConversions._ - -object ConfigUtils { - - case class SocketAddress(host: String, port: String) - - case class SystemConfig( - bindAddress: SocketAddress, - tcpAddress: SocketAddress, - seeds: List[ConfigValue], - roles: List[ConfigValue] - ) - - implicit class ConfigParser(config: Config) { - def parse(): SystemConfig = { - val bindHost = config.getString("akka.remote.netty.tcp.bind-hostname") - val bindPort = config.getString("akka.remote.netty.tcp.bind-port") - val bindAddress = SocketAddress(bindHost, bindPort) - - val tcpHost = config.getString("akka.remote.netty.tcp.hostname") - val tcpPort = config.getString("akka.remote.netty.tcp.port") - val tcpAddress = SocketAddress(tcpHost, tcpPort) - - val seeds = config.getList("akka.cluster.seed-nodes").toList - val roles = config.getList("akka.cluster.roles").toList - - SystemConfig(bindAddress = bindAddress, tcpAddress = tcpAddress, seeds, roles) - } - } -} diff --git a/mainproject/src/main/scala/com/raphtory/core/components/ClusterManagement/SeedActor.scala b/mainproject/src/main/scala/com/raphtory/core/components/ClusterManagement/SeedActor.scala index 7f50929895..63aff4ee4e 100644 --- a/mainproject/src/main/scala/com/raphtory/core/components/ClusterManagement/SeedActor.scala +++ b/mainproject/src/main/scala/com/raphtory/core/components/ClusterManagement/SeedActor.scala @@ -3,9 +3,8 @@ package com.raphtory.core.components.ClusterManagement import akka.actor.{Actor, ActorLogging} import akka.cluster.Cluster import akka.cluster.ClusterEvent._ -import com.raphtory.core.clustersetup.DocSvr -class SeedActor(svr: DocSvr) extends Actor with ActorLogging { +class SeedActor() extends Actor with ActorLogging { val cluster: Cluster = Cluster(context.system) override def preStart(): Unit = { @@ -28,24 +27,24 @@ class SeedActor(svr: DocSvr) extends Actor with ActorLogging { private def processMemberUpEvent(evt: MemberUp): Unit = { log.debug(s"SeedActor received [{}] event.", evt) - svr.nodes.synchronized { - svr.nodes += evt.member - } +// svr.nodes.synchronized { +// svr.nodes += evt.member +// } } private def processMemberRemovedEvent(evt: MemberRemoved): Unit = { log.debug(s"SeedActor received [{}] event.", evt) - svr.nodes.synchronized { - svr.nodes -= evt.member - } +// svr.nodes.synchronized { +// svr.nodes -= evt.member +// } } private def processMemberExitedEvent(evt: MemberExited): Unit = { log.debug(s"SeedActor received [{}] event.", evt) - svr.nodes.synchronized { - svr.nodes -= evt.member - } +// svr.nodes.synchronized { +// svr.nodes -= evt.member +// } } private def processUnreachableMemberEvent(evt: UnreachableMember): Unit = { From 2de45720ed43263b36696fd9fb7be6b67be43f46 Mon Sep 17 00:00:00 2001 From: Miratepuffin Date: Mon, 30 Nov 2020 02:48:34 +0000 Subject: [PATCH 45/58] Changed Go class (main server) to Raphtory Server --- mainproject/Build-Scripts/env-setter.sh | 4 +-- .../{Server.scala => RaphtoryServer.scala} | 33 +++++++------------ 2 files changed, 13 insertions(+), 24 deletions(-) rename mainproject/src/main/scala/com/raphtory/{Server.scala => RaphtoryServer.scala} (97%) diff --git a/mainproject/Build-Scripts/env-setter.sh b/mainproject/Build-Scripts/env-setter.sh index 2cbcf1716a..e029483bb1 100644 --- a/mainproject/Build-Scripts/env-setter.sh +++ b/mainproject/Build-Scripts/env-setter.sh @@ -13,7 +13,5 @@ echo "/////////////////////" #cd /node_exporter-1.0.1.linux-386 #./node_exporter >> /dev/null & -sleep 20 - cd /opt/docker/bin -go $1 +RaphtoryServer $1 diff --git a/mainproject/src/main/scala/com/raphtory/Server.scala b/mainproject/src/main/scala/com/raphtory/RaphtoryServer.scala similarity index 97% rename from mainproject/src/main/scala/com/raphtory/Server.scala rename to mainproject/src/main/scala/com/raphtory/RaphtoryServer.scala index e9a69d3277..8a9ce0ec2a 100644 --- a/mainproject/src/main/scala/com/raphtory/Server.scala +++ b/mainproject/src/main/scala/com/raphtory/RaphtoryServer.scala @@ -4,7 +4,6 @@ import java.lang.management.ManagementFactory import java.net.InetAddress import akka.actor.{ActorSystem, Address, ExtendedActorSystem, Props} -import akka.cluster.Member import akka.event.LoggingAdapter import akka.management.cluster.bootstrap.ClusterBootstrap import akka.management.javadsl.AkkaManagement @@ -21,26 +20,16 @@ import scala.language.postfixOps import scala.sys.process._ //main function -object Go extends App { +object RaphtoryServer extends App { // Kamon.init() //start tool logging - printJavaOptions() - - val conf = ConfigFactory.load() - - - - val docker = System.getenv().getOrDefault("DOCKER", "false").trim.toBoolean - val clusterSystemName: String = Utils.clusterSystemName val ssn: String = java.util.UUID.randomUUID.toString val partitionCount = sys.env("PARTITION_MIN").toInt val routerCount = sys.env("ROUTER_MIN").toInt - - /** Node lookup directory for registered members in the System */ - var nodes = Set.empty[Member] + val docker = System.getenv().getOrDefault("DOCKER", "false").trim.toBoolean args(0) match { case "seedNode" => seedNode() @@ -106,6 +95,16 @@ object Go extends App { RaphtoryGraph[Any](spoutPath,builderPath) } + def locateSeed(): String = + if (docker) { + while (!("nc seedNode 1600" !).equals(0)) { + println("Waiting for seednode to come online") + Thread.sleep(3000) + } + InetAddress.getByName("seedNode").getHostAddress() + ":1600" + } else "127.0.0.1" + + /** Initialise a new ActorSystem with configured name and seed nods * @@ -135,14 +134,6 @@ object Go extends App { actorSystem } - def locateSeed(): String = - if (docker) { - while (!("nc seedNode 1600" !).equals(0)) { - println("Waiting for seednode to come online") - Thread.sleep(3000) - } - InetAddress.getByName("seedNode").getHostAddress() + ":1600" - } else "127.0.0.1" case class SocketAddress(host: String, port: String) From 255408f170a1aa49aa8ba751642dd183aae9800f Mon Sep 17 00:00:00 2001 From: Miratepuffin Date: Mon, 30 Nov 2020 03:29:04 +0000 Subject: [PATCH 46/58] Testing single node deployment, all working --- mainproject/Build-Scripts/env-setter.sh | 2 +- mainproject/build.sbt | 2 +- startup/base.env | 10 ++++------ startup/base.yml | 4 ++-- startup/quick-test.sh | 0 startup/singlenode.yml | 18 ++++++++++++++++++ 6 files changed, 26 insertions(+), 10 deletions(-) mode change 100644 => 100755 startup/quick-test.sh create mode 100644 startup/singlenode.yml diff --git a/mainproject/Build-Scripts/env-setter.sh b/mainproject/Build-Scripts/env-setter.sh index e029483bb1..c5524295ee 100644 --- a/mainproject/Build-Scripts/env-setter.sh +++ b/mainproject/Build-Scripts/env-setter.sh @@ -14,4 +14,4 @@ echo "/////////////////////" #./node_exporter >> /dev/null & cd /opt/docker/bin -RaphtoryServer $1 +raphtory-server $1 diff --git a/mainproject/build.sbt b/mainproject/build.sbt index 4475ef105d..b5c07d8646 100644 --- a/mainproject/build.sbt +++ b/mainproject/build.sbt @@ -140,7 +140,7 @@ lazy val cluster = project .settings(dockerStuff: _*) .settings( mappings in Universal += - file(s"${baseDirectory.value}/../Build-Scripts/env-setter.sh") -> "bin/env-setter.sh" + file(s"${baseDirectory.value}/Build-Scripts/env-setter.sh") -> "bin/env-setter.sh" ) .settings(dockerEntrypoint := Seq("bash")) .settings( diff --git a/startup/base.env b/startup/base.env index 9b308b47e2..ecc5552284 100644 --- a/startup/base.env +++ b/startup/base.env @@ -1,12 +1,10 @@ COMPOSE_PROJECT_NAME=raphtory LOCAL=true DOCKER=true -SPOUTCLASS=com.raphtory.spouts.EtherFileReader -ROUTERCLASS=com.raphtory.routers.RouterCSV -FILE_SPOUT_DIRECTORY=/opt/docker/volume -FILE_SPOUT_FILENAME=example.csv -FILE_SPOUT_BLOCK_SIZE=50 -FILE_SPOUT_DROP_HEADER=false +SPOUT=com.raphtory.spouts.FileSpout +GRAPHBUILDER=com.raphtory.examples.gab.graphbuilders.GabUserGraphBuilder +FILE_SPOUT_FILENAME=gabNetwork500.csv +FILE_SPOUT_DROP_HEADER=true JAVA_OPTS=-Xms1G PARTITION_MIN=1 ROUTER_MIN=1 diff --git a/startup/base.yml b/startup/base.yml index 6e69cbde42..c657907c23 100644 --- a/startup/base.yml +++ b/startup/base.yml @@ -8,9 +8,9 @@ networks: services: raphtory: - image: tsukitsune/raphtory:latest + image: miratepuffin/raphtory:latest container_name: raphtory ports: - 8081:8081 - command: env-setter.sh singleNodeSetup + command: env-setter.sh local env_file: .env diff --git a/startup/quick-test.sh b/startup/quick-test.sh old mode 100644 new mode 100755 diff --git a/startup/singlenode.yml b/startup/singlenode.yml new file mode 100644 index 0000000000..cc832d2c89 --- /dev/null +++ b/startup/singlenode.yml @@ -0,0 +1,18 @@ +version: '3.3' + +networks: + default: + ipam: + config: + - subnet: 10.0.0.0/24 + +services: + raphtory: + image: miratepuffin/raphtory:latest + container_name: raphtory + ports: + - 8081:8081 + command: env-setter.sh local + env_file: .env + volumes: + - /Users/Mirate/github/newraphtory/Raphtory/startup:/opt/docker/volume From 3bff4288602e3c9c63bc403018e05b45ac68cd38 Mon Sep 17 00:00:00 2001 From: Miratepuffin Date: Mon, 30 Nov 2020 04:08:28 +0000 Subject: [PATCH 47/58] Initial tests for distributed seem to be working --- .../scala/com/raphtory/RaphtoryServer.scala | 6 +- startup/base.env | 1 - startup/cluster.yml | 66 +++++++++++++++++++ startup/quick-distro-test.sh | 10 +++ 4 files changed, 78 insertions(+), 5 deletions(-) create mode 100644 startup/cluster.yml create mode 100755 startup/quick-distro-test.sh diff --git a/mainproject/src/main/scala/com/raphtory/RaphtoryServer.scala b/mainproject/src/main/scala/com/raphtory/RaphtoryServer.scala index 8a9ce0ec2a..302e9489df 100644 --- a/mainproject/src/main/scala/com/raphtory/RaphtoryServer.scala +++ b/mainproject/src/main/scala/com/raphtory/RaphtoryServer.scala @@ -37,7 +37,7 @@ object RaphtoryServer extends App { case "partitionManager" => partition() case "spout" => spout() case "analysisManager" =>analysis() - case "clusterUp" => watchDog() + case "watchdog" => watchDog() case "local" => local() } @@ -54,9 +54,7 @@ object RaphtoryServer extends App { val builderPath = s"${sys.env.getOrElse("GRAPHBUILDER", "")}" val graphBuilder = Class.forName(builderPath).getConstructor().newInstance().asInstanceOf[GraphBuilder[Any]] - - val routerReplicator = RaphtoryReplicator.apply("Router", partitionCount, routerCount,graphBuilder) - system.actorOf(Props(routerReplicator), "Routers") + system.actorOf(Props(RaphtoryReplicator("Router", partitionCount, routerCount,graphBuilder)), "Routers") } def partition() = { diff --git a/startup/base.env b/startup/base.env index ecc5552284..b7e674997f 100644 --- a/startup/base.env +++ b/startup/base.env @@ -1,5 +1,4 @@ COMPOSE_PROJECT_NAME=raphtory -LOCAL=true DOCKER=true SPOUT=com.raphtory.spouts.FileSpout GRAPHBUILDER=com.raphtory.examples.gab.graphbuilders.GabUserGraphBuilder diff --git a/startup/cluster.yml b/startup/cluster.yml new file mode 100644 index 0000000000..d31cd137f6 --- /dev/null +++ b/startup/cluster.yml @@ -0,0 +1,66 @@ +version: '3.3' + + +services: + #### seedSetup #### + seedNode: + image: miratepuffin/raphtory:latest + command: env-setter.sh seedNode + env_file: .env + deploy: + endpoint_mode: dnsrr + replicas: 1 + + watchDog: + image: miratepuffin/raphtory:latest + depends_on: + - seedNode + command: env-setter.sh watchdog + env_file: .env + deploy: + endpoint_mode: vip + replicas: 1 + + partitionManager: + image: miratepuffin/raphtory:latest + depends_on: + - seedNode + - watchDog + command: env-setter.sh partitionManager + env_file: .env + deploy: + endpoint_mode: dnsrr + mode: global + + router: + image: miratepuffin/raphtory:latest + depends_on: + - seedNode + - watchDog + + command: env-setter.sh router + env_file: .env + deploy: + endpoint_mode: dnsrr + mode: global + + spout: + image: miratepuffin/raphtory:latest + depends_on: + - seedNode + - watchDog + command: env-setter.sh spout + env_file: .env + deploy: + replicas: 1 + + analysis: + image: miratepuffin/raphtory:latest + depends_on: + - seedNode + - watchDog + command: env-setter.sh analysisManager + env_file: .env + deploy: + replicas: 1 + \ No newline at end of file diff --git a/startup/quick-distro-test.sh b/startup/quick-distro-test.sh new file mode 100755 index 0000000000..bd3aac2e21 --- /dev/null +++ b/startup/quick-distro-test.sh @@ -0,0 +1,10 @@ +#/bin/bash + +cp base.env .env + +echo 'Starting up Raphtory ...' +docker-compose -f cluster.yml up --force-recreate + +if [ $? -ne 0 ]; then + echo Whoops! Something went wrong! +fi From c2efc12d639eb1ab73a2af71f5421a864101fa94 Mon Sep 17 00:00:00 2001 From: Miratepuffin Date: Mon, 30 Nov 2020 16:49:15 +0000 Subject: [PATCH 48/58] Refactored algorithms to be same level as spouts and core --- .../analysis/Algorithms => algorithms}/BinaryDefusion.scala | 2 +- .../Algorithms => algorithms}/CommunityOutlierDetection.scala | 2 +- .../Algorithms => algorithms}/ConnectedComponents.scala | 2 +- .../{core/analysis/Algorithms => algorithms}/DegreeBasic.scala | 2 +- .../analysis/Algorithms => algorithms}/DegreeDistribution.scala | 2 +- .../analysis/Algorithms => algorithms}/DegreeRanking.scala | 2 +- .../analysis/Algorithms => algorithms}/Distributions.scala | 2 +- .../Algorithms => algorithms}/GabWeightedPageRank.scala | 2 +- .../raphtory/{core/analysis/Algorithms => algorithms}/LPA.scala | 2 +- .../analysis/Algorithms => algorithms}/MultiLayerLPA.scala | 2 +- .../{core/analysis/Algorithms => algorithms}/PageRank.scala | 2 +- .../Algorithms => algorithms}/TemporalTriangleCount.scala | 2 +- .../analysis/Algorithms => algorithms}/TriangleCount.scala | 2 +- .../{core/analysis/Algorithms => algorithms}/WattsCascade.scala | 2 +- .../analysis/Algorithms => algorithms}/WeightedPageRank.scala | 2 +- .../{core/analysis/Algorithms => algorithms}/lotrExample.scala | 2 +- 16 files changed, 16 insertions(+), 16 deletions(-) rename mainproject/src/main/scala/com/raphtory/{core/analysis/Algorithms => algorithms}/BinaryDefusion.scala (97%) rename mainproject/src/main/scala/com/raphtory/{core/analysis/Algorithms => algorithms}/CommunityOutlierDetection.scala (98%) rename mainproject/src/main/scala/com/raphtory/{core/analysis/Algorithms => algorithms}/ConnectedComponents.scala (98%) rename mainproject/src/main/scala/com/raphtory/{core/analysis/Algorithms => algorithms}/DegreeBasic.scala (98%) rename mainproject/src/main/scala/com/raphtory/{core/analysis/Algorithms => algorithms}/DegreeDistribution.scala (98%) rename mainproject/src/main/scala/com/raphtory/{core/analysis/Algorithms => algorithms}/DegreeRanking.scala (98%) rename mainproject/src/main/scala/com/raphtory/{core/analysis/Algorithms => algorithms}/Distributions.scala (98%) rename mainproject/src/main/scala/com/raphtory/{core/analysis/Algorithms => algorithms}/GabWeightedPageRank.scala (98%) rename mainproject/src/main/scala/com/raphtory/{core/analysis/Algorithms => algorithms}/LPA.scala (98%) rename mainproject/src/main/scala/com/raphtory/{core/analysis/Algorithms => algorithms}/MultiLayerLPA.scala (98%) rename mainproject/src/main/scala/com/raphtory/{core/analysis/Algorithms => algorithms}/PageRank.scala (98%) rename mainproject/src/main/scala/com/raphtory/{core/analysis/Algorithms => algorithms}/TemporalTriangleCount.scala (98%) rename mainproject/src/main/scala/com/raphtory/{core/analysis/Algorithms => algorithms}/TriangleCount.scala (98%) rename mainproject/src/main/scala/com/raphtory/{core/analysis/Algorithms => algorithms}/WattsCascade.scala (98%) rename mainproject/src/main/scala/com/raphtory/{core/analysis/Algorithms => algorithms}/WeightedPageRank.scala (98%) rename mainproject/src/main/scala/com/raphtory/{core/analysis/Algorithms => algorithms}/lotrExample.scala (98%) diff --git a/mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/BinaryDefusion.scala b/mainproject/src/main/scala/com/raphtory/algorithms/BinaryDefusion.scala similarity index 97% rename from mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/BinaryDefusion.scala rename to mainproject/src/main/scala/com/raphtory/algorithms/BinaryDefusion.scala index fa260686b9..cca567de29 100644 --- a/mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/BinaryDefusion.scala +++ b/mainproject/src/main/scala/com/raphtory/algorithms/BinaryDefusion.scala @@ -1,4 +1,4 @@ -package com.raphtory.core.analysis.Algorithms +package com.raphtory.algorithms import com.raphtory.core.analysis.API.Analyser diff --git a/mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/CommunityOutlierDetection.scala b/mainproject/src/main/scala/com/raphtory/algorithms/CommunityOutlierDetection.scala similarity index 98% rename from mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/CommunityOutlierDetection.scala rename to mainproject/src/main/scala/com/raphtory/algorithms/CommunityOutlierDetection.scala index 534ff6571f..0cf68c0415 100644 --- a/mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/CommunityOutlierDetection.scala +++ b/mainproject/src/main/scala/com/raphtory/algorithms/CommunityOutlierDetection.scala @@ -1,4 +1,4 @@ -package com.raphtory.core.analysis.Algorithms +package com.raphtory.algorithms import com.raphtory.core.analysis.API.entityVisitors.VertexVisitor import com.raphtory.core.utils.Utils diff --git a/mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/ConnectedComponents.scala b/mainproject/src/main/scala/com/raphtory/algorithms/ConnectedComponents.scala similarity index 98% rename from mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/ConnectedComponents.scala rename to mainproject/src/main/scala/com/raphtory/algorithms/ConnectedComponents.scala index 25957260f8..04720bcc30 100644 --- a/mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/ConnectedComponents.scala +++ b/mainproject/src/main/scala/com/raphtory/algorithms/ConnectedComponents.scala @@ -1,4 +1,4 @@ -package com.raphtory.core.analysis.Algorithms +package com.raphtory.algorithms import com.raphtory.core.analysis.API.Analyser import com.raphtory.core.utils.Utils diff --git a/mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/DegreeBasic.scala b/mainproject/src/main/scala/com/raphtory/algorithms/DegreeBasic.scala similarity index 98% rename from mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/DegreeBasic.scala rename to mainproject/src/main/scala/com/raphtory/algorithms/DegreeBasic.scala index cf119aadf2..2cbf675640 100644 --- a/mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/DegreeBasic.scala +++ b/mainproject/src/main/scala/com/raphtory/algorithms/DegreeBasic.scala @@ -1,4 +1,4 @@ -package com.raphtory.core.analysis.Algorithms +package com.raphtory.algorithms import com.raphtory.core.analysis.API.Analyser import com.raphtory.core.utils.Utils diff --git a/mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/DegreeDistribution.scala b/mainproject/src/main/scala/com/raphtory/algorithms/DegreeDistribution.scala similarity index 98% rename from mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/DegreeDistribution.scala rename to mainproject/src/main/scala/com/raphtory/algorithms/DegreeDistribution.scala index e5b10a86d7..55692f13aa 100644 --- a/mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/DegreeDistribution.scala +++ b/mainproject/src/main/scala/com/raphtory/algorithms/DegreeDistribution.scala @@ -1,4 +1,4 @@ -package com.raphtory.core.analysis.Algorithms +package com.raphtory.algorithms import com.raphtory.core.analysis.API.Analyser import com.raphtory.core.utils.Utils diff --git a/mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/DegreeRanking.scala b/mainproject/src/main/scala/com/raphtory/algorithms/DegreeRanking.scala similarity index 98% rename from mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/DegreeRanking.scala rename to mainproject/src/main/scala/com/raphtory/algorithms/DegreeRanking.scala index 790aa8bc45..352d688922 100644 --- a/mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/DegreeRanking.scala +++ b/mainproject/src/main/scala/com/raphtory/algorithms/DegreeRanking.scala @@ -1,4 +1,4 @@ -package com.raphtory.core.analysis.Algorithms +package com.raphtory.algorithms import com.raphtory.core.analysis.API.Analyser import com.raphtory.core.utils.Utils diff --git a/mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/Distributions.scala b/mainproject/src/main/scala/com/raphtory/algorithms/Distributions.scala similarity index 98% rename from mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/Distributions.scala rename to mainproject/src/main/scala/com/raphtory/algorithms/Distributions.scala index 6ca6bc99f7..6ec3081197 100644 --- a/mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/Distributions.scala +++ b/mainproject/src/main/scala/com/raphtory/algorithms/Distributions.scala @@ -1,4 +1,4 @@ -package com.raphtory.core.analysis.Algorithms +package com.raphtory.algorithms import com.raphtory.core.analysis.API.Analyser import com.raphtory.core.utils.Utils diff --git a/mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/GabWeightedPageRank.scala b/mainproject/src/main/scala/com/raphtory/algorithms/GabWeightedPageRank.scala similarity index 98% rename from mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/GabWeightedPageRank.scala rename to mainproject/src/main/scala/com/raphtory/algorithms/GabWeightedPageRank.scala index 57226443cd..da9a922af9 100644 --- a/mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/GabWeightedPageRank.scala +++ b/mainproject/src/main/scala/com/raphtory/algorithms/GabWeightedPageRank.scala @@ -1,4 +1,4 @@ -package com.raphtory.core.analysis.Algorithms +package com.raphtory.algorithms import com.raphtory.core.analysis.API.Analyser import com.raphtory.core.utils.Utils diff --git a/mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/LPA.scala b/mainproject/src/main/scala/com/raphtory/algorithms/LPA.scala similarity index 98% rename from mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/LPA.scala rename to mainproject/src/main/scala/com/raphtory/algorithms/LPA.scala index 40b8469f34..478ea3363d 100644 --- a/mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/LPA.scala +++ b/mainproject/src/main/scala/com/raphtory/algorithms/LPA.scala @@ -1,4 +1,4 @@ -package com.raphtory.core.analysis.Algorithms +package com.raphtory.algorithms import com.raphtory.core.analysis.API.Analyser import com.raphtory.core.analysis.API.entityVisitors.VertexVisitor diff --git a/mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/MultiLayerLPA.scala b/mainproject/src/main/scala/com/raphtory/algorithms/MultiLayerLPA.scala similarity index 98% rename from mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/MultiLayerLPA.scala rename to mainproject/src/main/scala/com/raphtory/algorithms/MultiLayerLPA.scala index 37d7a1b296..7d4474591d 100644 --- a/mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/MultiLayerLPA.scala +++ b/mainproject/src/main/scala/com/raphtory/algorithms/MultiLayerLPA.scala @@ -1,4 +1,4 @@ -package com.raphtory.core.analysis.Algorithms +package com.raphtory.algorithms import scala.collection.mutable.ArrayBuffer diff --git a/mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/PageRank.scala b/mainproject/src/main/scala/com/raphtory/algorithms/PageRank.scala similarity index 98% rename from mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/PageRank.scala rename to mainproject/src/main/scala/com/raphtory/algorithms/PageRank.scala index b632713e65..19d14cb9a4 100644 --- a/mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/PageRank.scala +++ b/mainproject/src/main/scala/com/raphtory/algorithms/PageRank.scala @@ -1,4 +1,4 @@ -package com.raphtory.core.analysis.Algorithms +package com.raphtory.algorithms import com.raphtory.core.analysis.API.Analyser import com.raphtory.core.utils.Utils diff --git a/mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/TemporalTriangleCount.scala b/mainproject/src/main/scala/com/raphtory/algorithms/TemporalTriangleCount.scala similarity index 98% rename from mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/TemporalTriangleCount.scala rename to mainproject/src/main/scala/com/raphtory/algorithms/TemporalTriangleCount.scala index 38cfec8d13..cd276124a7 100644 --- a/mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/TemporalTriangleCount.scala +++ b/mainproject/src/main/scala/com/raphtory/algorithms/TemporalTriangleCount.scala @@ -1,4 +1,4 @@ -package com.raphtory.core.analysis.Algorithms +package com.raphtory.algorithms import com.raphtory.core.analysis.API.Analyser diff --git a/mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/TriangleCount.scala b/mainproject/src/main/scala/com/raphtory/algorithms/TriangleCount.scala similarity index 98% rename from mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/TriangleCount.scala rename to mainproject/src/main/scala/com/raphtory/algorithms/TriangleCount.scala index 89a62c452a..ca24a43a41 100644 --- a/mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/TriangleCount.scala +++ b/mainproject/src/main/scala/com/raphtory/algorithms/TriangleCount.scala @@ -1,4 +1,4 @@ -package com.raphtory.core.analysis.Algorithms +package com.raphtory.algorithms import com.raphtory.core.analysis.API.Analyser import com.raphtory.core.utils.Utils diff --git a/mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/WattsCascade.scala b/mainproject/src/main/scala/com/raphtory/algorithms/WattsCascade.scala similarity index 98% rename from mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/WattsCascade.scala rename to mainproject/src/main/scala/com/raphtory/algorithms/WattsCascade.scala index f15907cda4..7a9e9f3615 100644 --- a/mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/WattsCascade.scala +++ b/mainproject/src/main/scala/com/raphtory/algorithms/WattsCascade.scala @@ -1,4 +1,4 @@ -package com.raphtory.core.analysis.Algorithms +package com.raphtory.algorithms import com.raphtory.core.analysis.API.Analyser diff --git a/mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/WeightedPageRank.scala b/mainproject/src/main/scala/com/raphtory/algorithms/WeightedPageRank.scala similarity index 98% rename from mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/WeightedPageRank.scala rename to mainproject/src/main/scala/com/raphtory/algorithms/WeightedPageRank.scala index 3d47c2aa29..5d79c4f5aa 100644 --- a/mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/WeightedPageRank.scala +++ b/mainproject/src/main/scala/com/raphtory/algorithms/WeightedPageRank.scala @@ -1,4 +1,4 @@ -package com.raphtory.core.analysis.Algorithms +package com.raphtory.algorithms import com.raphtory.core.analysis.API.Analyser import com.raphtory.core.utils.Utils diff --git a/mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/lotrExample.scala b/mainproject/src/main/scala/com/raphtory/algorithms/lotrExample.scala similarity index 98% rename from mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/lotrExample.scala rename to mainproject/src/main/scala/com/raphtory/algorithms/lotrExample.scala index 712c676d37..0f36b7ec2b 100644 --- a/mainproject/src/main/scala/com/raphtory/core/analysis/Algorithms/lotrExample.scala +++ b/mainproject/src/main/scala/com/raphtory/algorithms/lotrExample.scala @@ -1,4 +1,4 @@ -package com.raphtory.core.analysis.Algorithms +package com.raphtory.algorithms import com.raphtory.core.analysis.API.Analyser From a17d808209d110fac34be20f35dcf47430b5b190 Mon Sep 17 00:00:00 2001 From: Miratepuffin Date: Mon, 30 Nov 2020 17:00:42 +0000 Subject: [PATCH 49/58] Moved EntityStorage into model package and minor utils cleanup --- .../raphtory/core/analysis/API/GraphLenses/GraphLens.scala | 2 +- .../raphtory/core/analysis/API/GraphLenses/ViewLens.scala | 2 +- .../raphtory/core/analysis/API/GraphLenses/WindowLens.scala | 2 +- .../components/ClusterManagement/RaphtoryReplicator.scala | 2 +- .../raphtory/core/components/PartitionManager/Reader.scala | 2 +- .../PartitionManager/Workers/IngestionWorker.scala | 2 +- .../components/PartitionManager/Workers/ReaderWorker.scala | 2 +- .../raphtory/core/components/PartitionManager/Writer.scala | 2 +- .../com/raphtory/core/{storage => model}/EntityStorage.scala | 3 +-- .../scala/com/raphtory/core/model/graphentities/Edge.scala | 2 +- .../scala/com/raphtory/core/model/graphentities/Entity.scala | 5 +++-- .../raphtory/core/model/graphentities/MutableProperty.scala | 5 +++-- .../scala/com/raphtory/core/model/graphentities/Vertex.scala | 2 +- .../main/scala/com/raphtory/core/utils/HistoryOrdering.scala | 5 ----- .../src/main/scala/com/raphtory/core/utils/Utils.scala | 4 ---- 15 files changed, 17 insertions(+), 25 deletions(-) rename mainproject/src/main/scala/com/raphtory/core/{storage => model}/EntityStorage.scala (99%) delete mode 100644 mainproject/src/main/scala/com/raphtory/core/utils/HistoryOrdering.scala diff --git a/mainproject/src/main/scala/com/raphtory/core/analysis/API/GraphLenses/GraphLens.scala b/mainproject/src/main/scala/com/raphtory/core/analysis/API/GraphLenses/GraphLens.scala index 0c96260cbc..f66d39f7e6 100644 --- a/mainproject/src/main/scala/com/raphtory/core/analysis/API/GraphLenses/GraphLens.scala +++ b/mainproject/src/main/scala/com/raphtory/core/analysis/API/GraphLenses/GraphLens.scala @@ -6,8 +6,8 @@ import akka.actor.ActorContext import com.raphtory.core.analysis.API.ManagerCount import com.raphtory.core.analysis.API.entityVisitors.VertexVisitor import com.raphtory.core.components.PartitionManager.Workers.ViewJob +import com.raphtory.core.model.EntityStorage import com.raphtory.core.model.graphentities.Vertex -import com.raphtory.core.storage.EntityStorage import scala.collection.parallel.ParIterable diff --git a/mainproject/src/main/scala/com/raphtory/core/analysis/API/GraphLenses/ViewLens.scala b/mainproject/src/main/scala/com/raphtory/core/analysis/API/GraphLenses/ViewLens.scala index ef6898b249..2e3ca07d6e 100644 --- a/mainproject/src/main/scala/com/raphtory/core/analysis/API/GraphLenses/ViewLens.scala +++ b/mainproject/src/main/scala/com/raphtory/core/analysis/API/GraphLenses/ViewLens.scala @@ -4,8 +4,8 @@ import akka.actor.ActorContext import com.raphtory.core.analysis.API.ManagerCount import com.raphtory.core.analysis.API.entityVisitors.VertexVisitor import com.raphtory.core.components.PartitionManager.Workers.ViewJob +import com.raphtory.core.model.EntityStorage import com.raphtory.core.model.graphentities.Vertex -import com.raphtory.core.storage.EntityStorage import kamon.Kamon import scala.collection.parallel.ParIterable diff --git a/mainproject/src/main/scala/com/raphtory/core/analysis/API/GraphLenses/WindowLens.scala b/mainproject/src/main/scala/com/raphtory/core/analysis/API/GraphLenses/WindowLens.scala index 47a8774e92..3ce251d258 100644 --- a/mainproject/src/main/scala/com/raphtory/core/analysis/API/GraphLenses/WindowLens.scala +++ b/mainproject/src/main/scala/com/raphtory/core/analysis/API/GraphLenses/WindowLens.scala @@ -4,8 +4,8 @@ import akka.actor.ActorContext import com.raphtory.core.analysis.API.ManagerCount import com.raphtory.core.analysis.API.entityVisitors.VertexVisitor import com.raphtory.core.components.PartitionManager.Workers.ViewJob +import com.raphtory.core.model.EntityStorage import com.raphtory.core.model.graphentities.Vertex -import com.raphtory.core.storage.EntityStorage import kamon.Kamon import scala.collection.mutable.ArrayBuffer diff --git a/mainproject/src/main/scala/com/raphtory/core/components/ClusterManagement/RaphtoryReplicator.scala b/mainproject/src/main/scala/com/raphtory/core/components/ClusterManagement/RaphtoryReplicator.scala index 71d39ca3b0..c9334cabab 100644 --- a/mainproject/src/main/scala/com/raphtory/core/components/ClusterManagement/RaphtoryReplicator.scala +++ b/mainproject/src/main/scala/com/raphtory/core/components/ClusterManagement/RaphtoryReplicator.scala @@ -8,8 +8,8 @@ import akka.util.Timeout import com.raphtory.core.components.PartitionManager.Workers.IngestionWorker import com.raphtory.core.components.PartitionManager.{Reader, Writer} import com.raphtory.core.components.Router.{GraphBuilder, RouterManager} +import com.raphtory.core.model.EntityStorage import com.raphtory.core.model.communication._ -import com.raphtory.core.storage.EntityStorage import com.raphtory.core.utils.{SchedulerUtil, Utils} import scala.collection.mutable diff --git a/mainproject/src/main/scala/com/raphtory/core/components/PartitionManager/Reader.scala b/mainproject/src/main/scala/com/raphtory/core/components/PartitionManager/Reader.scala index 0915932534..a155013f3c 100644 --- a/mainproject/src/main/scala/com/raphtory/core/components/PartitionManager/Reader.scala +++ b/mainproject/src/main/scala/com/raphtory/core/components/PartitionManager/Reader.scala @@ -4,8 +4,8 @@ import akka.actor.{Actor, ActorLogging, ActorRef, Props, Terminated} import akka.cluster.pubsub.{DistributedPubSub, DistributedPubSubMediator} import akka.cluster.pubsub.DistributedPubSubMediator.SubscribeAck import com.raphtory.core.components.PartitionManager.Workers.ReaderWorker +import com.raphtory.core.model.EntityStorage import com.raphtory.core.model.communication._ -import com.raphtory.core.storage.EntityStorage import com.raphtory.core.utils.Utils import scala.collection.parallel.mutable.ParTrieMap diff --git a/mainproject/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/IngestionWorker.scala b/mainproject/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/IngestionWorker.scala index 38d2d8e989..fc0dd8524e 100644 --- a/mainproject/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/IngestionWorker.scala +++ b/mainproject/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/IngestionWorker.scala @@ -4,8 +4,8 @@ import java.util.concurrent.atomic.AtomicInteger import akka.actor.{Actor, ActorLogging, ActorRef, Cancellable} import akka.cluster.pubsub.{DistributedPubSub, DistributedPubSubMediator} +import com.raphtory.core.model.EntityStorage import com.raphtory.core.model.communication._ -import com.raphtory.core.storage.EntityStorage import com.raphtory.core.utils.SchedulerUtil import kamon.Kamon diff --git a/mainproject/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/ReaderWorker.scala b/mainproject/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/ReaderWorker.scala index 8346474c67..004ce6b1c5 100644 --- a/mainproject/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/ReaderWorker.scala +++ b/mainproject/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/ReaderWorker.scala @@ -6,8 +6,8 @@ import akka.actor.{Actor, ActorLogging, ActorRef} import akka.cluster.pubsub.{DistributedPubSub, DistributedPubSubMediator} import com.raphtory.core.analysis.API.GraphLenses.{GraphLens, ViewLens, WindowLens} import com.raphtory.core.analysis.API.{Analyser, _} +import com.raphtory.core.model.EntityStorage import com.raphtory.core.model.communication._ -import com.raphtory.core.storage.EntityStorage import com.raphtory.core.utils.Utils import kamon.Kamon diff --git a/mainproject/src/main/scala/com/raphtory/core/components/PartitionManager/Writer.scala b/mainproject/src/main/scala/com/raphtory/core/components/PartitionManager/Writer.scala index 4ee5620edc..1a46b7f328 100644 --- a/mainproject/src/main/scala/com/raphtory/core/components/PartitionManager/Writer.scala +++ b/mainproject/src/main/scala/com/raphtory/core/components/PartitionManager/Writer.scala @@ -3,8 +3,8 @@ package com.raphtory.core.components.PartitionManager import akka.actor.SupervisorStrategy.Resume import akka.actor.{Actor, ActorLogging, ActorRef, Cancellable, OneForOneStrategy, Terminated} import akka.cluster.pubsub.{DistributedPubSub, DistributedPubSubMediator} +import com.raphtory.core.model.EntityStorage import com.raphtory.core.model.communication._ -import com.raphtory.core.storage.EntityStorage import com.raphtory.core.utils.{SchedulerUtil, Utils} import scala.collection.mutable diff --git a/mainproject/src/main/scala/com/raphtory/core/storage/EntityStorage.scala b/mainproject/src/main/scala/com/raphtory/core/model/EntityStorage.scala similarity index 99% rename from mainproject/src/main/scala/com/raphtory/core/storage/EntityStorage.scala rename to mainproject/src/main/scala/com/raphtory/core/model/EntityStorage.scala index 4c54c1cab2..fa5dc4a13b 100644 --- a/mainproject/src/main/scala/com/raphtory/core/storage/EntityStorage.scala +++ b/mainproject/src/main/scala/com/raphtory/core/model/EntityStorage.scala @@ -1,4 +1,4 @@ -package com.raphtory.core.storage +package com.raphtory.core.model import akka.actor.ActorRef import akka.cluster.pubsub.DistributedPubSubMediator @@ -29,7 +29,6 @@ class EntityStorage(partitionID:Int,workerID: Int) { var managerCount: Int = 1 var managerID: Int = 0 var mediator: ActorRef = null - var windowing: Boolean = Utils.windowing //stuff for compression and archiving var oldestTime: Long = Long.MaxValue var newestTime: Long = 0 diff --git a/mainproject/src/main/scala/com/raphtory/core/model/graphentities/Edge.scala b/mainproject/src/main/scala/com/raphtory/core/model/graphentities/Edge.scala index 55211070bd..4674a8ba63 100644 --- a/mainproject/src/main/scala/com/raphtory/core/model/graphentities/Edge.scala +++ b/mainproject/src/main/scala/com/raphtory/core/model/graphentities/Edge.scala @@ -1,6 +1,6 @@ package com.raphtory.core.model.graphentities -import com.raphtory.core.storage.EntityStorage +import com.raphtory.core.model.EntityStorage import scala.collection.mutable import scala.collection.parallel.mutable.ParTrieMap diff --git a/mainproject/src/main/scala/com/raphtory/core/model/graphentities/Entity.scala b/mainproject/src/main/scala/com/raphtory/core/model/graphentities/Entity.scala index c739ac14b7..5975b4dd9a 100644 --- a/mainproject/src/main/scala/com/raphtory/core/model/graphentities/Entity.scala +++ b/mainproject/src/main/scala/com/raphtory/core/model/graphentities/Entity.scala @@ -1,7 +1,5 @@ package com.raphtory.core.model.graphentities -import com.raphtory.core.utils.HistoryOrdering - import scala.collection.mutable import scala.collection.parallel.mutable.ParTrieMap @@ -20,6 +18,9 @@ abstract class Entity(val creationTime: Long, isInitialValue: Boolean) { var properties: ParTrieMap[String, Property] = ParTrieMap[String, Property]() // History of that entity + object HistoryOrdering extends Ordering[Long] { + def compare(key1: Long, key2: Long) = key2.compareTo(key1) + } var history: mutable.TreeMap[Long, Boolean] = mutable.TreeMap(creationTime -> isInitialValue)(HistoryOrdering) var oldestPoint: Long = creationTime diff --git a/mainproject/src/main/scala/com/raphtory/core/model/graphentities/MutableProperty.scala b/mainproject/src/main/scala/com/raphtory/core/model/graphentities/MutableProperty.scala index a0fb01a7bd..d6ebe3610b 100644 --- a/mainproject/src/main/scala/com/raphtory/core/model/graphentities/MutableProperty.scala +++ b/mainproject/src/main/scala/com/raphtory/core/model/graphentities/MutableProperty.scala @@ -1,7 +1,5 @@ package com.raphtory.core.model.graphentities -import com.raphtory.core.utils.HistoryOrdering - import scala.collection.mutable /** * @@ -11,6 +9,9 @@ import scala.collection.mutable * @param value Property value */ class MutableProperty(creationTime: Long, value: Any) extends Property { + object HistoryOrdering extends Ordering[Long] { + def compare(key1: Long, key2: Long) = key2.compareTo(key1) + } var previousState: mutable.TreeMap[Long, Any] = mutable.TreeMap()(HistoryOrdering) // add in the initial information update(creationTime, value) diff --git a/mainproject/src/main/scala/com/raphtory/core/model/graphentities/Vertex.scala b/mainproject/src/main/scala/com/raphtory/core/model/graphentities/Vertex.scala index d09fc655ae..8a7cbc9201 100644 --- a/mainproject/src/main/scala/com/raphtory/core/model/graphentities/Vertex.scala +++ b/mainproject/src/main/scala/com/raphtory/core/model/graphentities/Vertex.scala @@ -1,7 +1,7 @@ package com.raphtory.core.model.graphentities +import com.raphtory.core.model.EntityStorage import com.raphtory.core.model.communication.VertexMutliQueue -import com.raphtory.core.storage.EntityStorage import scala.collection.mutable import scala.collection.parallel.mutable.ParTrieMap diff --git a/mainproject/src/main/scala/com/raphtory/core/utils/HistoryOrdering.scala b/mainproject/src/main/scala/com/raphtory/core/utils/HistoryOrdering.scala deleted file mode 100644 index 80fb9822b1..0000000000 --- a/mainproject/src/main/scala/com/raphtory/core/utils/HistoryOrdering.scala +++ /dev/null @@ -1,5 +0,0 @@ -package com.raphtory.core.utils - -object HistoryOrdering extends Ordering[Long] { - def compare(key1: Long, key2: Long) = key2.compareTo(key1) -} diff --git a/mainproject/src/main/scala/com/raphtory/core/utils/Utils.scala b/mainproject/src/main/scala/com/raphtory/core/utils/Utils.scala index 3b3122620e..20bca47588 100644 --- a/mainproject/src/main/scala/com/raphtory/core/utils/Utils.scala +++ b/mainproject/src/main/scala/com/raphtory/core/utils/Utils.scala @@ -19,10 +19,6 @@ object Utils { val totalWorkers = 10 //must be power of 10 - val persistenceEnabled: Boolean = System.getenv().getOrDefault("SAVING", "false").trim.toBoolean - val compressing: Boolean = System.getenv().getOrDefault("COMPRESSING", "false").trim.toBoolean - val archiving: Boolean = System.getenv().getOrDefault("ARCHIVING", "false").trim.toBoolean - var windowing: Boolean = System.getenv().getOrDefault("WINDOWING", "false").trim.toBoolean var local: Boolean = System.getenv().getOrDefault("LOCAL", "false").trim.toBoolean def watchDogSelector(context: ActorContext, ip: String) = From 8f0c1d8f4456d278ec77bf3eec69cf49cfa0c592 Mon Sep 17 00:00:00 2001 From: Miratepuffin Date: Mon, 30 Nov 2020 17:57:52 +0000 Subject: [PATCH 50/58] Removed local variable and moved api to top level --- .../scala/com/raphtory/RaphtoryGraph.scala | 8 +++---- .../scala/com/raphtory/RaphtoryServer.scala | 8 +++---- .../raphtory/algorithms/BinaryDefusion.scala | 3 ++- .../CommunityOutlierDetection.scala | 2 +- .../algorithms/ConnectedComponents.scala | 2 +- .../com/raphtory/algorithms/DegreeBasic.scala | 2 +- .../algorithms/DegreeDistribution.scala | 2 +- .../raphtory/algorithms/DegreeRanking.scala | 2 +- .../raphtory/algorithms/Distributions.scala | 2 +- .../algorithms/GabWeightedPageRank.scala | 2 +- .../scala/com/raphtory/algorithms/LPA.scala | 4 ++-- .../com/raphtory/algorithms/PageRank.scala | 2 +- .../algorithms/TemporalTriangleCount.scala | 2 +- .../raphtory/algorithms/TriangleCount.scala | 2 +- .../raphtory/algorithms/WattsCascade.scala | 2 +- .../algorithms/WeightedPageRank.scala | 2 +- .../com/raphtory/algorithms/lotrExample.scala | 2 +- .../{core/analysis/API => api}/Analyser.scala | 4 ++-- .../serialisers => api}/S3Serialiser.scala | 3 ++- .../serialisers => api}/Serialiser.scala | 5 ++-- .../AnalysisManager}/AnalysisManager.scala | 10 ++++---- .../AnalysisManager}/AnalysisRestApi.scala | 2 +- .../AnalysisManager}/Tasks/AnalysisTask.scala | 24 +++++++++---------- .../LiveTasks/BWindowedLiveAnalysisTask.scala | 4 ++-- .../Tasks/LiveTasks/LiveAnalysisTask.scala | 6 ++--- .../LiveTasks/WindowedLiveAnalysisTask.scala | 4 ++-- .../BWindowedRangeAnalysisTask.scala | 4 ++-- .../Tasks/RangeTasks/RangeAnalysisTask.scala | 6 ++--- .../WindowedRangeAnalysisTask.scala | 4 ++-- .../ViewTasks/BWindowedViewAnalysisTask.scala | 4 ++-- .../Tasks/ViewTasks/ViewAnalysisTask.scala | 6 ++--- .../ViewTasks/WindowedViewAnalysisTask.scala | 4 ++-- .../RaphtoryReplicator.scala | 8 +++---- .../ClusterManagement/SeedActor.scala | 2 +- .../ClusterManagement/WatchDog.scala | 2 +- .../ClusterManagement/WatermarkManager.scala | 2 +- .../PartitionManager/Reader.scala | 4 ++-- .../Workers/IngestionWorker.scala | 2 +- .../Workers/ReaderWorker.scala | 6 ++--- .../PartitionManager/Writer.scala | 2 +- .../Router/GraphBuilder.scala | 2 +- .../Router/RouterManager.scala | 4 ++-- .../Router/RouterWorker.scala | 6 ++--- .../{components => actors}/Spout/Spout.scala | 2 +- .../Spout/SpoutAgent.scala | 4 ++-- .../analysis}/GraphLenses/GraphLens.scala | 8 +++---- .../analysis}/GraphLenses/ViewLens.scala | 8 +++---- .../analysis}/GraphLenses/WindowLens.scala | 8 +++---- .../entityVisitors/EdgeVisitor.scala | 8 +++---- .../entityVisitors/EntityVisitor.scala | 4 ++-- .../entityVisitors/VertexVisitor.scala | 8 +++---- .../communication/VertexMutliQueue.scala | 2 +- .../communication/raphtoryMessages.scala | 4 ++-- .../scala/com/raphtory/core/utils/Utils.scala | 6 ----- .../analysers/EthereumDegreeRanking.scala | 2 +- .../analysers/EthereumTaintTracking.scala | 2 +- .../analysers/TaintTrackExchangeStop.scala | 2 +- .../graphbuilders/BitcoinGraphBuilder.scala | 2 +- .../ChABClus2ClusGraphBuilder.scala | 2 +- .../ChABClus2ClusGraphBuilderNoprop.scala | 2 +- .../ChainalysisABGraphBuilder.scala | 2 +- .../graphbuilders/DashcoinGraphBuilder.scala | 2 +- .../EthereumGethGraphBuilder.scala | 2 +- .../EthereumKafkaGraphBuilder.scala | 2 +- .../EthereumTransactionGraphBuilder.scala | 2 +- .../FirehoseKafkaGraphBuilder.scala | 2 +- .../graphbuilders/LitecoinGraphBuilder.scala | 2 +- .../CitationGraphBuilder.scala | 2 +- .../gab/analysis/GabMostUsedTopics.scala | 2 +- .../gab/datasources/GabMongoSource.scala | 2 +- .../graphbuilders/GabPostGraphBuilder.scala | 2 +- .../graphbuilders/GabRawGraphBuilder.scala | 2 +- .../graphbuilders/GabUserGraphBuilder.scala | 2 +- .../ldbc/graphbuilders/LDBCGraphBuilder.scala | 2 +- .../graphbuilders/LDBCOldGraphBuilder.scala | 2 +- .../examples/lotr/LOTRGraphBuilder.scala | 2 +- .../raphtory/examples/lotr/LOTRSpout.scala | 2 +- .../test/actors/RandomGraphBuilder.scala | 2 +- .../examples/test/actors/RandomSpout.scala | 2 +- .../test/actors/TriangleGraphBuilder.scala | 2 +- .../examples/test/actors/TriangleSpout.scala | 2 +- .../TrackAndTraceGraphBuilder.scala | 2 +- .../examples/tsvnet/TSVGraphBuilder.scala | 2 +- .../CooccurrenceMatrixGraphBuilder.scala | 2 +- ...occurrenceMatrixGraphBuilderFiltered.scala | 2 +- .../serialisers/JSONSerialiser.scala | 5 ++-- .../serialisers/NetworkXSerialiser.scala | 5 ++-- .../scala/com/raphtory/spouts/FileSpout.scala | 2 +- .../com/raphtory/spouts/KafkaSpout.scala | 2 +- .../spouts/blockchain/BitcoinNodeSpout.scala | 2 +- .../spouts/blockchain/EthereumGethSpout.scala | 2 +- .../blockchain/EthereumPostgresSpout.scala | 2 +- 92 files changed, 159 insertions(+), 164 deletions(-) rename mainproject/src/main/scala/com/raphtory/{core/analysis/API => api}/Analyser.scala (95%) rename mainproject/src/main/scala/com/raphtory/{core/analysis/serialisers => api}/S3Serialiser.scala (96%) rename mainproject/src/main/scala/com/raphtory/{core/analysis/serialisers => api}/Serialiser.scala (92%) rename mainproject/src/main/scala/com/raphtory/core/{analysis => actors/AnalysisManager}/AnalysisManager.scala (95%) rename mainproject/src/main/scala/com/raphtory/core/{analysis => actors/AnalysisManager}/AnalysisRestApi.scala (99%) rename mainproject/src/main/scala/com/raphtory/core/{analysis => actors/AnalysisManager}/Tasks/AnalysisTask.scala (96%) rename mainproject/src/main/scala/com/raphtory/core/{analysis => actors/AnalysisManager}/Tasks/LiveTasks/BWindowedLiveAnalysisTask.scala (92%) rename mainproject/src/main/scala/com/raphtory/core/{analysis => actors/AnalysisManager}/Tasks/LiveTasks/LiveAnalysisTask.scala (95%) rename mainproject/src/main/scala/com/raphtory/core/{analysis => actors/AnalysisManager}/Tasks/LiveTasks/WindowedLiveAnalysisTask.scala (81%) rename mainproject/src/main/scala/com/raphtory/core/{analysis => actors/AnalysisManager}/Tasks/RangeTasks/BWindowedRangeAnalysisTask.scala (93%) rename mainproject/src/main/scala/com/raphtory/core/{analysis => actors/AnalysisManager}/Tasks/RangeTasks/RangeAnalysisTask.scala (89%) rename mainproject/src/main/scala/com/raphtory/core/{analysis => actors/AnalysisManager}/Tasks/RangeTasks/WindowedRangeAnalysisTask.scala (86%) rename mainproject/src/main/scala/com/raphtory/core/{analysis => actors/AnalysisManager}/Tasks/ViewTasks/BWindowedViewAnalysisTask.scala (92%) rename mainproject/src/main/scala/com/raphtory/core/{analysis => actors/AnalysisManager}/Tasks/ViewTasks/ViewAnalysisTask.scala (81%) rename mainproject/src/main/scala/com/raphtory/core/{analysis => actors/AnalysisManager}/Tasks/ViewTasks/WindowedViewAnalysisTask.scala (85%) rename mainproject/src/main/scala/com/raphtory/core/{components => actors}/ClusterManagement/RaphtoryReplicator.scala (95%) rename mainproject/src/main/scala/com/raphtory/core/{components => actors}/ClusterManagement/SeedActor.scala (96%) rename mainproject/src/main/scala/com/raphtory/core/{components => actors}/ClusterManagement/WatchDog.scala (99%) rename mainproject/src/main/scala/com/raphtory/core/{components => actors}/ClusterManagement/WatermarkManager.scala (97%) rename mainproject/src/main/scala/com/raphtory/core/{components => actors}/PartitionManager/Reader.scala (95%) rename mainproject/src/main/scala/com/raphtory/core/{components => actors}/PartitionManager/Workers/IngestionWorker.scala (99%) rename mainproject/src/main/scala/com/raphtory/core/{components => actors}/PartitionManager/Workers/ReaderWorker.scala (98%) rename mainproject/src/main/scala/com/raphtory/core/{components => actors}/PartitionManager/Writer.scala (98%) rename mainproject/src/main/scala/com/raphtory/core/{components => actors}/Router/GraphBuilder.scala (92%) rename mainproject/src/main/scala/com/raphtory/core/{components => actors}/Router/RouterManager.scala (95%) rename mainproject/src/main/scala/com/raphtory/core/{components => actors}/Router/RouterWorker.scala (96%) rename mainproject/src/main/scala/com/raphtory/core/{components => actors}/Spout/Spout.scala (84%) rename mainproject/src/main/scala/com/raphtory/core/{components => actors}/Spout/SpoutAgent.scala (97%) rename mainproject/src/main/scala/com/raphtory/core/{analysis/API => model/analysis}/GraphLenses/GraphLens.scala (83%) rename mainproject/src/main/scala/com/raphtory/core/{analysis/API => model/analysis}/GraphLenses/ViewLens.scala (91%) rename mainproject/src/main/scala/com/raphtory/core/{analysis/API => model/analysis}/GraphLenses/WindowLens.scala (91%) rename mainproject/src/main/scala/com/raphtory/core/{analysis/API => model/analysis}/entityVisitors/EdgeVisitor.scala (87%) rename mainproject/src/main/scala/com/raphtory/core/{analysis/API => model/analysis}/entityVisitors/EntityVisitor.scala (92%) rename mainproject/src/main/scala/com/raphtory/core/{analysis/API => model/analysis}/entityVisitors/VertexVisitor.scala (96%) rename mainproject/src/main/scala/com/raphtory/{core/analysis => }/serialisers/JSONSerialiser.scala (91%) rename mainproject/src/main/scala/com/raphtory/{core/analysis => }/serialisers/NetworkXSerialiser.scala (89%) diff --git a/mainproject/src/main/scala/com/raphtory/RaphtoryGraph.scala b/mainproject/src/main/scala/com/raphtory/RaphtoryGraph.scala index f3973bcb21..a18b791004 100644 --- a/mainproject/src/main/scala/com/raphtory/RaphtoryGraph.scala +++ b/mainproject/src/main/scala/com/raphtory/RaphtoryGraph.scala @@ -2,10 +2,10 @@ package com.raphtory import akka.actor.{ActorSystem, Props} import ch.qos.logback.classic.Level -import com.raphtory.core.analysis.{AnalysisManager, AnalysisRestApi} -import com.raphtory.core.components.ClusterManagement.{RaphtoryReplicator, WatchDog, WatermarkManager} -import com.raphtory.core.components.Router.GraphBuilder -import com.raphtory.core.components.Spout.{Spout, SpoutAgent} +import com.raphtory.core.actors.AnalysisManager.{AnalysisManager, AnalysisRestApi} +import com.raphtory.core.actors.ClusterManagement.{RaphtoryReplicator, WatchDog, WatermarkManager} +import com.raphtory.core.actors.Router.GraphBuilder +import com.raphtory.core.actors.Spout.{Spout, SpoutAgent} import kamon.Kamon import org.slf4j.LoggerFactory diff --git a/mainproject/src/main/scala/com/raphtory/RaphtoryServer.scala b/mainproject/src/main/scala/com/raphtory/RaphtoryServer.scala index 302e9489df..b797209f2b 100644 --- a/mainproject/src/main/scala/com/raphtory/RaphtoryServer.scala +++ b/mainproject/src/main/scala/com/raphtory/RaphtoryServer.scala @@ -7,10 +7,10 @@ import akka.actor.{ActorSystem, Address, ExtendedActorSystem, Props} import akka.event.LoggingAdapter import akka.management.cluster.bootstrap.ClusterBootstrap import akka.management.javadsl.AkkaManagement -import com.raphtory.core.analysis.{AnalysisManager, AnalysisRestApi} -import com.raphtory.core.components.ClusterManagement.{RaphtoryReplicator, SeedActor, WatchDog, WatermarkManager} -import com.raphtory.core.components.Router.GraphBuilder -import com.raphtory.core.components.Spout.{Spout, SpoutAgent} +import com.raphtory.core.actors.AnalysisManager.{AnalysisManager, AnalysisRestApi} +import com.raphtory.core.actors.ClusterManagement.{RaphtoryReplicator, SeedActor, WatchDog, WatermarkManager} +import com.raphtory.core.actors.Router.GraphBuilder +import com.raphtory.core.actors.Spout.{Spout, SpoutAgent} import com.raphtory.core.utils.Utils import com.typesafe.config.{Config, ConfigFactory, ConfigValue, ConfigValueFactory} diff --git a/mainproject/src/main/scala/com/raphtory/algorithms/BinaryDefusion.scala b/mainproject/src/main/scala/com/raphtory/algorithms/BinaryDefusion.scala index cca567de29..c7e562c40a 100644 --- a/mainproject/src/main/scala/com/raphtory/algorithms/BinaryDefusion.scala +++ b/mainproject/src/main/scala/com/raphtory/algorithms/BinaryDefusion.scala @@ -1,6 +1,7 @@ package com.raphtory.algorithms -import com.raphtory.core.analysis.API.Analyser + +import com.raphtory.api.Analyser import scala.collection.mutable.ArrayBuffer import scala.collection.parallel.immutable diff --git a/mainproject/src/main/scala/com/raphtory/algorithms/CommunityOutlierDetection.scala b/mainproject/src/main/scala/com/raphtory/algorithms/CommunityOutlierDetection.scala index 0cf68c0415..eca14dfc44 100644 --- a/mainproject/src/main/scala/com/raphtory/algorithms/CommunityOutlierDetection.scala +++ b/mainproject/src/main/scala/com/raphtory/algorithms/CommunityOutlierDetection.scala @@ -1,6 +1,6 @@ package com.raphtory.algorithms -import com.raphtory.core.analysis.API.entityVisitors.VertexVisitor +import com.raphtory.core.model.analysis.entityVisitors.VertexVisitor import com.raphtory.core.utils.Utils import scala.collection.mutable.ArrayBuffer diff --git a/mainproject/src/main/scala/com/raphtory/algorithms/ConnectedComponents.scala b/mainproject/src/main/scala/com/raphtory/algorithms/ConnectedComponents.scala index 04720bcc30..c48fb3339a 100644 --- a/mainproject/src/main/scala/com/raphtory/algorithms/ConnectedComponents.scala +++ b/mainproject/src/main/scala/com/raphtory/algorithms/ConnectedComponents.scala @@ -1,6 +1,6 @@ package com.raphtory.algorithms -import com.raphtory.core.analysis.API.Analyser +import com.raphtory.api.Analyser import com.raphtory.core.utils.Utils import scala.collection.mutable.ArrayBuffer diff --git a/mainproject/src/main/scala/com/raphtory/algorithms/DegreeBasic.scala b/mainproject/src/main/scala/com/raphtory/algorithms/DegreeBasic.scala index 2cbf675640..2ba56c7710 100644 --- a/mainproject/src/main/scala/com/raphtory/algorithms/DegreeBasic.scala +++ b/mainproject/src/main/scala/com/raphtory/algorithms/DegreeBasic.scala @@ -1,6 +1,6 @@ package com.raphtory.algorithms -import com.raphtory.core.analysis.API.Analyser +import com.raphtory.api.Analyser import com.raphtory.core.utils.Utils import scala.collection.mutable.ArrayBuffer diff --git a/mainproject/src/main/scala/com/raphtory/algorithms/DegreeDistribution.scala b/mainproject/src/main/scala/com/raphtory/algorithms/DegreeDistribution.scala index 55692f13aa..ae6d099bfb 100644 --- a/mainproject/src/main/scala/com/raphtory/algorithms/DegreeDistribution.scala +++ b/mainproject/src/main/scala/com/raphtory/algorithms/DegreeDistribution.scala @@ -1,6 +1,6 @@ package com.raphtory.algorithms -import com.raphtory.core.analysis.API.Analyser +import com.raphtory.api.Analyser import com.raphtory.core.utils.Utils import scala.collection.mutable.ArrayBuffer diff --git a/mainproject/src/main/scala/com/raphtory/algorithms/DegreeRanking.scala b/mainproject/src/main/scala/com/raphtory/algorithms/DegreeRanking.scala index 352d688922..479d3f9b0c 100644 --- a/mainproject/src/main/scala/com/raphtory/algorithms/DegreeRanking.scala +++ b/mainproject/src/main/scala/com/raphtory/algorithms/DegreeRanking.scala @@ -1,6 +1,6 @@ package com.raphtory.algorithms -import com.raphtory.core.analysis.API.Analyser +import com.raphtory.api.Analyser import com.raphtory.core.utils.Utils import scala.collection.mutable.ArrayBuffer diff --git a/mainproject/src/main/scala/com/raphtory/algorithms/Distributions.scala b/mainproject/src/main/scala/com/raphtory/algorithms/Distributions.scala index 6ec3081197..a5ce9b93f8 100644 --- a/mainproject/src/main/scala/com/raphtory/algorithms/Distributions.scala +++ b/mainproject/src/main/scala/com/raphtory/algorithms/Distributions.scala @@ -1,6 +1,6 @@ package com.raphtory.algorithms -import com.raphtory.core.analysis.API.Analyser +import com.raphtory.api.Analyser import com.raphtory.core.utils.Utils import scala.collection.mutable.ArrayBuffer diff --git a/mainproject/src/main/scala/com/raphtory/algorithms/GabWeightedPageRank.scala b/mainproject/src/main/scala/com/raphtory/algorithms/GabWeightedPageRank.scala index da9a922af9..bb05ad4b2d 100644 --- a/mainproject/src/main/scala/com/raphtory/algorithms/GabWeightedPageRank.scala +++ b/mainproject/src/main/scala/com/raphtory/algorithms/GabWeightedPageRank.scala @@ -1,6 +1,6 @@ package com.raphtory.algorithms -import com.raphtory.core.analysis.API.Analyser +import com.raphtory.api.Analyser import com.raphtory.core.utils.Utils import scala.collection.mutable.ArrayBuffer diff --git a/mainproject/src/main/scala/com/raphtory/algorithms/LPA.scala b/mainproject/src/main/scala/com/raphtory/algorithms/LPA.scala index 478ea3363d..422080f2bc 100644 --- a/mainproject/src/main/scala/com/raphtory/algorithms/LPA.scala +++ b/mainproject/src/main/scala/com/raphtory/algorithms/LPA.scala @@ -1,7 +1,7 @@ package com.raphtory.algorithms -import com.raphtory.core.analysis.API.Analyser -import com.raphtory.core.analysis.API.entityVisitors.VertexVisitor +import com.raphtory.api.Analyser +import com.raphtory.core.model.analysis.entityVisitors.VertexVisitor import scala.collection.mutable.ArrayBuffer import scala.collection.parallel.immutable diff --git a/mainproject/src/main/scala/com/raphtory/algorithms/PageRank.scala b/mainproject/src/main/scala/com/raphtory/algorithms/PageRank.scala index 19d14cb9a4..6764bfe061 100644 --- a/mainproject/src/main/scala/com/raphtory/algorithms/PageRank.scala +++ b/mainproject/src/main/scala/com/raphtory/algorithms/PageRank.scala @@ -1,6 +1,6 @@ package com.raphtory.algorithms -import com.raphtory.core.analysis.API.Analyser +import com.raphtory.api.Analyser import com.raphtory.core.utils.Utils import scala.collection.mutable.ArrayBuffer diff --git a/mainproject/src/main/scala/com/raphtory/algorithms/TemporalTriangleCount.scala b/mainproject/src/main/scala/com/raphtory/algorithms/TemporalTriangleCount.scala index cd276124a7..73da64e218 100644 --- a/mainproject/src/main/scala/com/raphtory/algorithms/TemporalTriangleCount.scala +++ b/mainproject/src/main/scala/com/raphtory/algorithms/TemporalTriangleCount.scala @@ -1,6 +1,6 @@ package com.raphtory.algorithms -import com.raphtory.core.analysis.API.Analyser +import com.raphtory.api.Analyser import scala.collection.mutable.ArrayBuffer import scala.collection.parallel.mutable.ParArray diff --git a/mainproject/src/main/scala/com/raphtory/algorithms/TriangleCount.scala b/mainproject/src/main/scala/com/raphtory/algorithms/TriangleCount.scala index ca24a43a41..dbf9e7d94e 100644 --- a/mainproject/src/main/scala/com/raphtory/algorithms/TriangleCount.scala +++ b/mainproject/src/main/scala/com/raphtory/algorithms/TriangleCount.scala @@ -1,6 +1,6 @@ package com.raphtory.algorithms -import com.raphtory.core.analysis.API.Analyser +import com.raphtory.api.Analyser import com.raphtory.core.utils.Utils import scala.collection.mutable.ArrayBuffer diff --git a/mainproject/src/main/scala/com/raphtory/algorithms/WattsCascade.scala b/mainproject/src/main/scala/com/raphtory/algorithms/WattsCascade.scala index 7a9e9f3615..508fee6122 100644 --- a/mainproject/src/main/scala/com/raphtory/algorithms/WattsCascade.scala +++ b/mainproject/src/main/scala/com/raphtory/algorithms/WattsCascade.scala @@ -1,6 +1,6 @@ package com.raphtory.algorithms -import com.raphtory.core.analysis.API.Analyser +import com.raphtory.api.Analyser import scala.collection.mutable.ArrayBuffer import scala.util.Random diff --git a/mainproject/src/main/scala/com/raphtory/algorithms/WeightedPageRank.scala b/mainproject/src/main/scala/com/raphtory/algorithms/WeightedPageRank.scala index 5d79c4f5aa..8ada0169a3 100644 --- a/mainproject/src/main/scala/com/raphtory/algorithms/WeightedPageRank.scala +++ b/mainproject/src/main/scala/com/raphtory/algorithms/WeightedPageRank.scala @@ -1,6 +1,6 @@ package com.raphtory.algorithms -import com.raphtory.core.analysis.API.Analyser +import com.raphtory.api.Analyser import com.raphtory.core.utils.Utils import scala.collection.mutable.ArrayBuffer diff --git a/mainproject/src/main/scala/com/raphtory/algorithms/lotrExample.scala b/mainproject/src/main/scala/com/raphtory/algorithms/lotrExample.scala index 0f36b7ec2b..15ac09341b 100644 --- a/mainproject/src/main/scala/com/raphtory/algorithms/lotrExample.scala +++ b/mainproject/src/main/scala/com/raphtory/algorithms/lotrExample.scala @@ -1,6 +1,6 @@ package com.raphtory.algorithms -import com.raphtory.core.analysis.API.Analyser +import com.raphtory.api.Analyser import scala.collection.mutable.ArrayBuffer import scala.collection.parallel.immutable diff --git a/mainproject/src/main/scala/com/raphtory/core/analysis/API/Analyser.scala b/mainproject/src/main/scala/com/raphtory/api/Analyser.scala similarity index 95% rename from mainproject/src/main/scala/com/raphtory/core/analysis/API/Analyser.scala rename to mainproject/src/main/scala/com/raphtory/api/Analyser.scala index 9844da7e88..574efc183d 100644 --- a/mainproject/src/main/scala/com/raphtory/core/analysis/API/Analyser.scala +++ b/mainproject/src/main/scala/com/raphtory/api/Analyser.scala @@ -1,7 +1,7 @@ -package com.raphtory.core.analysis.API +package com.raphtory.api import akka.actor.ActorContext -import com.raphtory.core.analysis.API.GraphLenses.GraphLens +import com.raphtory.core.model.analysis.GraphLenses.GraphLens import scala.collection.mutable import scala.collection.mutable.ArrayBuffer diff --git a/mainproject/src/main/scala/com/raphtory/core/analysis/serialisers/S3Serialiser.scala b/mainproject/src/main/scala/com/raphtory/api/S3Serialiser.scala similarity index 96% rename from mainproject/src/main/scala/com/raphtory/core/analysis/serialisers/S3Serialiser.scala rename to mainproject/src/main/scala/com/raphtory/api/S3Serialiser.scala index 3056ba26c3..b03c4a5c70 100644 --- a/mainproject/src/main/scala/com/raphtory/core/analysis/serialisers/S3Serialiser.scala +++ b/mainproject/src/main/scala/com/raphtory/api/S3Serialiser.scala @@ -1,4 +1,5 @@ -package com.raphtory.core.analysis.serialisers +package com.raphtory.api + import java.io.{BufferedWriter, File, FileWriter} import com.amazonaws.auth.{AWSStaticCredentialsProvider, BasicAWSCredentials} diff --git a/mainproject/src/main/scala/com/raphtory/core/analysis/serialisers/Serialiser.scala b/mainproject/src/main/scala/com/raphtory/api/Serialiser.scala similarity index 92% rename from mainproject/src/main/scala/com/raphtory/core/analysis/serialisers/Serialiser.scala rename to mainproject/src/main/scala/com/raphtory/api/Serialiser.scala index 2bcc49870a..ee2ba327e8 100644 --- a/mainproject/src/main/scala/com/raphtory/core/analysis/serialisers/Serialiser.scala +++ b/mainproject/src/main/scala/com/raphtory/api/Serialiser.scala @@ -1,9 +1,8 @@ -package com.raphtory.core.analysis.serialisers +package com.raphtory.api import java.io.{BufferedWriter, File, FileWriter} -import com.raphtory.core.analysis.API.Analyser -import com.raphtory.core.analysis.API.entityVisitors.{EdgeVisitor, VertexVisitor} +import com.raphtory.core.model.analysis.entityVisitors.{EdgeVisitor, VertexVisitor} import scala.collection.mutable.ArrayBuffer diff --git a/mainproject/src/main/scala/com/raphtory/core/analysis/AnalysisManager.scala b/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/AnalysisManager.scala similarity index 95% rename from mainproject/src/main/scala/com/raphtory/core/analysis/AnalysisManager.scala rename to mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/AnalysisManager.scala index f330ce4894..52806b630e 100644 --- a/mainproject/src/main/scala/com/raphtory/core/analysis/AnalysisManager.scala +++ b/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/AnalysisManager.scala @@ -1,13 +1,13 @@ -package com.raphtory.core.analysis +package com.raphtory.core.actors.AnalysisManager import akka.actor.{Actor, ActorRef, InvalidActorNameException, PoisonPill, Props} import akka.cluster.pubsub.{DistributedPubSub, DistributedPubSubMediator} import akka.pattern.ask import akka.util.Timeout -import com.raphtory.core.analysis.API.{Analyser, BlankAnalyser, LoadExternalAnalyser} -import com.raphtory.core.analysis.Tasks.LiveTasks.{BWindowedLiveAnalysisTask, LiveAnalysisTask, WindowedLiveAnalysisTask} -import com.raphtory.core.analysis.Tasks.RangeTasks.{BWindowedRangeAnalysisTask, RangeAnalysisTask, WindowedRangeAnalysisTask} -import com.raphtory.core.analysis.Tasks.ViewTasks.{BWindowedViewAnalysisTask, ViewAnalysisTask, WindowedViewAnalysisTask} +import com.raphtory.api.{Analyser, BlankAnalyser, LoadExternalAnalyser} +import com.raphtory.analysis.Tasks.LiveTasks.{BWindowedLiveAnalysisTask, LiveAnalysisTask, WindowedLiveAnalysisTask} +import com.raphtory.analysis.Tasks.RangeTasks.{BWindowedRangeAnalysisTask, RangeAnalysisTask, WindowedRangeAnalysisTask} +import com.raphtory.analysis.Tasks.ViewTasks.{BWindowedViewAnalysisTask, ViewAnalysisTask, WindowedViewAnalysisTask} import com.raphtory.core.model.communication.{ClusterStatusRequest, ClusterStatusResponse, _} import scala.collection.parallel.mutable.ParTrieMap diff --git a/mainproject/src/main/scala/com/raphtory/core/analysis/AnalysisRestApi.scala b/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/AnalysisRestApi.scala similarity index 99% rename from mainproject/src/main/scala/com/raphtory/core/analysis/AnalysisRestApi.scala rename to mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/AnalysisRestApi.scala index 2863251616..344d52d18f 100644 --- a/mainproject/src/main/scala/com/raphtory/core/analysis/AnalysisRestApi.scala +++ b/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/AnalysisRestApi.scala @@ -1,4 +1,4 @@ -package com.raphtory.core.analysis +package com.raphtory.core.actors.AnalysisManager import akka.actor.{ActorSystem, _} import akka.cluster.pubsub._ diff --git a/mainproject/src/main/scala/com/raphtory/core/analysis/Tasks/AnalysisTask.scala b/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/AnalysisTask.scala similarity index 96% rename from mainproject/src/main/scala/com/raphtory/core/analysis/Tasks/AnalysisTask.scala rename to mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/AnalysisTask.scala index 8d59f6cdb2..57fe22981d 100644 --- a/mainproject/src/main/scala/com/raphtory/core/analysis/Tasks/AnalysisTask.scala +++ b/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/AnalysisTask.scala @@ -1,4 +1,4 @@ -package com.raphtory.core.analysis.Tasks +package com.raphtory.analysis.Tasks import java.net.InetAddress @@ -7,9 +7,9 @@ import akka.cluster.pubsub.{DistributedPubSub, DistributedPubSubMediator} import com.mongodb.DBObject import com.mongodb.casbah.{MongoClient, MongoClientURI} import com.mongodb.util.JSON -import com.raphtory.core.analysis.API.Analyser -import com.raphtory.core.analysis.StartAnalysis -import com.raphtory.core.components.PartitionManager.Workers.ViewJob +import com.raphtory.api.Analyser +import com.raphtory.core.actors.AnalysisManager.StartAnalysis +import com.raphtory.core.actors.PartitionManager.Workers.ViewJob import com.raphtory.core.model.communication._ import com.raphtory.core.utils.Utils import kamon.Kamon @@ -22,7 +22,6 @@ abstract class AnalysisTask(jobID: String, args:Array[String], analyser: Analyse protected var currentSuperStep = 0 //SuperStep the algorithm is currently on implicit val executionContext = context.system.dispatchers.lookup("analysis-dispatcher") - private var local: Boolean = Utils.local val saveData = System.getenv().getOrDefault("ANALYSIS_SAVE_OUTPUT", "false").trim.toBoolean val mongoIP = System.getenv().getOrDefault("ANALYSIS_MONGO_HOST", "localhost").trim val mongoPort = System.getenv().getOrDefault("ANALYSIS_MONGO_PORT", "27017").trim @@ -84,14 +83,13 @@ abstract class AnalysisTask(jobID: String, args:Array[String], analyser: Analyse private def processOtherMessages(value: Any): Unit = println("Not handled message" + value.toString) protected def generateAnalyzer: Analyser = - if (local){ - try{ - Class.forName(analyser.getClass.getCanonicalName).getConstructor(classOf[Array[String]]).newInstance(args).asInstanceOf[Analyser] - } - catch { - case e:NoSuchMethodException => Class.forName(analyser.getClass.getCanonicalName).getConstructor().newInstance().asInstanceOf[Analyser] - } - } else analyser + try{ + Class.forName(analyser.getClass.getCanonicalName).getConstructor(classOf[Array[String]]).newInstance(args).asInstanceOf[Analyser] + } + catch { + case e:NoSuchMethodException => Class.forName(analyser.getClass.getCanonicalName).getConstructor().newInstance().asInstanceOf[Analyser] + } + final protected def getManagerCount: Int = managerCount final protected def getWorkerCount: Int = managerCount * Utils.totalWorkers diff --git a/mainproject/src/main/scala/com/raphtory/core/analysis/Tasks/LiveTasks/BWindowedLiveAnalysisTask.scala b/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/LiveTasks/BWindowedLiveAnalysisTask.scala similarity index 92% rename from mainproject/src/main/scala/com/raphtory/core/analysis/Tasks/LiveTasks/BWindowedLiveAnalysisTask.scala rename to mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/LiveTasks/BWindowedLiveAnalysisTask.scala index 3cd3947ad3..708b4f06c1 100644 --- a/mainproject/src/main/scala/com/raphtory/core/analysis/Tasks/LiveTasks/BWindowedLiveAnalysisTask.scala +++ b/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/LiveTasks/BWindowedLiveAnalysisTask.scala @@ -1,6 +1,6 @@ -package com.raphtory.core.analysis.Tasks.LiveTasks +package com.raphtory.analysis.Tasks.LiveTasks -import com.raphtory.core.analysis.API.Analyser +import com.raphtory.api.Analyser import com.raphtory.core.model.communication.AnalysisType import scala.collection.mutable.ArrayBuffer diff --git a/mainproject/src/main/scala/com/raphtory/core/analysis/Tasks/LiveTasks/LiveAnalysisTask.scala b/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/LiveTasks/LiveAnalysisTask.scala similarity index 95% rename from mainproject/src/main/scala/com/raphtory/core/analysis/Tasks/LiveTasks/LiveAnalysisTask.scala rename to mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/LiveTasks/LiveAnalysisTask.scala index 070d122328..29e0f88802 100644 --- a/mainproject/src/main/scala/com/raphtory/core/analysis/Tasks/LiveTasks/LiveAnalysisTask.scala +++ b/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/LiveTasks/LiveAnalysisTask.scala @@ -1,8 +1,8 @@ -package com.raphtory.core.analysis.Tasks.LiveTasks +package com.raphtory.analysis.Tasks.LiveTasks import akka.cluster.pubsub.DistributedPubSubMediator -import com.raphtory.core.analysis.API.Analyser -import com.raphtory.core.analysis.Tasks.AnalysisTask +import com.raphtory.api.Analyser +import com.raphtory.analysis.Tasks.AnalysisTask import com.raphtory.core.model.communication.{AnalysisType, Finish, Setup, TimeCheck} import com.raphtory.core.utils.Utils diff --git a/mainproject/src/main/scala/com/raphtory/core/analysis/Tasks/LiveTasks/WindowedLiveAnalysisTask.scala b/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/LiveTasks/WindowedLiveAnalysisTask.scala similarity index 81% rename from mainproject/src/main/scala/com/raphtory/core/analysis/Tasks/LiveTasks/WindowedLiveAnalysisTask.scala rename to mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/LiveTasks/WindowedLiveAnalysisTask.scala index 32f7811c82..d13a153e81 100644 --- a/mainproject/src/main/scala/com/raphtory/core/analysis/Tasks/LiveTasks/WindowedLiveAnalysisTask.scala +++ b/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/LiveTasks/WindowedLiveAnalysisTask.scala @@ -1,6 +1,6 @@ -package com.raphtory.core.analysis.Tasks.LiveTasks +package com.raphtory.analysis.Tasks.LiveTasks -import com.raphtory.core.analysis.API.Analyser +import com.raphtory.api.Analyser import com.raphtory.core.model.communication.AnalysisType class WindowedLiveAnalysisTask(managerCount:Int, jobID: String,args:Array[String], analyser: Analyser,repeatTime:Long,eventTime:Boolean, windowSize:Long,newAnalyser:Boolean,rawFile:String) diff --git a/mainproject/src/main/scala/com/raphtory/core/analysis/Tasks/RangeTasks/BWindowedRangeAnalysisTask.scala b/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/RangeTasks/BWindowedRangeAnalysisTask.scala similarity index 93% rename from mainproject/src/main/scala/com/raphtory/core/analysis/Tasks/RangeTasks/BWindowedRangeAnalysisTask.scala rename to mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/RangeTasks/BWindowedRangeAnalysisTask.scala index 04150628fb..db270a3900 100644 --- a/mainproject/src/main/scala/com/raphtory/core/analysis/Tasks/RangeTasks/BWindowedRangeAnalysisTask.scala +++ b/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/RangeTasks/BWindowedRangeAnalysisTask.scala @@ -1,6 +1,6 @@ -package com.raphtory.core.analysis.Tasks.RangeTasks +package com.raphtory.analysis.Tasks.RangeTasks -import com.raphtory.core.analysis.API.Analyser +import com.raphtory.api.Analyser import com.raphtory.core.model.communication.AnalysisType import scala.collection.mutable.ArrayBuffer diff --git a/mainproject/src/main/scala/com/raphtory/core/analysis/Tasks/RangeTasks/RangeAnalysisTask.scala b/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/RangeTasks/RangeAnalysisTask.scala similarity index 89% rename from mainproject/src/main/scala/com/raphtory/core/analysis/Tasks/RangeTasks/RangeAnalysisTask.scala rename to mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/RangeTasks/RangeAnalysisTask.scala index 6a26805dac..7dcf158a08 100644 --- a/mainproject/src/main/scala/com/raphtory/core/analysis/Tasks/RangeTasks/RangeAnalysisTask.scala +++ b/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/RangeTasks/RangeAnalysisTask.scala @@ -1,8 +1,8 @@ -package com.raphtory.core.analysis.Tasks.RangeTasks +package com.raphtory.analysis.Tasks.RangeTasks import akka.cluster.pubsub.DistributedPubSubMediator -import com.raphtory.core.analysis.API.Analyser -import com.raphtory.core.analysis.Tasks.AnalysisTask +import com.raphtory.api.Analyser +import com.raphtory.analysis.Tasks.AnalysisTask import com.raphtory.core.model.communication.{AnalyserPresentCheck, AnalysisType} import com.raphtory.core.utils.Utils diff --git a/mainproject/src/main/scala/com/raphtory/core/analysis/Tasks/RangeTasks/WindowedRangeAnalysisTask.scala b/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/RangeTasks/WindowedRangeAnalysisTask.scala similarity index 86% rename from mainproject/src/main/scala/com/raphtory/core/analysis/Tasks/RangeTasks/WindowedRangeAnalysisTask.scala rename to mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/RangeTasks/WindowedRangeAnalysisTask.scala index 175f40d02e..fb9301ceec 100644 --- a/mainproject/src/main/scala/com/raphtory/core/analysis/Tasks/RangeTasks/WindowedRangeAnalysisTask.scala +++ b/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/RangeTasks/WindowedRangeAnalysisTask.scala @@ -1,6 +1,6 @@ -package com.raphtory.core.analysis.Tasks.RangeTasks +package com.raphtory.analysis.Tasks.RangeTasks -import com.raphtory.core.analysis.API.Analyser +import com.raphtory.api.Analyser import com.raphtory.core.model.communication.AnalysisType class WindowedRangeAnalysisTask(managerCount:Int, jobID: String,args:Array[String], analyser: Analyser, start: Long, end: Long, jump: Long, window: Long,newAnalyser:Boolean,rawFile:String) diff --git a/mainproject/src/main/scala/com/raphtory/core/analysis/Tasks/ViewTasks/BWindowedViewAnalysisTask.scala b/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/ViewTasks/BWindowedViewAnalysisTask.scala similarity index 92% rename from mainproject/src/main/scala/com/raphtory/core/analysis/Tasks/ViewTasks/BWindowedViewAnalysisTask.scala rename to mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/ViewTasks/BWindowedViewAnalysisTask.scala index 684087de66..37cad86f68 100644 --- a/mainproject/src/main/scala/com/raphtory/core/analysis/Tasks/ViewTasks/BWindowedViewAnalysisTask.scala +++ b/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/ViewTasks/BWindowedViewAnalysisTask.scala @@ -1,6 +1,6 @@ -package com.raphtory.core.analysis.Tasks.ViewTasks +package com.raphtory.analysis.Tasks.ViewTasks -import com.raphtory.core.analysis.API.Analyser +import com.raphtory.api.Analyser import com.raphtory.core.model.communication.AnalysisType import scala.collection.mutable.ArrayBuffer diff --git a/mainproject/src/main/scala/com/raphtory/core/analysis/Tasks/ViewTasks/ViewAnalysisTask.scala b/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/ViewTasks/ViewAnalysisTask.scala similarity index 81% rename from mainproject/src/main/scala/com/raphtory/core/analysis/Tasks/ViewTasks/ViewAnalysisTask.scala rename to mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/ViewTasks/ViewAnalysisTask.scala index 086ef9c8b1..3b7eedf47b 100644 --- a/mainproject/src/main/scala/com/raphtory/core/analysis/Tasks/ViewTasks/ViewAnalysisTask.scala +++ b/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/ViewTasks/ViewAnalysisTask.scala @@ -1,7 +1,7 @@ -package com.raphtory.core.analysis.Tasks.ViewTasks +package com.raphtory.analysis.Tasks.ViewTasks -import com.raphtory.core.analysis.API.Analyser -import com.raphtory.core.analysis.Tasks.AnalysisTask +import com.raphtory.api.Analyser +import com.raphtory.analysis.Tasks.AnalysisTask import com.raphtory.core.model.communication.AnalysisType class ViewAnalysisTask(managerCount:Int, jobID: String,args:Array[String], analyser: Analyser, time: Long,newAnalyser:Boolean,rawFile:String) diff --git a/mainproject/src/main/scala/com/raphtory/core/analysis/Tasks/ViewTasks/WindowedViewAnalysisTask.scala b/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/ViewTasks/WindowedViewAnalysisTask.scala similarity index 85% rename from mainproject/src/main/scala/com/raphtory/core/analysis/Tasks/ViewTasks/WindowedViewAnalysisTask.scala rename to mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/ViewTasks/WindowedViewAnalysisTask.scala index fcb4b99ff4..dbdcc036e0 100644 --- a/mainproject/src/main/scala/com/raphtory/core/analysis/Tasks/ViewTasks/WindowedViewAnalysisTask.scala +++ b/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/ViewTasks/WindowedViewAnalysisTask.scala @@ -1,6 +1,6 @@ -package com.raphtory.core.analysis.Tasks.ViewTasks +package com.raphtory.analysis.Tasks.ViewTasks -import com.raphtory.core.analysis.API.Analyser +import com.raphtory.api.Analyser import com.raphtory.core.model.communication.AnalysisType class WindowedViewAnalysisTask(managerCount:Int, jobID: String,args:Array[String], analyser: Analyser, time: Long, window: Long,newAnalyser:Boolean,rawFile:String) diff --git a/mainproject/src/main/scala/com/raphtory/core/components/ClusterManagement/RaphtoryReplicator.scala b/mainproject/src/main/scala/com/raphtory/core/actors/ClusterManagement/RaphtoryReplicator.scala similarity index 95% rename from mainproject/src/main/scala/com/raphtory/core/components/ClusterManagement/RaphtoryReplicator.scala rename to mainproject/src/main/scala/com/raphtory/core/actors/ClusterManagement/RaphtoryReplicator.scala index c9334cabab..e01de9843c 100644 --- a/mainproject/src/main/scala/com/raphtory/core/components/ClusterManagement/RaphtoryReplicator.scala +++ b/mainproject/src/main/scala/com/raphtory/core/actors/ClusterManagement/RaphtoryReplicator.scala @@ -1,13 +1,13 @@ -package com.raphtory.core.components.ClusterManagement +package com.raphtory.core.actors.ClusterManagement import akka.actor.{Actor, ActorLogging, ActorRef, Cancellable, Props} import akka.cluster.pubsub.DistributedPubSubMediator.SubscribeAck import akka.cluster.pubsub.{DistributedPubSub, DistributedPubSubMediator} import akka.pattern.ask import akka.util.Timeout -import com.raphtory.core.components.PartitionManager.Workers.IngestionWorker -import com.raphtory.core.components.PartitionManager.{Reader, Writer} -import com.raphtory.core.components.Router.{GraphBuilder, RouterManager} +import com.raphtory.core.actors.PartitionManager.Workers.IngestionWorker +import com.raphtory.core.actors.PartitionManager.{Reader, Writer} +import com.raphtory.core.actors.Router.{GraphBuilder, RouterManager} import com.raphtory.core.model.EntityStorage import com.raphtory.core.model.communication._ import com.raphtory.core.utils.{SchedulerUtil, Utils} diff --git a/mainproject/src/main/scala/com/raphtory/core/components/ClusterManagement/SeedActor.scala b/mainproject/src/main/scala/com/raphtory/core/actors/ClusterManagement/SeedActor.scala similarity index 96% rename from mainproject/src/main/scala/com/raphtory/core/components/ClusterManagement/SeedActor.scala rename to mainproject/src/main/scala/com/raphtory/core/actors/ClusterManagement/SeedActor.scala index 63aff4ee4e..a1c8e17a12 100644 --- a/mainproject/src/main/scala/com/raphtory/core/components/ClusterManagement/SeedActor.scala +++ b/mainproject/src/main/scala/com/raphtory/core/actors/ClusterManagement/SeedActor.scala @@ -1,4 +1,4 @@ -package com.raphtory.core.components.ClusterManagement +package com.raphtory.core.actors.ClusterManagement import akka.actor.{Actor, ActorLogging} import akka.cluster.Cluster diff --git a/mainproject/src/main/scala/com/raphtory/core/components/ClusterManagement/WatchDog.scala b/mainproject/src/main/scala/com/raphtory/core/actors/ClusterManagement/WatchDog.scala similarity index 99% rename from mainproject/src/main/scala/com/raphtory/core/components/ClusterManagement/WatchDog.scala rename to mainproject/src/main/scala/com/raphtory/core/actors/ClusterManagement/WatchDog.scala index 995f11ba37..f51372685d 100644 --- a/mainproject/src/main/scala/com/raphtory/core/components/ClusterManagement/WatchDog.scala +++ b/mainproject/src/main/scala/com/raphtory/core/actors/ClusterManagement/WatchDog.scala @@ -1,4 +1,4 @@ -package com.raphtory.core.components.ClusterManagement +package com.raphtory.core.actors.ClusterManagement /** * Created by Mirate on 11/07/2017. diff --git a/mainproject/src/main/scala/com/raphtory/core/components/ClusterManagement/WatermarkManager.scala b/mainproject/src/main/scala/com/raphtory/core/actors/ClusterManagement/WatermarkManager.scala similarity index 97% rename from mainproject/src/main/scala/com/raphtory/core/components/ClusterManagement/WatermarkManager.scala rename to mainproject/src/main/scala/com/raphtory/core/actors/ClusterManagement/WatermarkManager.scala index a4a6323db6..e4fa09a902 100644 --- a/mainproject/src/main/scala/com/raphtory/core/components/ClusterManagement/WatermarkManager.scala +++ b/mainproject/src/main/scala/com/raphtory/core/actors/ClusterManagement/WatermarkManager.scala @@ -1,4 +1,4 @@ -package com.raphtory.core.components.ClusterManagement +package com.raphtory.core.actors.ClusterManagement import akka.actor.{Actor, ActorLogging, ActorRef} import akka.cluster.pubsub.{DistributedPubSub, DistributedPubSubMediator} diff --git a/mainproject/src/main/scala/com/raphtory/core/components/PartitionManager/Reader.scala b/mainproject/src/main/scala/com/raphtory/core/actors/PartitionManager/Reader.scala similarity index 95% rename from mainproject/src/main/scala/com/raphtory/core/components/PartitionManager/Reader.scala rename to mainproject/src/main/scala/com/raphtory/core/actors/PartitionManager/Reader.scala index a155013f3c..bd44370690 100644 --- a/mainproject/src/main/scala/com/raphtory/core/components/PartitionManager/Reader.scala +++ b/mainproject/src/main/scala/com/raphtory/core/actors/PartitionManager/Reader.scala @@ -1,9 +1,9 @@ -package com.raphtory.core.components.PartitionManager +package com.raphtory.core.actors.PartitionManager import akka.actor.{Actor, ActorLogging, ActorRef, Props, Terminated} import akka.cluster.pubsub.{DistributedPubSub, DistributedPubSubMediator} import akka.cluster.pubsub.DistributedPubSubMediator.SubscribeAck -import com.raphtory.core.components.PartitionManager.Workers.ReaderWorker +import com.raphtory.core.actors.PartitionManager.Workers.ReaderWorker import com.raphtory.core.model.EntityStorage import com.raphtory.core.model.communication._ import com.raphtory.core.utils.Utils diff --git a/mainproject/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/IngestionWorker.scala b/mainproject/src/main/scala/com/raphtory/core/actors/PartitionManager/Workers/IngestionWorker.scala similarity index 99% rename from mainproject/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/IngestionWorker.scala rename to mainproject/src/main/scala/com/raphtory/core/actors/PartitionManager/Workers/IngestionWorker.scala index fc0dd8524e..d698706f66 100644 --- a/mainproject/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/IngestionWorker.scala +++ b/mainproject/src/main/scala/com/raphtory/core/actors/PartitionManager/Workers/IngestionWorker.scala @@ -1,4 +1,4 @@ -package com.raphtory.core.components.PartitionManager.Workers +package com.raphtory.core.actors.PartitionManager.Workers import java.util.concurrent.atomic.AtomicInteger diff --git a/mainproject/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/ReaderWorker.scala b/mainproject/src/main/scala/com/raphtory/core/actors/PartitionManager/Workers/ReaderWorker.scala similarity index 98% rename from mainproject/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/ReaderWorker.scala rename to mainproject/src/main/scala/com/raphtory/core/actors/PartitionManager/Workers/ReaderWorker.scala index 004ce6b1c5..1dc332e03b 100644 --- a/mainproject/src/main/scala/com/raphtory/core/components/PartitionManager/Workers/ReaderWorker.scala +++ b/mainproject/src/main/scala/com/raphtory/core/actors/PartitionManager/Workers/ReaderWorker.scala @@ -1,11 +1,11 @@ -package com.raphtory.core.components.PartitionManager.Workers +package com.raphtory.core.actors.PartitionManager.Workers import java.util.concurrent.atomic.AtomicInteger import akka.actor.{Actor, ActorLogging, ActorRef} import akka.cluster.pubsub.{DistributedPubSub, DistributedPubSubMediator} -import com.raphtory.core.analysis.API.GraphLenses.{GraphLens, ViewLens, WindowLens} -import com.raphtory.core.analysis.API.{Analyser, _} +import com.raphtory.api.{Analyser, LoadExternalAnalyser, ManagerCount} +import com.raphtory.core.model.analysis.GraphLenses.{GraphLens, ViewLens, WindowLens} import com.raphtory.core.model.EntityStorage import com.raphtory.core.model.communication._ import com.raphtory.core.utils.Utils diff --git a/mainproject/src/main/scala/com/raphtory/core/components/PartitionManager/Writer.scala b/mainproject/src/main/scala/com/raphtory/core/actors/PartitionManager/Writer.scala similarity index 98% rename from mainproject/src/main/scala/com/raphtory/core/components/PartitionManager/Writer.scala rename to mainproject/src/main/scala/com/raphtory/core/actors/PartitionManager/Writer.scala index 1a46b7f328..008e24d3fd 100644 --- a/mainproject/src/main/scala/com/raphtory/core/components/PartitionManager/Writer.scala +++ b/mainproject/src/main/scala/com/raphtory/core/actors/PartitionManager/Writer.scala @@ -1,4 +1,4 @@ -package com.raphtory.core.components.PartitionManager +package com.raphtory.core.actors.PartitionManager import akka.actor.SupervisorStrategy.Resume import akka.actor.{Actor, ActorLogging, ActorRef, Cancellable, OneForOneStrategy, Terminated} diff --git a/mainproject/src/main/scala/com/raphtory/core/components/Router/GraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/core/actors/Router/GraphBuilder.scala similarity index 92% rename from mainproject/src/main/scala/com/raphtory/core/components/Router/GraphBuilder.scala rename to mainproject/src/main/scala/com/raphtory/core/actors/Router/GraphBuilder.scala index 23b860bb93..7fe2ba2dc5 100644 --- a/mainproject/src/main/scala/com/raphtory/core/components/Router/GraphBuilder.scala +++ b/mainproject/src/main/scala/com/raphtory/core/actors/Router/GraphBuilder.scala @@ -1,4 +1,4 @@ -package com.raphtory.core.components.Router +package com.raphtory.core.actors.Router import com.raphtory.core.model.communication.GraphUpdate diff --git a/mainproject/src/main/scala/com/raphtory/core/components/Router/RouterManager.scala b/mainproject/src/main/scala/com/raphtory/core/actors/Router/RouterManager.scala similarity index 95% rename from mainproject/src/main/scala/com/raphtory/core/components/Router/RouterManager.scala rename to mainproject/src/main/scala/com/raphtory/core/actors/Router/RouterManager.scala index 0e09aa82d5..da3de2d708 100644 --- a/mainproject/src/main/scala/com/raphtory/core/components/Router/RouterManager.scala +++ b/mainproject/src/main/scala/com/raphtory/core/actors/Router/RouterManager.scala @@ -1,8 +1,8 @@ -package com.raphtory.core.components.Router +package com.raphtory.core.actors.Router import akka.actor.{Actor, ActorLogging, ActorRef, Props} import akka.cluster.pubsub.{DistributedPubSub, DistributedPubSubMediator} -import com.raphtory.core.components.Router.RouterManager.Message.KeepAlive +import com.raphtory.core.actors.Router.RouterManager.Message.KeepAlive import com.raphtory.core.model.communication._ import scala.concurrent.ExecutionContext diff --git a/mainproject/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala b/mainproject/src/main/scala/com/raphtory/core/actors/Router/RouterWorker.scala similarity index 96% rename from mainproject/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala rename to mainproject/src/main/scala/com/raphtory/core/actors/Router/RouterWorker.scala index 1d10c54add..988dbfd7cc 100644 --- a/mainproject/src/main/scala/com/raphtory/core/components/Router/RouterWorker.scala +++ b/mainproject/src/main/scala/com/raphtory/core/actors/Router/RouterWorker.scala @@ -1,9 +1,9 @@ -package com.raphtory.core.components.Router +package com.raphtory.core.actors.Router import akka.actor.{Actor, ActorLogging} import akka.cluster.pubsub.{DistributedPubSub, DistributedPubSubMediator} -import com.raphtory.core.components.Router.RouterWorker.CommonMessage.TimeBroadcast -import com.raphtory.core.components.Spout.SpoutAgent.CommonMessage.{NoWork, SpoutOnline, WorkPlease} +import com.raphtory.core.actors.Router.RouterWorker.CommonMessage.TimeBroadcast +import com.raphtory.core.actors.Spout.SpoutAgent.CommonMessage.{NoWork, SpoutOnline, WorkPlease} import com.raphtory.core.model.communication._ import com.raphtory.core.utils.Utils import com.raphtory.core.utils.Utils.getManager diff --git a/mainproject/src/main/scala/com/raphtory/core/components/Spout/Spout.scala b/mainproject/src/main/scala/com/raphtory/core/actors/Spout/Spout.scala similarity index 84% rename from mainproject/src/main/scala/com/raphtory/core/components/Spout/Spout.scala rename to mainproject/src/main/scala/com/raphtory/core/actors/Spout/Spout.scala index fffa09dc3d..c4cd62ca2a 100644 --- a/mainproject/src/main/scala/com/raphtory/core/components/Spout/Spout.scala +++ b/mainproject/src/main/scala/com/raphtory/core/actors/Spout/Spout.scala @@ -1,4 +1,4 @@ -package com.raphtory.core.components.Spout +package com.raphtory.core.actors.Spout trait Spout[+T]{ diff --git a/mainproject/src/main/scala/com/raphtory/core/components/Spout/SpoutAgent.scala b/mainproject/src/main/scala/com/raphtory/core/actors/Spout/SpoutAgent.scala similarity index 97% rename from mainproject/src/main/scala/com/raphtory/core/components/Spout/SpoutAgent.scala rename to mainproject/src/main/scala/com/raphtory/core/actors/Spout/SpoutAgent.scala index b334a8e9b8..628f9fcda8 100644 --- a/mainproject/src/main/scala/com/raphtory/core/components/Spout/SpoutAgent.scala +++ b/mainproject/src/main/scala/com/raphtory/core/actors/Spout/SpoutAgent.scala @@ -1,8 +1,8 @@ -package com.raphtory.core.components.Spout +package com.raphtory.core.actors.Spout import akka.actor.{Actor, ActorLogging, ActorRef, Cancellable, Timers} import akka.cluster.pubsub.{DistributedPubSub, DistributedPubSubMediator} -import com.raphtory.core.components.Spout.SpoutAgent.CommonMessage._ +import com.raphtory.core.actors.Spout.SpoutAgent.CommonMessage._ import com.raphtory.core.model.communication._ import com.raphtory.core.utils.Utils import kamon.Kamon diff --git a/mainproject/src/main/scala/com/raphtory/core/analysis/API/GraphLenses/GraphLens.scala b/mainproject/src/main/scala/com/raphtory/core/model/analysis/GraphLenses/GraphLens.scala similarity index 83% rename from mainproject/src/main/scala/com/raphtory/core/analysis/API/GraphLenses/GraphLens.scala rename to mainproject/src/main/scala/com/raphtory/core/model/analysis/GraphLenses/GraphLens.scala index f66d39f7e6..6a4c707e97 100644 --- a/mainproject/src/main/scala/com/raphtory/core/analysis/API/GraphLenses/GraphLens.scala +++ b/mainproject/src/main/scala/com/raphtory/core/model/analysis/GraphLenses/GraphLens.scala @@ -1,12 +1,12 @@ -package com.raphtory.core.analysis.API.GraphLenses +package com.raphtory.core.model.analysis.GraphLenses import java.util.concurrent.atomic.AtomicInteger import akka.actor.ActorContext -import com.raphtory.core.analysis.API.ManagerCount -import com.raphtory.core.analysis.API.entityVisitors.VertexVisitor -import com.raphtory.core.components.PartitionManager.Workers.ViewJob +import com.raphtory.api.ManagerCount +import com.raphtory.core.actors.PartitionManager.Workers.ViewJob import com.raphtory.core.model.EntityStorage +import com.raphtory.core.model.analysis.entityVisitors.VertexVisitor import com.raphtory.core.model.graphentities.Vertex import scala.collection.parallel.ParIterable diff --git a/mainproject/src/main/scala/com/raphtory/core/analysis/API/GraphLenses/ViewLens.scala b/mainproject/src/main/scala/com/raphtory/core/model/analysis/GraphLenses/ViewLens.scala similarity index 91% rename from mainproject/src/main/scala/com/raphtory/core/analysis/API/GraphLenses/ViewLens.scala rename to mainproject/src/main/scala/com/raphtory/core/model/analysis/GraphLenses/ViewLens.scala index 2e3ca07d6e..e287a6e28d 100644 --- a/mainproject/src/main/scala/com/raphtory/core/analysis/API/GraphLenses/ViewLens.scala +++ b/mainproject/src/main/scala/com/raphtory/core/model/analysis/GraphLenses/ViewLens.scala @@ -1,10 +1,10 @@ -package com.raphtory.core.analysis.API.GraphLenses +package com.raphtory.core.model.analysis.GraphLenses import akka.actor.ActorContext -import com.raphtory.core.analysis.API.ManagerCount -import com.raphtory.core.analysis.API.entityVisitors.VertexVisitor -import com.raphtory.core.components.PartitionManager.Workers.ViewJob +import com.raphtory.api.ManagerCount +import com.raphtory.core.actors.PartitionManager.Workers.ViewJob import com.raphtory.core.model.EntityStorage +import com.raphtory.core.model.analysis.entityVisitors.VertexVisitor import com.raphtory.core.model.graphentities.Vertex import kamon.Kamon diff --git a/mainproject/src/main/scala/com/raphtory/core/analysis/API/GraphLenses/WindowLens.scala b/mainproject/src/main/scala/com/raphtory/core/model/analysis/GraphLenses/WindowLens.scala similarity index 91% rename from mainproject/src/main/scala/com/raphtory/core/analysis/API/GraphLenses/WindowLens.scala rename to mainproject/src/main/scala/com/raphtory/core/model/analysis/GraphLenses/WindowLens.scala index 3ce251d258..bffc311285 100644 --- a/mainproject/src/main/scala/com/raphtory/core/analysis/API/GraphLenses/WindowLens.scala +++ b/mainproject/src/main/scala/com/raphtory/core/model/analysis/GraphLenses/WindowLens.scala @@ -1,10 +1,10 @@ -package com.raphtory.core.analysis.API.GraphLenses +package com.raphtory.core.model.analysis.GraphLenses import akka.actor.ActorContext -import com.raphtory.core.analysis.API.ManagerCount -import com.raphtory.core.analysis.API.entityVisitors.VertexVisitor -import com.raphtory.core.components.PartitionManager.Workers.ViewJob +import com.raphtory.api.ManagerCount +import com.raphtory.core.actors.PartitionManager.Workers.ViewJob import com.raphtory.core.model.EntityStorage +import com.raphtory.core.model.analysis.entityVisitors.VertexVisitor import com.raphtory.core.model.graphentities.Vertex import kamon.Kamon diff --git a/mainproject/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/EdgeVisitor.scala b/mainproject/src/main/scala/com/raphtory/core/model/analysis/entityVisitors/EdgeVisitor.scala similarity index 87% rename from mainproject/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/EdgeVisitor.scala rename to mainproject/src/main/scala/com/raphtory/core/model/analysis/entityVisitors/EdgeVisitor.scala index 4593e61f6d..5def0b27a1 100644 --- a/mainproject/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/EdgeVisitor.scala +++ b/mainproject/src/main/scala/com/raphtory/core/model/analysis/entityVisitors/EdgeVisitor.scala @@ -1,10 +1,10 @@ -package com.raphtory.core.analysis.API.entityVisitors +package com.raphtory.core.model.analysis.entityVisitors import akka.actor.{ActorContext, ActorRef} import akka.cluster.pubsub.DistributedPubSubMediator -import com.raphtory.core.analysis.API.GraphLenses.GraphLens -import com.raphtory.core.analysis.API.ManagerCount -import com.raphtory.core.components.PartitionManager.Workers.ViewJob +import com.raphtory.api.ManagerCount +import com.raphtory.core.actors.PartitionManager.Workers.ViewJob +import com.raphtory.core.model.analysis.GraphLenses.GraphLens import com.raphtory.core.model.communication.{ImmutableProperty, VertexMessage} import com.raphtory.core.model.graphentities.{Edge, MutableProperty} import com.raphtory.core.utils.Utils diff --git a/mainproject/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/EntityVisitor.scala b/mainproject/src/main/scala/com/raphtory/core/model/analysis/entityVisitors/EntityVisitor.scala similarity index 92% rename from mainproject/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/EntityVisitor.scala rename to mainproject/src/main/scala/com/raphtory/core/model/analysis/entityVisitors/EntityVisitor.scala index 35f555febc..4d87656502 100644 --- a/mainproject/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/EntityVisitor.scala +++ b/mainproject/src/main/scala/com/raphtory/core/model/analysis/entityVisitors/EntityVisitor.scala @@ -1,6 +1,6 @@ -package com.raphtory.core.analysis.API.entityVisitors +package com.raphtory.core.model.analysis.entityVisitors -import com.raphtory.core.components.PartitionManager.Workers.ViewJob +import com.raphtory.core.actors.PartitionManager.Workers.ViewJob import com.raphtory.core.model.graphentities.Entity import scala.collection.mutable diff --git a/mainproject/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/VertexVisitor.scala b/mainproject/src/main/scala/com/raphtory/core/model/analysis/entityVisitors/VertexVisitor.scala similarity index 96% rename from mainproject/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/VertexVisitor.scala rename to mainproject/src/main/scala/com/raphtory/core/model/analysis/entityVisitors/VertexVisitor.scala index 2045cc9a08..72cfea277d 100644 --- a/mainproject/src/main/scala/com/raphtory/core/analysis/API/entityVisitors/VertexVisitor.scala +++ b/mainproject/src/main/scala/com/raphtory/core/model/analysis/entityVisitors/VertexVisitor.scala @@ -1,10 +1,10 @@ -package com.raphtory.core.analysis.API.entityVisitors +package com.raphtory.core.model.analysis.entityVisitors import akka.actor.{ActorContext, ActorRef} import akka.cluster.pubsub.{DistributedPubSub, DistributedPubSubMediator} -import com.raphtory.core.analysis.API.GraphLenses.GraphLens -import com.raphtory.core.analysis.API.ManagerCount -import com.raphtory.core.components.PartitionManager.Workers.ViewJob +import com.raphtory.api.ManagerCount +import com.raphtory.core.actors.PartitionManager.Workers.ViewJob +import com.raphtory.core.model.analysis.GraphLenses.GraphLens import com.raphtory.core.model.communication._ import com.raphtory.core.model.graphentities.{Edge, Vertex} import com.raphtory.core.utils.Utils diff --git a/mainproject/src/main/scala/com/raphtory/core/model/communication/VertexMutliQueue.scala b/mainproject/src/main/scala/com/raphtory/core/model/communication/VertexMutliQueue.scala index f36d791fb5..d891acdebd 100644 --- a/mainproject/src/main/scala/com/raphtory/core/model/communication/VertexMutliQueue.scala +++ b/mainproject/src/main/scala/com/raphtory/core/model/communication/VertexMutliQueue.scala @@ -1,6 +1,6 @@ package com.raphtory.core.model.communication -import com.raphtory.core.components.PartitionManager.Workers.ViewJob +import com.raphtory.core.actors.PartitionManager.Workers.ViewJob import scala.collection.mutable import scala.collection.parallel.mutable.ParTrieMap diff --git a/mainproject/src/main/scala/com/raphtory/core/model/communication/raphtoryMessages.scala b/mainproject/src/main/scala/com/raphtory/core/model/communication/raphtoryMessages.scala index 483dbffb13..3f1480a81a 100644 --- a/mainproject/src/main/scala/com/raphtory/core/model/communication/raphtoryMessages.scala +++ b/mainproject/src/main/scala/com/raphtory/core/model/communication/raphtoryMessages.scala @@ -1,7 +1,7 @@ package com.raphtory.core.model.communication -import com.raphtory.core.analysis.API.Analyser -import com.raphtory.core.components.PartitionManager.Workers.ViewJob +import com.raphtory.api.Analyser +import com.raphtory.core.actors.PartitionManager.Workers.ViewJob import com.raphtory.core.model.graphentities.Edge import scala.collection.mutable diff --git a/mainproject/src/main/scala/com/raphtory/core/utils/Utils.scala b/mainproject/src/main/scala/com/raphtory/core/utils/Utils.scala index 20bca47588..818d8d6d6b 100644 --- a/mainproject/src/main/scala/com/raphtory/core/utils/Utils.scala +++ b/mainproject/src/main/scala/com/raphtory/core/utils/Utils.scala @@ -19,12 +19,6 @@ object Utils { val totalWorkers = 10 //must be power of 10 - var local: Boolean = System.getenv().getOrDefault("LOCAL", "false").trim.toBoolean - - def watchDogSelector(context: ActorContext, ip: String) = - // IP $clusterSystemName@${InetAddress.getByName("watchDog").getHostAddress()} - context.actorSelection(s"akka.tcp://$ip:${config.getString("settings.bport")}/user/WatchDog") - def getPartition(ID: Long, managerCount: Int): Int = ((ID.abs % (managerCount * totalWorkers)) / totalWorkers).toInt def getWorker(ID: Long, managerCount: Int): Int = diff --git a/mainproject/src/main/scala/com/raphtory/examples/blockchain/analysers/EthereumDegreeRanking.scala b/mainproject/src/main/scala/com/raphtory/examples/blockchain/analysers/EthereumDegreeRanking.scala index b749cef3a6..e3ed98451c 100644 --- a/mainproject/src/main/scala/com/raphtory/examples/blockchain/analysers/EthereumDegreeRanking.scala +++ b/mainproject/src/main/scala/com/raphtory/examples/blockchain/analysers/EthereumDegreeRanking.scala @@ -1,6 +1,6 @@ package com.raphtory.examples.blockchain.analysers -import com.raphtory.core.analysis.API.Analyser +import com.raphtory.api.Analyser import com.raphtory.core.utils.Utils import scala.collection.mutable.ArrayBuffer diff --git a/mainproject/src/main/scala/com/raphtory/examples/blockchain/analysers/EthereumTaintTracking.scala b/mainproject/src/main/scala/com/raphtory/examples/blockchain/analysers/EthereumTaintTracking.scala index 6adb45cdd7..14e2ea0486 100644 --- a/mainproject/src/main/scala/com/raphtory/examples/blockchain/analysers/EthereumTaintTracking.scala +++ b/mainproject/src/main/scala/com/raphtory/examples/blockchain/analysers/EthereumTaintTracking.scala @@ -1,6 +1,6 @@ package com.raphtory.examples.blockchain.analysers -import com.raphtory.core.analysis.API.Analyser +import com.raphtory.api.Analyser import scala.collection.mutable.ArrayBuffer import scala.collection.parallel.immutable diff --git a/mainproject/src/main/scala/com/raphtory/examples/blockchain/analysers/TaintTrackExchangeStop.scala b/mainproject/src/main/scala/com/raphtory/examples/blockchain/analysers/TaintTrackExchangeStop.scala index efcf35b489..7322b29fde 100644 --- a/mainproject/src/main/scala/com/raphtory/examples/blockchain/analysers/TaintTrackExchangeStop.scala +++ b/mainproject/src/main/scala/com/raphtory/examples/blockchain/analysers/TaintTrackExchangeStop.scala @@ -1,6 +1,6 @@ package com.raphtory.examples.blockchain.analysers -import com.raphtory.core.analysis.API.Analyser +import com.raphtory.api.Analyser import com.raphtory.core.utils.Utils import scala.collection.mutable.ArrayBuffer diff --git a/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/BitcoinGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/BitcoinGraphBuilder.scala index e820a064d3..3ac8c333bf 100644 --- a/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/BitcoinGraphBuilder.scala +++ b/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/BitcoinGraphBuilder.scala @@ -1,6 +1,6 @@ package com.raphtory.examples.blockchain.graphbuilders -import com.raphtory.core.components.Router.GraphBuilder +import com.raphtory.core.actors.Router.GraphBuilder import com.raphtory.core.model.communication._ import com.raphtory.spouts.blockchain.BitcoinTransaction import spray.json.JsArray diff --git a/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChABClus2ClusGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChABClus2ClusGraphBuilder.scala index 13af351514..67ec99031e 100644 --- a/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChABClus2ClusGraphBuilder.scala +++ b/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChABClus2ClusGraphBuilder.scala @@ -1,6 +1,6 @@ package com.raphtory.examples.blockchain.graphbuilders -import com.raphtory.core.components.Router.GraphBuilder +import com.raphtory.core.actors.Router.GraphBuilder import com.raphtory.core.model.communication.{Type, _} class ChABClus2ClusGraphBuilder extends GraphBuilder[String]{ diff --git a/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChABClus2ClusGraphBuilderNoprop.scala b/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChABClus2ClusGraphBuilderNoprop.scala index 759d0401f2..af35782408 100644 --- a/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChABClus2ClusGraphBuilderNoprop.scala +++ b/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChABClus2ClusGraphBuilderNoprop.scala @@ -1,6 +1,6 @@ package com.raphtory.examples.blockchain.graphbuilders -import com.raphtory.core.components.Router.GraphBuilder +import com.raphtory.core.actors.Router.GraphBuilder import com.raphtory.core.model.communication.{Type, _} class ChABClus2ClusGraphBuilderNoprop extends GraphBuilder[String] { diff --git a/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChainalysisABGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChainalysisABGraphBuilder.scala index 6b6d74351c..47dcb2d76d 100644 --- a/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChainalysisABGraphBuilder.scala +++ b/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChainalysisABGraphBuilder.scala @@ -1,6 +1,6 @@ package com.raphtory.examples.blockchain.graphbuilders -import com.raphtory.core.components.Router.GraphBuilder +import com.raphtory.core.actors.Router.GraphBuilder import com.raphtory.core.model.communication.{Type, _} class ChainalysisABGraphBuilder extends GraphBuilder[String] { diff --git a/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/DashcoinGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/DashcoinGraphBuilder.scala index 7ad4cb7306..6285724b0d 100644 --- a/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/DashcoinGraphBuilder.scala +++ b/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/DashcoinGraphBuilder.scala @@ -1,6 +1,6 @@ package com.raphtory.examples.blockchain.graphbuilders -import com.raphtory.core.components.Router.GraphBuilder +import com.raphtory.core.actors.Router.GraphBuilder import com.raphtory.core.model.communication._ import com.raphtory.spouts.blockchain.BitcoinTransaction import spray.json.JsArray diff --git a/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/EthereumGethGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/EthereumGethGraphBuilder.scala index 22a6bfb349..d6de53b74c 100644 --- a/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/EthereumGethGraphBuilder.scala +++ b/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/EthereumGethGraphBuilder.scala @@ -1,6 +1,6 @@ package com.raphtory.examples.blockchain.graphbuilders -import com.raphtory.core.components.Router.GraphBuilder +import com.raphtory.core.actors.Router.GraphBuilder import com.raphtory.core.model.communication._ class EthereumGethGraphBuilder extends GraphBuilder[String] { def hexToInt(hex: String) = Integer.parseInt(hex.drop(2), 16) diff --git a/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/EthereumKafkaGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/EthereumKafkaGraphBuilder.scala index 3ad0387929..5177ad12c1 100644 --- a/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/EthereumKafkaGraphBuilder.scala +++ b/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/EthereumKafkaGraphBuilder.scala @@ -1,6 +1,6 @@ package com.raphtory.examples.blockchain.graphbuilders -import com.raphtory.core.components.Router.GraphBuilder +import com.raphtory.core.actors.Router.GraphBuilder import com.raphtory.core.model.communication._ class EthereumKafkaGraphBuilder extends GraphBuilder[String] { def hexToInt(hex: String) = Integer.parseInt(hex.drop(2), 16) diff --git a/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/EthereumTransactionGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/EthereumTransactionGraphBuilder.scala index 673e8b6ec1..d57f61f559 100644 --- a/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/EthereumTransactionGraphBuilder.scala +++ b/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/EthereumTransactionGraphBuilder.scala @@ -1,6 +1,6 @@ package com.raphtory.examples.blockchain.graphbuilders -import com.raphtory.core.components.Router.GraphBuilder +import com.raphtory.core.actors.Router.GraphBuilder import com.raphtory.core.model.communication.{EdgeAddWithProperties, Properties, StringProperty, VertexAddWithProperties} import scala.util.hashing.MurmurHash3 diff --git a/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/FirehoseKafkaGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/FirehoseKafkaGraphBuilder.scala index 8f94d5863f..1cee339848 100644 --- a/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/FirehoseKafkaGraphBuilder.scala +++ b/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/FirehoseKafkaGraphBuilder.scala @@ -1,6 +1,6 @@ package com.raphtory.examples.blockchain.graphbuilders -import com.raphtory.core.components.Router.GraphBuilder +import com.raphtory.core.actors.Router.GraphBuilder import com.raphtory.core.model.communication._ import scala.util.Random diff --git a/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/LitecoinGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/LitecoinGraphBuilder.scala index e09ea5827a..3fe58183b4 100644 --- a/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/LitecoinGraphBuilder.scala +++ b/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/LitecoinGraphBuilder.scala @@ -1,6 +1,6 @@ package com.raphtory.examples.blockchain.graphbuilders -import com.raphtory.core.components.Router.GraphBuilder +import com.raphtory.core.actors.Router.GraphBuilder import com.raphtory.core.model.communication._ import com.raphtory.spouts.blockchain.BitcoinTransaction import spray.json.JsArray diff --git a/mainproject/src/main/scala/com/raphtory/examples/citationNetwork/CitationGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/citationNetwork/CitationGraphBuilder.scala index 3a15e85d9d..74a9fc6993 100644 --- a/mainproject/src/main/scala/com/raphtory/examples/citationNetwork/CitationGraphBuilder.scala +++ b/mainproject/src/main/scala/com/raphtory/examples/citationNetwork/CitationGraphBuilder.scala @@ -1,6 +1,6 @@ package com.raphtory.examples.citationNetwork -import com.raphtory.core.components.Router.GraphBuilder +import com.raphtory.core.actors.Router.GraphBuilder import com.raphtory.core.model.communication._ class CitationGraphBuilder extends GraphBuilder[String] { diff --git a/mainproject/src/main/scala/com/raphtory/examples/gab/analysis/GabMostUsedTopics.scala b/mainproject/src/main/scala/com/raphtory/examples/gab/analysis/GabMostUsedTopics.scala index 191d6fdbfe..377c4e3b98 100644 --- a/mainproject/src/main/scala/com/raphtory/examples/gab/analysis/GabMostUsedTopics.scala +++ b/mainproject/src/main/scala/com/raphtory/examples/gab/analysis/GabMostUsedTopics.scala @@ -1,6 +1,6 @@ package com.raphtory.examples.gab.analysis -import com.raphtory.core.analysis.API.Analyser +import com.raphtory.api.Analyser import scala.collection.mutable.ArrayBuffer diff --git a/mainproject/src/main/scala/com/raphtory/examples/gab/datasources/GabMongoSource.scala b/mainproject/src/main/scala/com/raphtory/examples/gab/datasources/GabMongoSource.scala index 84de1acff8..7b93d296f1 100644 --- a/mainproject/src/main/scala/com/raphtory/examples/gab/datasources/GabMongoSource.scala +++ b/mainproject/src/main/scala/com/raphtory/examples/gab/datasources/GabMongoSource.scala @@ -3,7 +3,7 @@ package com.raphtory.examples.gab.datasources import akka.actor.Cancellable import ch.qos.logback.classic.Level import com.mongodb.casbah.Imports.{MongoConnection, _} -import com.raphtory.core.components.Spout.Spout +import com.raphtory.core.actors.Spout.Spout import org.slf4j.LoggerFactory import scala.collection.mutable diff --git a/mainproject/src/main/scala/com/raphtory/examples/gab/graphbuilders/GabPostGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/gab/graphbuilders/GabPostGraphBuilder.scala index bf81b5cf86..8fb279cede 100644 --- a/mainproject/src/main/scala/com/raphtory/examples/gab/graphbuilders/GabPostGraphBuilder.scala +++ b/mainproject/src/main/scala/com/raphtory/examples/gab/graphbuilders/GabPostGraphBuilder.scala @@ -2,7 +2,7 @@ package com.raphtory.examples.gab.graphbuilders import java.text.SimpleDateFormat -import com.raphtory.core.components.Router.GraphBuilder +import com.raphtory.core.actors.Router.GraphBuilder import com.raphtory.core.model.communication._ // The lines sent by the Gab mining spout are read and processed accordingly. diff --git a/mainproject/src/main/scala/com/raphtory/examples/gab/graphbuilders/GabRawGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/gab/graphbuilders/GabRawGraphBuilder.scala index c001f69561..7de92e95ed 100644 --- a/mainproject/src/main/scala/com/raphtory/examples/gab/graphbuilders/GabRawGraphBuilder.scala +++ b/mainproject/src/main/scala/com/raphtory/examples/gab/graphbuilders/GabRawGraphBuilder.scala @@ -2,7 +2,7 @@ package com.raphtory.examples.gab.graphbuilders import java.time.OffsetDateTime -import com.raphtory.core.components.Router.GraphBuilder +import com.raphtory.core.actors.Router.GraphBuilder import com.raphtory.core.model.communication.{EdgeAddWithProperties, Properties, StringProperty, VertexAddWithProperties} import com.raphtory.examples.gab.rawgraphmodel.GabPost import spray.json._ diff --git a/mainproject/src/main/scala/com/raphtory/examples/gab/graphbuilders/GabUserGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/gab/graphbuilders/GabUserGraphBuilder.scala index d85da967e8..0a513e06b8 100644 --- a/mainproject/src/main/scala/com/raphtory/examples/gab/graphbuilders/GabUserGraphBuilder.scala +++ b/mainproject/src/main/scala/com/raphtory/examples/gab/graphbuilders/GabUserGraphBuilder.scala @@ -2,7 +2,7 @@ package com.raphtory.examples.gab.graphbuilders import java.text.SimpleDateFormat -import com.raphtory.core.components.Router.GraphBuilder +import com.raphtory.core.actors.Router.GraphBuilder import com.raphtory.core.model.communication.{Type, _} // The lines sent by the Gab mining spout are read and processed accordingly. diff --git a/mainproject/src/main/scala/com/raphtory/examples/ldbc/graphbuilders/LDBCGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/ldbc/graphbuilders/LDBCGraphBuilder.scala index 1ed0993e2f..a7fe0def3b 100644 --- a/mainproject/src/main/scala/com/raphtory/examples/ldbc/graphbuilders/LDBCGraphBuilder.scala +++ b/mainproject/src/main/scala/com/raphtory/examples/ldbc/graphbuilders/LDBCGraphBuilder.scala @@ -2,7 +2,7 @@ package com.raphtory.examples.ldbc.graphbuilders import java.text.SimpleDateFormat -import com.raphtory.core.components.Router.GraphBuilder +import com.raphtory.core.actors.Router.GraphBuilder import com.raphtory.core.model.communication._ class LDBCGraphBuilder extends GraphBuilder[String] { diff --git a/mainproject/src/main/scala/com/raphtory/examples/ldbc/graphbuilders/LDBCOldGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/ldbc/graphbuilders/LDBCOldGraphBuilder.scala index 9a026ddec1..14c0fd7944 100644 --- a/mainproject/src/main/scala/com/raphtory/examples/ldbc/graphbuilders/LDBCOldGraphBuilder.scala +++ b/mainproject/src/main/scala/com/raphtory/examples/ldbc/graphbuilders/LDBCOldGraphBuilder.scala @@ -2,7 +2,7 @@ package com.raphtory.examples.ldbc.graphbuilders import java.text.SimpleDateFormat -import com.raphtory.core.components.Router.GraphBuilder +import com.raphtory.core.actors.Router.GraphBuilder import com.raphtory.core.model.communication.{EdgeAdd, Type, VertexAdd} class LDBCOldGraphBuilder extends GraphBuilder[String] { diff --git a/mainproject/src/main/scala/com/raphtory/examples/lotr/LOTRGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/lotr/LOTRGraphBuilder.scala index ca57d39535..96cbffedc0 100644 --- a/mainproject/src/main/scala/com/raphtory/examples/lotr/LOTRGraphBuilder.scala +++ b/mainproject/src/main/scala/com/raphtory/examples/lotr/LOTRGraphBuilder.scala @@ -1,6 +1,6 @@ package com.raphtory.examples.lotr -import com.raphtory.core.components.Router.GraphBuilder +import com.raphtory.core.actors.Router.GraphBuilder import com.raphtory.core.model.communication._ class LOTRGraphBuilder extends GraphBuilder[String]{ diff --git a/mainproject/src/main/scala/com/raphtory/examples/lotr/LOTRSpout.scala b/mainproject/src/main/scala/com/raphtory/examples/lotr/LOTRSpout.scala index 058396eea6..9a4029f4b2 100644 --- a/mainproject/src/main/scala/com/raphtory/examples/lotr/LOTRSpout.scala +++ b/mainproject/src/main/scala/com/raphtory/examples/lotr/LOTRSpout.scala @@ -1,6 +1,6 @@ package com.raphtory.examples.lotr -import com.raphtory.core.components.Spout.Spout +import com.raphtory.core.actors.Spout.Spout import scala.collection.mutable diff --git a/mainproject/src/main/scala/com/raphtory/examples/test/actors/RandomGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/test/actors/RandomGraphBuilder.scala index e6631d6290..d852e6af56 100644 --- a/mainproject/src/main/scala/com/raphtory/examples/test/actors/RandomGraphBuilder.scala +++ b/mainproject/src/main/scala/com/raphtory/examples/test/actors/RandomGraphBuilder.scala @@ -1,6 +1,6 @@ package com.raphtory.examples.test.actors -import com.raphtory.core.components.Router.GraphBuilder +import com.raphtory.core.actors.Router.GraphBuilder import com.raphtory.core.model.communication.{GraphUpdate, _} import spray.json._ diff --git a/mainproject/src/main/scala/com/raphtory/examples/test/actors/RandomSpout.scala b/mainproject/src/main/scala/com/raphtory/examples/test/actors/RandomSpout.scala index b7d160027f..a3217beef0 100644 --- a/mainproject/src/main/scala/com/raphtory/examples/test/actors/RandomSpout.scala +++ b/mainproject/src/main/scala/com/raphtory/examples/test/actors/RandomSpout.scala @@ -1,6 +1,6 @@ package com.raphtory.examples.test.actors -import com.raphtory.core.components.Spout.Spout +import com.raphtory.core.actors.Spout.Spout import scala.language.postfixOps import scala.util.Random diff --git a/mainproject/src/main/scala/com/raphtory/examples/test/actors/TriangleGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/test/actors/TriangleGraphBuilder.scala index 6e15beba69..5626cec18d 100644 --- a/mainproject/src/main/scala/com/raphtory/examples/test/actors/TriangleGraphBuilder.scala +++ b/mainproject/src/main/scala/com/raphtory/examples/test/actors/TriangleGraphBuilder.scala @@ -1,6 +1,6 @@ package com.raphtory.examples.test.actors -import com.raphtory.core.components.Router.GraphBuilder +import com.raphtory.core.actors.Router.GraphBuilder import com.raphtory.core.model.communication.EdgeAdd class TriangleGraphBuilder extends GraphBuilder[String] { diff --git a/mainproject/src/main/scala/com/raphtory/examples/test/actors/TriangleSpout.scala b/mainproject/src/main/scala/com/raphtory/examples/test/actors/TriangleSpout.scala index 5e5e2171d2..b8a45a7b5c 100644 --- a/mainproject/src/main/scala/com/raphtory/examples/test/actors/TriangleSpout.scala +++ b/mainproject/src/main/scala/com/raphtory/examples/test/actors/TriangleSpout.scala @@ -1,6 +1,6 @@ package com.raphtory.examples.test.actors -import com.raphtory.core.components.Spout.Spout +import com.raphtory.core.actors.Spout.Spout import scala.collection.mutable.Queue diff --git a/mainproject/src/main/scala/com/raphtory/examples/trackAndTrace/TrackAndTraceGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/trackAndTrace/TrackAndTraceGraphBuilder.scala index 47b5a27bec..ffa979e7df 100644 --- a/mainproject/src/main/scala/com/raphtory/examples/trackAndTrace/TrackAndTraceGraphBuilder.scala +++ b/mainproject/src/main/scala/com/raphtory/examples/trackAndTrace/TrackAndTraceGraphBuilder.scala @@ -1,6 +1,6 @@ package com.raphtory.examples.trackAndTrace -import com.raphtory.core.components.Router.GraphBuilder +import com.raphtory.core.actors.Router.GraphBuilder import com.raphtory.core.model.communication.{Type, _} class TrackAndTraceGraphBuilder extends GraphBuilder[String] { diff --git a/mainproject/src/main/scala/com/raphtory/examples/tsvnet/TSVGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/tsvnet/TSVGraphBuilder.scala index a7bcb4f367..4fda7b5fef 100644 --- a/mainproject/src/main/scala/com/raphtory/examples/tsvnet/TSVGraphBuilder.scala +++ b/mainproject/src/main/scala/com/raphtory/examples/tsvnet/TSVGraphBuilder.scala @@ -1,6 +1,6 @@ package com.raphtory.examples.tsvnet -import com.raphtory.core.components.Router.GraphBuilder +import com.raphtory.core.actors.Router.GraphBuilder import com.raphtory.core.model.communication.{EdgeAdd, Type, VertexAdd} /** Spout for network datasets of the form SRC_NODE_ID DEST_NODE_ID TIMESTAMP */ diff --git a/mainproject/src/main/scala/com/raphtory/examples/wordSemantic/graphbuilders/CooccurrenceMatrixGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/wordSemantic/graphbuilders/CooccurrenceMatrixGraphBuilder.scala index 15ddd258f2..247be1a7f4 100644 --- a/mainproject/src/main/scala/com/raphtory/examples/wordSemantic/graphbuilders/CooccurrenceMatrixGraphBuilder.scala +++ b/mainproject/src/main/scala/com/raphtory/examples/wordSemantic/graphbuilders/CooccurrenceMatrixGraphBuilder.scala @@ -1,6 +1,6 @@ package com.raphtory.examples.wordSemantic.graphbuilders -import com.raphtory.core.components.Router.GraphBuilder +import com.raphtory.core.actors.Router.GraphBuilder import com.raphtory.core.model.communication._ class CooccurrenceMatrixGraphBuilder extends GraphBuilder[String] { diff --git a/mainproject/src/main/scala/com/raphtory/examples/wordSemantic/graphbuilders/CooccurrenceMatrixGraphBuilderFiltered.scala b/mainproject/src/main/scala/com/raphtory/examples/wordSemantic/graphbuilders/CooccurrenceMatrixGraphBuilderFiltered.scala index 3a424840ae..3e15adb318 100644 --- a/mainproject/src/main/scala/com/raphtory/examples/wordSemantic/graphbuilders/CooccurrenceMatrixGraphBuilderFiltered.scala +++ b/mainproject/src/main/scala/com/raphtory/examples/wordSemantic/graphbuilders/CooccurrenceMatrixGraphBuilderFiltered.scala @@ -1,6 +1,6 @@ package com.raphtory.examples.wordSemantic.graphbuilders -import com.raphtory.core.components.Router.GraphBuilder +import com.raphtory.core.actors.Router.GraphBuilder import com.raphtory.core.model.communication._ class CooccurrenceMatrixGraphBuilderFiltered extends GraphBuilder[String] { diff --git a/mainproject/src/main/scala/com/raphtory/core/analysis/serialisers/JSONSerialiser.scala b/mainproject/src/main/scala/com/raphtory/serialisers/JSONSerialiser.scala similarity index 91% rename from mainproject/src/main/scala/com/raphtory/core/analysis/serialisers/JSONSerialiser.scala rename to mainproject/src/main/scala/com/raphtory/serialisers/JSONSerialiser.scala index 2dcaa563b4..2ec2335e27 100644 --- a/mainproject/src/main/scala/com/raphtory/core/analysis/serialisers/JSONSerialiser.scala +++ b/mainproject/src/main/scala/com/raphtory/serialisers/JSONSerialiser.scala @@ -1,6 +1,7 @@ -package com.raphtory.core.analysis.serialisers +package com.raphtory.serialisers -import com.raphtory.core.analysis.API.entityVisitors.{EdgeVisitor, VertexVisitor} +import com.raphtory.api.Serialiser +import com.raphtory.core.model.analysis.entityVisitors.{EdgeVisitor, VertexVisitor} import scala.collection.parallel.mutable.ParTrieMap diff --git a/mainproject/src/main/scala/com/raphtory/core/analysis/serialisers/NetworkXSerialiser.scala b/mainproject/src/main/scala/com/raphtory/serialisers/NetworkXSerialiser.scala similarity index 89% rename from mainproject/src/main/scala/com/raphtory/core/analysis/serialisers/NetworkXSerialiser.scala rename to mainproject/src/main/scala/com/raphtory/serialisers/NetworkXSerialiser.scala index 388c7f9c73..6a91a1c144 100644 --- a/mainproject/src/main/scala/com/raphtory/core/analysis/serialisers/NetworkXSerialiser.scala +++ b/mainproject/src/main/scala/com/raphtory/serialisers/NetworkXSerialiser.scala @@ -1,5 +1,6 @@ -package com.raphtory.core.analysis.serialisers -import com.raphtory.core.analysis.API.entityVisitors.{EdgeVisitor, VertexVisitor} +package com.raphtory.serialisers +import com.raphtory.api.Serialiser +import com.raphtory.core.model.analysis.entityVisitors.{EdgeVisitor, VertexVisitor} class NetworkXSerialiser extends Serialiser { diff --git a/mainproject/src/main/scala/com/raphtory/spouts/FileSpout.scala b/mainproject/src/main/scala/com/raphtory/spouts/FileSpout.scala index c242d3931c..e3aa0a593e 100644 --- a/mainproject/src/main/scala/com/raphtory/spouts/FileSpout.scala +++ b/mainproject/src/main/scala/com/raphtory/spouts/FileSpout.scala @@ -3,7 +3,7 @@ package com.raphtory.spouts import java.io._ import java.util.zip.GZIPInputStream -import com.raphtory.core.components.Spout.Spout +import com.raphtory.core.actors.Spout.Spout import com.typesafe.scalalogging.LazyLogging class FileSpout extends Spout[String] { diff --git a/mainproject/src/main/scala/com/raphtory/spouts/KafkaSpout.scala b/mainproject/src/main/scala/com/raphtory/spouts/KafkaSpout.scala index 4361676ff6..b6c13fa136 100644 --- a/mainproject/src/main/scala/com/raphtory/spouts/KafkaSpout.scala +++ b/mainproject/src/main/scala/com/raphtory/spouts/KafkaSpout.scala @@ -3,7 +3,7 @@ package com.raphtory.spouts import java.util import java.util.Properties -import com.raphtory.core.components.Spout.Spout +import com.raphtory.core.actors.Spout.Spout import org.apache.kafka.clients.consumer.KafkaConsumer import scala.annotation.tailrec diff --git a/mainproject/src/main/scala/com/raphtory/spouts/blockchain/BitcoinNodeSpout.scala b/mainproject/src/main/scala/com/raphtory/spouts/blockchain/BitcoinNodeSpout.scala index 78693514ba..c1eb4a9f6f 100644 --- a/mainproject/src/main/scala/com/raphtory/spouts/blockchain/BitcoinNodeSpout.scala +++ b/mainproject/src/main/scala/com/raphtory/spouts/blockchain/BitcoinNodeSpout.scala @@ -2,7 +2,7 @@ package com.raphtory.spouts.blockchain // import java.io.{File, PrintWriter} -import com.raphtory.core.components.Spout.Spout +import com.raphtory.core.actors.Spout.Spout import scalaj.http.{Http, HttpRequest} import spray.json._ diff --git a/mainproject/src/main/scala/com/raphtory/spouts/blockchain/EthereumGethSpout.scala b/mainproject/src/main/scala/com/raphtory/spouts/blockchain/EthereumGethSpout.scala index 398f306ccb..541e5bc89b 100644 --- a/mainproject/src/main/scala/com/raphtory/spouts/blockchain/EthereumGethSpout.scala +++ b/mainproject/src/main/scala/com/raphtory/spouts/blockchain/EthereumGethSpout.scala @@ -3,7 +3,7 @@ package com.raphtory.spouts.blockchain import java.net.InetAddress import java.util.NoSuchElementException -import com.raphtory.core.components.Spout.Spout +import com.raphtory.core.actors.Spout.Spout import com.raphtory.core.utils.Utils import scalaj.http.{Http, HttpRequest} import spray.json.DefaultJsonProtocol._ diff --git a/mainproject/src/main/scala/com/raphtory/spouts/blockchain/EthereumPostgresSpout.scala b/mainproject/src/main/scala/com/raphtory/spouts/blockchain/EthereumPostgresSpout.scala index d091fb6be1..01f92d7382 100644 --- a/mainproject/src/main/scala/com/raphtory/spouts/blockchain/EthereumPostgresSpout.scala +++ b/mainproject/src/main/scala/com/raphtory/spouts/blockchain/EthereumPostgresSpout.scala @@ -1,7 +1,7 @@ package com.raphtory.spouts.blockchain import cats.effect.{Blocker, IO} -import com.raphtory.core.components.Spout.Spout +import com.raphtory.core.actors.Spout.Spout import doobie.implicits._ import doobie.util.ExecutionContexts import doobie.util.transactor.Transactor From 96ed86ced2cd1f40188dda46ddb9b7b478483f32 Mon Sep 17 00:00:00 2001 From: imanehaf Date: Mon, 30 Nov 2020 18:56:24 +0000 Subject: [PATCH 51/58] Added logger properties --- mainproject/src/main/resources/logback.xml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 mainproject/src/main/resources/logback.xml diff --git a/mainproject/src/main/resources/logback.xml b/mainproject/src/main/resources/logback.xml new file mode 100644 index 0000000000..0e6f347552 --- /dev/null +++ b/mainproject/src/main/resources/logback.xml @@ -0,0 +1,15 @@ + + + + + + %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n + + + + + + + + + From 23940ab77ce91f8b31c1986e27e6855634c8b5db Mon Sep 17 00:00:00 2001 From: Miratepuffin Date: Mon, 30 Nov 2020 23:38:32 +0000 Subject: [PATCH 52/58] Minor refactor to the vertex message queues for thread safety --- .../core/model/communication/VertexMutliQueue.scala | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/mainproject/src/main/scala/com/raphtory/core/model/communication/VertexMutliQueue.scala b/mainproject/src/main/scala/com/raphtory/core/model/communication/VertexMutliQueue.scala index d891acdebd..4262c2dff1 100644 --- a/mainproject/src/main/scala/com/raphtory/core/model/communication/VertexMutliQueue.scala +++ b/mainproject/src/main/scala/com/raphtory/core/model/communication/VertexMutliQueue.scala @@ -23,16 +23,15 @@ class VertexMutliQueue { def clearQueue(jobID: ViewJob, superStep: Int) = { val queueMap = if (superStep % 2 == 0) evenMessageQueueMap else oddMessageQueueMap - queueMap(jobID) = mutable.ArrayBuffer[Any]() + queueMap.update(jobID, mutable.ArrayBuffer[Any]()) } - def clearQueues(jobID: ViewJob) = { evenMessageQueueMap.get(jobID) match { - case Some(q) => evenMessageQueueMap(jobID) = mutable.ArrayBuffer[Any]() + case Some(_) => evenMessageQueueMap.update(jobID, mutable.ArrayBuffer[Any]()) case None => } oddMessageQueueMap.get(jobID) match { - case Some(q) => oddMessageQueueMap(jobID) = mutable.ArrayBuffer[Any]() + case Some(_) => oddMessageQueueMap.update(jobID, mutable.ArrayBuffer[Any]()) case None => } } From c6b1e64352bd04c63b8a0407d8b20cde9d3a3349 Mon Sep 17 00:00:00 2001 From: Miratepuffin Date: Tue, 1 Dec 2020 00:21:14 +0000 Subject: [PATCH 53/58] Removed the Utils folder, creating new RaphtoryActor trait which all actors now inherit from --- .../scala/com/raphtory/RaphtoryServer.scala | 3 +- .../CommunityOutlierDetection.scala | 5 +- .../algorithms/ConnectedComponents.scala | 5 +- .../com/raphtory/algorithms/DegreeBasic.scala | 5 +- .../algorithms/DegreeDistribution.scala | 3 +- .../raphtory/algorithms/DegreeRanking.scala | 5 +- .../raphtory/algorithms/Distributions.scala | 5 +- .../algorithms/GabWeightedPageRank.scala | 3 +- .../com/raphtory/algorithms/PageRank.scala | 3 +- .../raphtory/algorithms/TriangleCount.scala | 3 +- .../algorithms/WeightedPageRank.scala | 3 +- .../scala/com/raphtory/api/Analyser.scala | 24 ++++ .../AnalysisManager/AnalysisManager.scala | 3 +- .../AnalysisManager/Tasks/AnalysisTask.scala | 45 +++--- .../Tasks/LiveTasks/LiveAnalysisTask.scala | 9 +- .../Tasks/RangeTasks/RangeAnalysisTask.scala | 3 +- .../RaphtoryReplicator.scala | 13 +- .../actors/ClusterManagement/SeedActor.scala | 3 +- .../actors/ClusterManagement/WatchDog.scala | 23 ++-- .../ClusterManagement/WatermarkManager.scala | 6 +- .../core/actors/PartitionManager/Reader.scala | 11 +- .../Workers/IngestionWorker.scala | 8 +- .../Workers/ReaderWorker.scala | 6 +- .../core/actors/PartitionManager/Writer.scala | 14 +- .../raphtory/core/actors/RaphtoryActor.scala | 97 +++++++++++++ .../core/actors/Router/RouterManager.scala | 4 +- .../core/actors/Router/RouterWorker.scala | 23 ++-- .../core/actors/Spout/SpoutAgent.scala | 6 +- .../raphtory/core/model/EntityStorage.scala | 12 +- .../analysis/entityVisitors/EdgeVisitor.scala | 3 +- .../entityVisitors/EntityVisitor.scala | 8 ++ .../entityVisitors/VertexVisitor.scala | 3 +- .../raphtory/core/utils/SchedulerUtil.scala | 49 ------- .../scala/com/raphtory/core/utils/Utils.scala | 128 ------------------ .../analysers/EthereumDegreeRanking.scala | 3 +- .../analysers/TaintTrackExchangeStop.scala | 7 +- .../spouts/blockchain/EthereumGethSpout.scala | 8 +- 37 files changed, 254 insertions(+), 308 deletions(-) create mode 100644 mainproject/src/main/scala/com/raphtory/core/actors/RaphtoryActor.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/core/utils/SchedulerUtil.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/core/utils/Utils.scala diff --git a/mainproject/src/main/scala/com/raphtory/RaphtoryServer.scala b/mainproject/src/main/scala/com/raphtory/RaphtoryServer.scala index b797209f2b..3d16b5c576 100644 --- a/mainproject/src/main/scala/com/raphtory/RaphtoryServer.scala +++ b/mainproject/src/main/scala/com/raphtory/RaphtoryServer.scala @@ -11,7 +11,6 @@ import com.raphtory.core.actors.AnalysisManager.{AnalysisManager, AnalysisRestAp import com.raphtory.core.actors.ClusterManagement.{RaphtoryReplicator, SeedActor, WatchDog, WatermarkManager} import com.raphtory.core.actors.Router.GraphBuilder import com.raphtory.core.actors.Spout.{Spout, SpoutAgent} -import com.raphtory.core.utils.Utils import com.typesafe.config.{Config, ConfigFactory, ConfigValue, ConfigValueFactory} import scala.collection.JavaConversions @@ -24,7 +23,7 @@ object RaphtoryServer extends App { // Kamon.init() //start tool logging printJavaOptions() val conf = ConfigFactory.load() - val clusterSystemName: String = Utils.clusterSystemName + val clusterSystemName = "Raphtory" val ssn: String = java.util.UUID.randomUUID.toString val partitionCount = sys.env("PARTITION_MIN").toInt diff --git a/mainproject/src/main/scala/com/raphtory/algorithms/CommunityOutlierDetection.scala b/mainproject/src/main/scala/com/raphtory/algorithms/CommunityOutlierDetection.scala index eca14dfc44..afa88d0f2d 100644 --- a/mainproject/src/main/scala/com/raphtory/algorithms/CommunityOutlierDetection.scala +++ b/mainproject/src/main/scala/com/raphtory/algorithms/CommunityOutlierDetection.scala @@ -1,7 +1,6 @@ package com.raphtory.algorithms import com.raphtory.core.model.analysis.entityVisitors.VertexVisitor -import com.raphtory.core.utils.Utils import scala.collection.mutable.ArrayBuffer import scala.collection.parallel.immutable @@ -31,7 +30,7 @@ class CommunityOutlierDetection(args:Array[String]) extends LPA(args) { val total = outliers.length val proportion = total/endResults.length.toFloat val text = s"""{"time":$timestamp,"total":$total,"top5":[${top.mkString(",")}],"outliers":{${sortedstr.take(topnum).mkString(",")}},"proportion":$proportion,"viewTime":$viewCompleteTime}""" - Utils.writeLines(output_file, text, "{\"views\":[") + writeLines(output_file, text, "{\"views\":[") println(text) publishData(text) } @@ -47,7 +46,7 @@ class CommunityOutlierDetection(args:Array[String]) extends LPA(args) { val total = outliers.length val proportion = total/endResults.length.toFloat val text = s"""{"time":$timestamp,"windowsize":$windowSize,"total":$total,"top5":[${top.mkString(",")}],"outliers":{${sortedstr.take(topnum).mkString(",")}},"proportion":$proportion,"viewTime":$viewCompleteTime},""" - Utils.writeLines(output_file, text, "{\"views\":[") + writeLines(output_file, text, "{\"views\":[") println(text) publishData(text) } diff --git a/mainproject/src/main/scala/com/raphtory/algorithms/ConnectedComponents.scala b/mainproject/src/main/scala/com/raphtory/algorithms/ConnectedComponents.scala index c48fb3339a..7fa212ad23 100644 --- a/mainproject/src/main/scala/com/raphtory/algorithms/ConnectedComponents.scala +++ b/mainproject/src/main/scala/com/raphtory/algorithms/ConnectedComponents.scala @@ -1,7 +1,6 @@ package com.raphtory.algorithms import com.raphtory.api.Analyser -import com.raphtory.core.utils.Utils import scala.collection.mutable.ArrayBuffer import scala.collection.parallel.immutable @@ -37,7 +36,7 @@ class ConnectedComponents(args:Array[String]) extends Analyser(args){ var output_file = output_folder + "/" + System.getenv().getOrDefault("OUTPUT_FILE","ConnectedComponents.json").trim println(text) - Utils.writeLines(output_file, text, "{\"views\":[") + writeLines(output_file, text, "{\"views\":[") publishData(text) } @@ -47,7 +46,7 @@ class ConnectedComponents(args:Array[String]) extends Analyser(args){ var output_folder = System.getenv().getOrDefault("OUTPUT_FOLDER", "/app").trim var output_file = output_folder + "/" + System.getenv().getOrDefault("OUTPUT_FILE","ConnectedComponents.json").trim val text = s"""{"time":$timestamp,"windowsize":$windowSize,"top5":[${er.top5.mkString(",")}],"total":${er.total},"totalIslands":${er.totalIslands},"proportion":${er.proportion},"clustersGT2":${er.totalGT2},"viewTime":$viewCompleteTime}""" - Utils.writeLines(output_file, text, "{\"views\":[") + writeLines(output_file, text, "{\"views\":[") println(text) //publishData(text) diff --git a/mainproject/src/main/scala/com/raphtory/algorithms/DegreeBasic.scala b/mainproject/src/main/scala/com/raphtory/algorithms/DegreeBasic.scala index 2ba56c7710..8d55ede71f 100644 --- a/mainproject/src/main/scala/com/raphtory/algorithms/DegreeBasic.scala +++ b/mainproject/src/main/scala/com/raphtory/algorithms/DegreeBasic.scala @@ -1,7 +1,6 @@ package com.raphtory.algorithms import com.raphtory.api.Analyser -import com.raphtory.core.utils.Utils import scala.collection.mutable.ArrayBuffer @@ -44,7 +43,7 @@ val startTime = System.currentTimeMillis() val text = s"""{"time":$timestamp,"vertices":$totalVert,"edges":$totalEdge,"degree":$degree}""" var output_folder = System.getenv().getOrDefault("OUTPUT_FOLDER", "/app").trim var output_file = output_folder + "/" + System.getenv().getOrDefault("OUTPUT_FILE","DegreeBasic.json").trim - Utils.writeLines(output_file, text, "") + writeLines(output_file, text, "") println(text) publishData(text) } @@ -66,7 +65,7 @@ val startTime = System.currentTimeMillis() case e: ArithmeticException => 0 } val text = s"""{"time":$timestamp,"windowsize":$windowSize,"vertices":$totalVert,"edges":$totalEdge,"degree":$degree},""" - Utils.writeLines(output_file, text, "") + writeLines(output_file, text, "") println(text) publishData(text) diff --git a/mainproject/src/main/scala/com/raphtory/algorithms/DegreeDistribution.scala b/mainproject/src/main/scala/com/raphtory/algorithms/DegreeDistribution.scala index ae6d099bfb..812af7105b 100644 --- a/mainproject/src/main/scala/com/raphtory/algorithms/DegreeDistribution.scala +++ b/mainproject/src/main/scala/com/raphtory/algorithms/DegreeDistribution.scala @@ -1,7 +1,6 @@ package com.raphtory.algorithms import com.raphtory.api.Analyser -import com.raphtory.core.utils.Utils import scala.collection.mutable.ArrayBuffer @@ -59,7 +58,7 @@ class DegreeDistribution(args:Array[String]) extends Analyser(args){ val meanDegSq = if (totalVert > 0) endResults.map(x => x._5/totalVert.toDouble).sum else 0.0 val text = s"""{"time":$timestamp,"windowsize":$windowSize,"vertices":$totalVert,"maxDeg":$maxDeg,"avgSquaredDeg":$meanDegSq,"avgSquaredInDeg":$meanInDegSq,"avgSquaredOutDeg":$meanOutDegSq,"viewTime":$viewCompleteTime}""" - Utils.writeLines(output_file, text, "{\"views\":[") + writeLines(output_file, text, "{\"views\":[") println(text) //publishData(text) } diff --git a/mainproject/src/main/scala/com/raphtory/algorithms/DegreeRanking.scala b/mainproject/src/main/scala/com/raphtory/algorithms/DegreeRanking.scala index 479d3f9b0c..cd8b628df3 100644 --- a/mainproject/src/main/scala/com/raphtory/algorithms/DegreeRanking.scala +++ b/mainproject/src/main/scala/com/raphtory/algorithms/DegreeRanking.scala @@ -1,7 +1,6 @@ package com.raphtory.algorithms import com.raphtory.api.Analyser -import com.raphtory.core.utils.Utils import scala.collection.mutable.ArrayBuffer @@ -59,7 +58,7 @@ class DegreeRanking(args:Array[String]) extends Analyser(args){ val text = s"""{"time":$timeStamp,"vertices":$totalVert,"edges":$totalEdge,"degree":$degree,"bestusers":$bestUserArray,"viewTime":$viewCompleteTime,"concatTime":${System .currentTimeMillis() - startTime}},""" - // Utils.writeLines(output_file, text, "{\"views\":[") + // writeLines(output_file, text, "{\"views\":[") println(text) publishData(text) } @@ -92,7 +91,7 @@ class DegreeRanking(args:Array[String]) extends Analyser(args){ val text = s"""{"time":$timestamp,"windowsize":$windowSize,"vertices":$totalVert,"edges":$totalEdge,"degree":$degree,"bestusers":$bestUserArray,"viewTime":$viewCompleteTime,"concatTime":${System .currentTimeMillis() - startTime}}""" - Utils.writeLines(output_file, text, "{\"views\":[") + writeLines(output_file, text, "{\"views\":[") println(text) //publishData(text) } diff --git a/mainproject/src/main/scala/com/raphtory/algorithms/Distributions.scala b/mainproject/src/main/scala/com/raphtory/algorithms/Distributions.scala index a5ce9b93f8..ef65cdfea3 100644 --- a/mainproject/src/main/scala/com/raphtory/algorithms/Distributions.scala +++ b/mainproject/src/main/scala/com/raphtory/algorithms/Distributions.scala @@ -1,7 +1,6 @@ package com.raphtory.algorithms import com.raphtory.api.Analyser -import com.raphtory.core.utils.Utils import scala.collection.mutable.ArrayBuffer import scala.collection.parallel.immutable @@ -51,7 +50,7 @@ class Distributions(args:Array[String]) extends Analyser(args){ val text = s"""{"time":$timeStamp,"degDist":$degDistArr,"weightDist":$weightDistArr, "edgeDist":$edgeDistArr,"viewTime":$viewCompleteTime}""" println(text) publishData(text) - Utils.writeLines(output_file, text, "{\"views\":[") + writeLines(output_file, text, "{\"views\":[") } override def processWindowResults(results: ArrayBuffer[Any], timestamp: Long, windowSize: Long, viewCompleteTime: Long): Unit = { @@ -67,7 +66,7 @@ class Distributions(args:Array[String]) extends Analyser(args){ val text = s"""{"time":$timestamp,"windowsize":$windowSize,"degDist":$degDistArr,"weightDist":$weightDistArr, "edgeDist":$edgeDistArr,"viewTime":$viewCompleteTime}""" println(text) publishData(text) - Utils.writeLines(output_file, text, "{\"views\":[") + writeLines(output_file, text, "{\"views\":[") } def extractData(results: ArrayBuffer[Any]): extractedData = { diff --git a/mainproject/src/main/scala/com/raphtory/algorithms/GabWeightedPageRank.scala b/mainproject/src/main/scala/com/raphtory/algorithms/GabWeightedPageRank.scala index bb05ad4b2d..b6e8a949d2 100644 --- a/mainproject/src/main/scala/com/raphtory/algorithms/GabWeightedPageRank.scala +++ b/mainproject/src/main/scala/com/raphtory/algorithms/GabWeightedPageRank.scala @@ -1,7 +1,6 @@ package com.raphtory.algorithms import com.raphtory.api.Analyser -import com.raphtory.core.utils.Utils import scala.collection.mutable.ArrayBuffer @@ -92,7 +91,7 @@ class GabWeightedPageRank(args:Array[String]) extends Analyser(args) { .map(x => s"""{"id":${x._1},"pagerank":${x._2}}""").mkString("[",",","]") val text = s"""{"time":$timestamp,"windowsize":$windowSize,"vertices":$totalVert,"bestusers":$bestUsers,"viewTime":$viewCompleteTime}""" - Utils.writeLines(output_file, text, "{\"views\":[") + writeLines(output_file, text, "{\"views\":[") println(text) //publishData(text) } diff --git a/mainproject/src/main/scala/com/raphtory/algorithms/PageRank.scala b/mainproject/src/main/scala/com/raphtory/algorithms/PageRank.scala index 6764bfe061..18680ae362 100644 --- a/mainproject/src/main/scala/com/raphtory/algorithms/PageRank.scala +++ b/mainproject/src/main/scala/com/raphtory/algorithms/PageRank.scala @@ -1,7 +1,6 @@ package com.raphtory.algorithms import com.raphtory.api.Analyser -import com.raphtory.core.utils.Utils import scala.collection.mutable.ArrayBuffer @@ -87,7 +86,7 @@ class PageRank(args:Array[String]) extends Analyser(args) { .map(x => s"""{"id":${x._1},"pagerank":${x._2}}""").mkString("[",",","]") val text = s"""{"time":$timestamp,"windowsize":$windowSize,"vertices":$totalVert,"bestusers":$bestUsers,"viewTime":$viewCompleteTime}""" - Utils.writeLines(output_file, text, "{\"views\":[") + writeLines(output_file, text, "{\"views\":[") println(text) //publishData(text) } diff --git a/mainproject/src/main/scala/com/raphtory/algorithms/TriangleCount.scala b/mainproject/src/main/scala/com/raphtory/algorithms/TriangleCount.scala index dbf9e7d94e..0ffce00511 100644 --- a/mainproject/src/main/scala/com/raphtory/algorithms/TriangleCount.scala +++ b/mainproject/src/main/scala/com/raphtory/algorithms/TriangleCount.scala @@ -1,7 +1,6 @@ package com.raphtory.algorithms import com.raphtory.api.Analyser -import com.raphtory.core.utils.Utils import scala.collection.mutable.ArrayBuffer @@ -79,7 +78,7 @@ class TriangleCount(args:Array[String]) extends Analyser(args) { val text = s"""{"time":$timestamp,"windowsize":$windowSize,"totTriangles":$totalTri,"avgCluster":$avgCluster,"viewTime":$viewCompleteTime,"concatTime":${System .currentTimeMillis() - startTime}}""" - Utils.writeLines(output_file, text, "{\"views\":[") + writeLines(output_file, text, "{\"views\":[") publishData(text) println(text) } diff --git a/mainproject/src/main/scala/com/raphtory/algorithms/WeightedPageRank.scala b/mainproject/src/main/scala/com/raphtory/algorithms/WeightedPageRank.scala index 8ada0169a3..f11a293639 100644 --- a/mainproject/src/main/scala/com/raphtory/algorithms/WeightedPageRank.scala +++ b/mainproject/src/main/scala/com/raphtory/algorithms/WeightedPageRank.scala @@ -1,7 +1,6 @@ package com.raphtory.algorithms import com.raphtory.api.Analyser -import com.raphtory.core.utils.Utils import scala.collection.mutable.ArrayBuffer @@ -89,7 +88,7 @@ class WeightedPageRank(args:Array[String]) extends Analyser(args) { .map(x => s"""{"id":${x._1},"pagerank":${x._2}}""").mkString("[",",","]") val text = s"""{"time":$timestamp,"windowsize":$windowSize,"vertices":$totalVert,"bestusers":$bestUsers,"viewTime":$viewCompleteTime}""" - Utils.writeLines(output_file, text, "{\"views\":[") + writeLines(output_file, text, "{\"views\":[") println(text) publishData(text) } diff --git a/mainproject/src/main/scala/com/raphtory/api/Analyser.scala b/mainproject/src/main/scala/com/raphtory/api/Analyser.scala index 574efc183d..b55fbaf90e 100644 --- a/mainproject/src/main/scala/com/raphtory/api/Analyser.scala +++ b/mainproject/src/main/scala/com/raphtory/api/Analyser.scala @@ -1,5 +1,7 @@ package com.raphtory.api +import java.io.{BufferedWriter, File, FileWriter} + import akka.actor.ActorContext import com.raphtory.core.model.analysis.GraphLenses.GraphLens @@ -53,4 +55,26 @@ abstract class Analyser(args:Array[String]) extends java.io.Serializable { def processWindowResults(results: ArrayBuffer[Any], timestamp: Long, windowSize: Long, viewCompleteTime: Long): Unit = processResults(results, timestamp: Long, viewCompleteTime: Long) + //TODO THIS IS A CURSED FUNCTION AND SHOULD BE DESTROYED + def writeLines(fileName: String, line: String, header: String): Unit = { + val f = new File(fileName) + if (!f.exists()) { + f.createNewFile() + val file = new FileWriter(fileName, true) + var bw = new BufferedWriter(file) + bw.write(header) + bw.newLine() + bw.write(line) + bw.newLine() + bw.flush() + } else { + val file = new FileWriter(fileName, true) + var bw = new BufferedWriter(file) + bw.write(line) + bw.newLine() + bw.flush() + } + + } + } diff --git a/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/AnalysisManager.scala b/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/AnalysisManager.scala index 52806b630e..9591b262f1 100644 --- a/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/AnalysisManager.scala +++ b/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/AnalysisManager.scala @@ -8,6 +8,7 @@ import com.raphtory.api.{Analyser, BlankAnalyser, LoadExternalAnalyser} import com.raphtory.analysis.Tasks.LiveTasks.{BWindowedLiveAnalysisTask, LiveAnalysisTask, WindowedLiveAnalysisTask} import com.raphtory.analysis.Tasks.RangeTasks.{BWindowedRangeAnalysisTask, RangeAnalysisTask, WindowedRangeAnalysisTask} import com.raphtory.analysis.Tasks.ViewTasks.{BWindowedViewAnalysisTask, ViewAnalysisTask, WindowedViewAnalysisTask} +import com.raphtory.core.actors.RaphtoryActor import com.raphtory.core.model.communication.{ClusterStatusRequest, ClusterStatusResponse, _} import scala.collection.parallel.mutable.ParTrieMap @@ -15,7 +16,7 @@ import scala.concurrent.Await import scala.concurrent.duration.{Duration, _} import scala.language.postfixOps case class StartAnalysis() -class AnalysisManager() extends Actor{ +class AnalysisManager() extends RaphtoryActor{ implicit val executionContext = context.system.dispatchers.lookup("misc-dispatcher") implicit val timeout: Timeout = 10.seconds final protected val mediator = DistributedPubSub(context.system).mediator diff --git a/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/AnalysisTask.scala b/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/AnalysisTask.scala index 57fe22981d..5114661583 100644 --- a/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/AnalysisTask.scala +++ b/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/AnalysisTask.scala @@ -10,15 +10,15 @@ import com.mongodb.util.JSON import com.raphtory.api.Analyser import com.raphtory.core.actors.AnalysisManager.StartAnalysis import com.raphtory.core.actors.PartitionManager.Workers.ViewJob +import com.raphtory.core.actors.RaphtoryActor import com.raphtory.core.model.communication._ -import com.raphtory.core.utils.Utils import kamon.Kamon import scala.collection.mutable import scala.collection.mutable.ArrayBuffer import scala.concurrent.duration._ -abstract class AnalysisTask(jobID: String, args:Array[String], analyser: Analyser, managerCount:Int,newAnalyser: Boolean,rawFile:String) extends Actor { +abstract class AnalysisTask(jobID: String, args:Array[String], analyser: Analyser, managerCount:Int,newAnalyser: Boolean,rawFile:String) extends RaphtoryActor { protected var currentSuperStep = 0 //SuperStep the algorithm is currently on implicit val executionContext = context.system.dispatchers.lookup("analysis-dispatcher") @@ -91,7 +91,7 @@ abstract class AnalysisTask(jobID: String, args:Array[String], analyser: Analyse } final protected def getManagerCount: Int = managerCount - final protected def getWorkerCount: Int = managerCount * Utils.totalWorkers + final protected def getWorkerCount: Int = managerCount * totalWorkers protected def processResults(timeStamp:Long) = { analyser.processResults(results, timeStamp,viewCompleteTime()) @@ -135,8 +135,7 @@ abstract class AnalysisTask(jobID: String, args:Array[String], analyser: Analyse } mediator ! DistributedPubSubMediator.Put(self) - mediator ! DistributedPubSubMediator.Subscribe(Utils.partitionsTopic, self) - mediator ! DistributedPubSubMediator.Subscribe(Utils.liveAnalysisTopic, self) + mediator ! DistributedPubSubMediator.Subscribe(partitionsTopic, self) override def preStart() { context.system.scheduler.scheduleOnce(Duration(1, MILLISECONDS), self, StartAnalysis()) @@ -161,7 +160,7 @@ abstract class AnalysisTask(jobID: String, args:Array[String], analyser: Analyse def requestResults() =sender() ! ResultsForApiPI(analyser.getPublishedData()) def startAnalysis() = { - for (worker <- Utils.getAllReaders(managerCount)) + for (worker <- getAllReaders(managerCount)) mediator ! DistributedPubSubMediator.Send(worker, ReaderWorkersOnline(), false) } @@ -170,12 +169,12 @@ abstract class AnalysisTask(jobID: String, args:Array[String], analyser: Analyse ReaderACKS += 1 if (ReaderACKS == getManagerCount) { if (newAnalyser) { - for (worker <- Utils.getAllReaderWorkers(managerCount)) + for (worker <- getAllReaderWorkers(managerCount)) mediator ! DistributedPubSubMediator .Send(worker, CompileNewAnalyser(rawFile,args,jobID), false) } else { - for (worker <- Utils.getAllReaders(managerCount)) + for (worker <- getAllReaders(managerCount)) mediator ! DistributedPubSubMediator .Send(worker, AnalyserPresentCheck(this.generateAnalyzer.getClass.getName.replace("$", "")), false) } @@ -186,13 +185,13 @@ abstract class AnalysisTask(jobID: String, args:Array[String], analyser: Analyse ReaderAnalyserACKS += 1 if(newAnalyser){ if (ReaderAnalyserACKS == getWorkerCount) { - for (worker <- Utils.getAllReaderWorkers(managerCount)) + for (worker <- getAllReaderWorkers(managerCount)) mediator ! DistributedPubSubMediator.Send(worker, TimeCheck(timestamp), false) } } else{ if (ReaderAnalyserACKS == getManagerCount) { - for (worker <- Utils.getAllReaderWorkers(managerCount)) + for (worker <- getAllReaderWorkers(managerCount)) mediator ! DistributedPubSubMediator.Send(worker, TimeCheck(timestamp), false) } } @@ -209,13 +208,13 @@ abstract class AnalysisTask(jobID: String, args:Array[String], analyser: Analyse if (TimeOKFlag) { viewTimeCurrentTimer = System.currentTimeMillis() if (analyser.defineMaxSteps() > 1) - for (worker <- Utils.getAllReaderWorkers(managerCount)) + for (worker <- getAllReaderWorkers(managerCount)) if(newAnalyser) mediator ! DistributedPubSubMediator.Send(worker, SetupNewAnalyser(jobID, args, currentSuperStep, timestamp, analysisType(), windowSize(), windowSet()),false) else mediator ! DistributedPubSubMediator.Send(worker, Setup(this.generateAnalyzer, jobID, args, currentSuperStep, timestamp, analysisType(), windowSize(), windowSet()),false) else - for (worker <- Utils.getAllReaderWorkers(managerCount)) + for (worker <- getAllReaderWorkers(managerCount)) if(newAnalyser) mediator ! DistributedPubSubMediator.Send(worker, FinishNewAnalyser(jobID, args, currentSuperStep, timestamp, analysisType(), windowSize(), windowSet()), false) else @@ -231,7 +230,7 @@ abstract class AnalysisTask(jobID: String, args:Array[String], analyser: Analyse } def timeRecheck() = { - for (worker <- Utils.getAllReaderWorkers(managerCount)) + for (worker <- getAllReaderWorkers(managerCount)) mediator ! DistributedPubSubMediator.Send(worker, TimeCheck(timestamp), false) } @@ -253,7 +252,7 @@ abstract class AnalysisTask(jobID: String, args:Array[String], analyser: Analyse if (debug) println(s"$workersFinishedSuperStep / $getWorkerCount : $currentSuperStep / $steps") if (workersFinishedSuperStep == getWorkerCount) if (currentSuperStep == steps || this.voteToHalt) - for (worker <- Utils.getAllReaderWorkers(managerCount)) + for (worker <- getAllReaderWorkers(managerCount)) if(newAnalyser) mediator ! DistributedPubSubMediator.Send(worker, FinishNewAnalyser(jobID, args, currentSuperStep, timestamp, analysisType(), windowSize(), windowSet()), false) else @@ -287,16 +286,16 @@ abstract class AnalysisTask(jobID: String, args:Array[String], analyser: Analyse if (totalSentMessages == 0) { currentSuperStep += 1 if (newAnalyser) - for (worker <- Utils.getAllReaderWorkers(managerCount)) + for (worker <- getAllReaderWorkers(managerCount)) mediator ! DistributedPubSubMediator.Send(worker, NextStepNewAnalyser(jobID, args, currentSuperStep, timestamp, analysisType: AnalysisType.Value, windowSize(), windowSet()), false) else - for (worker <- Utils.getAllReaderWorkers(managerCount)) + for (worker <- getAllReaderWorkers(managerCount)) mediator ! DistributedPubSubMediator.Send(worker, NextStep(this.generateAnalyzer, jobID, args, currentSuperStep, timestamp, analysisType: AnalysisType.Value, windowSize(), windowSet()), false) } else { totalSentMessages = 0 totalReceivedMessages = 0 - for (worker <- Utils.getAllReaderWorkers(managerCount)) + for (worker <- getAllReaderWorkers(managerCount)) mediator ! DistributedPubSubMediator.Send(worker, CheckMessages(ViewJob(jobID,timestamp(),-1),currentSuperStep), false) } @@ -311,10 +310,10 @@ abstract class AnalysisTask(jobID: String, args:Array[String], analyser: Analyse totalSentMessages = 0 totalReceivedMessages = 0 if (newAnalyser) - for (worker <- Utils.getAllReaderWorkers(managerCount)) + for (worker <- getAllReaderWorkers(managerCount)) mediator ! DistributedPubSubMediator.Send(worker, NextStepNewAnalyser(jobID, args, currentSuperStep, timestamp, analysisType: AnalysisType.Value, windowSize(), windowSet()), false) else - for (worker <- Utils.getAllReaderWorkers(managerCount)) + for (worker <- getAllReaderWorkers(managerCount)) mediator ! DistributedPubSubMediator.Send(worker, NextStep(this.generateAnalyzer, jobID, args, currentSuperStep, timestamp, analysisType: AnalysisType.Value, windowSize(), windowSet()), false) } else { @@ -323,10 +322,10 @@ abstract class AnalysisTask(jobID: String, args:Array[String], analyser: Analyse totalReceivedMessages = 0 totalSentMessages = 0 if (newAnalyser) - for (worker <- Utils.getAllReaderWorkers(managerCount)) + for (worker <- getAllReaderWorkers(managerCount)) mediator ! DistributedPubSubMediator.Send(worker, CheckMessages(ViewJob(jobID,timestamp(),-1),currentSuperStep),false) else - for (worker <- Utils.getAllReaderWorkers(managerCount)) + for (worker <- getAllReaderWorkers(managerCount)) mediator ! DistributedPubSubMediator.Send(worker, CheckMessages(ViewJob(jobID,timestamp(),-1),currentSuperStep),false) } } @@ -346,12 +345,12 @@ abstract class AnalysisTask(jobID: String, args:Array[String], analyser: Analyse def killme() = self.tell(PoisonPill.getInstance,self) + + def failedToCompile(stackTrace: String): Unit = println(s"$sender failed to compiled, stacktrace returned: \n $stackTrace") - - } // diff --git a/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/LiveTasks/LiveAnalysisTask.scala b/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/LiveTasks/LiveAnalysisTask.scala index 29e0f88802..37da69e1ab 100644 --- a/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/LiveTasks/LiveAnalysisTask.scala +++ b/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/LiveTasks/LiveAnalysisTask.scala @@ -4,7 +4,6 @@ import akka.cluster.pubsub.DistributedPubSubMediator import com.raphtory.api.Analyser import com.raphtory.analysis.Tasks.AnalysisTask import com.raphtory.core.model.communication.{AnalysisType, Finish, Setup, TimeCheck} -import com.raphtory.core.utils.Utils import scala.collection.mutable import scala.concurrent.duration.{Duration, MILLISECONDS} @@ -40,12 +39,12 @@ class LiveAnalysisTask(managerCount:Int, jobID: String, args:Array[String],analy if(eventTime){ //if its event time then we wait for the repeat time to be represented in the storage currentTimestamp=liveTime+repeatTime - for (worker <- Utils.getAllReaderWorkers(managerCount)) + for (worker <- getAllReaderWorkers(managerCount)) mediator ! DistributedPubSubMediator.Send(worker, TimeCheck(timestamp()), false) } else{ currentTimestamp=liveTime - for (worker <- Utils.getAllReaderWorkers(managerCount)) + for (worker <- getAllReaderWorkers(managerCount)) mediator ! DistributedPubSubMediator.Send(worker, TimeCheck(timestamp), false) } } @@ -62,7 +61,7 @@ class LiveAnalysisTask(managerCount:Int, jobID: String, args:Array[String],analy if (TimeOKFlag) { setLiveTime() if (analyser.defineMaxSteps() > 1) - for (worker <- Utils.getAllReaderWorkers(managerCount)) + for (worker <- getAllReaderWorkers(managerCount)) mediator ! DistributedPubSubMediator.Send( worker, Setup( @@ -78,7 +77,7 @@ class LiveAnalysisTask(managerCount:Int, jobID: String, args:Array[String],analy false ) else - for (worker <- Utils.getAllReaderWorkers(managerCount)) + for (worker <- getAllReaderWorkers(managerCount)) mediator ! DistributedPubSubMediator.Send( worker, Finish( diff --git a/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/RangeTasks/RangeAnalysisTask.scala b/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/RangeTasks/RangeAnalysisTask.scala index 7dcf158a08..acf050123b 100644 --- a/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/RangeTasks/RangeAnalysisTask.scala +++ b/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/RangeTasks/RangeAnalysisTask.scala @@ -4,7 +4,6 @@ import akka.cluster.pubsub.DistributedPubSubMediator import com.raphtory.api.Analyser import com.raphtory.analysis.Tasks.AnalysisTask import com.raphtory.core.model.communication.{AnalyserPresentCheck, AnalysisType} -import com.raphtory.core.utils.Utils class RangeAnalysisTask(managerCount:Int, jobID: String, args:Array[String],analyser: Analyser, start: Long, end: Long, jump: Long,newAnalyser:Boolean,rawFile:String) extends AnalysisTask(jobID: String,args, analyser,managerCount,newAnalyser,rawFile) { @@ -23,7 +22,7 @@ class RangeAnalysisTask(managerCount:Int, jobID: String, args:Array[String],anal if (currentTimestamp > end) currentTimestamp = end - for (worker <- Utils.getAllReaders(managerCount)) + for (worker <- getAllReaders(managerCount)) mediator ! DistributedPubSubMediator .Send(worker, AnalyserPresentCheck(this.generateAnalyzer.getClass.getName.replace("$", "")), false) } diff --git a/mainproject/src/main/scala/com/raphtory/core/actors/ClusterManagement/RaphtoryReplicator.scala b/mainproject/src/main/scala/com/raphtory/core/actors/ClusterManagement/RaphtoryReplicator.scala index e01de9843c..f6ded105bb 100644 --- a/mainproject/src/main/scala/com/raphtory/core/actors/ClusterManagement/RaphtoryReplicator.scala +++ b/mainproject/src/main/scala/com/raphtory/core/actors/ClusterManagement/RaphtoryReplicator.scala @@ -7,10 +7,10 @@ import akka.pattern.ask import akka.util.Timeout import com.raphtory.core.actors.PartitionManager.Workers.IngestionWorker import com.raphtory.core.actors.PartitionManager.{Reader, Writer} +import com.raphtory.core.actors.RaphtoryActor import com.raphtory.core.actors.Router.{GraphBuilder, RouterManager} import com.raphtory.core.model.EntityStorage import com.raphtory.core.model.communication._ -import com.raphtory.core.utils.{SchedulerUtil, Utils} import scala.collection.mutable import scala.collection.parallel.mutable.ParTrieMap @@ -27,8 +27,7 @@ object RaphtoryReplicator { } class RaphtoryReplicator[T](actorType: String, initialManagerCount: Int, initialRouterCount:Int, graphBuilder: GraphBuilder[T]) - extends Actor - with ActorLogging { + extends RaphtoryActor { // TODO Make implicit timeouts as secondary (curried), optional implicit parameter implicit val timeout: Timeout = 10.seconds @@ -42,7 +41,7 @@ class RaphtoryReplicator[T](actorType: String, initialManagerCount: Int, initial val mediator: ActorRef = DistributedPubSub(context.system).mediator mediator ! DistributedPubSubMediator.Put(self) - mediator ! DistributedPubSubMediator.Subscribe(Utils.partitionsTopic, self) + mediator ! DistributedPubSubMediator.Subscribe(partitionsTopic, self) override def preStart(): Unit = { log.debug("Replicator [{}] is being started.") @@ -53,7 +52,7 @@ class RaphtoryReplicator[T](actorType: String, initialManagerCount: Int, initial override def postStop(): Unit = { val allTasksCancelled = scheduledTaskMap.forall { case (key, task) => - SchedulerUtil.cancelTask(key, task) + cancelTask(key, task) } if (!allTasksCancelled) log.warning("Failed to cancel all scheduled tasks post stop.") @@ -127,7 +126,7 @@ class RaphtoryReplicator[T](actorType: String, initialManagerCount: Int, initial var workers: ParTrieMap[Int, ActorRef] = new ParTrieMap[Int, ActorRef]() var storages: ParTrieMap[Int, EntityStorage] = new ParTrieMap[Int, EntityStorage]() - for (index <- 0 until Utils.totalWorkers) { + for (index <- 0 until totalWorkers) { val storage = new EntityStorage(assignedId,index) storages.put(index, storage) @@ -159,7 +158,7 @@ class RaphtoryReplicator[T](actorType: String, initialManagerCount: Int, initial log.debug("Preparing to schedule tasks in Replicator.") val tickCancellable = - SchedulerUtil.scheduleTask(initialDelay = 2 seconds, interval = 5 seconds, receiver = self, message = "tick") + scheduleTask(initialDelay = 2 seconds, interval = 5 seconds, receiver = self, message = "tick") scheduledTaskMap.put("tick", tickCancellable) } } diff --git a/mainproject/src/main/scala/com/raphtory/core/actors/ClusterManagement/SeedActor.scala b/mainproject/src/main/scala/com/raphtory/core/actors/ClusterManagement/SeedActor.scala index a1c8e17a12..7cb1467302 100644 --- a/mainproject/src/main/scala/com/raphtory/core/actors/ClusterManagement/SeedActor.scala +++ b/mainproject/src/main/scala/com/raphtory/core/actors/ClusterManagement/SeedActor.scala @@ -3,8 +3,9 @@ package com.raphtory.core.actors.ClusterManagement import akka.actor.{Actor, ActorLogging} import akka.cluster.Cluster import akka.cluster.ClusterEvent._ +import com.raphtory.core.actors.RaphtoryActor -class SeedActor() extends Actor with ActorLogging { +class SeedActor() extends RaphtoryActor { val cluster: Cluster = Cluster(context.system) override def preStart(): Unit = { diff --git a/mainproject/src/main/scala/com/raphtory/core/actors/ClusterManagement/WatchDog.scala b/mainproject/src/main/scala/com/raphtory/core/actors/ClusterManagement/WatchDog.scala index f51372685d..9d16825cf7 100644 --- a/mainproject/src/main/scala/com/raphtory/core/actors/ClusterManagement/WatchDog.scala +++ b/mainproject/src/main/scala/com/raphtory/core/actors/ClusterManagement/WatchDog.scala @@ -3,16 +3,18 @@ package com.raphtory.core.actors.ClusterManagement /** * Created by Mirate on 11/07/2017. */ +import java.text.SimpleDateFormat + import akka.actor.{Actor, ActorLogging, ActorRef, Cancellable} import akka.cluster.pubsub.{DistributedPubSub, DistributedPubSubMediator} +import com.raphtory.core.actors.RaphtoryActor import com.raphtory.core.model.communication._ -import com.raphtory.core.utils.{SchedulerUtil, Utils} import scala.collection.concurrent.TrieMap import scala.collection.mutable import scala.concurrent.duration._ -class WatchDog(managerCount: Int, minimumRouters: Int) extends Actor with ActorLogging { +class WatchDog(managerCount: Int, minimumRouters: Int) extends RaphtoryActor { private val scheduledTaskMap: mutable.HashMap[String, Cancellable] = mutable.HashMap[String, Cancellable]() implicit val executionContext = context.system.dispatchers.lookup("misc-dispatcher") @@ -37,7 +39,7 @@ class WatchDog(managerCount: Int, minimumRouters: Int) extends Actor with ActorL override def postStop(): Unit = { val allTasksCancelled = scheduledTaskMap.forall { case (key, task) => - SchedulerUtil.cancelTask(key, task) + cancelTask(key, task) } if (!allTasksCancelled) log.warning("Failed to cancel all scheduled tasks post stop.") @@ -80,7 +82,7 @@ class WatchDog(managerCount: Int, minimumRouters: Int) extends Actor with ActorL private def processRefreshManagerCountMessage(msg: String): Unit = { log.debug(s"WatchDog received [{}] message.", msg) - mediator ! DistributedPubSubMediator.Publish(Utils.partitionsTopic, PartitionsCount(pmCounter)) + mediator ! DistributedPubSubMediator.Publish(partitionsTopic, PartitionsCount(pmCounter)) } private def processClusterStatusRequest(req: ClusterStatusRequest): Unit = { @@ -115,7 +117,7 @@ class WatchDog(managerCount: Int, minimumRouters: Int) extends Actor with ActorL pmCounter += 1 log.debug("Propagating the new total partition managers [{}] to all the subscribers.", pmCounter) - mediator ! DistributedPubSubMediator.Publish(Utils.partitionsTopic, PartitionsCount(pmCounter)) + mediator ! DistributedPubSubMediator.Publish(partitionsTopic, PartitionsCount(pmCounter)) } private def processRequestRouterIdRequest(req: RequestRouterId): Unit = { @@ -129,7 +131,7 @@ class WatchDog(managerCount: Int, minimumRouters: Int) extends Actor with ActorL map.foreach { case (pmId, startTime) => if (startTime + maxTime <= System.currentTimeMillis()) - log.debug("Partition manager [{}] not responding since [{}].", pmId, Utils.unixToTimeStamp(startTime)) + log.debug("Partition manager [{}] not responding since [{}].", pmId, unixToTimeStamp(startTime)) } private def mapHandler(id: Int, map: TrieMap[Int, Long], mapType: String): Unit = { @@ -142,7 +144,7 @@ class WatchDog(managerCount: Int, minimumRouters: Int) extends Actor with ActorL "The [{}] for id [{}] has started. Keep alive will be sent at [{}].", mapType, id, - Utils.nowTimeStamp() + nowTimeStamp() ) } } @@ -150,12 +152,13 @@ class WatchDog(managerCount: Int, minimumRouters: Int) extends Actor with ActorL private def scheduleTasks(): Unit = { log.debug("Preparing to schedule tasks in WatchDog.") - val tickCancellable = SchedulerUtil.scheduleTask(2 seconds, 10 seconds, self, "tick") + val tickCancellable = scheduleTask(2 seconds, 10 seconds, self, "tick") scheduledTaskMap.put("tick", tickCancellable) val refreshManagerCountCancellable = - SchedulerUtil.scheduleTask(3 minutes, 1 minute, self, "refreshManagerCount") + scheduleTask(3 minutes, 1 minute, self, "refreshManagerCount") scheduledTaskMap.put("refreshManagerCount", refreshManagerCountCancellable) } - + def nowTimeStamp() = new SimpleDateFormat("dd-MM hh:mm:ss").format(System.currentTimeMillis()) + def unixToTimeStamp(unixTime: Long) = new SimpleDateFormat("dd-MM hh:mm:ss").format(unixTime) } diff --git a/mainproject/src/main/scala/com/raphtory/core/actors/ClusterManagement/WatermarkManager.scala b/mainproject/src/main/scala/com/raphtory/core/actors/ClusterManagement/WatermarkManager.scala index e4fa09a902..4ce5979280 100644 --- a/mainproject/src/main/scala/com/raphtory/core/actors/ClusterManagement/WatermarkManager.scala +++ b/mainproject/src/main/scala/com/raphtory/core/actors/ClusterManagement/WatermarkManager.scala @@ -2,8 +2,8 @@ package com.raphtory.core.actors.ClusterManagement import akka.actor.{Actor, ActorLogging, ActorRef} import akka.cluster.pubsub.{DistributedPubSub, DistributedPubSubMediator} +import com.raphtory.core.actors.RaphtoryActor import com.raphtory.core.model.communication.{UpdateArrivalTime, WatermarkTime} -import com.raphtory.core.utils.Utils import kamon.Kamon import scala.collection.mutable @@ -13,7 +13,7 @@ case class queueItem(wallclock:Long,timestamp:Long)extends Ordered[queueItem] { def compare(that: queueItem): Int = (that.timestamp-this.timestamp).toInt } -class WatermarkManager(managerCount: Int) extends Actor with ActorLogging { +class WatermarkManager(managerCount: Int) extends RaphtoryActor { val spoutWallClock = Kamon.histogram("Raphtory_Wall_Clock").withTag("Actor","Watchdog") val safeTime = Kamon.gauge("Raphtory_Safe_Time").withTag("actor",s"WatermarkManager") @@ -34,7 +34,7 @@ class WatermarkManager(managerCount: Int) extends Actor with ActorLogging { val currentTime = System.currentTimeMillis() safeMessageMap put(sender().toString(),u.time) counter +=1 - if(counter%(Utils.totalWorkers*managerCount)==0) { + if(counter%(totalWorkers*managerCount)==0) { val watermark = safeMessageMap.map(x=>x._2).min safeTime.update(watermark) while((watermarkqueue nonEmpty) && (watermarkqueue.head.timestamp<= watermark)) { diff --git a/mainproject/src/main/scala/com/raphtory/core/actors/PartitionManager/Reader.scala b/mainproject/src/main/scala/com/raphtory/core/actors/PartitionManager/Reader.scala index bd44370690..acc1938959 100644 --- a/mainproject/src/main/scala/com/raphtory/core/actors/PartitionManager/Reader.scala +++ b/mainproject/src/main/scala/com/raphtory/core/actors/PartitionManager/Reader.scala @@ -4,9 +4,9 @@ import akka.actor.{Actor, ActorLogging, ActorRef, Props, Terminated} import akka.cluster.pubsub.{DistributedPubSub, DistributedPubSubMediator} import akka.cluster.pubsub.DistributedPubSubMediator.SubscribeAck import com.raphtory.core.actors.PartitionManager.Workers.ReaderWorker +import com.raphtory.core.actors.RaphtoryActor import com.raphtory.core.model.EntityStorage import com.raphtory.core.model.communication._ -import com.raphtory.core.utils.Utils import scala.collection.parallel.mutable.ParTrieMap import scala.util.Try @@ -16,9 +16,7 @@ class Reader( test: Boolean, managerCountVal: Int, storage: ParTrieMap[Int, EntityStorage], - workerCount: Int = Utils.totalWorkers -) extends Actor - with ActorLogging { +) extends RaphtoryActor { implicit var managerCount: Int = managerCountVal @@ -28,12 +26,11 @@ class Reader( val mediator: ActorRef = DistributedPubSub(context.system).mediator mediator ! DistributedPubSubMediator.Put(self) - mediator ! DistributedPubSubMediator.Subscribe(Utils.readersTopic, self) var readers: ParTrieMap[Int, ActorRef] = new ParTrieMap[Int, ActorRef]() - for (i <- 0 until workerCount) { - log.debug("Initialising [{}] worker children for Reader [{}}.", workerCount, managerId) + for (i <- 0 until totalWorkers) { + log.debug("Initialising [{}] worker children for Reader [{}}.", totalWorkers, managerId) // create threads for writing val child = context.system.actorOf( diff --git a/mainproject/src/main/scala/com/raphtory/core/actors/PartitionManager/Workers/IngestionWorker.scala b/mainproject/src/main/scala/com/raphtory/core/actors/PartitionManager/Workers/IngestionWorker.scala index d698706f66..c6ea9895a5 100644 --- a/mainproject/src/main/scala/com/raphtory/core/actors/PartitionManager/Workers/IngestionWorker.scala +++ b/mainproject/src/main/scala/com/raphtory/core/actors/PartitionManager/Workers/IngestionWorker.scala @@ -4,9 +4,9 @@ import java.util.concurrent.atomic.AtomicInteger import akka.actor.{Actor, ActorLogging, ActorRef, Cancellable} import akka.cluster.pubsub.{DistributedPubSub, DistributedPubSubMediator} +import com.raphtory.core.actors.RaphtoryActor import com.raphtory.core.model.EntityStorage import com.raphtory.core.model.communication._ -import com.raphtory.core.utils.SchedulerUtil import kamon.Kamon import scala.collection.mutable @@ -18,7 +18,7 @@ case class queueItem(routerEpoch:Int,timestamp:Long)extends Ordered[queueItem] { } // TODO Re-add compression (removed during commit Log-Revamp) -class IngestionWorker(workerId: Int,partitionID:Int, storage: EntityStorage) extends Actor with ActorLogging { +class IngestionWorker(workerId: Int,partitionID:Int, storage: EntityStorage) extends RaphtoryActor { val mediator: ActorRef = DistributedPubSub(context.system).mediator mediator ! DistributedPubSubMediator.Put(self) @@ -324,14 +324,14 @@ class IngestionWorker(workerId: Int,partitionID:Int, storage: EntityStorage) ext override def postStop() { val allTasksCancelled = scheduledTaskMap.forall { case (key, task) => - SchedulerUtil.cancelTask(key, task) + cancelTask(key, task) } if (!allTasksCancelled) log.warning("Failed to cancel all scheduled tasks post stop.") } private def scheduleTasks(): Unit = { log.debug("Preparing to schedule tasks in Spout.") val watermarkCancellable = - SchedulerUtil.scheduleTask(initialDelay = 10 seconds, interval = 5 second, receiver = self, message = "watermark") + scheduleTask(initialDelay = 10 seconds, interval = 5 second, receiver = self, message = "watermark") scheduledTaskMap.put("watermark", watermarkCancellable) } diff --git a/mainproject/src/main/scala/com/raphtory/core/actors/PartitionManager/Workers/ReaderWorker.scala b/mainproject/src/main/scala/com/raphtory/core/actors/PartitionManager/Workers/ReaderWorker.scala index 1dc332e03b..eafc0170e4 100644 --- a/mainproject/src/main/scala/com/raphtory/core/actors/PartitionManager/Workers/ReaderWorker.scala +++ b/mainproject/src/main/scala/com/raphtory/core/actors/PartitionManager/Workers/ReaderWorker.scala @@ -5,10 +5,10 @@ import java.util.concurrent.atomic.AtomicInteger import akka.actor.{Actor, ActorLogging, ActorRef} import akka.cluster.pubsub.{DistributedPubSub, DistributedPubSubMediator} import com.raphtory.api.{Analyser, LoadExternalAnalyser, ManagerCount} +import com.raphtory.core.actors.RaphtoryActor import com.raphtory.core.model.analysis.GraphLenses.{GraphLens, ViewLens, WindowLens} import com.raphtory.core.model.EntityStorage import com.raphtory.core.model.communication._ -import com.raphtory.core.utils.Utils import kamon.Kamon import scala.collection.concurrent.TrieMap @@ -18,7 +18,7 @@ import scala.collection.parallel.mutable.ParTrieMap case class ViewJob(jobID:String,timestamp:Long,window:Long) -class ReaderWorker(managerCountVal: Int, managerID: Int, workerId: Int, storage: EntityStorage) extends Actor with ActorLogging { +class ReaderWorker(managerCountVal: Int, managerID: Int, workerId: Int, storage: EntityStorage) extends RaphtoryActor { implicit var managerCount: ManagerCount = ManagerCount(managerCountVal) val analyserMap: TrieMap[String,LoadExternalAnalyser] = TrieMap[String,LoadExternalAnalyser]() @@ -28,8 +28,6 @@ class ReaderWorker(managerCountVal: Int, managerID: Int, workerId: Int, storage: val mediator: ActorRef = DistributedPubSub(context.system).mediator mediator ! DistributedPubSubMediator.Put(self) - mediator ! DistributedPubSubMediator.Subscribe(Utils.readersWorkerTopic, self) - override def preStart(): Unit = log.debug("ReaderWorker [{}] belonging to Reader [{}] is being started.", workerId, managerID) diff --git a/mainproject/src/main/scala/com/raphtory/core/actors/PartitionManager/Writer.scala b/mainproject/src/main/scala/com/raphtory/core/actors/PartitionManager/Writer.scala index 008e24d3fd..961f3411bb 100644 --- a/mainproject/src/main/scala/com/raphtory/core/actors/PartitionManager/Writer.scala +++ b/mainproject/src/main/scala/com/raphtory/core/actors/PartitionManager/Writer.scala @@ -3,9 +3,9 @@ package com.raphtory.core.actors.PartitionManager import akka.actor.SupervisorStrategy.Resume import akka.actor.{Actor, ActorLogging, ActorRef, Cancellable, OneForOneStrategy, Terminated} import akka.cluster.pubsub.{DistributedPubSub, DistributedPubSubMediator} +import com.raphtory.core.actors.RaphtoryActor import com.raphtory.core.model.EntityStorage import com.raphtory.core.model.communication._ -import com.raphtory.core.utils.{SchedulerUtil, Utils} import scala.collection.mutable import scala.collection.parallel.mutable.ParTrieMap @@ -23,15 +23,14 @@ class Writer( managerCountVal: Int, workers: ParTrieMap[Int, ActorRef], storage: ParTrieMap[Int, EntityStorage] -) extends Actor - with ActorLogging { +) extends RaphtoryActor { private val scheduledTaskMap: mutable.HashMap[String, Cancellable] = mutable.HashMap[String, Cancellable]() implicit val executionContext = context.system.dispatchers.lookup("misc-dispatcher") // Id which refers to the partitions position in the graph manager map val managerId: Int = id - val children: Int = Utils.totalWorkers + val children: Int = totalWorkers var lastLogTime: Long = System.currentTimeMillis() / 1000 // should the handled messages be printed to terminal @@ -69,7 +68,7 @@ class Writer( override def postStop(): Unit = { val allTasksCancelled = scheduledTaskMap.forall { case (key, task) => - SchedulerUtil.cancelTask(key, task) + cancelTask(key, task) } if (!allTasksCancelled) log.warning("Failed to cancel all scheduled tasks post stop.") @@ -122,12 +121,11 @@ class Writer( log.debug("Preparing to schedule tasks in Writer [{}].", managerId) val countCancellable = - SchedulerUtil.scheduleTask(initialDelay = 10 seconds, interval = 1 seconds, receiver = self, message = "count") + scheduleTask(initialDelay = 10 seconds, interval = 1 seconds, receiver = self, message = "count") scheduledTaskMap.put("count", countCancellable) val keepAliveCancellable = - SchedulerUtil - .scheduleTask(initialDelay = 10 seconds, interval = 10 seconds, receiver = self, message = "keep_alive") + scheduleTask(initialDelay = 10 seconds, interval = 10 seconds, receiver = self, message = "keep_alive") scheduledTaskMap.put("keep_alive", keepAliveCancellable) } } diff --git a/mainproject/src/main/scala/com/raphtory/core/actors/RaphtoryActor.scala b/mainproject/src/main/scala/com/raphtory/core/actors/RaphtoryActor.scala new file mode 100644 index 0000000000..94bc2faaa3 --- /dev/null +++ b/mainproject/src/main/scala/com/raphtory/core/actors/RaphtoryActor.scala @@ -0,0 +1,97 @@ +package com.raphtory.core.actors + +import akka.actor.{Actor, ActorContext, ActorLogging, ActorRef, Cancellable, Timers} + +import scala.collection.mutable +import scala.concurrent.ExecutionContext +import scala.concurrent.duration.FiniteDuration + +trait RaphtoryActor extends Actor with ActorLogging with Timers { + + val partitionsTopic = "/partitionsCount" + val totalWorkers = 10 //must be power of 10 + + //get the partition a vertex is stored in + def checkDst(dstID: Long, managerCount: Int, managerID: Int): Boolean = ((dstID.abs % (managerCount * totalWorkers)) / totalWorkers).toInt == managerID //check if destination is also local + def checkWorker(dstID: Long, managerCount: Int, workerID: Int): Boolean = ((dstID.abs % (managerCount * totalWorkers)) % totalWorkers).toInt == workerID //check if destination is also local + + def getManager(srcId: Long, managerCount: Int): String = { + val mod = srcId.abs % (managerCount * totalWorkers) + val manager = mod / totalWorkers + val worker = mod % totalWorkers + s"/user/Manager_${manager}_child_$worker" + } + + def getReader(srcId: Long, managerCount: Int): String = { + val mod = srcId.abs % (managerCount * totalWorkers) + val manager = mod / totalWorkers + val worker = mod % totalWorkers + s"/user/Manager_${manager}_reader_$worker" + } + + def getAllRouterWorkers(managerCount: Int): Array[String] = { + val workers = mutable.ArrayBuffer[String]() + for (i <- 0 until managerCount) + for (j <- 0 until totalWorkers) + workers += s"/user/router/router_${i}_Worker_$j" + workers.toArray + } + + + def getAllReaders(managerCount: Int): Array[String] = { + val workers = mutable.ArrayBuffer[String]() + for (i <- 0 until managerCount) + workers += s"/user/ManagerReader_$i" + workers.toArray + } + + def getAllReaderWorkers(managerCount: Int): Array[String] = { + val workers = mutable.ArrayBuffer[String]() + for (i <- 0 until managerCount) + for (j <- 0 until totalWorkers) + workers += s"/user/Manager_${i}_reader_$j" + workers.toArray + } + + def scheduleTask(initialDelay: FiniteDuration, interval: FiniteDuration, receiver: ActorRef, message: Any)( + implicit context: ActorContext, + executor: ExecutionContext, + sender: ActorRef = Actor.noSender + ): Cancellable = { + val scheduler = context.system.scheduler + + val cancellable = scheduler.schedule(initialDelay, interval, receiver, message)(executor,self) + context.system.log.debug("The message [{}] has been scheduled for send to [{}].", message, receiver.path) + + cancellable + } + + def scheduleTaskOnce( + delay: FiniteDuration, + receiver: ActorRef, + message: Any + )(implicit context: ActorContext, executor: ExecutionContext, sender: ActorRef = Actor.noSender): Cancellable = { + val scheduler = context.system.scheduler + + val cancellable = scheduler.scheduleOnce(delay, receiver, message)(executor,self) + context.system.log.debug("The message [{}] has been scheduled for send to [{}].", message, receiver.path) + + cancellable + } + + def cancelTask(key: String, task: Cancellable)(implicit context: ActorContext): Boolean = { + task.cancel() + + val isCancelled = task.isCancelled + + if (isCancelled) + context.system.log.debug("The task [{}] has been cancelled.", key) + else + context.system.log.debug("Failed to cancel the task [{}].", key) + + isCancelled + } + + + +} diff --git a/mainproject/src/main/scala/com/raphtory/core/actors/Router/RouterManager.scala b/mainproject/src/main/scala/com/raphtory/core/actors/Router/RouterManager.scala index da3de2d708..1319f5817e 100644 --- a/mainproject/src/main/scala/com/raphtory/core/actors/Router/RouterManager.scala +++ b/mainproject/src/main/scala/com/raphtory/core/actors/Router/RouterManager.scala @@ -2,6 +2,7 @@ package com.raphtory.core.actors.Router import akka.actor.{Actor, ActorLogging, ActorRef, Props} import akka.cluster.pubsub.{DistributedPubSub, DistributedPubSubMediator} +import com.raphtory.core.actors.RaphtoryActor import com.raphtory.core.actors.Router.RouterManager.Message.KeepAlive import com.raphtory.core.model.communication._ @@ -15,8 +16,7 @@ import scala.language.postfixOps // TODO break object RouterManager { getProps = { routerManager(arg1, arg2...): Props }} // todo slave type should be Props final case class RouterManager[T](routerId: Int, initialManagerCount: Int, initialRouterCount:Int, graphBuilder: GraphBuilder[T]) - extends Actor - with ActorLogging { + extends RaphtoryActor { implicit val executionContext: ExecutionContext = context.system.dispatcher private val childrenNumber = 10 diff --git a/mainproject/src/main/scala/com/raphtory/core/actors/Router/RouterWorker.scala b/mainproject/src/main/scala/com/raphtory/core/actors/Router/RouterWorker.scala index 988dbfd7cc..dc71383f10 100644 --- a/mainproject/src/main/scala/com/raphtory/core/actors/Router/RouterWorker.scala +++ b/mainproject/src/main/scala/com/raphtory/core/actors/Router/RouterWorker.scala @@ -2,13 +2,13 @@ package com.raphtory.core.actors.Router import akka.actor.{Actor, ActorLogging} import akka.cluster.pubsub.{DistributedPubSub, DistributedPubSubMediator} +import com.raphtory.core.actors.RaphtoryActor import com.raphtory.core.actors.Router.RouterWorker.CommonMessage.TimeBroadcast import com.raphtory.core.actors.Spout.SpoutAgent.CommonMessage.{NoWork, SpoutOnline, WorkPlease} import com.raphtory.core.model.communication._ -import com.raphtory.core.utils.Utils -import com.raphtory.core.utils.Utils.getManager import kamon.Kamon +import scala.collection.mutable import scala.collection.parallel.mutable.ParTrieMap import scala.concurrent.ExecutionContext import scala.concurrent.duration._ @@ -17,8 +17,7 @@ import scala.concurrent.duration._ // e.g. BlockChainRouter val name = "Blockchain Router" // Log.debug that read 'Router' should then read 'Blockchain Router' class RouterWorker[T](val graphBuilder: GraphBuilder[T],val routerId: Int, val workerID: Int, val initialManagerCount: Int,val initialRouterCount:Int) - extends Actor - with ActorLogging { + extends RaphtoryActor { implicit val executionContext: ExecutionContext = context.system.dispatcher println(s"Router $routerId $workerID with $initialManagerCount $initialRouterCount") private val messageIDs = ParTrieMap[String, Int]() @@ -66,7 +65,7 @@ class RouterWorker[T](val graphBuilder: GraphBuilder[T],val routerId: Int, val w context.become(work(managerCount, wallClock, newNewestTime)) context.sender() ! WorkPlease case TimeBroadcast => { - Utils.getAllWriterWorkers(managerCount).foreach { workerPath => + getAllWriterWorkers(managerCount).foreach { workerPath => mediator ! DistributedPubSubMediator.Send( workerPath, RouterWorkerTimeSync(newestTime, s"${routerId}_$workerID", getMessageIDForWriter(workerPath)), @@ -78,7 +77,7 @@ class RouterWorker[T](val graphBuilder: GraphBuilder[T],val routerId: Int, val w //println(s"Router $routerId $workerID with $newestTime ${messageIDs.mkString("[",",","]")}") } case DataFinished => { - Utils.getAllRouterWorkers(initialRouterCount).foreach { workerPath => + getAllRouterWorkers(initialRouterCount).foreach { workerPath => mediator ! DistributedPubSubMediator.Send( workerPath, DataFinishedSync(newestTime), @@ -90,7 +89,7 @@ class RouterWorker[T](val graphBuilder: GraphBuilder[T],val routerId: Int, val w case DataFinishedSync(time) => { if (time >= newestTime) { println(s"Router $routerId $workerID ${time}") - Utils.getAllWriterWorkers(managerCount).foreach { workerPath => + getAllWriterWorkers(managerCount).foreach { workerPath => mediator ! DistributedPubSubMediator.Send( workerPath, RouterWorkerTimeSync(time, s"${routerId}_$workerID", getMessageIDForWriter(workerPath)), @@ -100,7 +99,7 @@ class RouterWorker[T](val graphBuilder: GraphBuilder[T],val routerId: Int, val w context.become(work(managerCount, trackedTime, time)) } else { - Utils.getAllRouterWorkers(initialRouterCount).foreach { workerPath => + getAllRouterWorkers(initialRouterCount).foreach { workerPath => mediator ! DistributedPubSubMediator.Send( workerPath, DataFinishedSync(newestTime), @@ -168,6 +167,14 @@ class RouterWorker[T](val graphBuilder: GraphBuilder[T],val routerId: Int, val w messageIDs put (path, 1) 0 } + + def getAllWriterWorkers(managerCount: Int): Array[String] = { + val workers = mutable.ArrayBuffer[String]() + for (i <- 0 until managerCount) + for (j <- 0 until totalWorkers) + workers += s"/user/Manager_${i}_child_$j" + workers.toArray + } } object RouterWorker { diff --git a/mainproject/src/main/scala/com/raphtory/core/actors/Spout/SpoutAgent.scala b/mainproject/src/main/scala/com/raphtory/core/actors/Spout/SpoutAgent.scala index 628f9fcda8..478b7c2c32 100644 --- a/mainproject/src/main/scala/com/raphtory/core/actors/Spout/SpoutAgent.scala +++ b/mainproject/src/main/scala/com/raphtory/core/actors/Spout/SpoutAgent.scala @@ -2,9 +2,9 @@ package com.raphtory.core.actors.Spout import akka.actor.{Actor, ActorLogging, ActorRef, Cancellable, Timers} import akka.cluster.pubsub.{DistributedPubSub, DistributedPubSubMediator} +import com.raphtory.core.actors.RaphtoryActor import com.raphtory.core.actors.Spout.SpoutAgent.CommonMessage._ import com.raphtory.core.model.communication._ -import com.raphtory.core.utils.Utils import kamon.Kamon import scala.concurrent.ExecutionContext @@ -13,7 +13,7 @@ import scala.language.postfixOps -class SpoutAgent(datasource:Spout[Any]) extends Actor with ActorLogging with Timers { +class SpoutAgent(datasource:Spout[Any]) extends RaphtoryActor { // todo: wvv should assign the dispatcher when create the actor implicit val executionContext: ExecutionContext = context.system.dispatchers.lookup("spout-dispatcher") //implicit val executionContext: ExecutionContext = context.system.dispatcher @@ -79,7 +79,7 @@ class SpoutAgent(datasource:Spout[Any]) extends Actor with ActorLogging with Tim datasource.setupDataSource() partitionManagers=pmCount routers=roCount - Utils.getAllRouterWorkers(roCount).foreach { workerPath => + getAllRouterWorkers(roCount).foreach { workerPath => mediator ! DistributedPubSubMediator.Send( workerPath, SpoutOnline, diff --git a/mainproject/src/main/scala/com/raphtory/core/model/EntityStorage.scala b/mainproject/src/main/scala/com/raphtory/core/model/EntityStorage.scala index fa5dc4a13b..303d66cb87 100644 --- a/mainproject/src/main/scala/com/raphtory/core/model/EntityStorage.scala +++ b/mainproject/src/main/scala/com/raphtory/core/model/EntityStorage.scala @@ -4,7 +4,6 @@ import akka.actor.ActorRef import akka.cluster.pubsub.DistributedPubSubMediator import com.raphtory.core.model.communication._ import com.raphtory.core.model.graphentities.{Edge, Entity, SplitEdge, Vertex} -import com.raphtory.core.utils.Utils import kamon.Kamon import scala.collection.mutable @@ -17,7 +16,6 @@ import scala.collection.parallel.mutable.ParTrieMap //TODO What happens when an edge which has been archived gets readded class EntityStorage(partitionID:Int,workerID: Int) { - import com.raphtory.core.utils.Utils.{checkDst, checkWorker, getManager} val debug = System.getenv().getOrDefault("DEBUG", "false").trim.toBoolean /** @@ -448,4 +446,14 @@ class EntityStorage(partitionID:Int,workerID: Int) { case None => /*todo Should this happen*/ } } + + //TODO these are placed here until YanYangs changes can be integrated + def getManager(srcId: Long, managerCount: Int): String = { + val mod = srcId.abs % (managerCount * 10) + val manager = mod / 10 + val worker = mod % 10 + s"/user/Manager_${manager}_child_$worker" + } + def checkDst(dstID: Long, managerCount: Int, managerID: Int): Boolean = ((dstID.abs % (managerCount * 10)) / 10).toInt == managerID //check if destination is also local + def checkWorker(dstID: Long, managerCount: Int, workerID: Int): Boolean = ((dstID.abs % (managerCount * 10)) % 10).toInt == workerID //check if destination is also local } diff --git a/mainproject/src/main/scala/com/raphtory/core/model/analysis/entityVisitors/EdgeVisitor.scala b/mainproject/src/main/scala/com/raphtory/core/model/analysis/entityVisitors/EdgeVisitor.scala index 5def0b27a1..fa78a1bada 100644 --- a/mainproject/src/main/scala/com/raphtory/core/model/analysis/entityVisitors/EdgeVisitor.scala +++ b/mainproject/src/main/scala/com/raphtory/core/model/analysis/entityVisitors/EdgeVisitor.scala @@ -7,7 +7,6 @@ import com.raphtory.core.actors.PartitionManager.Workers.ViewJob import com.raphtory.core.model.analysis.GraphLenses.GraphLens import com.raphtory.core.model.communication.{ImmutableProperty, VertexMessage} import com.raphtory.core.model.graphentities.{Edge, MutableProperty} -import com.raphtory.core.utils.Utils import scala.collection.mutable @@ -22,7 +21,7 @@ class EdgeVisitor(edge:Edge,id:Long,viewJob:ViewJob,superStep:Int,view:GraphLens def send(data: Any): Unit = { val message = VertexMessage(id, viewJob, superStep, data) view.recordMessage() - mediator ! DistributedPubSubMediator.Send(Utils.getReader(id, managerCount.count), message, false) + mediator ! DistributedPubSubMediator.Send(getReader(id, managerCount.count), message, false) } //TODO edge properties diff --git a/mainproject/src/main/scala/com/raphtory/core/model/analysis/entityVisitors/EntityVisitor.scala b/mainproject/src/main/scala/com/raphtory/core/model/analysis/entityVisitors/EntityVisitor.scala index 4d87656502..228c9120f1 100644 --- a/mainproject/src/main/scala/com/raphtory/core/model/analysis/entityVisitors/EntityVisitor.scala +++ b/mainproject/src/main/scala/com/raphtory/core/model/analysis/entityVisitors/EntityVisitor.scala @@ -48,4 +48,12 @@ abstract class EntityVisitor(entity:Entity,viewJob:ViewJob) { } } + + //TODO only here for temp, needs to be removed + protected def getReader(srcId: Long, managerCount: Int): String = { + val mod = srcId.abs % (managerCount * 10) + val manager = mod / 10 + val worker = mod % 10 + s"/user/Manager_${manager}_reader_$worker" + } } diff --git a/mainproject/src/main/scala/com/raphtory/core/model/analysis/entityVisitors/VertexVisitor.scala b/mainproject/src/main/scala/com/raphtory/core/model/analysis/entityVisitors/VertexVisitor.scala index 72cfea277d..e1c7193b63 100644 --- a/mainproject/src/main/scala/com/raphtory/core/model/analysis/entityVisitors/VertexVisitor.scala +++ b/mainproject/src/main/scala/com/raphtory/core/model/analysis/entityVisitors/VertexVisitor.scala @@ -7,7 +7,6 @@ import com.raphtory.core.actors.PartitionManager.Workers.ViewJob import com.raphtory.core.model.analysis.GraphLenses.GraphLens import com.raphtory.core.model.communication._ import com.raphtory.core.model.graphentities.{Edge, Vertex} -import com.raphtory.core.utils.Utils import scala.collection.parallel.mutable.ParIterable import scala.reflect.ClassTag @@ -135,7 +134,7 @@ class VertexVisitor(v: Vertex, viewJob:ViewJob, superStep: Int, view: GraphLens) def messageNeighbour(vertexID: Long, data: Any): Unit = { val message = VertexMessage(vertexID, viewJob, superStep, data) view.recordMessage() - mediator ! DistributedPubSubMediator.Send(Utils.getReader(vertexID, managerCount.count), message, false) + mediator ! DistributedPubSubMediator.Send(getReader(vertexID, managerCount.count), message, false) } def messageAllOutgoingNeighbors(message: Any): Unit = diff --git a/mainproject/src/main/scala/com/raphtory/core/utils/SchedulerUtil.scala b/mainproject/src/main/scala/com/raphtory/core/utils/SchedulerUtil.scala deleted file mode 100644 index 0f2d93eb23..0000000000 --- a/mainproject/src/main/scala/com/raphtory/core/utils/SchedulerUtil.scala +++ /dev/null @@ -1,49 +0,0 @@ -package com.raphtory.core.utils - -import akka.actor.{Actor, ActorContext, ActorRef, Cancellable} - -import scala.concurrent.ExecutionContext -import scala.concurrent.duration.FiniteDuration - -object SchedulerUtil { - - def scheduleTask(initialDelay: FiniteDuration, interval: FiniteDuration, receiver: ActorRef, message: Any)( - implicit context: ActorContext, - executor: ExecutionContext, - sender: ActorRef = Actor.noSender - ): Cancellable = { - val scheduler = context.system.scheduler - - val cancellable = scheduler.schedule(initialDelay, interval, receiver, message) - context.system.log.debug("The message [{}] has been scheduled for send to [{}].", message, receiver.path) - - cancellable - } - - def scheduleTaskOnce( - delay: FiniteDuration, - receiver: ActorRef, - message: Any - )(implicit context: ActorContext, executor: ExecutionContext, sender: ActorRef = Actor.noSender): Cancellable = { - val scheduler = context.system.scheduler - - val cancellable = scheduler.scheduleOnce(delay, receiver, message) - context.system.log.debug("The message [{}] has been scheduled for send to [{}].", message, receiver.path) - - cancellable - } - - def cancelTask(key: String, task: Cancellable)(implicit context: ActorContext): Boolean = { - task.cancel() - - val isCancelled = task.isCancelled - - if (isCancelled) - context.system.log.debug("The task [{}] has been cancelled.", key) - else - context.system.log.debug("Failed to cancel the task [{}].", key) - - isCancelled - } - -} diff --git a/mainproject/src/main/scala/com/raphtory/core/utils/Utils.scala b/mainproject/src/main/scala/com/raphtory/core/utils/Utils.scala deleted file mode 100644 index 818d8d6d6b..0000000000 --- a/mainproject/src/main/scala/com/raphtory/core/utils/Utils.scala +++ /dev/null @@ -1,128 +0,0 @@ -package com.raphtory.core.utils - -import java.io._ -import java.text.SimpleDateFormat - -import akka.actor.ActorContext -import com.typesafe.config.ConfigFactory -import org.apache.commons.lang.StringEscapeUtils - -import scala.collection.mutable - -object Utils { - val clusterSystemName = "dockerexp" - val config = ConfigFactory.load - val partitionsTopic = "/partitionsCount" - val readersTopic = "/readers" - val readersWorkerTopic = "/readerWorkers" - val liveAnalysisTopic = "/liveanalysis" - - val totalWorkers = 10 //must be power of 10 - - def getPartition(ID: Long, managerCount: Int): Int = - ((ID.abs % (managerCount * totalWorkers)) / totalWorkers).toInt - def getWorker(ID: Long, managerCount: Int): Int = - ((ID.abs % (managerCount * totalWorkers)) % totalWorkers).toInt - //get the partition a vertex is stored in - def checkDst(dstID: Long, managerCount: Int, managerID: Int): Boolean = - getPartition(dstID, managerCount) == managerID //check if destination is also local - def checkWorker(dstID: Long, managerCount: Int, workerID: Int): Boolean = - getWorker(dstID, managerCount) == workerID //check if destination is also local - - def getManager(srcId: Long, managerCount: Int): String = { - val mod = srcId.abs % (managerCount * totalWorkers) - val manager = mod / totalWorkers - val worker = mod % totalWorkers - s"/user/Manager_${manager}_child_$worker" - } - - def getReader(srcId: Long, managerCount: Int): String = { - val mod = srcId.abs % (managerCount * totalWorkers) - val manager = mod / totalWorkers - val worker = mod % totalWorkers - s"/user/Manager_${manager}_reader_$worker" - } - def getReaderInt(srcId: Long, managerCount: Int): (Long, Long) = { - val mod = srcId.abs % (managerCount * totalWorkers) - val manager = mod / totalWorkers - val worker = mod % totalWorkers - (manager, worker) - } - - def getAllReaders(managerCount: Int): Array[String] = { - val workers = mutable.ArrayBuffer[String]() - for (i <- 0 until managerCount) - workers += s"/user/ManagerReader_$i" - workers.toArray - } - - def getAllReaderWorkers(managerCount: Int): Array[String] = { - val workers = mutable.ArrayBuffer[String]() - for (i <- 0 until managerCount) - for (j <- 0 until totalWorkers) - workers += s"/user/Manager_${i}_reader_$j" - workers.toArray - } - - def getAllWriterWorkers(managerCount: Int): Array[String] = { - val workers = mutable.ArrayBuffer[String]() - for (i <- 0 until managerCount) - for (j <- 0 until totalWorkers) - workers += s"/user/Manager_${i}_child_$j" - workers.toArray - } - - def getAllRouterWorkers(managerCount: Int): Array[String] = { - val workers = mutable.ArrayBuffer[String]() - for (i <- 0 until managerCount) - for (j <- 0 until totalWorkers) - workers += s"/user/router/router_${i}_Worker_$j" - workers.toArray - } - - def createHistory(history: mutable.TreeMap[Long, Boolean]): String = { - if (history.size == 0) - return "" - var s = "{" - for ((k, v) <- history) - s = s + s"$k : $v, " - s.dropRight(2) + "}" - } - - def createPropHistory(history: mutable.TreeMap[Long, String]): String = { - if (history.size == 0) - return "" - var s = "{" - for ((k, v) <- history) - s = s + s"$k : '${StringEscapeUtils.escapeJava(v.replaceAll("'", ""))}', " - s.dropRight(2) + "}" - } - - def nowTimeStamp() = new SimpleDateFormat("dd-MM hh:mm:ss").format(System.currentTimeMillis()) - def unixToTimeStamp(unixTime: Long) = new SimpleDateFormat("dd-MM hh:mm:ss").format(unixTime) - - def writeLines(fileName: String, line: String, header: String): Unit = { - val f = new File(fileName) - if (!f.exists()) { - f.createNewFile() - val file = new FileWriter(fileName, true) - var bw = new BufferedWriter(file) - bw.write(header) - bw.newLine() - bw.write(line) - bw.newLine() - bw.flush() - } else { - val file = new FileWriter(fileName, true) - var bw = new BufferedWriter(file) - bw.write(line) - bw.newLine() - bw.flush() - } - - } - - def IPRegex = - "\\b(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\\b" - -} diff --git a/mainproject/src/main/scala/com/raphtory/examples/blockchain/analysers/EthereumDegreeRanking.scala b/mainproject/src/main/scala/com/raphtory/examples/blockchain/analysers/EthereumDegreeRanking.scala index e3ed98451c..23f11250c0 100644 --- a/mainproject/src/main/scala/com/raphtory/examples/blockchain/analysers/EthereumDegreeRanking.scala +++ b/mainproject/src/main/scala/com/raphtory/examples/blockchain/analysers/EthereumDegreeRanking.scala @@ -1,7 +1,6 @@ package com.raphtory.examples.blockchain.analysers import com.raphtory.api.Analyser -import com.raphtory.core.utils.Utils import scala.collection.mutable.ArrayBuffer @@ -78,7 +77,7 @@ class EthereumDegreeRanking(args:Array[String]) extends Analyser(args) { val text = s"""{"time":$timestamp,"windowsize":$windowSize,"vertices":$totalVert,"edges":$totalEdge,"degree":$degree,"bestusers":$bestUserArray,"viewTime":$viewCompleteTime,"concatTime":${System .currentTimeMillis() - startTime}},""" - Utils.writeLines(output_file, text, "{\"views\":[") + writeLines(output_file, text, "{\"views\":[") println(text) } diff --git a/mainproject/src/main/scala/com/raphtory/examples/blockchain/analysers/TaintTrackExchangeStop.scala b/mainproject/src/main/scala/com/raphtory/examples/blockchain/analysers/TaintTrackExchangeStop.scala index 7322b29fde..616457182a 100644 --- a/mainproject/src/main/scala/com/raphtory/examples/blockchain/analysers/TaintTrackExchangeStop.scala +++ b/mainproject/src/main/scala/com/raphtory/examples/blockchain/analysers/TaintTrackExchangeStop.scala @@ -1,7 +1,6 @@ package com.raphtory.examples.blockchain.analysers import com.raphtory.api.Analyser -import com.raphtory.core.utils.Utils import scala.collection.mutable.ArrayBuffer import scala.collection.parallel.immutable @@ -108,11 +107,11 @@ class TaintTrackExchangeStop(args:Array[String]) extends Analyser(args) { data+=s"""{"infected":"${elem._1}","block":${elem._2},"infector":"${elem._3}","exchange":"${elem._4}","value":${elem._5}},""" data+="]}" //publishData(data) -// Utils.writeLines("/opt/docker/output.csv", data, "{\"views\":[") +// writeLines("/opt/docker/output.csv", data, "{\"views\":[") //print(data) - Utils.writeLines("/app/output.csv", data, "") + writeLines("/app/output.csv", data, "") - //for (elem <- endResults) { Utils.writeLines(s"/Users/mirate/Documents/phd/etheroutput/block${timeStamp}.csv", s"${elem._1},${elem._2},${elem._3},", "")} + //for (elem <- endResults) { writeLines(s"/Users/mirate/Documents/phd/etheroutput/block${timeStamp}.csv", s"${elem._1},${elem._2},${elem._3},", "")} } } diff --git a/mainproject/src/main/scala/com/raphtory/spouts/blockchain/EthereumGethSpout.scala b/mainproject/src/main/scala/com/raphtory/spouts/blockchain/EthereumGethSpout.scala index 541e5bc89b..afd0cd63f0 100644 --- a/mainproject/src/main/scala/com/raphtory/spouts/blockchain/EthereumGethSpout.scala +++ b/mainproject/src/main/scala/com/raphtory/spouts/blockchain/EthereumGethSpout.scala @@ -4,7 +4,6 @@ import java.net.InetAddress import java.util.NoSuchElementException import com.raphtory.core.actors.Spout.Spout -import com.raphtory.core.utils.Utils import scalaj.http.{Http, HttpRequest} import spray.json.DefaultJsonProtocol._ import spray.json._ @@ -21,7 +20,8 @@ class EthereumGethSpout extends Spout[String] { val nodeIP = System.getenv().getOrDefault("SPOUT_ETHEREUM_IP_ADDRESS", "127.0.0.1").trim val nodePort = System.getenv().getOrDefault("SPOUT_ETHEREUM_PORT", "8545").trim // val nodePort = System.getenv().getOrDefault("SPOUT_ETHEREUM_PORT", "30303").trim - +def IPRegex = + "\\b(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\\b" print(currentBlock) val queue = mutable.Queue[Option[String]]() @@ -42,7 +42,7 @@ class EthereumGethSpout extends Spout[String] { implicit val EthFormat = jsonFormat14(EthResult) implicit val EthTransactionFormat = jsonFormat3(EthTransaction) - if (nodeIP.matches(Utils.IPRegex)) + if (nodeIP.matches(IPRegex)) println(s"Connecting to Ethereum RPC \n Address:$nodeIP \n Port:$nodePort") else println(s"Connecting to Ethereum RPC \n Address:${hostname2Ip(nodeIP)} \n Port:$nodePort") @@ -79,7 +79,7 @@ class EthereumGethSpout extends Spout[String] { def executeBatchRequest(data: String) = requestBatch(data).execute().body.toString def requestBatch(data: String): HttpRequest = baseRequest.postData(data) def requestBuilder() = - if (nodeIP.matches(Utils.IPRegex)) + if (nodeIP.matches(IPRegex)) Http("http://" + nodeIP + ":" + nodePort).header("content-type", "application/json") else Http("http://" + hostname2Ip(nodeIP) + ":" + nodePort).header("content-type", "application/json") From 0979b043813696125bb1a507f4476b9af5d534b7 Mon Sep 17 00:00:00 2001 From: Miratepuffin Date: Tue, 1 Dec 2020 01:12:51 +0000 Subject: [PATCH 54/58] Added basic query API for local tests within the RaphtoryGraph, meaning you dont have to deal with the curl commands --- .../scala/com/raphtory/RaphtoryGraph.scala | 40 ++++++++++++++++++- .../algorithms/ConnectedComponents.scala | 5 +++ .../com/raphtory/algorithms/DegreeBasic.scala | 4 ++ .../examples/lotr/LOTRDeployment.scala | 12 +++++- 4 files changed, 59 insertions(+), 2 deletions(-) diff --git a/mainproject/src/main/scala/com/raphtory/RaphtoryGraph.scala b/mainproject/src/main/scala/com/raphtory/RaphtoryGraph.scala index a18b791004..d3ddf25a4c 100644 --- a/mainproject/src/main/scala/com/raphtory/RaphtoryGraph.scala +++ b/mainproject/src/main/scala/com/raphtory/RaphtoryGraph.scala @@ -2,10 +2,12 @@ package com.raphtory import akka.actor.{ActorSystem, Props} import ch.qos.logback.classic.Level +import com.raphtory.api.Analyser import com.raphtory.core.actors.AnalysisManager.{AnalysisManager, AnalysisRestApi} import com.raphtory.core.actors.ClusterManagement.{RaphtoryReplicator, WatchDog, WatermarkManager} import com.raphtory.core.actors.Router.GraphBuilder import com.raphtory.core.actors.Spout.{Spout, SpoutAgent} +import com.raphtory.core.model.communication.{LiveAnalysisRequest, RangeAnalysisRequest, ViewAnalysisRequest} import kamon.Kamon import org.slf4j.LoggerFactory @@ -37,6 +39,42 @@ class RaphtoryGraph[T](spout: Spout[T], graphBuilder: GraphBuilder[T]) { system.actorOf(Props(RaphtoryReplicator.apply("Router", partitionNumber, minimumRouters,graphBuilder)), s"Routers") system.actorOf(Props(RaphtoryReplicator("Partition Manager", partitionNumber,minimumRouters)), s"PartitionManager") - system.actorOf(Props[AnalysisManager], s"AnalysisManager") + val analysisManager = system.actorOf(Props[AnalysisManager], s"AnalysisManager") AnalysisRestApi(system) + + //TODO tidy these, but will be done with full analysis Overhall + def rangeQuery(analyser:Analyser,start:Long,end:Long,increment:Long,args:Array[String]):Unit = { + analysisManager ! RangeAnalysisRequest(analyser.getClass.getCanonicalName,start,end,increment,"false",0L,Array[Long](),args,"") + } + + def rangeQuery(analyser:Analyser,start:Long,end:Long,increment:Long,window:Long,args:Array[String]):Unit = { + analysisManager ! RangeAnalysisRequest(analyser.getClass.getCanonicalName,start,end,increment,"true",window,Array[Long](),args,"") + } + def rangeQuery(analyser:Analyser,start:Long,end:Long,increment:Long,windowBatch:Array[Long],args:Array[String]):Unit = { + analysisManager ! RangeAnalysisRequest(analyser.getClass.getCanonicalName,start,end,increment,"batched",0L,windowBatch,args,"") + } + + def viewQuery(analyser:Analyser,timestamp:Long,args:Array[String]):Unit = { + analysisManager ! ViewAnalysisRequest(analyser.getClass.getCanonicalName,timestamp,"false",0L,Array[Long](),args,"") + } + + def viewQuery(analyser:Analyser,timestamp:Long,window:Long,args:Array[String]):Unit = { + analysisManager ! ViewAnalysisRequest(analyser.getClass.getCanonicalName,timestamp,"true",window,Array[Long](),args,"") + } + def viewQuery(analyser:Analyser,timestamp:Long,windowBatch:Array[Long],args:Array[String]):Unit = { + analysisManager ! ViewAnalysisRequest(analyser.getClass.getCanonicalName,timestamp,"batched",0L,windowBatch,args,"") + } + + def liveQuery(analyser:Analyser,repeat:Long,eventTime:Boolean,args:Array[String]):Unit = { + analysisManager ! LiveAnalysisRequest(analyser.getClass.getCanonicalName,repeat,eventTime,"false",0L,Array[Long](),args,"") + } + + def liveQuery(analyser:Analyser,repeat:Long,eventTime:Boolean,window:Long,args:Array[String]):Unit = { + analysisManager ! LiveAnalysisRequest(analyser.getClass.getCanonicalName,repeat,eventTime,"true",window,Array[Long](),args,"") + } + + def liveQuery(analyser:Analyser,repeat:Long,eventTime:Boolean,windowBatch:Array[Long],args:Array[String]):Unit = { + analysisManager ! LiveAnalysisRequest(analyser.getClass.getCanonicalName,repeat,eventTime,"batched",0L,windowBatch,args,"") + } + } diff --git a/mainproject/src/main/scala/com/raphtory/algorithms/ConnectedComponents.scala b/mainproject/src/main/scala/com/raphtory/algorithms/ConnectedComponents.scala index 7fa212ad23..13bbde4052 100644 --- a/mainproject/src/main/scala/com/raphtory/algorithms/ConnectedComponents.scala +++ b/mainproject/src/main/scala/com/raphtory/algorithms/ConnectedComponents.scala @@ -4,6 +4,11 @@ import com.raphtory.api.Analyser import scala.collection.mutable.ArrayBuffer import scala.collection.parallel.immutable + +object ConnectedComponents { + def apply():ConnectedComponents = new ConnectedComponents(Array()) +} + class ConnectedComponents(args:Array[String]) extends Analyser(args){ override def setup(): Unit = diff --git a/mainproject/src/main/scala/com/raphtory/algorithms/DegreeBasic.scala b/mainproject/src/main/scala/com/raphtory/algorithms/DegreeBasic.scala index 8d55ede71f..8e8de069b0 100644 --- a/mainproject/src/main/scala/com/raphtory/algorithms/DegreeBasic.scala +++ b/mainproject/src/main/scala/com/raphtory/algorithms/DegreeBasic.scala @@ -4,6 +4,10 @@ import com.raphtory.api.Analyser import scala.collection.mutable.ArrayBuffer +object DegreeBasic{ + def apply() = new DegreeBasic(Array()) +} + class DegreeBasic(args:Array[String]) extends Analyser(args){ object sortOrdering extends Ordering[Int] { def compare(key1: Int, key2: Int) = key2.compareTo(key1) diff --git a/mainproject/src/main/scala/com/raphtory/examples/lotr/LOTRDeployment.scala b/mainproject/src/main/scala/com/raphtory/examples/lotr/LOTRDeployment.scala index 3f363d516d..ab4880df12 100644 --- a/mainproject/src/main/scala/com/raphtory/examples/lotr/LOTRDeployment.scala +++ b/mainproject/src/main/scala/com/raphtory/examples/lotr/LOTRDeployment.scala @@ -1,9 +1,19 @@ package com.raphtory.examples.lotr import com.raphtory.RaphtoryGraph +import com.raphtory.algorithms.{ConnectedComponents, DegreeBasic} object LOTRDeployment extends App{ val source = new LOTRSpout() val builder = new LOTRGraphBuilder() - RaphtoryGraph[String](source,builder) + val rg = RaphtoryGraph[String](source,builder) + val arguments = Array[String]() + + rg.rangeQuery(ConnectedComponents(),start = 1,end = 32674,increment = 100,arguments) + rg.rangeQuery(ConnectedComponents(),start = 1,end = 32674,increment = 100,window=100,arguments) + rg.rangeQuery(ConnectedComponents(),start = 1,end = 32674,increment = 100,windowBatch=Array(100,50,10),arguments) + + rg.viewQuery(DegreeBasic(),timestamp = 10000,arguments) + rg.viewQuery(DegreeBasic(),timestamp = 10000,window=100,arguments) + rg.viewQuery(DegreeBasic(),timestamp = 10000,windowBatch=Array(100,50,10),arguments) } From 2cad40fe2d915528933a79bb53515a969a54ec27 Mon Sep 17 00:00:00 2001 From: Miratepuffin Date: Tue, 1 Dec 2020 02:01:25 +0000 Subject: [PATCH 55/58] I have broke out the examples into a seperate project. Closing issue #104 --- .../analysers/EthereumDegreeRanking.scala | 84 - .../analysers/EthereumTaintTracking.scala | 98 - .../analysers/TaintTrackExchangeStop.scala | 117 - .../graphbuilders/BitcoinGraphBuilder.scala | 96 - .../ChABClus2ClusGraphBuilder.scala | 57 - .../ChABClus2ClusGraphBuilderNoprop.scala | 40 - .../ChABClus2ClusRouter.scala.save | 60 - .../ChainalysisABGraphBuilder.scala | 62 - .../graphbuilders/DashcoinGraphBuilder.scala | 112 - .../EthereumGethGraphBuilder.scala | 58 - .../EthereumKafkaGraphBuilder.scala | 68 - .../EthereumTransactionGraphBuilder.scala | 43 - .../FirehoseKafkaGraphBuilder.scala | 59 - .../graphbuilders/LitecoinGraphBuilder.scala | 120 - .../CitationGraphBuilder.scala | 40 - .../gab/analysis/GabMostUsedTopics.scala | 58 - .../gab/datasources/GabMongoSource.scala | 61 - .../graphbuilders/GabPostGraphBuilder.scala | 52 - .../graphbuilders/GabRawGraphBuilder.scala | 133 - .../graphbuilders/GabUserGraphBuilder.scala | 54 - .../gab/rawgraphmodel/GabEntityType.scala | 7 - .../gab/rawgraphmodel/GabJsonProtocol.scala | 105 - .../examples/gab/rawgraphmodel/GabPost.scala | 85 - .../ldbc/graphbuilders/LDBCGraphBuilder.scala | 38 - .../graphbuilders/LDBCOldGraphBuilder.scala | 38 - .../examples/lotr/LOTRDeployment.scala | 19 - .../examples/lotr/LOTRGraphBuilder.scala | 26 - .../raphtory/examples/lotr/LOTRSpout.scala | 27 - .../scala/com/raphtory/examples/lotr/lotr.csv | 2649 ----------------- .../test/actors/RandomGraphBuilder.scala | 95 - .../examples/test/actors/RandomSpout.scala | 94 - .../test/actors/TriangleGraphBuilder.scala | 14 - .../examples/test/actors/TriangleSpout.scala | 23 - .../TrackAndTraceGraphBuilder.scala | 101 - .../examples/tsvnet/SamplerSpout.scala | 56 - .../examples/tsvnet/TSVGraphBuilder.scala | 28 - .../CooccurrenceMatrixGraphBuilder.scala | 40 - ...occurrenceMatrixGraphBuilderFiltered.scala | 38 - .../spouts/CooccurrenceMatrixSpout.scala | 126 - .../CooccurrenceMatrixSpoutFiltered.scala | 53 - 40 files changed, 5134 deletions(-) delete mode 100644 mainproject/src/main/scala/com/raphtory/examples/blockchain/analysers/EthereumDegreeRanking.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/examples/blockchain/analysers/EthereumTaintTracking.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/examples/blockchain/analysers/TaintTrackExchangeStop.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/BitcoinGraphBuilder.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChABClus2ClusGraphBuilder.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChABClus2ClusGraphBuilderNoprop.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChABClus2ClusRouter.scala.save delete mode 100644 mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChainalysisABGraphBuilder.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/DashcoinGraphBuilder.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/EthereumGethGraphBuilder.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/EthereumKafkaGraphBuilder.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/EthereumTransactionGraphBuilder.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/FirehoseKafkaGraphBuilder.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/LitecoinGraphBuilder.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/examples/citationNetwork/CitationGraphBuilder.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/examples/gab/analysis/GabMostUsedTopics.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/examples/gab/datasources/GabMongoSource.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/examples/gab/graphbuilders/GabPostGraphBuilder.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/examples/gab/graphbuilders/GabRawGraphBuilder.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/examples/gab/graphbuilders/GabUserGraphBuilder.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/examples/gab/rawgraphmodel/GabEntityType.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/examples/gab/rawgraphmodel/GabJsonProtocol.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/examples/gab/rawgraphmodel/GabPost.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/examples/ldbc/graphbuilders/LDBCGraphBuilder.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/examples/ldbc/graphbuilders/LDBCOldGraphBuilder.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/examples/lotr/LOTRDeployment.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/examples/lotr/LOTRGraphBuilder.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/examples/lotr/LOTRSpout.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/examples/lotr/lotr.csv delete mode 100644 mainproject/src/main/scala/com/raphtory/examples/test/actors/RandomGraphBuilder.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/examples/test/actors/RandomSpout.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/examples/test/actors/TriangleGraphBuilder.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/examples/test/actors/TriangleSpout.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/examples/trackAndTrace/TrackAndTraceGraphBuilder.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/examples/tsvnet/SamplerSpout.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/examples/tsvnet/TSVGraphBuilder.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/examples/wordSemantic/graphbuilders/CooccurrenceMatrixGraphBuilder.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/examples/wordSemantic/graphbuilders/CooccurrenceMatrixGraphBuilderFiltered.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/examples/wordSemantic/spouts/CooccurrenceMatrixSpout.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/examples/wordSemantic/spouts/CooccurrenceMatrixSpoutFiltered.scala diff --git a/mainproject/src/main/scala/com/raphtory/examples/blockchain/analysers/EthereumDegreeRanking.scala b/mainproject/src/main/scala/com/raphtory/examples/blockchain/analysers/EthereumDegreeRanking.scala deleted file mode 100644 index 23f11250c0..0000000000 --- a/mainproject/src/main/scala/com/raphtory/examples/blockchain/analysers/EthereumDegreeRanking.scala +++ /dev/null @@ -1,84 +0,0 @@ -package com.raphtory.examples.blockchain.analysers - -import com.raphtory.api.Analyser - -import scala.collection.mutable.ArrayBuffer - -class EthereumDegreeRanking(args:Array[String]) extends Analyser(args) { - object sortOrdering extends Ordering[Int] { - def compare(key1: Int, key2: Int) = key2.compareTo(key1) - } - - override def setup(): Unit = {} - override def analyse(): Unit = {} - override def returnResults(): Any = { - val degree = view.getVertices().map { vertex => - val outDegree = vertex.getOutEdges.size - val inDegree = vertex.getIncEdges.size - val walletID = vertex.getPropertyValue("id").get - (walletID, outDegree, inDegree) - } - val totalV = degree.size - val totalOut = degree.map(x => x._2).sum - val totalIn = degree.map(x => x._3).sum - val topUsers = degree.toArray.sortBy(x => x._3)(sortOrdering).take(20) - (totalV, totalOut, totalIn, topUsers) - } - - override def defineMaxSteps(): Int = 1 - - override def processResults(results: ArrayBuffer[Any], timestamp: Long, viewCompleteTime: Long): Unit = { - val endResults = results.asInstanceOf[ArrayBuffer[(Int, Int, Int, Array[(Int, Int, Int)])]] - val totalVert = endResults.map(x => x._1).sum - val totalEdge = endResults.map(x => x._3).sum - val degree = - try totalEdge / totalVert - catch { case e: ArithmeticException => 0 } - var bestUserArray = "[" - val bestUsers = endResults - .map(x => x._4) - .flatten - .sortBy(x => x._3)(sortOrdering) - .take(20) - .map(x => s"""{"id":${x._1},"indegree":${x._3},"outdegree":${x._2}}""") - .foreach(x => bestUserArray += x + ",") - bestUserArray = if (bestUserArray.length > 1) bestUserArray.dropRight(1) + "]" else bestUserArray + "]" - val text = - s"""{"time":$timestamp,"vertices":$totalVert,"edges":$totalEdge,"degree":$degree,"bestusers":$bestUserArray,"viewTime":$viewCompleteTime},""" - println(text) - } - - - - override def processWindowResults( - results: ArrayBuffer[Any], - timestamp: Long, - windowSize: Long, - viewCompleteTime: Long - ): Unit = { - val endResults = results.asInstanceOf[ArrayBuffer[(Int, Int, Int, Array[(Int, Int, Int)])]] - var output_file = System.getenv().getOrDefault("GAB_PROJECT_OUTPUT", "/app/defout.csv").trim - val startTime = System.currentTimeMillis() - val totalVert = endResults.map(x => x._1).sum - val totalEdge = endResults.map(x => x._3).sum - - val degree = - try totalEdge / totalVert - catch { case e: ArithmeticException => 0 } - var bestUserArray = "[" - val bestUsers = endResults - .map(x => x._4) - .flatten - .sortBy(x => x._3)(sortOrdering) - .take(20) - .map(x => s"""{"id":${x._1},"indegree":${x._3},"outdegree":${x._2}}""") - .foreach(x => bestUserArray += x + ",") - bestUserArray = if (bestUserArray.length > 1) bestUserArray.dropRight(1) + "]" else bestUserArray + "]" - val text = - s"""{"time":$timestamp,"windowsize":$windowSize,"vertices":$totalVert,"edges":$totalEdge,"degree":$degree,"bestusers":$bestUserArray,"viewTime":$viewCompleteTime,"concatTime":${System - .currentTimeMillis() - startTime}},""" - writeLines(output_file, text, "{\"views\":[") - println(text) - } - -} diff --git a/mainproject/src/main/scala/com/raphtory/examples/blockchain/analysers/EthereumTaintTracking.scala b/mainproject/src/main/scala/com/raphtory/examples/blockchain/analysers/EthereumTaintTracking.scala deleted file mode 100644 index 14e2ea0486..0000000000 --- a/mainproject/src/main/scala/com/raphtory/examples/blockchain/analysers/EthereumTaintTracking.scala +++ /dev/null @@ -1,98 +0,0 @@ -package com.raphtory.examples.blockchain.analysers - -import com.raphtory.api.Analyser - -import scala.collection.mutable.ArrayBuffer -import scala.collection.parallel.immutable - -class EthereumTaintTracking(args:Array[String]) extends Analyser(args) { - //val infectedNode =System.getenv().getOrDefault("ETHEREUM_TAINT_TRACKING_BAD_ACTOR", "0xa09871aeadf4994ca12f5c0b6056bbd1d343c029").trim - //val infectionStartingBlock =System.getenv().getOrDefault("ETHEREUM_TAINT_TRACKING_START_BLOCK", "9007863").trim.toLongя - //val infectedNode =System.getenv().getOrDefault("ETHEREUM_TAINT_TRACKING_BAD_ACTOR", "0x52bc44d5378309EE2abF1539BF71dE1b7d7bE3b5").trim.toLowerCase.asInstanceOf[String] - // val infectionStartingBlock =System.getenv().getOrDefault("ETHEREUM_TAINT_TRACKING_START_BLOCK", "4000000").trim.toLong - val infectedNode = args(0).trim.toLowerCase - val infectionStartingBlock = args(1).trim.toLong - override def setup(): Unit = - view.getVertices().foreach { vertex => - val walletID = vertex.getPropertyValue("id").get.asInstanceOf[String] - val numberOfHops = 0 - val nodeDegree = 1 - if(walletID equals infectedNode) { - vertex.getOrSetState("infected", infectionStartingBlock) - vertex.getOrSetState("infectedBy", "Start") - vertex.getOrSetState("numberOfHops", 0) - vertex.getOrSetState("nodeDegree", 0) - vertex.getOutEdgesAfter(infectionStartingBlock).foreach { neighbour => - neighbour.send((walletID,neighbour.firstActivityAfter(infectionStartingBlock), numberOfHops, nodeDegree)) - } - } - } - - override def analyse(): Unit = - view.getMessagedVertices().foreach { vertex => - var infectionBlock = infectionStartingBlock - var infector = infectedNode - var numberOfHops = 1 - var nodeDegree = 0 - // send a pair (string, long) - // set a new long to this one - // add number of hops - val queue = vertex.messageQueue[(String,Long,Int, Int)] - println(queue) - try{ - // it is extracting the minimum along the second line of mappers - infectionBlock = queue.map(x=>x._2).min - println(infectionBlock) - - numberOfHops = queue.map(x=>x._3).min - numberOfHops = numberOfHops + 1 - - nodeDegree = queue.map(x=>x._2).length - } - catch { - case e: UnsupportedOperationException => print(e) - } - - infector = queue.filter(x=>x._2==infectionBlock).head._1 //todo check if multiple - println(infector) - - - //if (vertex.containsCompValue("infected")) - // vertex.voteToHalt() //already infected - //else { - val walletID = vertex.getPropertyValue("id").get.asInstanceOf[String] - vertex.getOrSetState("infected", infectionBlock) - vertex.getOrSetState("infectedBy",infector) - vertex.getOrSetState("numberOfHops", numberOfHops) - vertex.getOrSetState("nodeDegree", nodeDegree) - vertex.getOutEdgesAfter(infectionBlock).foreach { neighbour => - neighbour.send((walletID,neighbour.firstActivityAfter(infectionBlock), numberOfHops, nodeDegree)) - } - //} - } - - override def returnResults(): Any = - view - .getVertices().map { vertex => - if (vertex.containsState("infected")) - (vertex.getPropertyValue("id").get.asInstanceOf[String], vertex.getState[Long]("infected"),vertex.getState[String]("infectedBy")) - else - ("", -1L,"") - - - } - .filter(f => f._2 >= 0).par - - override def defineMaxSteps(): Int = 100 - - override def processResults(results: ArrayBuffer[Any], timeStamp: Long, viewCompleteTime: Long): Unit = { - val endResults = results.asInstanceOf[ArrayBuffer[immutable.ParIterable[(String, Long,String, Int, Int)]]].flatten - var data = s"{block:$timeStamp,edges:[" - for (elem <- endResults) - data+=s"""{"infected":"${elem._1}","block":"${elem._2}","infector":"${elem._3}", "numberOfHops":"${elem._4}", "nodeDegree":"${elem._5}",}""" - data+="]}" - publishData(data) - - } - -} diff --git a/mainproject/src/main/scala/com/raphtory/examples/blockchain/analysers/TaintTrackExchangeStop.scala b/mainproject/src/main/scala/com/raphtory/examples/blockchain/analysers/TaintTrackExchangeStop.scala deleted file mode 100644 index 616457182a..0000000000 --- a/mainproject/src/main/scala/com/raphtory/examples/blockchain/analysers/TaintTrackExchangeStop.scala +++ /dev/null @@ -1,117 +0,0 @@ -package com.raphtory.examples.blockchain.analysers - -import com.raphtory.api.Analyser - -import scala.collection.mutable.ArrayBuffer -import scala.collection.parallel.immutable - -class TaintTrackExchangeStop(args:Array[String]) extends Analyser(args) { - //val infectedNode =System.getenv().getOrDefault("ETHEREUM_TAINT_TRACKING_BAD_ACTOR", "0xa09871aeadf4994ca12f5c0b6056bbd1d343c029").trim - //val infectionStartingBlock =System.getenv().getOrDefault("ETHEREUM_TAINT_TRACKING_START_BLOCK", "9007863").trim.toLong - //val infectedNode =System.getenv().getOrDefault("ETHEREUM_TAINT_TRACKING_BAD_ACTOR", "0x52bc44d5378309EE2abF1539BF71dE1b7d7bE3b5").trim.toLowerCase.asInstanceOf[String] - //val infectionStartingBlock =System.getenv().getOrDefault("ETHEREUM_TAINT_TRACKING_START_BLOCK", "4000000").trim.toLong -//"0x9A207194cbED9F229694fdF5a28caab59157920D" - - val infectedNode = args(0).trim.toLowerCase - val infectionStartingBlock = args(1).trim.toLong - val listOfExchanges = Array("0x83053C32b7819F420dcFed2D218335fe430Fe3b5","0x05f51aab068caa6ab7eeb672f88c180f67f17ec7","0x2ddd202174a72514ed522e77972b461b03155525","0xf0c80fb9fb22bef8269cb6feb9a51130288a671f","0x94597850916a49b3b152ee374e97260b99249f5b","0x4df5f3610e2471095a130d7d934d551f3dde01ed","0xadb72986ead16bdbc99208086bd431c1aa38938e","0x7a10ec7d68a048bdae36a70e93532d31423170fa","0xce1bf8e51f8b39e51c6184e059786d1c0eaf360f","0xf73c3c65bde10bf26c2e1763104e609a41702efe","0x0bb5de248dbbd31ee6c402c3c4a70293024acf74","0xed5cdb0d02152046e6f234ad578613831b9184d4","0xa30d8157911ef23c46c0eb71889efe6a648a41f7","0x6eff3372fa352b239bb24ff91b423a572347000d","0xf7793d27a1b76cdf14db7c83e82c772cf7c92910","0xcce8d59affdd93be338fc77fa0a298c2cb65da59","0x3f5ce5fbfe3e9af3971dd833d26ba9b5c936f0be","0xd551234ae421e3bcba99a0da6d736074f22192ff","0x564286362092d8e7936f0549571a803b203aaced","0x0681d8db095565fe8a346fa0277bffde9c0edbbf","0xfe9e8709d3215310075d67e3ed32a380ccf451c8","0x4e9ce36e442e55ecd9025b9a6e0d88485d628a67","0xbe0eb53f46cd790cd13851d5eff43d12404d33e8","0xf977814e90da44bfa03b6295a0616a897441acec","0x001866ae5b3de6caa5a51543fd9fb64f524f5478","0xab83d182f3485cf1d6ccdd34c7cfef95b4c08da4","0x4b729cf402cfcffd057e254924b32241aedc1795","0x7c49e1c0e33f3efb57d64b7690fa287c8d15b90a","0x5d375281582791a38e0348915fa9cbc6139e9c2a","0xdf5021a4c1401f1125cd347e394d977630e17cf7","0x28ebe764b8f9a853509840645216d3c2c0fd774b","0x1151314c646ce4e0efd76d1af4760ae66a9fe30f","0x742d35cc6634c0532925a3b844bc454e4438f44e","0x876eabf441b2ee5b5b0554fd502a8e0600950cfa","0xab7c74abc0c4d48d1bdad5dcb26153fc8780f83e","0xc6cde7c39eb2f0f0095f41570af89efc2c1ea828","0x88d34944cf554e9cccf4a24292d891f620e9c94f","0x186549a4ae594fc1f70ba4cffdac714b405be3f9","0xd273bd546b11bd60214a2f9d71f22a088aafe31b","0x3052cd6bf951449a984fe4b5a38b46aef9455c8e","0x2140efd7ba31169c69dfff6cdc66c542f0211825","0xa0ff1e0f30b5dda2dc01e7e828290bc72b71e57d","0xc1da8f69e4881efe341600620268934ef01a3e63","0xb4460b75254ce0563bb68ec219208344c7ea838c","0x15878e87c685f866edfaf454be6dc06fa517b35b","0x31d03f07178bcd74f9099afebd23b0ae30184ab5","0xed48dc0628789c2956b1e41726d062a86ec45bff","0x3fbe1f8fc5ddb27d428aa60f661eaaab0d2000ce","0xe79eef9b9388a4ff70ed7ec5bccd5b928ebb8bd1","0x03bdf69b1322d623836afbd27679a1c0afa067e9","0x4b1a99467a284cc690e3237bc69105956816f762","0x986a2fca9eda0e06fbf7839b89bfc006ee2a23dd","0x00bdb5699745f5b860228c8f939abf1b9ae374ed","0x1522900b6dafac587d499a862861c0869be6e428","0x9a9bed3eb03e386d66f8a29dc67dc29bbb1ccb72","0x059799f2261d37b829c2850cee67b5b975432271","0x4c766def136f59f6494f0969b1355882080cf8e0","0xc5b611f502a0dcf6c3188fd494061ae29b2baa4f","0x1b8a38ea02ceda9440e00c1aeba26ee2dc570423","0xfbb1b73c4f0bda4f67dca266ce6ef42f520fbb98","0xe94b04a0fed112f3664e45adb2b8915693dd5ff3","0x66f820a414680b5bcda5eeca5dea238543f42054","0xaa90b4aae74cee41e004bc45e45a427406c4dcae","0xf8d04a720520d0bcbc722b1d21ca194aa22699f2","0xfb9f7f41319157ac5c5dccae308a63a4337ad5d9","0x00cdc153aa8894d08207719fe921fff964f28ba3","0x007174732705604bbbf77038332dc52fd5a5000c","0x1c00d840ccaa67c494109f46e55cfeb2d8562f5c","0x73957709695e73fd175582105c44743cf0fb6f2f","0xd7c866d0d536937bf9123e02f7c052446588189f","0x4dc98c79a52968a6c20ce9a7a08d5e8d1c2d5605","0x88988d6ef12d7084e34814b9edafa01ae0d05082","0x72bcfa6932feacd91cb2ea44b0731ed8ae04d0d3","0x7a56f645dcb513d0326cbaa048e9106ff6d4cd5f","0xfd648cc72f1b4e71cbdda7a0a91fe34d32abd656","0x96fc4553a00c117c5b0bed950dd625d1c16dc894","0x8958618332df62af93053cb9c535e26462c959b0","0xb726da4fbdc3e4dbda97bb20998cf899b0e727e0","0x9539e0b14021a43cde41d9d45dc34969be9c7cb0","0x33683b94334eebc9bd3ea85ddbda4a86fb461405","0xb6ba1931e4e74fd080587688f6db10e830f810d5","0xb9ee1e551f538a464e8f8c41e9904498505b49b0","0x4b01721f0244e7c5b5f63c20942850e447f5a5ee","0xd4bddf5e3d0435d7a6214a0b949c7bb58621f37c","0xf2d4766ad705e3a5c9ba5b0436b473085f82f82f","0xa270f3ad1a7a82e6a3157f12a900f1e25bc4fbfd","0x167a9333bf582556f35bd4d16a7e80e191aa6476","0x1e2fcfd26d36183f1a5d90f0e6296915b02bcb40","0xd0808da05cc71a9f308d330bc9c5c81bbc26fc59","0x0d6b5a54f940bf3d52e438cab785981aaefdf40c","0xd1560b3984b7481cd9a8f40435a53c860187174d","0x521db06bf657ed1d6c98553a70319a8ddbac75a3","0x5baeac0a0417a05733884852aa068b706967e790","0x2984581ece53a4390d1f568673cf693139c97049","0xe17ee7b3c676701c66b395a35f0df4c2276a344e","0xf1c525a488a848b58b95d79da48c21ce434290f7","0x8d76166c22658a144c0211d87abf152e6a2d9d95","0xd3808c5d48903be1490989f3fce2a2b3890e8eb6","0x1fd6267f0d86f62d88172b998390afee2a1f54b6","0xd7b9a9b2f665849c4071ad5af77d8c76aa30fb32","0x94fe3ad91dacba8ec4b82f56ff7c122181f1535d","0x915d7915f2b469bb654a7d903a5d4417cb8ea7df","0x4e5b2e1dc63f6b91cb6cd759936495434c7e972f","0x0021845f4c2604c58f9ba5b7bff58d16a2ab372c","0x2faf487a4414fe77e2327f0bf4ae2a264a776ad2","0x0d0707963952f2fba59dd06f2b425ace40b492fe","0x7793cd85c11a924478d358d49b05b37e91b5810f","0x1c4b70a3968436b9a0a9cf5205c787eb81bb558c","0xd793281182a0e3e023116004778f45c29fc14f19","0x9f5ca0012b9b72e8f3db57092a6f26bf4f13dc69","0xd24400ae8bfebb18ca49be86258a3c749cf46853","0x6fc82a5fe25a5cdb58bc74600a40a69c065263f8","0x61edcdf5bb737adffe5043706e7c5bb1f1a56eea","0x07ee55aa48bb72dcc6e9d78256648910de513eca","0x9fb01a2584aac5aae3fab1ed25f86c5269b32999","0x9c67e141c0472115aa1b98bd0088418be68fd249","0x59a5208b32e627891c389ebafc644145224006e8","0xa12431d0b9db640034b0cdfceef9cce161e62be4","0x274f3c32c90517975e29dfc209a23f315c1e5fc7","0x8533a0bd9310eb63e7cc8e1116c18a3d67b1976a","0xab5c66752a9e8167967685f1450532fb96d5d24f","0xe93381fb4c4f14bda253907b18fad305d799241a","0xfa4b5be3f2f84f56703c42eb22142744e95a2c58","0x46705dfff24256421a05d056c29e81bdc09723b8","0x32598293906b5b17c27d657db3ad2c9b3f3e4265","0x5861b8446a2f6e19a067874c133f04c578928727","0x926fc576b7facf6ae2d08ee2d4734c134a743988","0xeec606a66edb6f497662ea31b5eb1610da87ab5f","0x7ef35bb398e0416b81b019fea395219b65c52164","0x229b5c097f9b35009ca1321ad2034d4b3d5070f6","0xd8a83b72377476d0a66683cde20a8aad0b628713","0x6748f50f686bfbca6fe8ad62b22228b87f31ff2b","0x90e9ddd9d8d5ae4e3763d0cf856c97594dea7325","0x18916e1a2933cb349145a280473a5de8eb6630cb","0x6f48a3e70f0251d1e83a989e62aaa2281a6d5380","0xf056f435ba0cc4fcd2f1b17e3766549ffc404b94","0x137ad9c4777e1d36e4b605e745e8f37b2b62e9c5","0x5401dbf7da53e1c9dbf484e3d69505815f2f5e6e","0x034f854b44d28e26386c1bc37ff9b20c6380b00d","0x0577a79cfc63bbc0df38833ff4c4a3bf2095b404","0x0c6c34cdd915845376fb5407e0895196c9dd4eec","0x794d28ac31bcb136294761a556b68d2634094153","0xfdb16996831753d5331ff813c29a93c76834a0ad","0xfd54078badd5653571726c3370afb127351a6f26","0xb4cd0386d2db86f30c1a11c2b8c4f4185c1dade9","0x4d77a1144dc74f26838b69391a6d3b1e403d0990","0x28ffe35688ffffd0659aee2e34778b0ae4e193ad","0xcac725bef4f114f728cbcfd744a731c2a463c3fc","0xeee28d484628d41a82d01e21d12e2e78d69920da","0x5c985e89dde482efe97ea9f1950ad149eb73829b","0xdc76cd25977e0a5ae17155770273ad58648900d3","0xadb2b42f6bd96f5c65920b9ac88619dce4166f94","0xa8660c8ffd6d578f657b72c0c811284aef0b735e","0x1062a747393198f70f71ec65a582423dba7e5ab3","0x3c11c3025ce387d76c2eddf1493ec55a8cc2a0f7","0x51836a753e344257b361519e948ffcaf5fb8d521","0x9cbadd5ce7e14742f70414a6dcbd4e7bb8712719","0xb1a34309af7f29b4195a6b589737f86e14597ddc","0x2910543af39aba0cd09dbb2d50200b3e800a63d2","0x0a869d79a7052c7f1b55a8ebabbea3420f0d1e13","0xe853c56864a2ebe4576a807d26fdc4a0ada51919","0x267be1c1d684f78cb4f6a176c4911b741e4ffdc0","0xfa52274dd61e1643d2205169732f29114bc240b3","0xe8a0e282e6a3e8023465accd47fae39dd5db010b","0x629a7144235259336ea2694167f3c8b856edd7dc","0x30b71d015f60e2f959743038ce0aaec9b4c1ea44","0x2b5634c42055806a59e9107ed44d43c426e58258","0x689c56aef474df92d44a1b70850f808488f9769c","0xea81ce54a0afa10a027f65503bd52fba83d745b8","0x0861fca546225fbf8806986d211c8398f7457734","0x7891b20c690605f4e370d6944c8a5dbfac5a451c","0x1b6c1a0e20af81b922cb454c3e52408496ee7201","0x8271b2e8cbe29396e9563229030c89679b9470db","0x5e575279bf9f4acf0a130c186861454247394c06","0xedbb72e6b3cf66a792bff7faac5ea769fe810517","0x243bec9256c9a3469da22103891465b47583d9f1","0xe03c23519e18d64f144d2800e30e81b0065c48b5","0x75e89d5979e4f6fba9f97c104c2f0afb3f1dcb88","0x0211f3cedbef3143223d3acf0e589747933e8527","0xae7006588d03bd15d6954e3084a7e644596bc251","0x6cc5f688a315f3dc28a7781717a9a798a59fda7b","0x236f9f97e0e62388479bf9e5ba4889e46b0273c3","0xa7efae728d2936e78bda97dc267687568dd593f3","0x03e3ff995863828554282e80870b489cc31dc8bc","0xaeec6f5aca72f3a005af1b3420ab8c8c7009bac8","0xbd8ef191caa1571e8ad4619ae894e07a75de0c35","0x2bb97b6cf6ffe53576032c11711d59bd056830ee","0xfb90501083a3b6af766c8da35d3dde01eb0d2a68","0xabc74170f3cb8ab352820c39cc1d1e05ce9e41d3","0xd4dcd2459bb78d7a645aa7e196857d421b10d93f","0x32be343b94f860124dc4fee278fdcbd38c102d88","0x209c4784ab1e8183cf58ca33cb740efbf3fc18ef","0xb794f5ea0ba39494ce839613fffba74279579268","0xa910f92acdaf488fa6ef02174fb86208ad7722ba","0xaa9fa73dfe17ecaa2c89b39f0bb2779613c5fc3b","0x2fa2bc2ce6a4f92952921a4caa46b3727d24a1ec","0x31a2feb9b5d3b5f4e76c71d6c92fc46ebb3cb1c1","0x6b71834d65c5c4d8ed158d54b47e6ea4ff4e5437","0x48d466b7c0d32b61e8a82cd2bcf060f7c3f966df","0x0536806df512d6cdde913cf95c9886f65b1d3462","0x8d451ae5ee8f557a9ce7a9d7be8a8cb40002d5cb","0xbd2ec7c608a06fe975dbdca729e84dedb34ecc21","0xc0e30823e5e628df8bc9bf2636a347e1512f0ecb","0x65f9b2e4d7aaeb40ffea8c6f5844d5ad7da257e0","0x36b01066b7fa4a0fdb2968ea0256c848e9135674","0xab11204cfeaccffa63c2d23aef2ea9accdb0a0d5","0x6795cf8eb25585eadc356ae32ac6641016c550f2","0xfbf2173154f7625713be22e0504404ebfe021eae","0x6f803466bcd17f44fa18975bf7c509ba64bf3825","0xead6be34ce315940264519f250d8160f369fa5cd","0xd344539efe31f8b6de983a0cab4fb721fc69c547","0x5ca39c42f4dee3a5ba8fec3ad4902157d48700bf","0x2a048d9a8ffdd239f063b09854976c3049ae659c","0xb8cf411b956b3f9013c1d0ac8c909b086218207c","0x2819c144d5946404c0516b6f817a960db37d4929","0x120a270bbc009644e35f0bb6ab13f95b8199c4ad","0x9e6316f44baeeee5d41a1070516cc5fa47baf227","0x70faa28a6b8d6829a4b1e649d26ec9a2a39ba413","0x563b377a956c80d77a7c613a9343699ad6123911","0xd3273eba07248020bf98a8b560ec1576a612102f","0x3b0bc51ab9de1e5b7b6e34e5b960285805c41736","0xeed16856d551569d134530ee3967ec79995e2051","0x324cc2c9fb379ea7a0d1c0862c3b48ca28d174a4","0x91f6d99b232153cb655ad3e0d05e13ef505f6cd5","0xe855283086fbee485aecf2084345a91424c23954","0xa96b536eef496e21f5432fd258b6f78cf3673f74","0xea3a46bd1dbd0620d80037f70d0bf7c7dc5a837c","0xed8204345a0cf4639d2db61a4877128fe5cf7599","0x3613ef1125a078ef96ffc898c4ec28d73c5b8c52","0x0a73573cf2903d2d8305b1ecb9e9730186a312ae","0x0068eb681ec52dbd9944517d785727310b494575","0xb2cc3cdd53fc9a1aeaf3a68edeba2736238ddc5d","0x1119aaefb02bf12b84d28a5d8ea48ec3c90ef1db","0x2f1233ec3a4930fd95874291db7da9e90dfb2f03","0x390de26d772d2e2005c6d1d24afc902bae37a4bb","0xba826fec90cefdf6706858e5fbafcb27a290fbe0","0x5e032243d507c743b061ef021e2ec7fcc6d3ab89","0xb436c96c6de1f50a160ed307317c275424dbe4f2","0xf5bec430576ff1b82e44ddb5a1c93f6f9d0884f3","0x8f3ab2c3b651382b07a76653d2be9eb4b87e1630","0xd94c9ff168dc6aebf9b6cc86deff54f3fb0afc33","0x42da8a05cb7ed9a43572b5ba1b8f82a0a6e263dc","0x700f6912e5753e91ea3fae877a2374a2db1245d7","0x60d0cc2ae15859f69bf74dadb8ae3bd58434976b") - .map(x=>x.toLowerCase()) - //args(2).split(",").map(x=>x.toLowerCase) - - override def setup(): Unit = view.getVertices().foreach { vertex => - val walletID = vertex.getPropertyValue("id").get.asInstanceOf[String] - if(walletID contains infectedNode) { - vertex.getOrSetState("infected", infectionStartingBlock) - vertex.getOrSetState("infectedBy", "Start") - vertex.getOutEdgesAfter(infectionStartingBlock).foreach { neighbour => - neighbour.send((walletID,neighbour.firstActivityAfter(infectionStartingBlock),vertex.ID())) - } - } - } - - override def analyse(): Unit = - view.getMessagedVertices().foreach { vertex => - try{ - var infectionBlock = infectionStartingBlock - var infector = infectedNode - - val queue = vertex.messageQueue[(String,Long,Long)] - infectionBlock = queue.map(x=>x._2).min - val tuple = queue.filter(x=>x._2==infectionBlock).head //todo check if multiple - infector = tuple._1 - - val walletID = vertex.getPropertyValue("id").get.asInstanceOf[String] - - if(vertex.containsState("infected")){ - if(infectionBlockx.asInstanceOf[Double]).mkString(",")) - vertex.setState("sent",vertex.getInEdge(tuple._3).get.getPropertyValuesAfter("value",infectionBlock).get.map(x=>x.asInstanceOf[Double]).sum) - vertex.setState("infected", infectionBlock) - vertex.setState("infectedBy",infector) - } - else{ - vertex.setState("infected", infectionBlock) - vertex.setState("infectedBy",infector) - vertex.getOutEdgesAfter(infectionBlock).take(100).foreach { neighbour => - neighbour.send((walletID,neighbour.firstActivityAfter(infectionBlock),vertex.ID())) - } - } - } - } - else{ - if(listOfExchanges contains walletID){ - vertex.setState("exchangeHit", true) - vertex.setState("sent",vertex.getInEdge(tuple._3).get.getPropertyValuesAfter("value",infectionBlock).get.map(x=>x.asInstanceOf[Double]).sum) - vertex.setState("infected", infectionBlock) - vertex.setState("infectedBy",infector) - } - else{ - vertex.setState("infected", infectionBlock) - vertex.setState("infectedBy",infector) - vertex.getOutEdgesAfter(infectionBlock).take(100).foreach { neighbour => - neighbour.send((walletID,neighbour.firstActivityAfter(infectionBlock),vertex.ID())) - } - } - } - }catch {case e:Exception => } - } - - - - override def returnResults(): Any = { - val tosend = view - .getVertices() - .map { vertex => - if (vertex.containsState("exchangeHit")) - (vertex.getPropertyValue("id").get.asInstanceOf[String], vertex.getState[Long]("infected"),vertex.getState[String]("infectedBy"),true,vertex.getState[Double]("sent")) - else if (vertex.containsState("infected")) - (vertex.getPropertyValue("id").get.asInstanceOf[String], vertex.getState[Long]("infected"),vertex.getState[String]("infectedBy"),false,0) - else - ("", -1L,"",false,-1) - - } - .filter(f => f._2 >= 0) - println(s"worker $workerID finished") - tosend - } - - override def defineMaxSteps(): Int = 100 - - override def processResults(results: ArrayBuffer[Any], timeStamp: Long, viewCompleteTime: Long): Unit = { - println("howdy") - val endResults = results.asInstanceOf[ArrayBuffer[immutable.ParIterable[(String, Long,String,Boolean,Double)]]].flatten - var data = s"""{"block":$timeStamp,"edges":[""" - for (elem <- endResults) - data+=s"""{"infected":"${elem._1}","block":${elem._2},"infector":"${elem._3}","exchange":"${elem._4}","value":${elem._5}},""" - data+="]}" - //publishData(data) -// writeLines("/opt/docker/output.csv", data, "{\"views\":[") - //print(data) - writeLines("/app/output.csv", data, "") - - //for (elem <- endResults) { writeLines(s"/Users/mirate/Documents/phd/etheroutput/block${timeStamp}.csv", s"${elem._1},${elem._2},${elem._3},", "")} - - } -} diff --git a/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/BitcoinGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/BitcoinGraphBuilder.scala deleted file mode 100644 index 3ac8c333bf..0000000000 --- a/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/BitcoinGraphBuilder.scala +++ /dev/null @@ -1,96 +0,0 @@ -package com.raphtory.examples.blockchain.graphbuilders - -import com.raphtory.core.actors.Router.GraphBuilder -import com.raphtory.core.model.communication._ -import com.raphtory.spouts.blockchain.BitcoinTransaction -import spray.json.JsArray - -class BitcoinGraphBuilder extends GraphBuilder[BitcoinTransaction]{ - - override def parseTuple(tuple:BitcoinTransaction) = { - - val transaction = tuple.transaction - val time = tuple.time - val blockID = tuple.blockID - val block = tuple.block - val timeAsString = time.toString - val timeAsLong = timeAsString.toLong - - val txid = transaction.asJsObject.fields("txid").toString() - val vins = transaction.asJsObject.fields("vin") - val vouts = transaction.asJsObject.fields("vout") - var total: Double = 0 - - for (vout <- vouts.asInstanceOf[JsArray].elements) { - val voutOBJ = vout.asJsObject() - var value = voutOBJ.fields("value").toString - total += value.toDouble - val n = voutOBJ.fields("n").toString - val scriptpubkey = voutOBJ.fields("scriptPubKey").asJsObject() - - var address = "nulldata" - if (scriptpubkey.fields.contains("addresses")) - address = scriptpubkey.fields("addresses").asInstanceOf[JsArray].elements(0).toString - else value = "0" //TODO deal with people burning money - - //println(s"Edge $timeAsLong, ${txid.hashCode}, ${address.hashCode}, $n, $value") - //creates vertex for the receiving wallet - - sendUpdate(VertexAddWithProperties( - msgTime = timeAsLong, - srcID = address.hashCode, - properties = Properties(StringProperty("type", "address"), - StringProperty("address", address)))) - //creates edge between the transaction and the wallet - sendUpdate(EdgeAddWithProperties( - msgTime = timeAsLong, - srcID = txid.hashCode, - dstID = address.hashCode, - properties = Properties(StringProperty("n", n), StringProperty("value", value)))) - - } - sendUpdate(VertexAddWithProperties( - msgTime = timeAsLong, - srcID = txid.hashCode, - properties = Properties( - StringProperty("type", "transaction"), - StringProperty("time", timeAsString), - StringProperty("id", txid), - StringProperty("total", total.toString), - StringProperty("blockhash", blockID.toString), - StringProperty("block", block.toString) - ))) - - if (vins.toString().contains("coinbase")) { - //creates the coingen node //TODO change so only added once - sendUpdate(VertexAddWithProperties( - msgTime = timeAsLong, - srcID = "coingen".hashCode, - properties = Properties(StringProperty("type", "coingen")))) - - - //creates edge between coingen and the transaction - sendUpdate(EdgeAdd(msgTime = timeAsLong, srcID = "coingen".hashCode, dstID = txid.hashCode)) - } else - for (vin <- vins.asInstanceOf[JsArray].elements) { - val vinOBJ = vin.asJsObject() - val prevVout = vinOBJ.fields("vout").toString - val prevtxid = vinOBJ.fields("txid").toString - //no need to create node for prevtxid as should already exist - //creates edge between the prev transaction and current transaction - sendUpdate(EdgeAddWithProperties( - msgTime = timeAsLong, - srcID = prevtxid.hashCode, - dstID = txid.hashCode, - properties = Properties(StringProperty("vout", prevVout)))) - - } - } - -} -//sendCommand(s"""" {"VertexAdd":{ "messageID":$time , "srcID":${address.hashCode}, "properties":{"type":"address", "address":$address} }}"""") -//sendCommand(s"""" {"EdgeAdd":{ "messageID":$time , "srcID":${txid.hashCode} , "dstID":${address.hashCode} , "properties":{"n": $n, "value":$value}}}"""") -//sendCommand(s"""" {"VertexAdd":{ "messageID":$time , "srcID":${txid.hashCode} , "properties":{"type":"transaction", "time":$time, "id:$txid, "total": $total,"blockhash":$blockID}}}"""") -//sendCommand(s"""" {"VertexAdd":{ "messageID":$time , "srcID":${"coingen".hashCode} ,"properties":{"type":"coingen"}}}"""") -//sendCommand(s"""" {"EdgeAdd":{ "messageID":$time , "srcID":${"coingen".hashCode}, "dstID":${txid.hashCode}}}"""") -////sendCommand(s"""" {"EdgeAdd":{ "messageID":$time , "srcID":${prevtxid.hashCode}, "dstID":${txid.hashCode}, "properties":{"vout":$prevVout}}}"""") diff --git a/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChABClus2ClusGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChABClus2ClusGraphBuilder.scala deleted file mode 100644 index 67ec99031e..0000000000 --- a/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChABClus2ClusGraphBuilder.scala +++ /dev/null @@ -1,57 +0,0 @@ -package com.raphtory.examples.blockchain.graphbuilders - -import com.raphtory.core.actors.Router.GraphBuilder -import com.raphtory.core.model.communication.{Type, _} - -class ChABClus2ClusGraphBuilder extends GraphBuilder[String]{ - - override def parseTuple(tuple: String) = { - val dp = formatLine(tuple.split(",").map(_.trim)) - val transactionTime = dp.time - val srcClusterId = dp.srcCluster - val dstClusterId = dp.dstCluster - val transactionId = dp.txid - val btcAmount = dp.amount - val usdAmount = dp.usd - - - sendUpdate(VertexAdd(msgTime = transactionTime, srcID = srcClusterId, Type("Cluster"))) - sendUpdate((VertexAdd(msgTime = transactionTime, srcID = dstClusterId, Type("Cluster")))) - - sendUpdate( - EdgeAddWithProperties(msgTime = transactionTime, - srcID = srcClusterId, - dstID = dstClusterId, - Properties(DoubleProperty("BitCoin", btcAmount), - DoubleProperty("USD", usdAmount), - DoubleProperty("Transaction", transactionId)), - Type("Transfer") - ) - ) - } - - //converts the line into a case class which has all of the data via the correct name and type - def formatLine(line: Array[String]): Datapoint = - Datapoint( - line(1).toDouble / 100000000, //Amount of transaction in BTC - line(2).toLong, //ID of destination cluster - line(3).toLong, //ID of source cluster - line(4).toLong * 1000, //Time of transaction in seconds (milli in Raph) - line(5).toLong, //ID of transaction, can be similar for many records - line(6).toDouble / 100000 //Amount of transaction in USD - - ) - - def longCheck(data: String): Option[Long] = if (data equals "") None else Some(data.toLong) - - - case class Datapoint( - amount: Double, //Amount of transaction in Satoshi - dstCluster: Long, //ID of destination cluster - srcCluster: Long, //ID of source cluster - time: Long, //Time of transaction in seconds - txid: Long, //ID of transaction, can be similar for many records - usd: Double //Amount of transaction in USD - ) - -} diff --git a/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChABClus2ClusGraphBuilderNoprop.scala b/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChABClus2ClusGraphBuilderNoprop.scala deleted file mode 100644 index af35782408..0000000000 --- a/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChABClus2ClusGraphBuilderNoprop.scala +++ /dev/null @@ -1,40 +0,0 @@ -package com.raphtory.examples.blockchain.graphbuilders - -import com.raphtory.core.actors.Router.GraphBuilder -import com.raphtory.core.model.communication.{Type, _} - -class ChABClus2ClusGraphBuilderNoprop extends GraphBuilder[String] { - - override def parseTuple(tuple: String) = { - val dp = formatLine(tuple.split(",").map(_.trim)) - val transactionTime = dp.time - val srcClusterId = dp.srcCluster - val dstClusterId = dp.dstCluster - sendUpdate(VertexAdd(msgTime = transactionTime, srcID = srcClusterId, Type("Cluster"))) - sendUpdate(VertexAdd(msgTime = transactionTime, srcID = dstClusterId, Type("Cluster"))) - sendUpdate(EdgeAdd(msgTime = transactionTime, srcID = srcClusterId, dstID = dstClusterId, Type("Transfer"))) - } - - //converts the line into a case class which has all of the data via the correct name and type - def formatLine(line: Array[String]): Datapoint = - Datapoint( - line(1).toDouble / 100000000, //Amount of transaction in BTC - line(2).toLong, //ID of destination cluster - line(3).toLong, //ID of source cluster - line(4).toLong * 1000, //Time of transaction in seconds (milli in Raph) - line(5).toLong, //ID of transaction, can be similar for many records - line(6).toDouble / 100000 //Amount of transaction in USD - ) - - def longCheck(data: String): Option[Long] = if (data equals "") None else Some(data.toLong) - - - case class Datapoint( - amount: Double, //Amount of transaction in Satoshi - dstCluster: Long, //ID of destination cluster - srcCluster: Long, //ID of source cluster - time: Long, //Time of transaction in seconds - txid: Long, //ID of transaction, can be similar for many records - usd: Double //Amount of transaction in USD - ) -} diff --git a/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChABClus2ClusRouter.scala.save b/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChABClus2ClusRouter.scala.save deleted file mode 100644 index 6d716f3b2b..0000000000 --- a/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChABClus2ClusRouter.scala.save +++ /dev/null @@ -1,60 +0,0 @@ -package com.raphtory.examples.blockchain.routers - -import com.raphtory.core.components.Router.RouterWorker -import com.raphtory.core.model.communication.Type -import com.raphtory.core.model.communication._ - -class ChABClus2ClusRouter(override val routerId: Int,override val workerID:Int, override val initialManagerCount: Int) extends RouterWorker { - val JUMP = 50 - - def parseTuple(record: Any): Unit = { - val lines = record.asInstanceOf[String].split("\n").map(_.trim) - for(line <- lines){ - val dp = formatLine(line.split(",").map(_.trim)) - val transactionTime = dp.time - val srcClusterId = dp.srcCluster - val dstClusterId = dp.dstCluster - val transactionId = dp.txid - val btcAmount = dp.amount - val usdAmount = dp.usd - - sendGraphUpdate(VertexAdd(msgTime = transactionTime, srcID = srcClusterId, Type("Cluster"))) - sendGraphUpdate(VertexAdd(msgTime = transactionTime, srcID = dstClusterId, Type("Cluster"))) - - sendGraphUpdate( - EdgeAddWithProperties(msgTime = transactionTime, - srcID = srcClusterId, - dstID = dstClusterId, - Properties(DoubleProperty("BitCoin", btcAmount), - DoubleProperty("USD", usdAmount), - DoubleProperty("Transaction", transactionId)), - Type("Transfer") - ) - ) - } - - } - - //converts the line into a case class which has all of the data via the correct name and type - def formatLine(line: Array[String]): Datapoint = - Datapoint( - line(1).toDouble / 100000000, //Amount of transaction in BTC - line(2).toLong, //ID of destination cluster - line(3).toLong, //ID of source cluster - line(4).toLong * 1000, //Time of transaction in seconds (milli in Raph) - line(5).toLong, //ID of transaction, can be similar for many records - line(6).toDouble / 100000 //Amount of transaction in USD - ) - - def longCheck(data: String): Option[Long] = if (data equals "") None else Some(data.toLong) - - - case class Datapoint( - amount: Double, //Amount of transaction in Satoshi - dstCluster: Long, //ID of destination cluster - srcCluster: Long, //ID of source cluster - time: Long, //Time of transaction in seconds - txid: Long, //ID of transaction, can be similar for many records - usd: Double //Amount of transaction in USD - ) -} diff --git a/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChainalysisABGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChainalysisABGraphBuilder.scala deleted file mode 100644 index 47dcb2d76d..0000000000 --- a/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/ChainalysisABGraphBuilder.scala +++ /dev/null @@ -1,62 +0,0 @@ -package com.raphtory.examples.blockchain.graphbuilders - -import com.raphtory.core.actors.Router.GraphBuilder -import com.raphtory.core.model.communication.{Type, _} - -class ChainalysisABGraphBuilder extends GraphBuilder[String] { - - override def parseTuple(tuple: String) = { - val dp = formatLine(tuple.split(",").map(_.trim)) - val transactionTime = dp.time - val srcClusterId = dp.srcCluster - val dstClusterId = dp.dstCluster - val transactionId = dp.txid - val btcAmount = dp.amount - val usdAmount = dp.usd - - sendUpdate(VertexAdd(msgTime = transactionTime, srcID = srcClusterId, Type("Cluster"))) - sendUpdate(VertexAdd(msgTime = transactionTime, srcID = dstClusterId, Type("Cluster"))) - sendUpdate(VertexAdd(msgTime = transactionTime, srcID = transactionId, Type("Transaction"))) - - sendUpdate( - EdgeAddWithProperties(msgTime = transactionTime, - srcID = srcClusterId, - dstID = transactionId, - Properties(DoubleProperty("BitCoin", btcAmount), DoubleProperty("USD",usdAmount)), - Type("Incoming Payment") - ) - ) - sendUpdate( - EdgeAddWithProperties(msgTime = transactionTime, - srcID = transactionId, - dstID = dstClusterId, - Properties(DoubleProperty("BitCoin", btcAmount), DoubleProperty("USD",usdAmount)), - Type("Outgoing Payment") - ) - ) - } - - //converts the line into a case class which has all of the data via the correct name and type - def formatLine(line: Array[String]): Datapoint = - Datapoint( - line(1).toDouble / 100000000, //Amount of transaction in BTC - line(2).toLong, //ID of destination cluster - line(3).toLong, //ID of source cluster - line(4).toLong * 1000, //Time of transaction in seconds (milli in Raph) - line(5).toLong, //ID of transaction, can be similar for many records - line(6).toDouble / 100000 //Amount of transaction in USD - - ) - - def longCheck(data: String): Option[Long] = if (data equals "") None else Some(data.toLong) - - - case class Datapoint( - amount: Double, //Amount of transaction in Satoshi - dstCluster: Long, //ID of destination cluster - srcCluster: Long, //ID of source cluster - time: Long, //Time of transaction in seconds - txid: Long, //ID of transaction, can be similar for many records - usd: Double //Amount of transaction in USD - ) -} diff --git a/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/DashcoinGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/DashcoinGraphBuilder.scala deleted file mode 100644 index 6285724b0d..0000000000 --- a/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/DashcoinGraphBuilder.scala +++ /dev/null @@ -1,112 +0,0 @@ -package com.raphtory.examples.blockchain.graphbuilders - -import com.raphtory.core.actors.Router.GraphBuilder -import com.raphtory.core.model.communication._ -import com.raphtory.spouts.blockchain.BitcoinTransaction -import spray.json.JsArray - -import scala.util.hashing.MurmurHash3 - -class DashcoinGraphBuilder extends GraphBuilder[BitcoinTransaction] { - - override def parseTuple(tuple: BitcoinTransaction) = { - val transaction = tuple.transaction - val time = tuple.time - val blockID = tuple.blockID - val block = tuple.block - val timeAsString = time.toString - val timeAsLong = (timeAsString.toLong) * 1000 - - val txid = transaction.asJsObject.fields("txid").toString() - val vins = transaction.asJsObject.fields("vin") - val vouts = transaction.asJsObject.fields("vout") - val locktime = transaction.asJsObject.fields("locktime") - val version = transaction.asJsObject.fields("version") - var total: Double = 0 - for (vout <- vouts.asInstanceOf[JsArray].elements) { - val voutOBJ = vout.asJsObject() - var value = voutOBJ.fields("value").toString - total += value.toDouble - val n = voutOBJ.fields("n").toString - val scriptpubkey = voutOBJ.fields("scriptPubKey").asJsObject() - - var address = "nulldata" - val outputType = scriptpubkey.fields("type").toString - - if (scriptpubkey.fields.contains("addresses")) - address = scriptpubkey.fields("addresses").asInstanceOf[JsArray].elements(0).toString - else value = "0" //TODO deal with people burning money - - //creates vertex for the receiving wallet - sendUpdate( - VertexAddWithProperties( - msgTime = timeAsLong, - srcID = MurmurHash3.stringHash(address), - properties = Properties( - //("type","address"), - StringProperty("address", address), - StringProperty("outputType", outputType) - ) - ) - ) - //creates edge between the transaction and the wallet - sendUpdate( - EdgeAddWithProperties( - msgTime = timeAsLong, - srcID = MurmurHash3.stringHash(txid), - dstID = MurmurHash3.stringHash(address), - properties = Properties(StringProperty("n", n), StringProperty("value", value)) - ) - ) - - } - sendUpdate( - VertexAddWithProperties( - msgTime = timeAsLong, - srcID = MurmurHash3.stringHash(txid), - properties = Properties( - //("type","transaction"), - StringProperty("id", txid), - StringProperty("total", total.toString), - //("lockTime",locktime.toString), - //("version",version.toString), - // ("blockhash",blockID.toString), - StringProperty("block", block.toString) - ) - ) - ) - - if (vins.toString().contains("coinbase")) { - //creates the coingen node - //toPartitionManager(VertexAddWithProperties(msgTime = timeAsLong, srcID = MurmurHash3.stringHash("coingen"), properties = Map[String,String](("type","coingen")))) - sendUpdate(VertexAdd(msgTime = timeAsLong, srcID = MurmurHash3.stringHash("coingen"))) - - //creates edge between coingen and the transaction - sendUpdate( - EdgeAdd( - msgTime = timeAsLong, - srcID = MurmurHash3.stringHash("coingen"), - dstID = MurmurHash3.stringHash(txid) - ) - ) - } else - for (vin <- vins.asInstanceOf[JsArray].elements) { - val vinOBJ = vin.asJsObject() - val prevVout = vinOBJ.fields("vout").toString - val prevtxid = vinOBJ.fields("txid").toString - val sequence = vinOBJ.fields("sequence").toString - //no need to create node for prevtxid as should already exist - //creates edge between the prev transaction and current transaction - sendUpdate( - EdgeAddWithProperties( - msgTime = timeAsLong, - srcID = MurmurHash3.stringHash(prevtxid), - dstID = MurmurHash3.stringHash(txid), - properties = Properties(StringProperty("vout", prevVout), StringProperty("sequence", sequence)) - ) - ) - - } - } - -} diff --git a/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/EthereumGethGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/EthereumGethGraphBuilder.scala deleted file mode 100644 index d6de53b74c..0000000000 --- a/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/EthereumGethGraphBuilder.scala +++ /dev/null @@ -1,58 +0,0 @@ -package com.raphtory.examples.blockchain.graphbuilders - -import com.raphtory.core.actors.Router.GraphBuilder -import com.raphtory.core.model.communication._ -class EthereumGethGraphBuilder extends GraphBuilder[String] { - def hexToInt(hex: String) = Integer.parseInt(hex.drop(2), 16) - - - - override def parseTuple(tuple: String) = { - print(tuple) - val transaction = tuple.split(",") - val blockNumber = hexToInt(transaction(0)) - - val from = transaction(1).replaceAll("\"", "").toLowerCase - val to = transaction(2).replaceAll("\"", "").toLowerCase - val sent = transaction(3).replaceAll("\"", "") - val sourceNode = assignID(from) //hash the id to get a vertex ID - val destinationNode = assignID(to) //hash the id to get a vertex ID - - - sendUpdate( - VertexAddWithProperties(blockNumber, sourceNode, properties = Properties(ImmutableProperty("id", from))) - ) - sendUpdate( - VertexAddWithProperties(blockNumber, destinationNode, properties = Properties(ImmutableProperty("id", to))) - ) - sendUpdate( - EdgeAddWithProperties( - blockNumber, - sourceNode, - destinationNode, - properties = Properties(StringProperty("value", sent)) - ) - ) - } -} - -//{ -// "jsonrpc": "2.0", -// "id": 100, -// "result": { -// "blockHash": "0x4e3a3754410177e6937ef1f84bba68ea139e8d1a2258c5f85db9f1cd715a1bdd", -// "blockNumber": "0xb443", -// "from": "0xa1e4380a3b1f749673e270229993ee55f35663b4", -// "gas": "0x5208", -// "gasPrice": "0x2d79883d2000", -// "hash": "0x5c504ed432cb51138bcf09aa5e8a410dd4a1e204ef84bfed1be16dfba1b22060", -// "input": "0x", -// "nonce": "0x0", -// "to": "0x5df9b87991262f6ba471f09758cde1c0fc1de734", -// "transactionIndex": "0x0", -// "value": "0x7a69", -// "v": "0x1c", -// "r": "0x88ff6cf0fefd94db46111149ae4bfc179e9b94721fffd821d38d16464b3f71d0", -// "s": "0x45e0aff800961cfce805daef7016b9b675c137a6a41a548f7b60a3484c06a33a" -// } -//} diff --git a/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/EthereumKafkaGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/EthereumKafkaGraphBuilder.scala deleted file mode 100644 index 5177ad12c1..0000000000 --- a/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/EthereumKafkaGraphBuilder.scala +++ /dev/null @@ -1,68 +0,0 @@ -package com.raphtory.examples.blockchain.graphbuilders - -import com.raphtory.core.actors.Router.GraphBuilder -import com.raphtory.core.model.communication._ -class EthereumKafkaGraphBuilder extends GraphBuilder[String] { - def hexToInt(hex: String) = Integer.parseInt(hex.drop(2), 16) - override def parseTuple(tuple: String):Unit = { - - val transaction = tuple.split(",") - - if(transaction(1).equals("block_number")) return - - - print(transaction) -// if(transaction(2).equals("block_number")) return -// val blockNumber = transaction(2).toInt -// -// val from = transaction(1).replaceAll("\"", "").toLowerCase -// val to = transaction(2).replaceAll("\"", "").toLowerCase -// val sent = transaction(5).replaceAll("\"", "") - - if(transaction(2).equals("block_number")) return - val blockNumber = transaction(2).toInt - - val from = transaction(4).replaceAll("\"", "").toLowerCase - val to = transaction(5).replaceAll("\"", "").toLowerCase - val sent = transaction(9).replaceAll("\"", "") - - val sourceNode = assignID(from) //hash the id to get a vertex ID - val destinationNode = assignID(to) //hash the id to get a vertex ID - - sendUpdate( - VertexAddWithProperties(blockNumber, sourceNode, properties = Properties(ImmutableProperty("id", from))) - ) - sendUpdate( - VertexAddWithProperties(blockNumber, destinationNode, properties = Properties(ImmutableProperty("id", to))) - ) - sendUpdate( - EdgeAddWithProperties( - blockNumber, - sourceNode, - destinationNode, - properties = Properties(StringProperty("value", sent)) - ) - ) - } -} - -//{ -// "jsonrpc": "2.0", -// "id": 100, -// "result": { -// "blockHash": "0x4e3a3754410177e6937ef1f84bba68ea139e8d1a2258c5f85db9f1cd715a1bdd", -// "blockNumber": "0xb443", -// "from": "0xa1e4380a3b1f749673e270229993ee55f35663b4", -// "gas": "0x5208", -// "gasPrice": "0x2d79883d2000", -// "hash": "0x5c504ed432cb51138bcf09aa5e8a410dd4a1e204ef84bfed1be16dfba1b22060", -// "input": "0x", -// "nonce": "0x0", -// "to": "0x5df9b87991262f6ba471f09758cde1c0fc1de734", -// "transactionIndex": "0x0", -// "value": "0x7a69", -// "v": "0x1c", -// "r": "0x88ff6cf0fefd94db46111149ae4bfc179e9b94721fffd821d38d16464b3f71d0", -// "s": "0x45e0aff800961cfce805daef7016b9b675c137a6a41a548f7b60a3484c06a33a" -// } -//} diff --git a/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/EthereumTransactionGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/EthereumTransactionGraphBuilder.scala deleted file mode 100644 index d57f61f559..0000000000 --- a/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/EthereumTransactionGraphBuilder.scala +++ /dev/null @@ -1,43 +0,0 @@ -package com.raphtory.examples.blockchain.graphbuilders - -import com.raphtory.core.actors.Router.GraphBuilder -import com.raphtory.core.model.communication.{EdgeAddWithProperties, Properties, StringProperty, VertexAddWithProperties} - -import scala.util.hashing.MurmurHash3 - -class EthereumTransactionGraphBuilder extends GraphBuilder[String]{ - - override def parseTuple(tuple: String):Unit = { - val components = tuple.drop(1).dropRight(1).split(",") - val creationDate = components(3).toLong * 1000 //seconds to miliseconds - val sourceNode = MurmurHash3.stringHash(components(0)) //hash the id to get a vertex ID - sendUpdate( - VertexAddWithProperties(creationDate, sourceNode, Properties(StringProperty("id", components(0)))) - ) //create the source node and add the wallet ID as a property - if (components(1).nonEmpty) { //money being sent to an actual user - val targetNode = MurmurHash3.stringHash(components(1)) //hash the id of the to wallet to get a vertex ID - sendUpdate( - VertexAddWithProperties(creationDate, targetNode, Properties(StringProperty("id", components(1)))) - ) //create the destination vertex - sendUpdate( - EdgeAddWithProperties( - creationDate, - sourceNode, - targetNode, - Properties(StringProperty("id", components(2))) - ) - ) //create the edge between them adding the value as a property - } else { //burnt cash - val targetNode = MurmurHash3.stringHash("null") - sendUpdate(VertexAddWithProperties(creationDate, targetNode, Properties(StringProperty("id", "null")))) - sendUpdate( - EdgeAddWithProperties( - creationDate, - sourceNode, - targetNode, - Properties(StringProperty("value", components(2))) - ) - ) - } - } -} diff --git a/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/FirehoseKafkaGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/FirehoseKafkaGraphBuilder.scala deleted file mode 100644 index 1cee339848..0000000000 --- a/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/FirehoseKafkaGraphBuilder.scala +++ /dev/null @@ -1,59 +0,0 @@ -package com.raphtory.examples.blockchain.graphbuilders - -import com.raphtory.core.actors.Router.GraphBuilder -import com.raphtory.core.model.communication._ - -import scala.util.Random -class FirehoseKafkaGraphBuilder extends GraphBuilder[String] { - var DELETEPERCENT = System.getenv().getOrDefault("ETHER_DELETE_PERCENT", "0").trim.toDouble/100 - var DELETESEED = System.getenv().getOrDefault("ETHER_DELETE_SEED", "123").trim.toInt - val random = new Random(DELETESEED) - def hexToInt(hex: String) = Integer.parseInt(hex.drop(2), 16) - - override def parseTuple(tuple: String): Unit = { - //if(value.toString.contains("0xa09871aeadf4994ca12f5c0b6056bbd1d343c029")) println(value.toString) - val transaction = tuple.split(",") - if(transaction(1).equals("block_number")) return - val blockNumber = transaction(2).toInt - - val from = transaction(4).replaceAll("\"", "").toLowerCase - val to = transaction(5).replaceAll("\"", "").toLowerCase - val sent = (BigDecimal(transaction(6).replaceAll("\"", ""))/BigDecimal("1000000000000000000")).toDouble - val sourceNode = assignID(from) //hash the id to get a vertex ID - val destinationNode = assignID(to) //hash the id to get a vertex ID - //if(from.contains("0xa09871aeadf4994ca12f5c0b6056bbd1d343c029".toLowerCase())) println(from) - //if(to.contains("0xa09871aeadf4994ca12f5c0b6056bbd1d343c029".toLowerCase())) println(to) - sendUpdate(VertexAddWithProperties(blockNumber, sourceNode, properties = Properties(ImmutableProperty("id", from)))) -// if(random.nextDouble()<=DELETEPERCENT) -// sendGraphUpdate(VertexDelete(blockNumber+1,sourceNode)) - - sendUpdate(VertexAddWithProperties(blockNumber, destinationNode, properties = Properties(ImmutableProperty("id", to)))) -// if(random.nextDouble()<=DELETEPERCENT) -// sendGraphUpdate(VertexDelete(blockNumber+1,destinationNode)) - - sendUpdate(EdgeAddWithProperties(blockNumber, sourceNode, destinationNode,properties = Properties(DoubleProperty("value", sent)))) -// if(random.nextDouble()<=DELETEPERCENT) -// sendGraphUpdate(EdgeDelete(blockNumber+1,sourceNode,destinationNode)) - } -} - -//{ -// "jsonrpc": "2.0", -// "id": 100, -// "result": { -// "blockHash": "0x4e3a3754410177e6937ef1f84bba68ea139e8d1a2258c5f85db9f1cd715a1bdd", -// "blockNumber": "0xb443", -// "from": "0xa1e4380a3b1f749673e270229993ee55f35663b4", -// "gas": "0x5208", -// "gasPrice": "0x2d79883d2000", -// "hash": "0x5c504ed432cb51138bcf09aa5e8a410dd4a1e204ef84bfed1be16dfba1b22060", -// "input": "0x", -// "nonce": "0x0", -// "to": "0x5df9b87991262f6ba471f09758cde1c0fc1de734", -// "transactionIndex": "0x0", -// "value": "0x7a69", -// "v": "0x1c", -// "r": "0x88ff6cf0fefd94db46111149ae4bfc179e9b94721fffd821d38d16464b3f71d0", -// "s": "0x45e0aff800961cfce805daef7016b9b675c137a6a41a548f7b60a3484c06a33a" -// } -//} diff --git a/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/LitecoinGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/LitecoinGraphBuilder.scala deleted file mode 100644 index 3fe58183b4..0000000000 --- a/mainproject/src/main/scala/com/raphtory/examples/blockchain/graphbuilders/LitecoinGraphBuilder.scala +++ /dev/null @@ -1,120 +0,0 @@ -package com.raphtory.examples.blockchain.graphbuilders - -import com.raphtory.core.actors.Router.GraphBuilder -import com.raphtory.core.model.communication._ -import com.raphtory.spouts.blockchain.BitcoinTransaction -import spray.json.JsArray - -import scala.util.hashing.MurmurHash3 - - -class LitecoinGraphBuilder extends GraphBuilder[BitcoinTransaction]{ - - override def parseTuple(tuple:BitcoinTransaction) = { - val transaction = tuple.transaction - val time = tuple.time - val blockID = tuple.blockID - val block = tuple.block - val timeAsString = time.toString - val timeAsLong = (timeAsString.toLong) * 1000 - - val txid = transaction.asJsObject.fields("txid").toString() - val vins = transaction.asJsObject.fields("vin") - val vouts = transaction.asJsObject.fields("vout") - val locktime = transaction.asJsObject.fields("locktime") - val version = transaction.asJsObject.fields("version") - var total: Double = 0 - for (vout <- vouts.asInstanceOf[JsArray].elements) { - val voutOBJ = vout.asJsObject() - var value = voutOBJ.fields("value").toString - total += value.toDouble - val n = voutOBJ.fields("n").toString - val scriptpubkey = voutOBJ.fields("scriptPubKey").asJsObject() - - var address = "nulldata" - val outputType = scriptpubkey.fields("type").toString - - if (scriptpubkey.fields.contains("addresses")) - address = scriptpubkey.fields("addresses").asInstanceOf[JsArray].elements(0).toString - else value = "0" //TODO deal with people burning money - - //creates vertex for the receiving wallet - sendUpdate( - VertexAddWithProperties( - msgTime = timeAsLong, - srcID = MurmurHash3.stringHash(address), - Properties( - StringProperty("type", "address"), - StringProperty("address", address), - StringProperty("outputType", outputType) - ) - ) - ) - //creates edge between the transaction and the wallet - sendUpdate( - EdgeAddWithProperties( - msgTime = timeAsLong, - srcID = MurmurHash3.stringHash(txid), - dstID = MurmurHash3.stringHash(address), - Properties(StringProperty("n", n), StringProperty("value", value)) - ) - ) - - } - sendUpdate( - VertexAddWithProperties( - msgTime = timeAsLong, - srcID = MurmurHash3.stringHash(txid), - Properties( - StringProperty("type", "transaction"), - StringProperty("time", timeAsString), - StringProperty("id", txid), - StringProperty("total", total.toString), - StringProperty("lockTime", locktime.toString), - StringProperty("version", version.toString), - StringProperty("blockhash", blockID.toString), - StringProperty("block", block.toString) - ) - ) - ) - - if (vins.toString().contains("coinbase")) { - //creates the coingen node - sendUpdate( - VertexAddWithProperties( - msgTime = timeAsLong, - srcID = MurmurHash3.stringHash("coingen"), - Properties(StringProperty("type", "coingen")) - ) - ) - - //creates edge between coingen and the transaction - sendUpdate( - EdgeAdd( - msgTime = timeAsLong, - srcID = MurmurHash3.stringHash("coingen"), - dstID = MurmurHash3.stringHash(txid) - ) - ) - } else { - for (vin <- vins.asInstanceOf[JsArray].elements) { - val vinOBJ = vin.asJsObject() - val prevVout = vinOBJ.fields("vout").toString - val prevtxid = vinOBJ.fields("txid").toString - val sequence = vinOBJ.fields("sequence").toString - //no need to create node for prevtxid as should already exist - //creates edge between the prev transaction and current transaction - sendUpdate( - EdgeAddWithProperties( - msgTime = timeAsLong, - srcID = MurmurHash3.stringHash(prevtxid), - dstID = MurmurHash3.stringHash(txid), - Properties(StringProperty("vout", prevVout), StringProperty("sequence", sequence)) - ) - ) - } - - } - } - -} diff --git a/mainproject/src/main/scala/com/raphtory/examples/citationNetwork/CitationGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/citationNetwork/CitationGraphBuilder.scala deleted file mode 100644 index 74a9fc6993..0000000000 --- a/mainproject/src/main/scala/com/raphtory/examples/citationNetwork/CitationGraphBuilder.scala +++ /dev/null @@ -1,40 +0,0 @@ -package com.raphtory.examples.citationNetwork - -import com.raphtory.core.actors.Router.GraphBuilder -import com.raphtory.core.model.communication._ - -class CitationGraphBuilder extends GraphBuilder[String] { - override def parseTuple(tuple: String) = { - val fileLine = tuple.split(",").map(_.trim) //take the tuple and split on , as we are only interested in the first 4 fields - // title_paper,year,volume,title,pages,number,journal,author,ENTRYTYPE,ID - val sourceTitle = fileLine(0) - val destinationTitle = fileLine(3) - val destinationYear = fileLine(1).toLong - val sourceYear = 2020 //toBeCreated - - val sourceID = assignID(sourceTitle) - val destinationID = assignID(destinationTitle) - //create sourceNode - sendUpdate(VertexAddWithProperties( - sourceYear, //when it happened ?? - sourceID, // the id of the node - Properties(ImmutableProperty("title", sourceTitle)), //properties for the node - Type("Publication")) //node type - ) - //create destinationNode - sendUpdate(VertexAddWithProperties( - destinationYear, - destinationID, - Properties(ImmutableProperty("title", destinationTitle)), - Type("Publication")) - ) - - //create edge - sendUpdate(EdgeAdd( - sourceYear, //time of edge ?? - sourceID, //source of edge - destinationID, //destination of edge - Type("Cited") // edge type - )) - } -} diff --git a/mainproject/src/main/scala/com/raphtory/examples/gab/analysis/GabMostUsedTopics.scala b/mainproject/src/main/scala/com/raphtory/examples/gab/analysis/GabMostUsedTopics.scala deleted file mode 100644 index 377c4e3b98..0000000000 --- a/mainproject/src/main/scala/com/raphtory/examples/gab/analysis/GabMostUsedTopics.scala +++ /dev/null @@ -1,58 +0,0 @@ -package com.raphtory.examples.gab.analysis - -import com.raphtory.api.Analyser - -import scala.collection.mutable.ArrayBuffer - -class GabMostUsedTopics(args:Array[String]) extends Analyser(args){ - private var epsilon = 1 - private val dumplingFactor = 0.85f - private var firstStep = true - - override def setup(): Unit = {} - - override def analyse(): Unit = { - //println("Analyzing") - var results = ArrayBuffer[(String, Int, String)]() - view.getVertices().foreach { vertex => - if (vertex.getPropertyValue("type").getOrElse("no type").equals("topic")) { - val ingoingNeighbors = vertex.getIncEdges.size - results.synchronized { - vertex.getPropertyValue("id") match { - case None => - case Some(id) => - results +:= (id.toString, ingoingNeighbors, vertex - .getPropertyValue("title") - .getOrElse("no title") - .toString) - } - if (results.size > 10) - results = results.sortBy(_._2)(Ordering[Int].reverse).take(10) - } - } - } - //println("Sending step end") - results - } - - override def defineMaxSteps(): Int = 1 - - override def processResults(results: ArrayBuffer[Any], timeStamp: Long, viewCompleteTime: Long): Unit = { - println() - println("Current top topics") - results - .asInstanceOf[ArrayBuffer[ArrayBuffer[(String, Int, String)]]] - .flatten - .sortBy(f => f._2)(Ordering[Int].reverse) - .foreach(topic => println(s"Topic: ${topic._3} with ID ${topic._1} and total uses of ${topic._2}")) - println() - } - override def processWindowResults( - results: ArrayBuffer[Any], - timestamp: Long, - windowSize: Long, - viewCompleteTime: Long - ): Unit = {} - - override def returnResults(): Any = ??? -} diff --git a/mainproject/src/main/scala/com/raphtory/examples/gab/datasources/GabMongoSource.scala b/mainproject/src/main/scala/com/raphtory/examples/gab/datasources/GabMongoSource.scala deleted file mode 100644 index 7b93d296f1..0000000000 --- a/mainproject/src/main/scala/com/raphtory/examples/gab/datasources/GabMongoSource.scala +++ /dev/null @@ -1,61 +0,0 @@ -package com.raphtory.examples.gab.datasources - -import akka.actor.Cancellable -import ch.qos.logback.classic.Level -import com.mongodb.casbah.Imports.{MongoConnection, _} -import com.raphtory.core.actors.Spout.Spout -import org.slf4j.LoggerFactory - -import scala.collection.mutable -import scala.language.postfixOps - -final class GabMongoSpout extends Spout[String] { - - //private val redis = new RedisClient("moe", 6379) - //private val redisKey = "gab-posts" - private var sched: Cancellable = null - - //val options: MongoClientOptions = MongoClientOptions.builder.addCommandListener(new LoggingClusterListener).build() - //ddClusterListener(new LoggingClusterListener).build - private val mongoConn = MongoConnection("138.37.32.67", 27017) - private val mongoColl = mongoConn("gab")("posts") - private var window = 1000 - private var postMin = 0 - private var postMax = 1001 - - - - val root = LoggerFactory.getLogger(org.slf4j.Logger.ROOT_LOGGER_NAME).asInstanceOf[ch.qos.logback.classic.Logger] - root.setLevel(Level.ERROR) - - var queue = mutable.Queue[Option[String]]() - - override def generateData(): Option[String] = { - if(queue.isEmpty) - getNextPosts() - queue.dequeue() - - } - - - private def getNextPosts() = { - - for (x <- mongoColl.find("_id" $lt postMax $gt postMin)) - try { - val data = x.get("data").toString.drop(2).dropRight(1).replaceAll("""\\"""", "").replaceAll("""\\""", "") - queue += Some(data) - } catch { - case e: Throwable => - println("Cannot parse record") - } - postMin += window - postMax += window - } - - override def setupDataSource(): Unit = {} - - - - override def closeDataSource(): Unit = {} -} -//redis-server --dir /home/moe/ben/gab --dbfilename gab.rdb --daemonize yes diff --git a/mainproject/src/main/scala/com/raphtory/examples/gab/graphbuilders/GabPostGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/gab/graphbuilders/GabPostGraphBuilder.scala deleted file mode 100644 index 8fb279cede..0000000000 --- a/mainproject/src/main/scala/com/raphtory/examples/gab/graphbuilders/GabPostGraphBuilder.scala +++ /dev/null @@ -1,52 +0,0 @@ -package com.raphtory.examples.gab.graphbuilders - -import java.text.SimpleDateFormat - -import com.raphtory.core.actors.Router.GraphBuilder -import com.raphtory.core.model.communication._ - -// The lines sent by the Gab mining spout are read and processed accordingly. -//In this router we needed to transform the data that was sent by the spout by turning it into a epoch value (long value) -// in order to be accepted by Raphtory to create the corresponding entity. -// for the gab dataset, a filter is applied to exclude those lines in where the parent post id ir the parent user id -// is equal to -1. Columns 1 and 4 correspond to the postId and parentPostid in the file. -// Then either the vertex or the edge are created accordingly. - -class GabPostGraphBuilder extends GraphBuilder[String] { - - override def parseTuple(tuple: String) = { - val fileLine = tuple.split(";").map(_.trim) - //user wise -// val sourceNode=fileLine(2).toInt -// val targetNode=fileLine(5).toInt - //comment wise - val sourceNode = fileLine(1).toInt - val targetNode = fileLine(4).toInt - - if (targetNode > 0) { - val creationDate = dateToUnixTime(timestamp = fileLine(0).slice(0, 19)) - sendUpdate(VertexAdd(creationDate, sourceNode)) - sendUpdate(VertexAdd(creationDate, targetNode)) - sendUpdate(EdgeAdd(creationDate, sourceNode, targetNode)) - } - } - - def dateToUnixTime(timestamp: => String): Long = { - //if(timestamp == null) return null; - // println("TIME FUNC: "+ timestamp) - //val sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'+'HH:mm") - val sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss") - //println(sdf) - val dt = sdf.parse(timestamp) - //println(dt) - val epoch = dt.getTime - //println("*******EPOCH: "+epoch) - epoch -//======= -// val epoch = dt.getTime -// // println(epoch) -// epoch -//>>>>>>> upstream/master - - } -} diff --git a/mainproject/src/main/scala/com/raphtory/examples/gab/graphbuilders/GabRawGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/gab/graphbuilders/GabRawGraphBuilder.scala deleted file mode 100644 index 7de92e95ed..0000000000 --- a/mainproject/src/main/scala/com/raphtory/examples/gab/graphbuilders/GabRawGraphBuilder.scala +++ /dev/null @@ -1,133 +0,0 @@ -package com.raphtory.examples.gab.graphbuilders - -import java.time.OffsetDateTime - -import com.raphtory.core.actors.Router.GraphBuilder -import com.raphtory.core.model.communication.{EdgeAddWithProperties, Properties, StringProperty, VertexAddWithProperties} -import com.raphtory.examples.gab.rawgraphmodel.GabPost -import spray.json._ - -/** - * The Graph Manager is the top level actor in this system (under the stream) - * which tracks all the graph partitions - passing commands processed by the 'command processor' actors - * to the correct partition - */ -/** - * The Command Processor takes string message from Kafka and translates them into - * the correct case Class which can then be passed to the graph manager - * which will then pass it to the graph partition dealing with the associated vertex - */ -final class GabRawGraphBuilder extends GraphBuilder[String] { - - import com.raphtory.examples.gab.rawgraphmodel.GabJsonProtocol._ - - private val nullStr = "null" - - override def parseTuple(tuple: String) = { - try { - val command = tuple - val post = command.parseJson.convertTo[GabPost] - sendPostToPartitions(post) - } catch { - case e: Exception => println("Could not parse post") - } - - //val parsedOBJ: Command = command.parseJson.convertTo[Command] - //val manager = getManager(parsedOBJ.value.srcId, getManagerCount) - //mediator ! DistributedPubSubMediator.Send(manager, parsedOBJ.value, false) - - def sendPostToPartitions(post: GabPost, recursiveCall: Boolean = false, parent: Int = 0): Unit = { - val postUUID = post.id.get.toInt - val timestamp = OffsetDateTime.parse(post.created_at.get).toEpochSecond - sendUpdate( - VertexAddWithProperties( - timestamp, - postUUID, - Properties( - StringProperty("user", post.user match { - case Some(u) => u.id.toString - case None => nullStr - }), - StringProperty("likeCount", post.like_count match { - case Some(likeCount) => likeCount.toString - case None => nullStr - }), - StringProperty("score", post.score match { - case Some(score) => score.toString - case None => nullStr - }), - StringProperty("topic", post.topic match { - case Some(topic) => topic.id - case None => nullStr - }), - StringProperty("type", "post") - ) - ) - ) - - post.user match { - case Some(user) => - val userUUID: Int = "user".hashCode() + user.id //TODO improve in case of clashes - sendUpdate( - VertexAddWithProperties( - timestamp, - userUUID, - Properties( - StringProperty("type", "user"), - StringProperty("id", user.id.toString), - StringProperty("name", user.name), - StringProperty("username", user.username), - StringProperty("verified", user.verified.toString) - ) - ) - ) - - sendUpdate( - EdgeAddWithProperties(timestamp, userUUID, postUUID, Properties((StringProperty("type", "userToPost")))) - ) - sendUpdate( - EdgeAddWithProperties(timestamp, postUUID, userUUID, Properties(StringProperty("type", "postToUser"))) - ) - case None => - } - - post.topic match { - case Some(topic) => - val topicUUID: Int = Math.pow(2, 24).toInt + (topic.id.hashCode()) - sendUpdate( - VertexAddWithProperties( - timestamp, - topicUUID, - Properties( - StringProperty("created_at", topic.created_at), - StringProperty("category", topic.category.toString), - StringProperty("title", topic.title.getOrElse("null")), - StringProperty("type", "topic"), - StringProperty("id", topic.id) - ) - ) - ) - - sendUpdate( - EdgeAddWithProperties(timestamp, postUUID, topicUUID, Properties(StringProperty("type", "postToTopic"))) - ) - case None => - } - - // Edge from child to parent post - if (recursiveCall && parent != 0) - sendUpdate( - EdgeAddWithProperties(timestamp, postUUID, parent, Properties(StringProperty("type", "childToParent"))) - ) - post.parent match { - case Some(p) => - if (!recursiveCall) // Allow only one recursion per post - //println("Found parent post: Recursion!") - sendPostToPartitions(p, true, postUUID) - case None => - } - - } - } -} - diff --git a/mainproject/src/main/scala/com/raphtory/examples/gab/graphbuilders/GabUserGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/gab/graphbuilders/GabUserGraphBuilder.scala deleted file mode 100644 index 0a513e06b8..0000000000 --- a/mainproject/src/main/scala/com/raphtory/examples/gab/graphbuilders/GabUserGraphBuilder.scala +++ /dev/null @@ -1,54 +0,0 @@ -package com.raphtory.examples.gab.graphbuilders - -import java.text.SimpleDateFormat - -import com.raphtory.core.actors.Router.GraphBuilder -import com.raphtory.core.model.communication.{Type, _} - -// The lines sent by the Gab mining spout are read and processed accordingly. -//In this router we needed to transform the data that was sent by the spout by turning it into a epoch value (long value) -// in order to be accepted by Raphtory to create the corresponding entity. -// for the gab dataset, a filter is applied to exclude those lines in where the parent post id ir the parent user id -// is equal to -1. Columns 2 and 5 correspond to the userid and parentUserid in the file. -// Then either the vertex or the edge are created accordingly. - -class GabUserGraphBuilder extends GraphBuilder[String] { - - override def parseTuple(tuple: String) = { - val fileLine = tuple.split(";").map(_.trim) - //user wise - val sourceNode = fileLine(2).toInt - val targetNode = fileLine(5).toInt - //comment wise - // val sourceNode=fileLine(1).toInt - //val targetNode=fileLine(4).toInt - if (targetNode > 0 && targetNode != sourceNode) { - val creationDate = dateToUnixTime(timestamp = fileLine(0).slice(0, 19)) - sendUpdate(VertexAdd(creationDate, sourceNode, Type("User"))) - sendUpdate(VertexAdd(creationDate, targetNode, Type("User"))) - sendUpdate(EdgeAdd(creationDate, sourceNode, targetNode, Type("User to User"))) -// sendGraphUpdate(VertexAddWithProperties(creationDate, sourceNode, Properties(StringProperty("test1","value1"),StringProperty("test2","Value2")),Type("User"))) -// sendGraphUpdate(VertexAddWithProperties(creationDate, targetNode, Properties(StringProperty("test1","value1"),StringProperty("test2","Value2")),Type("User"))) -// sendGraphUpdate(EdgeAddWithProperties(creationDate, sourceNode, targetNode, Properties(StringProperty("test1","value1"),StringProperty("test2","Value2")),Type("User To User"))) - } - } - - def dateToUnixTime(timestamp: => String): Long = { - //if(timestamp == null) return null; - // println("TIME FUNC: "+ timestamp) - //val sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'+'HH:mm") - val sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss") - //println(sdf) - val dt = sdf.parse(timestamp) - //println(dt) - val epoch = dt.getTime - //println("*******EPOCH: "+epoch) - epoch - //======= - // val epoch = dt.getTime - // // println(epoch) - // epoch - //>>>>>>> upstream/master - - } -} diff --git a/mainproject/src/main/scala/com/raphtory/examples/gab/rawgraphmodel/GabEntityType.scala b/mainproject/src/main/scala/com/raphtory/examples/gab/rawgraphmodel/GabEntityType.scala deleted file mode 100644 index 45625261ed..0000000000 --- a/mainproject/src/main/scala/com/raphtory/examples/gab/rawgraphmodel/GabEntityType.scala +++ /dev/null @@ -1,7 +0,0 @@ -package com.raphtory.examples.gab.rawgraphmodel - -object GabEntityType extends Enumeration { - val post: Value = Value("post") - val user: Value = Value("user") - val topic: Value = Value("topic") -} diff --git a/mainproject/src/main/scala/com/raphtory/examples/gab/rawgraphmodel/GabJsonProtocol.scala b/mainproject/src/main/scala/com/raphtory/examples/gab/rawgraphmodel/GabJsonProtocol.scala deleted file mode 100644 index 945cace120..0000000000 --- a/mainproject/src/main/scala/com/raphtory/examples/gab/rawgraphmodel/GabJsonProtocol.scala +++ /dev/null @@ -1,105 +0,0 @@ -package com.raphtory.examples.gab.rawgraphmodel - -import spray.json._ - -object GabJsonProtocol extends DefaultJsonProtocol { - - implicit val gabEmbedFormat = jsonFormat2(GabEmbed) - implicit val gabMediaFormat = jsonFormat5(GabMedia) - implicit val gabAttachmentFormat = jsonFormat2(GabAttachment) - implicit val gabCategoryDetailsFormat = jsonFormat4(GabCategoryDetails) - implicit val gabCategotyUserFormat = jsonFormat10(GabUser) - implicit val gabTopicFormat = jsonFormat5(GabTopic) - implicit val gabRepliesFormat = jsonFormat1(GabReplies) - - implicit object GabPostJsonFormat extends RootJsonFormat[GabPost] { - // TODO Writer method - - def getRawField(field: String)(implicit jsObj: JsObject): Option[JsValue] = - jsObj.getFields(field).headOption - - def getField(field: String)(implicit jsObj: JsObject): Option[String] = - getRawField(field) match { - case Some(s) => Some(s.toString()) - case None => None - } - - def getBoolean(field: String)(implicit jsObj: JsObject): Option[Boolean] = - getField(field) match { - case Some(s) => Some(s.toBoolean) - case None => None - } - - def getInt(field: String)(implicit jsObj: JsObject): Option[Int] = - getField(field) match { - case Some(s) => Some(s.toInt) - case None => None - } - - def getLong(field: String)(implicit jsObj: JsObject): Option[Long] = - getField(field) match { - case Some(s) => Some(s.toLong) - case None => None - } - - def write(p: GabPost) = JsString("TODO") - - def read(value: JsValue) = { - implicit val jsObj = value.asJsObject - - new GabPost( - getLong("id"), - getField("created_at") match { - case Some(s) => Some(s.replaceAll("\"", "")) - case None => None - }, - getField("revised_at") match { - case Some(s) => Some(s.replaceAll("\"", "")) - case None => None - }, - getBoolean("edited"), - /*getField("body"), - getField("body_html"), - getField("body_html_summary"), - getField("body_html_summary_truncated").toBoolean,*/ - getBoolean("only_emoji"), - getBoolean("liked"), - getBoolean("disliked"), - getBoolean("bookmarked"), - getBoolean("repost"), - getInt("score"), - getInt("like_count"), - getInt("dislike_count"), - getInt("reply_count"), - getInt("repost_count"), - getBoolean("is_quote"), - getBoolean("is_reply"), - getBoolean("is_replies_disabled"), - //getRawField("embed").convertTo[GabEmbed], - //getRawField("attachment").convertTo[GabAttachment], - //getInt("category"), - /*tRawField("category_details") match { - case Some(c) => Some(c.convertTo[GabCategoryDetails]) - case None => None - },*/ - getField("language"), - getBoolean("nsfw"), - getBoolean("is_premium"), - getBoolean("is_locked"), - getRawField("user") match { - case Some(u) => Some(u.convertTo[GabUser]) - case None => None - }, - getRawField("topic") match { - case Some(t) => Some(t.convertTo[GabTopic]) - case None => None - }, - getRawField("parent") match { - case Some(p) => Some(p.convertTo[GabPost]) - case None => None - } - ) - } - } - -} diff --git a/mainproject/src/main/scala/com/raphtory/examples/gab/rawgraphmodel/GabPost.scala b/mainproject/src/main/scala/com/raphtory/examples/gab/rawgraphmodel/GabPost.scala deleted file mode 100644 index 4ce57dbf25..0000000000 --- a/mainproject/src/main/scala/com/raphtory/examples/gab/rawgraphmodel/GabPost.scala +++ /dev/null @@ -1,85 +0,0 @@ -package com.raphtory.examples.gab.rawgraphmodel - -case class GabPost( - id: Option[Long], - created_at: Option[String], - revised_at: Option[String], - edit: Option[Boolean], - /*body : Option[String], - body_html : Option[String], - body_html_summary : Option[String], - body_html_summary_trucated : Option[Boolean],*/ - only_emojy: Option[Boolean], - liked: Option[Boolean], - disliked: Option[Boolean], - bookmarked: Option[Boolean], - repost: Option[Boolean], - score: Option[Int], - like_count: Option[Int], - dislike_count: Option[Int], - reply_count: Option[Int], - repost_count: Option[Int], - is_quote: Option[Boolean], - is_reply: Option[Boolean], - is_replies_disabled: Option[Boolean], - //embed : Option[GabEmbed], - //attachment : Option[GabAttachment], - //category : Option[Int], - //categoryDetails: Option[GabCategoryDetails], - language: Option[String], - nsfw: Option[Boolean], - is_premium: Option[Boolean], - is_locked: Option[Boolean], - user: Option[GabUser], - topic: Option[GabTopic], - parent: Option[GabPost] -) - -case class GabEmbed( - html: Option[String], - iframe: Option[String] -) - -case class GabAttachment( - type_ : String, - value: Seq[GabMedia] -) - -case class GabMedia( - id: String, - url_thumbnail: String, - url_full: String, - width: Int, - height: Int -) -case class GabCategoryDetails( - title: String, - slug: String, - value: Int, - emoji: String -) - -case class GabUser( - id: Int, - name: String, - username: String, - picture_url: String, - verified: Boolean, - is_donor: Boolean, - is_investor: Boolean, - is_pro: Boolean, - is_private: Boolean, - is_premium: Boolean -) - -case class GabTopic( - id: String, - created_at: String, - is_featured: Option[Boolean], - title: Option[String], - category: Option[Long] -) - -case class GabReplies( - data: String -) diff --git a/mainproject/src/main/scala/com/raphtory/examples/ldbc/graphbuilders/LDBCGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/ldbc/graphbuilders/LDBCGraphBuilder.scala deleted file mode 100644 index a7fe0def3b..0000000000 --- a/mainproject/src/main/scala/com/raphtory/examples/ldbc/graphbuilders/LDBCGraphBuilder.scala +++ /dev/null @@ -1,38 +0,0 @@ -package com.raphtory.examples.ldbc.graphbuilders - -import java.text.SimpleDateFormat - -import com.raphtory.core.actors.Router.GraphBuilder -import com.raphtory.core.model.communication._ - -class LDBCGraphBuilder extends GraphBuilder[String] { - override def parseTuple(tuple: String) = { - val fileLine = tuple.asInstanceOf[String].split("\\|") - val date = fileLine(1).substring(0, 10) + fileLine(1).substring(11, 23); //extract the day of the event - val date2 = fileLine(2).substring(0, 10) + fileLine(1).substring(11, 23); //extract the day of the event - val creationDate: Long = new SimpleDateFormat("yyyy-MM-ddHH:mm:ss.SSS").parse(date).getTime() - val deletionDate: Long = new SimpleDateFormat("yyyy-MM-ddHH:mm:ss.SSS").parse(date2).getTime() - val vertexDeletion = sys.env.getOrElse("LDBC_VERTEX_DELETION", "false").trim.toBoolean - val edgeDeletion = sys.env.getOrElse("LDBC_EDGE_DELETION", "false").trim.toBoolean - fileLine(0) match { - case "person" => - sendUpdate(VertexAdd(creationDate, assignID("person" + fileLine(3)), Type("person"))) - //sendGraphUpdate(VertexAdd(creationDate, fileLine(3).toLong,Type("person"))) - if(vertexDeletion) - sendUpdate(VertexDelete(deletionDate, assignID("person" + fileLine(3)))) - case "person_knows_person" => - //sendGraphUpdate(EdgeAdd(creationDate, fileLine(3).toLong,fileLine(4).toLong,Type("person_knows_person"))) - sendUpdate( - EdgeAdd( - creationDate, - assignID("person" + fileLine(3)), - assignID("person" + fileLine(4)), - Type("person_knows_person") - ) - ) - if(edgeDeletion) - sendUpdate(EdgeDelete(deletionDate, assignID("person"+fileLine(3)),assignID("person"+fileLine(4)))) - } - } -} -//2012-11-01T09:28:01.185+00:00|2019-07-22T11:24:24.362+00:00|35184372093644|Jose|Garcia|female|1988-05-20|111.68.47.44|Firefox diff --git a/mainproject/src/main/scala/com/raphtory/examples/ldbc/graphbuilders/LDBCOldGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/ldbc/graphbuilders/LDBCOldGraphBuilder.scala deleted file mode 100644 index 14c0fd7944..0000000000 --- a/mainproject/src/main/scala/com/raphtory/examples/ldbc/graphbuilders/LDBCOldGraphBuilder.scala +++ /dev/null @@ -1,38 +0,0 @@ -package com.raphtory.examples.ldbc.graphbuilders - -import java.text.SimpleDateFormat - -import com.raphtory.core.actors.Router.GraphBuilder -import com.raphtory.core.model.communication.{EdgeAdd, Type, VertexAdd} - -class LDBCOldGraphBuilder extends GraphBuilder[String] { - override def parseTuple(tuple: String) = { - - val fileLine = tuple.asInstanceOf[String].split("\\|") - //val deletionDate:Long = new SimpleDateFormat("yyyy-MM-ddHH:mm:ss.SSS").parse(date2).getTime() - fileLine(0) match { - case "person" => - val date = fileLine(6).substring(0, 10) + fileLine(5).substring(11, 23); //extract the day of the event - //val date2 = fileLine(2).substring(0, 10) + fileLine(1).substring(11, 23); //extract the day of the event - val creationDate: Long = new SimpleDateFormat("yyyy-MM-ddHH:mm:ss.SSS").parse(date).getTime() - sendUpdate(VertexAdd(creationDate, assignID("person" + fileLine(1)), Type("person"))) - //sendGraphUpdate(VertexAdd(creationDate, fileLine(3).toLong,Type("person"))) - // sendGraphUpdate(VertexDelete(deletionDate, assignID("person"+fileLine(3)))) - case "person_knows_person" => - val date = fileLine(3).substring(0, 10) + fileLine(3).substring(11, 23); //extract the day of the event - //val date2 = fileLine(2).substring(0, 10) + fileLine(1).substring(11, 23); //extract the day of the event - val creationDate: Long = new SimpleDateFormat("yyyy-MM-ddHH:mm:ss.SSS").parse(date).getTime() - //sendGraphUpdate(EdgeAdd(creationDate, fileLine(3).toLong,fileLine(4).toLong,Type("person_knows_person"))) - sendUpdate( - EdgeAdd( - creationDate, - assignID("person" + fileLine(1)), - assignID("person" + fileLine(2)), - Type("person_knows_person") - ) - ) - //sendGraphUpdate(EdgeDelete(deletionDate, assignID("person"+fileLine(3)),assignID("person"+fileLine(4)))) - } - } -} -//2012-11-01T09:28:01.185+00:00|2019-07-22T11:24:24.362+00:00|35184372093644|Jose|Garcia|female|1988-05-20|111.68.47.44|Firefox diff --git a/mainproject/src/main/scala/com/raphtory/examples/lotr/LOTRDeployment.scala b/mainproject/src/main/scala/com/raphtory/examples/lotr/LOTRDeployment.scala deleted file mode 100644 index ab4880df12..0000000000 --- a/mainproject/src/main/scala/com/raphtory/examples/lotr/LOTRDeployment.scala +++ /dev/null @@ -1,19 +0,0 @@ -package com.raphtory.examples.lotr - -import com.raphtory.RaphtoryGraph -import com.raphtory.algorithms.{ConnectedComponents, DegreeBasic} - -object LOTRDeployment extends App{ - val source = new LOTRSpout() - val builder = new LOTRGraphBuilder() - val rg = RaphtoryGraph[String](source,builder) - val arguments = Array[String]() - - rg.rangeQuery(ConnectedComponents(),start = 1,end = 32674,increment = 100,arguments) - rg.rangeQuery(ConnectedComponents(),start = 1,end = 32674,increment = 100,window=100,arguments) - rg.rangeQuery(ConnectedComponents(),start = 1,end = 32674,increment = 100,windowBatch=Array(100,50,10),arguments) - - rg.viewQuery(DegreeBasic(),timestamp = 10000,arguments) - rg.viewQuery(DegreeBasic(),timestamp = 10000,window=100,arguments) - rg.viewQuery(DegreeBasic(),timestamp = 10000,windowBatch=Array(100,50,10),arguments) -} diff --git a/mainproject/src/main/scala/com/raphtory/examples/lotr/LOTRGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/lotr/LOTRGraphBuilder.scala deleted file mode 100644 index 96cbffedc0..0000000000 --- a/mainproject/src/main/scala/com/raphtory/examples/lotr/LOTRGraphBuilder.scala +++ /dev/null @@ -1,26 +0,0 @@ -package com.raphtory.examples.lotr - -import com.raphtory.core.actors.Router.GraphBuilder -import com.raphtory.core.model.communication._ - -class LOTRGraphBuilder extends GraphBuilder[String]{ - - override def parseTuple(tuple: String) = { - - val fileLine = tuple.split(",").map(_.trim) - val sourceNode = fileLine(0) - val srcID = assignID(sourceNode) - - val targetNode = fileLine(1) - val tarID = assignID(targetNode) - - val timeStamp = fileLine(2).toLong - - sendUpdate(VertexAddWithProperties(timeStamp, srcID, - Properties(ImmutableProperty("name",sourceNode)),Type("Character"))) - sendUpdate(VertexAddWithProperties(timeStamp, tarID, - Properties(ImmutableProperty("name",targetNode)),Type("Character"))) - - sendUpdate(EdgeAdd(timeStamp,srcID,tarID, Type("Character Co-occurence"))) - } -} diff --git a/mainproject/src/main/scala/com/raphtory/examples/lotr/LOTRSpout.scala b/mainproject/src/main/scala/com/raphtory/examples/lotr/LOTRSpout.scala deleted file mode 100644 index 9a4029f4b2..0000000000 --- a/mainproject/src/main/scala/com/raphtory/examples/lotr/LOTRSpout.scala +++ /dev/null @@ -1,27 +0,0 @@ -package com.raphtory.examples.lotr - -import com.raphtory.core.actors.Spout.Spout - -import scala.collection.mutable - - -class LOTRSpout extends Spout[String] { - - val fileQueue = mutable.Queue[String]() - - override def setupDataSource(): Unit = { - fileQueue++= - scala.io.Source.fromFile("src/main/scala/com/raphtory/examples/lotr/lotr.csv") - .getLines - }//no setup - - override def generateData(): Option[String] = { - if(fileQueue isEmpty){ - dataSourceComplete() - None - } - else - Some(fileQueue.dequeue()) - } - override def closeDataSource(): Unit = {}//no file closure already done -} diff --git a/mainproject/src/main/scala/com/raphtory/examples/lotr/lotr.csv b/mainproject/src/main/scala/com/raphtory/examples/lotr/lotr.csv deleted file mode 100644 index 897532fcf5..0000000000 --- a/mainproject/src/main/scala/com/raphtory/examples/lotr/lotr.csv +++ /dev/null @@ -1,2649 +0,0 @@ -Gandalf,Elrond,33 -Frodo,Bilbo,114 -Blanco,Marcho,146 -Frodo,Bilbo,205 -Thorin,Gandalf,270 -Thorin,Bilbo,270 -Gandalf,Bilbo,270 -Gollum,Bilbo,286 -Gollum,Bilbo,306 -Gollum,Bilbo,308 -Bilbo,Elrond,317 -Frodo,Samwise,319 -Gandalf,Bilbo,320 -Gollum,Bilbo,324 -Frodo,Gandalf,329 -Peregrin,Elessar,356 -Arwen,Aragorn,358 -Barahir,Faramir,359 -Bilbo,Findegil,360 -Meriadoc,Peregrin,363 -Peregrin,Elendil,368 -Galadriel,Celeborn,374 -Frodo,Bilbo,387 -Frodo,Bilbo,388 -Frodo,Bilbo,389 -Frodo,Bilbo,390 -Frodo,Bilbo,393 -Frodo,Bilbo,399 -Hamfast,Bilbo,402 -Gandalf,Bilbo,483 -Gandalf,Bilbo,543 -Frodo,Gandalf,555 -Frodo,Bilbo,555 -Gandalf,Bilbo,555 -Frodo,Bilbo,562 -Gandalf,Bilbo,730 -Frodo,Bilbo,808 -Frodo,Bilbo,815 -Gandalf,Bilbo,843 -Frodo,Gandalf,861 -Frodo,Bilbo,898 -Frodo,Merry,929 -Frodo,Merry,930 -Frodo,Bilbo,942 -Frodo,Bilbo,944 -Odo,Frodo,959 -Frodo,Bilbo,1038 -Meriadoc,Peregrin,1042 -Meriadoc,Merry,1042 -Meriadoc,Pippin,1042 -Meriadoc,Fredegar,1042 -Peregrin,Merry,1042 -Peregrin,Pippin,1042 -Peregrin,Fredegar,1042 -Merry,Pippin,1042 -Merry,Fredegar,1042 -Pippin,Fredegar,1042 -Pippin,Merry,1044 -Pippin,Bilbo,1044 -Merry,Bilbo,1044 -Frodo,Gandalf,1056 -Sam,Halfast,1089 -Frodo,Bilbo,1106 -Frodo,Gandalf,1130 -Gandalf,Bilbo,1135 -Gandalf,Bilbo,1156 -Frodo,Gandalf,1160 -Frodo,Saruman,1185 -Gandalf,Bilbo,1229 -Frodo,Gandalf,1234 -Frodo,Gandalf,1241 -Isildur,Elendil,1309 -Isildur,Gil-galad,1309 -Isildur,Sauron,1309 -Elendil,Gil-galad,1309 -Elendil,Sauron,1309 -Gil-galad,Sauron,1309 -Sméagol,Déagol,1324 -Sméagol,Déagol,1330 -Sméagol,Déagol,1331 -Sméagol,Déagol,1336 -Frodo,Gollum,1356 -Frodo,Gollum,1359 -Frodo,Gandalf,1390 -Frodo,Gollum,1400 -Frodo,Bilbo,1401 -Isildur,Gollum,1407 -Isildur,Déagol,1407 -Gollum,Déagol,1407 -Frodo,Gandalf,1417 -Isildur,Elendil,1420 -Frodo,Gollum,1429 -Gollum,Déagol,1441 -Gollum,Bilbo,1459 -Gandalf,Bilbo,1469 -Frodo,Bilbo,1484 -Frodo,Gollum,1484 -Bilbo,Gollum,1484 -Frodo,Bilbo,1567 -Frodo,Gollum,1567 -Bilbo,Gollum,1567 -Gandalf,Bilbo,1654 -Frodo,Gandalf,1656 -Sam,Gandalf,1677 -Sam,Frodo,1687 -Sam,Frodo,1705 -Sam,Gandalf,1710 -Sam,Gandalf,1711 -Sam,Frodo,1714 -Gandalf,Bilbo,1721 -Frodo,Gandalf,1741 -Gandalf,Bilbo,1748 -Sam,Elrond,1756 -Frodo,Gandalf,1783 -Frodo,Gandalf,1785 -Pippin,Frodo,1789 -Pippin,Merry,1789 -Pippin,Fredegar,1789 -Frodo,Merry,1789 -Frodo,Fredegar,1789 -Merry,Fredegar,1789 -Frodo,Gandalf,1792 -Frodo,Bilbo,1802 -Frodo,Gandalf,1804 -Merry,Fredegar,1806 -Frodo,Gandalf,1809 -Lobelia,Frodo,1816 -Lobelia,Lotho,1816 -Frodo,Lotho,1816 -Pippin,Sam,1825 -Lobelia,Sam,1826 -Lobelia,Frodo,1826 -Sam,Frodo,1826 -Pippin,Sam,1829 -Sam,Frodo,1869 -Sam,Frodo,1871 -Peregrin,Bilbo,1879 -Pippin,Sam,1888 -Pippin,Frodo,1951 -Pippin,Sam,1955 -Pippin,Bilbo,1981 -Frodo,Gandalf,1999 -Pippin,Frodo,2024 -Pippin,Sam,2024 -Frodo,Sam,2024 -Frodo,Gandalf,2056 -Frodo,Bilbo,2083 -Pippin,Frodo,2110 -Pippin,Sam,2110 -Frodo,Sam,2110 -Frodo,Bilbo,2175 -Sam,Frodo,2204 -Pippin,Frodo,2242 -Frodo,Bilbo,2246 -Frodo,Bilbo,2250 -Sam,Frodo,2254 -Sam,Gandalf,2254 -Frodo,Gandalf,2254 -Sam,Frodo,2318 -Sam,Frodo,2358 -Sam,Frodo,2393 -Pippin,Sam,2446 -Pippin,Frodo,2488 -Maggot,Pippin,2490 -Pippin,Frodo,2511 -Pippin,Sam,2511 -Frodo,Sam,2511 -Pippin,Sam,2513 -Maggot,Pippin,2519 -Pippin,Peregrin,2521 -Sam,Frodo,2541 -Maggot,Frodo,2607 -Frodo,Bilbo,2613 -Frodo,Peregrin,2628 -Pippin,Frodo,2633 -Pippin,Sam,2633 -Frodo,Sam,2633 -Pippin,Sam,2659 -Sam,Frodo,2668 -Frodo,Merry,2694 -Pippin,Frodo,2742 -Pippin,Merry,2742 -Pippin,Sam,2742 -Frodo,Merry,2742 -Frodo,Sam,2742 -Merry,Sam,2742 -Maggot,Frodo,2756 -Meriadoc,Frodo,2786 -Pippin,Bilbo,2794 -Maggot,Frodo,2821 -Pippin,Frodo,2829 -Maggot,Merry,2834 -Pippin,Frodo,2835 -Maggot,Frodo,2837 -Maggot,Bilbo,2837 -Frodo,Bilbo,2837 -Maggot,Merry,2838 -Pippin,Merry,2845 -Frodo,Merry,2850 -Pippin,Frodo,2857 -Sam,Frodo,2882 -Sam,Merry,2919 -Sam,Frodo,2922 -Frodo,Gandalf,2925 -Sam,Frodo,2938 -Pippin,Merry,2949 -Gandalf,Merry,2972 -Frodo,Gandalf,2999 -Gandalf,Fredegar,3060 -Pippin,Merry,3239 -Sam,Frodo,3265 -Pippin,Frodo,3271 -Pippin,Sam,3271 -Frodo,Sam,3271 -Sam,Frodo,3278 -Pippin,Sam,3282 -Pippin,Merry,3291 -Sam,Frodo,3316 -Sam,Frodo,3337 -Sam,Frodo,3345 -Sam,Merry,3356 -Frodo,Goldberry,3461 -Tom,Goldberry,3482 -Tom,Goldberry,3499 -Tom,Goldberry,3514 -Tom,Frodo,3526 -Pippin,Merry,3544 -Tom,Goldberry,3616 -Tom,Goldberry,3686 -Tom,Frodo,3702 -Tom,Frodo,3703 -Tom,Gandalf,3703 -Tom,Bilbo,3703 -Frodo,Gandalf,3703 -Frodo,Bilbo,3703 -Gandalf,Bilbo,3703 -Tom,Frodo,3705 -Tom,Frodo,3714 -Tom,Gandalf,3718 -Frodo,Merry,3721 -Tom,Frodo,3733 -Pippin,Sam,3837 -Pippin,Merry,3837 -Sam,Merry,3837 -Frodo,Gandalf,3914 -Frodo,Bilbo,3914 -Gandalf,Bilbo,3914 -Pippin,Sam,3919 -Pippin,Merry,3919 -Sam,Merry,3919 -Pippin,Sam,3938 -Pippin,Merry,3938 -Sam,Merry,3938 -Frodo,Merry,3946 -Tom,Frodo,3960 -Pippin,Sam,3976 -Pippin,Merry,3976 -Sam,Merry,3976 -Tom,Frodo,3988 -Tom,Merry,4037 -Pippin,Sam,4046 -Pippin,Merry,4046 -Sam,Merry,4046 -Tom,Goldberry,4060 -Butterbur,Barliman,4126 -Tom,Goldberry,4131 -Butterbur,Barliman,4261 -Sam,Frodo,4269 -Nob,Butterbur,4332 -Pippin,Frodo,4342 -Pippin,Sam,4342 -Frodo,Sam,4342 -Butterbur,Barliman,4353 -Pippin,Sam,4380 -Frodo,Butterbur,4389 -Frodo,Bilbo,4418 -Butterbur,Bilbo,4421 -Pippin,Sam,4482 -Pippin,Frodo,4565 -Pippin,Sam,4565 -Frodo,Sam,4565 -Sam,Frodo,4620 -Pippin,Frodo,4629 -Pippin,Sam,4629 -Frodo,Sam,4629 -Sam,Frodo,4692 -Sam,Frodo,4697 -Nob,Butterbur,4714 -Pippin,Sam,4734 -Pippin,Butterbur,4736 -Frodo,Butterbur,4749 -Pippin,Frodo,4902 -Pippin,Sam,4902 -Frodo,Sam,4902 -Frodo,Gandalf,4910 -Pippin,Gandalf,4926 -Pippin,Sam,4930 -Nob,Merry,5012 -Frodo,Merry,5089 -Nob,Merry,5090 -Nob,Gandalf,5090 -Merry,Gandalf,5090 -Bob,Bill,5228 -Bob,Bill,5229 -Butterbur,Merry,5242 -Tom,Butterbur,5249 -Nob,Butterbur,5268 -Nob,Bob,5268 -Butterbur,Bob,5268 -Pippin,Frodo,5274 -Pippin,Merry,5274 -Pippin,Sam,5274 -Frodo,Merry,5274 -Frodo,Sam,5274 -Merry,Sam,5274 -Nob,Bob,5276 -Pippin,Frodo,5410 -Elendil,Gil-galad,5434 -Pippin,Sam,5468 -Frodo,Merry,5470 -Merry,Gandalf,5500 -Frodo,Merry,5534 -Sam,Peregrin,5538 -Pippin,Gandalf,5544 -Merry,Gil-galad,5615 -Pippin,Frodo,5619 -Pippin,Merry,5619 -Frodo,Merry,5619 -Frodo,Gandalf,5620 -Lúthien,Beren,5653 -Lúthien,Barahir,5653 -Beren,Barahir,5653 -Beren,Lúthien,5654 -Beren,Thingol,5654 -Lúthien,Thingol,5654 -Beren,Barahir,5657 -Beren,Thingol,5657 -Barahir,Thingol,5657 -Beren,Lúthien,5660 -Beren,Sauron,5660 -Beren,Thingol,5660 -Lúthien,Sauron,5660 -Lúthien,Thingol,5660 -Sauron,Thingol,5660 -Beren,Thingol,5667 -Beren,Elwing,5667 -Beren,Eärendil,5667 -Beren,Lúthien,5667 -Beren,Dior,5667 -Thingol,Elwing,5667 -Thingol,Eärendil,5667 -Thingol,Lúthien,5667 -Thingol,Dior,5667 -Elwing,Eärendil,5667 -Elwing,Lúthien,5667 -Elwing,Dior,5667 -Eärendil,Lúthien,5667 -Eärendil,Dior,5667 -Lúthien,Dior,5667 -Sam,Merry,5682 -Pippin,Frodo,5683 -Pippin,Merry,5713 -Sam,Frodo,5714 -Sam,Frodo,5752 -Pippin,Merry,5758 -Frodo,Gandalf,5775 -Sam,Merry,5999 -Pippin,Merry,6010 -Sam,Frodo,6011 -Pippin,Merry,6012 -Sam,Frodo,6020 -Frodo,Bilbo,6037 -Frodo,Bilbo,6044 -Gandalf,Bilbo,6118 -Frodo,Bilbo,6126 -Frodo,Bilbo,6127 -Glorfindel,Elrond,6161 -Glorfindel,Frodo,6195 -Elrond,Aragorn,6200 -Glorfindel,Frodo,6220 -Pippin,Sam,6225 -Pippin,Merry,6225 -Sam,Merry,6225 -Glorfindel,Frodo,6267 -Frodo,Lúthien,6325 -Frodo,Gandalf,6381 -Butterbur,Bill,6429 -Gandalf,Barliman,6431 -Frodo,Gandalf,6531 -Glorfindel,Aragorn,6549 -Frodo,Gandalf,6578 -Glorfindel,Elrond,6583 -Sam,Frodo,6628 -Glorfindel,Gandalf,6660 -Glorfindel,Elrond,6660 -Gandalf,Elrond,6660 -Glorfindel,Frodo,6661 -Glorfindel,Gandalf,6661 -Glorfindel,Elrond,6661 -Frodo,Gandalf,6661 -Frodo,Elrond,6661 -Gandalf,Elrond,6661 -Frodo,Elrond,6669 -Arwen,Frodo,6673 -Arwen,Lúthien,6673 -Arwen,Elrond,6673 -Frodo,Lúthien,6673 -Frodo,Elrond,6673 -Lúthien,Elrond,6673 -Frodo,Elrond,6676 -Pippin,Frodo,6682 -Pippin,Merry,6682 -Frodo,Merry,6682 -Thorin,Glóin,6693 -Gandalf,Elrond,6701 -Frodo,Glóin,6709 -Frodo,Beorn,6710 -Frodo,Grimbeorn,6710 -Beorn,Grimbeorn,6710 -Bard,Bain,6717 -Bard,Brand,6717 -Bain,Brand,6717 -Dwalin,Bombur,6728 -Dwalin,Glóin,6728 -Dwalin,Bofur,6728 -Bombur,Glóin,6728 -Bombur,Bofur,6728 -Glóin,Bofur,6728 -Ori,Balin,6730 -Ori,Óin,6730 -Balin,Óin,6730 -Frodo,Glóin,6736 -Smaug,Bilbo,6747 -Frodo,Glóin,6748 -Arwen,Elrond,6754 -Frodo,Gandalf,6757 -Sam,Frodo,6810 -Sam,Bilbo,6810 -Frodo,Bilbo,6810 -Gandalf,Elrond,6833 -Gandalf,Bilbo,6835 -Frodo,Bilbo,6855 -Frodo,Bilbo,6935 -Eärendil,Elrond,6969 -Frodo,Bilbo,6987 -Frodo,Aragorn,6994 -Arwen,Frodo,6995 -Sam,Bilbo,7006 -Sam,Bilbo,7009 -Frodo,Elrond,7018 -Gandalf,Bilbo,7024 -Gandalf,Elrond,7037 -Sam,Frodo,7040 -Sam,Bilbo,7040 -Frodo,Bilbo,7040 -Frodo,Gandalf,7041 -Glorfindel,Glóin,7047 -Glorfindel,Frodo,7047 -Glóin,Frodo,7047 -Frodo,Elrond,7048 -Gimli,Glóin,7051 -Elrond,Glorfindel,7052 -Elrond,Galdor,7052 -Elrond,Erestor,7052 -Elrond,Círdan,7052 -Glorfindel,Galdor,7052 -Glorfindel,Erestor,7052 -Glorfindel,Círdan,7052 -Galdor,Erestor,7052 -Galdor,Círdan,7052 -Erestor,Círdan,7052 -Legolas,Thranduil,7053 -Frodo,Bilbo,7058 -Gandalf,Elrond,7059 -Gandalf,Boromir,7059 -Elrond,Boromir,7059 -Frodo,Glóin,7064 -Balin,Óin,7077 -Balin,Ori,7077 -Óin,Ori,7077 -Elrond,Sauron,7127 -Elrond,Sauron,7128 -Isildur,Elendil,7134 -Isildur,Anárion,7134 -Elendil,Anárion,7134 -Elendil,Gil-galad,7135 -Elendil,Sauron,7135 -Gil-galad,Sauron,7135 -Gil-galad,Elrond,7142 -Elwing,Eärendil,7145 -Elwing,Lúthien,7145 -Elwing,Dior,7145 -Eärendil,Lúthien,7145 -Eärendil,Dior,7145 -Lúthien,Dior,7145 -Elendil,Gil-galad,7148 -Isildur,Elendil,7149 -Isildur,Gil-galad,7149 -Isildur,Sauron,7149 -Elendil,Gil-galad,7149 -Elendil,Sauron,7149 -Gil-galad,Sauron,7149 -Gil-galad,Círdan,7162 -Isildur,Valandil,7170 -Isildur,Elendil,7170 -Isildur,Ohtar,7170 -Valandil,Elendil,7170 -Valandil,Ohtar,7170 -Elendil,Ohtar,7170 -Isildur,Elendil,7179 -Isildur,Gil-galad,7179 -Isildur,Anárion,7179 -Elendil,Gil-galad,7179 -Elendil,Anárion,7179 -Gil-galad,Anárion,7179 -Meneldil,Anárion,7192 -Elrond,Aragorn,7235 -Isildur,Elendil,7241 -Isildur,Elrond,7241 -Isildur,Aragorn,7241 -Elendil,Elrond,7241 -Elendil,Aragorn,7241 -Elrond,Aragorn,7241 -Frodo,Bilbo,7270 -Aragorn,Boromir,7279 -Isildur,Elendil,7281 -Isildur,Denethor,7281 -Elendil,Denethor,7281 -Gollum,Bilbo,7330 -Frodo,Bilbo,7336 -Galdor,Elrond,7353 -Frodo,Gandalf,7356 -Frodo,Galdor,7356 -Frodo,Glóin,7356 -Gandalf,Galdor,7356 -Gandalf,Glóin,7356 -Galdor,Glóin,7356 -Gollum,Gandalf,7391 -Isildur,Aragorn,7404 -Gollum,Aragorn,7405 -Isildur,Gandalf,7407 -Isildur,Saruman,7434 -Isildur,Boromir,7434 -Saruman,Boromir,7434 -Isildur,Elendil,7440 -Gil-galad,Sauron,7448 -Celebrimbor,Sauron,7454 -Gollum,Denethor,7455 -Gollum,Aragorn,7455 -Denethor,Aragorn,7455 -Gandalf,Elrond,7484 -Sméagol,Gollum,7509 -Legolas,Glóin,7525 -Saruman,Radagast,7581 -Gandalf,Saruman,7599 -Saruman,Sauron,7714 -Saruman,Radagast,7739 -Sauron,Aragorn,7769 -Glorfindel,Galdor,7904 -Gandalf,Elrond,7911 -Glorfindel,Saruman,7913 -Glorfindel,Erestor,7932 -Frodo,Boromir,7948 -Balin,Thrór,7992 -Glorfindel,Sauron,8028 -Bilbo,Boromir,8060 -Gandalf,Bilbo,8062 -Frodo,Elrond,8097 -Beren,Túrin,8105 -Beren,Hador,8105 -Beren,Húrin,8105 -Túrin,Hador,8105 -Túrin,Húrin,8105 -Hador,Húrin,8105 -Pippin,Frodo,8117 -Pippin,Merry,8117 -Pippin,Sam,8117 -Frodo,Merry,8117 -Frodo,Sam,8117 -Merry,Sam,8117 -Legolas,Gollum,8145 -Sam,Frodo,8150 -Sam,Bilbo,8150 -Frodo,Bilbo,8150 -Elrond,Thranduil,8157 -Elrond,Aragorn,8158 -Frodo,Gandalf,8183 -Frodo,Gandalf,8190 -Frodo,Bilbo,8190 -Gandalf,Bilbo,8190 -Gimli,Legolas,8255 -Gimli,Glóin,8255 -Legolas,Glóin,8255 -Isildur,Aragorn,8257 -Frodo,Boromir,8262 -Pippin,Frodo,8276 -Pippin,Gandalf,8276 -Frodo,Gandalf,8276 -Pippin,Elrond,8286 -Elendil,Aragorn,8292 -Gandalf,Elrond,8295 -Gandalf,Aragorn,8295 -Elrond,Aragorn,8295 -Frodo,Bilbo,8296 -Pippin,Merry,8297 -Pippin,Frodo,8297 -Pippin,Beren,8297 -Pippin,Bilbo,8297 -Pippin,Sam,8297 -Pippin,Lúthien,8297 -Merry,Frodo,8297 -Merry,Beren,8297 -Merry,Bilbo,8297 -Merry,Sam,8297 -Merry,Lúthien,8297 -Frodo,Beren,8297 -Frodo,Bilbo,8297 -Frodo,Sam,8297 -Frodo,Lúthien,8297 -Beren,Bilbo,8297 -Beren,Sam,8297 -Beren,Lúthien,8297 -Bilbo,Sam,8297 -Bilbo,Lúthien,8297 -Sam,Lúthien,8297 -Frodo,Bilbo,8300 -Frodo,Bilbo,8336 -Frodo,Bilbo,8341 -Elrond,Boromir,8373 -Frodo,Bilbo,8379 -Thorin,Gandalf,8380 -Sam,Bill,8384 -Pippin,Sam,8386 -Frodo,Bilbo,8390 -Elrond,Aragorn,8391 -Sam,Frodo,8397 -Gandalf,Elrond,8405 -Gandalf,Aragorn,8438 -Frodo,Gandalf,8459 -Gimli,Legolas,8473 -Sam,Aragorn,8525 -Sam,Aragorn,8534 -Sam,Aragorn,8538 -Sam,Frodo,8563 -Frodo,Gandalf,8598 -Frodo,Aragorn,8598 -Gandalf,Aragorn,8598 -Gandalf,Aragorn,8600 -Gandalf,Aragorn,8614 -Frodo,Aragorn,8619 -Gandalf,Aragorn,8647 -Gimli,Sauron,8686 -Frodo,Merry,8690 -Frodo,Bilbo,8710 -Gandalf,Boromir,8712 -Legolas,Gandalf,8765 -Aragorn,Boromir,8779 -Aragorn,Boromir,8791 -Aragorn,Boromir,8796 -Merry,Aragorn,8822 -Pippin,Legolas,8827 -Pippin,Merry,8827 -Legolas,Merry,8827 -Sam,Boromir,8828 -Gimli,Gandalf,8829 -Gimli,Bill,8829 -Gandalf,Bill,8829 -Frodo,Aragorn,8830 -Pippin,Sam,8866 -Pippin,Merry,8866 -Sam,Merry,8866 -Aragorn,Boromir,8867 -Frodo,Gandalf,8871 -Gandalf,Boromir,8892 -Gandalf,Sauron,8912 -Balin,Fundin,8919 -Sam,Frodo,8952 -Pippin,Sam,8978 -Pippin,Elrond,8978 -Sam,Elrond,8978 -Pippin,Sam,8982 -Gandalf,Aragorn,9006 -Boromir,Aragorn,9018 -Gandalf,Boromir,9059 -Gimli,Frodo,9098 -Gimli,Gandalf,9098 -Frodo,Gandalf,9098 -Sam,Bill,9134 -Gimli,Legolas,9154 -Gandalf,Bill,9162 -Sam,Frodo,9169 -Sam,Bill,9179 -Gandalf,Fëanor,9210 -Gandalf,Aragorn,9240 -Peregrin,Gandalf,9258 -Sam,Bill,9282 -Gimli,Gandalf,9316 -Sam,Frodo,9329 -Sam,Frodo,9334 -Sam,Frodo,9343 -Sam,Gandalf,9343 -Frodo,Gandalf,9343 -Sam,Frodo,9350 -Gimli,Gandalf,9374 -Legolas,Sam,9402 -Legolas,Frodo,9402 -Legolas,Boromir,9402 -Sam,Frodo,9402 -Sam,Boromir,9402 -Frodo,Boromir,9402 -Gimli,Gandalf,9410 -Gimli,Glóin,9413 -Gimli,Gandalf,9417 -Gimli,Legolas,9444 -Gimli,Boromir,9444 -Gimli,Aragorn,9444 -Legolas,Boromir,9444 -Legolas,Aragorn,9444 -Boromir,Aragorn,9444 -Pippin,Gandalf,9468 -Pippin,Merry,9468 -Gandalf,Merry,9468 -Merry,Aragorn,9478 -Thorin,Bilbo,9605 -Gandalf,Daeron,9670 -Balin,Frodo,9676 -Balin,Bilbo,9676 -Frodo,Bilbo,9676 -Gimli,Frodo,9687 -Ori,Gimli,9717 -Balin,Fundin,9755 -Balin,Gimli,9852 -Aragorn,Boromir,9853 -Pippin,Merry,9861 -Frodo,Aragorn,9864 -Aragorn,Boromir,9870 -Pippin,Frodo,9877 -Pippin,Merry,9877 -Pippin,Aragorn,9877 -Frodo,Merry,9877 -Frodo,Aragorn,9877 -Merry,Aragorn,9877 -Balin,Gimli,9878 -Balin,Legolas,9878 -Gimli,Legolas,9878 -Sam,Frodo,9901 -Frodo,Gandalf,9903 -Gandalf,Bilbo,9982 -Gandalf,Boromir,10098 -Gandalf,Aragorn,10098 -Boromir,Aragorn,10098 -Aragorn,Boromir,10144 -Aragorn,Boromir,10152 -Sam,Frodo,10155 -Sam,Frodo,10235 -Pippin,Sam,10251 -Sam,Frodo,10273 -Sam,Aragorn,10273 -Frodo,Aragorn,10273 -Legolas,Aragorn,10279 -Aragorn,Boromir,10280 -Gimli,Frodo,10298 -Gimli,Sam,10298 -Gimli,Aragorn,10298 -Frodo,Sam,10298 -Frodo,Aragorn,10298 -Sam,Aragorn,10298 -Sam,Aragorn,10299 -Merry,Bilbo,10334 -Sam,Frodo,10356 -Sam,Aragorn,10356 -Frodo,Aragorn,10356 -Gimli,Frodo,10360 -Gimli,Aragorn,10478 -Legolas,Sam,10515 -Legolas,Frodo,10515 -Sam,Frodo,10515 -Legolas,Frodo,10518 -Sam,Frodo,10521 -Orophin,Rúmil,10531 -Haldir,Aragorn,10540 -Frodo,Elrond,10550 -Legolas,Aragorn,10555 -Haldir,Pippin,10568 -Haldir,Legolas,10568 -Haldir,Merry,10568 -Pippin,Legolas,10568 -Pippin,Merry,10568 -Legolas,Merry,10568 -Legolas,Frodo,10591 -Haldir,Rúmil,10645 -Haldir,Amroth,10795 -Frodo,Aragorn,10843 -Frodo,Aragorn,10845 -Frodo,Amroth,10850 -Galadriel,Celeborn,10863 -Haldir,Frodo,10869 -Haldir,Galadriel,10879 -Haldir,Celeborn,10879 -Galadriel,Celeborn,10879 -Legolas,Frodo,10883 -Galadriel,Celeborn,10892 -Haldir,Frodo,10896 -Gimli,Glóin,10909 -Haldir,Legolas,10933 -Balin,Celeborn,10938 -Balin,Aragorn,10938 -Celeborn,Aragorn,10938 -Frodo,Galadriel,10964 -Legolas,Aragorn,10978 -Frodo,Boromir,11009 -Sam,Frodo,11054 -Sam,Frodo,11062 -Sam,Frodo,11095 -Sam,Frodo,11140 -Frodo,Gandalf,11189 -Frodo,Gandalf,11192 -Frodo,Saruman,11192 -Gandalf,Saruman,11192 -Frodo,Galadriel,11248 -Frodo,Gandalf,11279 -Legolas,Boromir,11333 -Legolas,Aragorn,11333 -Boromir,Aragorn,11333 -Celeborn,Aragorn,11344 -Legolas,Galadriel,11359 -Frodo,Aragorn,11362 -Gandalf,Boromir,11363 -Elendil,Sauron,11364 -Frodo,Gandalf,11365 -Frodo,Boromir,11365 -Gandalf,Boromir,11365 -Frodo,Boromir,11367 -Frodo,Boromir,11375 -Frodo,Boromir,11380 -Frodo,Aragorn,11380 -Boromir,Aragorn,11380 -Pippin,Sam,11382 -Pippin,Merry,11382 -Sam,Merry,11382 -Pippin,Merry,11471 -Pippin,Frodo,11471 -Pippin,Boromir,11471 -Pippin,Gimli,11471 -Pippin,Sam,11471 -Pippin,Aragorn,11471 -Pippin,Legolas,11471 -Merry,Frodo,11471 -Merry,Boromir,11471 -Merry,Gimli,11471 -Merry,Sam,11471 -Merry,Aragorn,11471 -Merry,Legolas,11471 -Frodo,Boromir,11471 -Frodo,Gimli,11471 -Frodo,Sam,11471 -Frodo,Aragorn,11471 -Frodo,Legolas,11471 -Boromir,Gimli,11471 -Boromir,Sam,11471 -Boromir,Aragorn,11471 -Boromir,Legolas,11471 -Gimli,Sam,11471 -Gimli,Aragorn,11471 -Gimli,Legolas,11471 -Sam,Aragorn,11471 -Sam,Legolas,11471 -Aragorn,Legolas,11471 -Galadriel,Celeborn,11486 -Aragorn,Boromir,11516 -Galadriel,Celeborn,11528 -Celeborn,Galadriel,11539 -Celeborn,Aragorn,11539 -Galadriel,Aragorn,11539 -Elendil,Elessar,11549 -Arwen,Celebrían,11551 -Pippin,Merry,11553 -Pippin,Boromir,11553 -Merry,Boromir,11553 -Gimli,Galadriel,11566 -Gimli,Glóin,11572 -Gimli,Galadriel,11575 -Gimli,Glóin,11589 -Gimli,Glóin,11649 -Gimli,Glóin,11655 -Sam,Frodo,11680 -Gimli,Legolas,11730 -Pippin,Merry,11731 -Pippin,Boromir,11731 -Pippin,Aragorn,11731 -Merry,Boromir,11731 -Merry,Aragorn,11731 -Boromir,Aragorn,11731 -Pippin,Frodo,11732 -Frodo,Aragorn,11736 -Sam,Frodo,11739 -Sam,Frodo,11744 -Sam,Frodo,11771 -Sam,Frodo,11794 -Gollum,Aragorn,11841 -Merry,Aragorn,11895 -Sam,Frodo,11903 -Legolas,Galadriel,11943 -Frodo,Boromir,11956 -Frodo,Boromir,12025 -Frodo,Aragorn,12025 -Boromir,Aragorn,12025 -Legolas,Aragorn,12030 -Legolas,Frodo,12042 -Legolas,Aragorn,12042 -Frodo,Aragorn,12042 -Pippin,Merry,12063 -Aragorn,Boromir,12066 -Sam,Frodo,12117 -Isildur,Anárion,12126 -Isildur,Valandil,12127 -Isildur,Elendil,12127 -Isildur,Elessar,12127 -Valandil,Elendil,12127 -Valandil,Elessar,12127 -Elendil,Elessar,12127 -Frodo,Aragorn,12205 -Sam,Frodo,12226 -Sam,Boromir,12226 -Frodo,Boromir,12226 -Gandalf,Bilbo,12236 -Frodo,Boromir,12260 -Frodo,Boromir,12321 -Pippin,Sam,12469 -Pippin,Merry,12469 -Sam,Merry,12469 -Denethor,Elrond,12500 -Gimli,Sam,12520 -Legolas,Meriadoc,12521 -Legolas,Peregrin,12521 -Legolas,Boromir,12521 -Meriadoc,Peregrin,12521 -Meriadoc,Boromir,12521 -Peregrin,Boromir,12521 -Sam,Aragorn,12563 -Aragorn,Boromir,12601 -Frodo,Boromir,12615 -Pippin,Frodo,12634 -Pippin,Merry,12634 -Frodo,Merry,12634 -Gimli,Legolas,12637 -Sam,Aragorn,12645 -Sam,Frodo,12702 -Sam,Frodo,12730 -Sam,Frodo,12736 -Sam,Frodo,12769 -Samwise,Boromir,12805 -Samwise,Aragorn,12805 -Boromir,Aragorn,12805 -Gimli,Legolas,12876 -Gimli,Legolas,12878 -Legolas,Aragorn,12881 -Pippin,Frodo,12893 -Pippin,Merry,12893 -Pippin,Sam,12893 -Frodo,Merry,12893 -Frodo,Sam,12893 -Merry,Sam,12893 -Gimli,Sauron,12935 -Gandalf,Saruman,12946 -Gimli,Legolas,12961 -Denethor,Boromir,12986 -Frodo,Aragorn,13052 -Gimli,Frodo,13056 -Legolas,Aragorn,13191 -Gimli,Legolas,13238 -Gimli,Legolas,13309 -Gimli,Legolas,13429 -Gimli,Aragorn,13429 -Legolas,Aragorn,13429 -Saruman,Sauron,13474 -Gimli,Gandalf,13478 -Aragorn,Boromir,13479 -Gimli,Legolas,13530 -Sauron,Aragorn,13562 -Isildur,Elendil,13572 -Isildur,Elessar,13572 -Isildur,Aragorn,13572 -Elendil,Elessar,13572 -Elendil,Aragorn,13572 -Elessar,Aragorn,13572 -Gimli,Legolas,13576 -Legolas,Aragorn,13578 -Denethor,Boromir,13585 -Sauron,Thengel,13588 -Denethor,Sauron,13644 -Gandalf,Saruman,13655 -Gandalf,Shadowfax,13660 -Gandalf,Shadowfax,13663 -Gandalf,Aragorn,13663 -Shadowfax,Aragorn,13663 -Gimli,Legolas,13739 -Gimli,Legolas,13794 -Pippin,Merry,13839 -Legolas,Gandalf,13845 -Gimli,Gandalf,13846 -Gandalf,Aragorn,13848 -Legolas,Aragorn,13870 -Celeborn,Aragorn,13882 -Gimli,Saruman,13908 -Legolas,Aragorn,13909 -Gimli,Legolas,13949 -Gimli,Aragorn,13949 -Legolas,Aragorn,13949 -Pippin,Frodo,13952 -Pippin,Merry,13967 -Gandalf,Elrond,13986 -Pippin,Merry,14151 -Pippin,Merry,14177 -Pippin,Merry,14236 -Pippin,Merry,14331 -Frodo,Gandalf,14364 -Gandalf,Shadowfax,14365 -Pippin,Merry,14394 -Pippin,Merry,14436 -Pippin,Merry,14468 -Pippin,Merry,14544 -Pippin,Merry,14621 -Meriadoc,Merry,14799 -Pippin,Peregrin,14800 -Pippin,Merry,14804 -Pippin,Merry,15032 -Tom,Galadriel,15038 -Tom,Elrond,15038 -Galadriel,Elrond,15038 -Treebeard,Gandalf,15049 -Treebeard,Saruman,15049 -Gandalf,Saruman,15049 -Sam,Gandalf,15050 -Gandalf,Galadriel,15059 -Pippin,Merry,15258 -Treebeard,Merry,15265 -Treebeard,Pippin,15265 -Merry,Pippin,15265 -Pippin,Merry,15267 -Pippin,Merry,15292 -Treebeard,Pippin,15294 -Treebeard,Merry,15298 -Treebeard,Pippin,15298 -Merry,Pippin,15298 -Pippin,Merry,15312 -Pippin,Merry,15338 -Sam,Frodo,15374 -Legolas,Aragorn,15600 -Pippin,Merry,15612 -Pippin,Merry,15619 -Pippin,Boromir,15619 -Merry,Boromir,15619 -Gimli,Legolas,15669 -Gimli,Aragorn,15669 -Legolas,Aragorn,15669 -Pippin,Merry,15672 -Aragorn,Boromir,15901 -Pippin,Merry,15958 -Gandalf,Saruman,16001 -Gandalf,Saruman,16002 -Treebeard,Gandalf,16018 -Pippin,Merry,16036 -Gimli,Saruman,16040 -Legolas,Aragorn,16046 -Gandalf,Aragorn,16109 -Elessar,Aragorn,16187 -Gandalf,Shadowfax,16250 -Legolas,Aragorn,16259 -Gimli,Shadowfax,16260 -Gandalf,Shadowfax,16268 -Gimli,Legolas,16293 -Gimli,Gandalf,16293 -Gimli,Aragorn,16293 -Legolas,Gandalf,16293 -Legolas,Aragorn,16293 -Gandalf,Aragorn,16293 -Gimli,Gandalf,16298 -Gimli,Legolas,16410 -Elendil,Thengel,16455 -Elendil,Aragorn,16455 -Thengel,Aragorn,16455 -Denethor,Aragorn,16456 -Gandalf,Wormtongue,16556 -Gandalf,Thengel,16571 -Gimli,Gandalf,16579 -Gandalf,Wormtongue,16663 -Gandalf,Wormtongue,16851 -Gandalf,Saruman,16910 -Legolas,Aragorn,16949 -Gimli,Gandalf,16951 -Gimli,Legolas,17015 -Gandalf,Shadowfax,17026 -Legolas,Aragorn,17034 -Gimli,Galadriel,17041 -Gandalf,Shadowfax,17059 -Legolas,Gandalf,17094 -Helm,Erkenbrand,17122 -Helm,Erkenbrand,17178 -Gandalf,Wormtongue,17194 -Helm,Gandalf,17196 -Helm,Erkenbrand,17196 -Gandalf,Erkenbrand,17196 -Legolas,Aragorn,17207 -Helm,Erkenbrand,17239 -Helm,Gamling,17258 -Legolas,Aragorn,17457 -Gamling,Saruman,17476 -Gandalf,Aragorn,17581 -Elendil,Aragorn,17650 -Legolas,Erkenbrand,17707 -Legolas,Aragorn,17707 -Erkenbrand,Aragorn,17707 -Gimli,Gamling,17710 -Gandalf,Saruman,17770 -Gimli,Legolas,17785 -Gimli,Gandalf,17785 -Gimli,Aragorn,17785 -Legolas,Gandalf,17785 -Legolas,Aragorn,17785 -Gandalf,Aragorn,17785 -Gimli,Legolas,17820 -Gimli,Gandalf,17820 -Legolas,Gandalf,17820 -Legolas,Gandalf,17821 -Gimli,Legolas,17828 -Legolas,Helm,17848 -Legolas,Galadriel,17857 -Gimli,Legolas,17881 -Gandalf,Shadowfax,17989 -Shadowfax,Gandalf,18002 -Grimbold,Erkenbrand,18006 -Helm,Saruman,18010 -Saruman,Wormtongue,18116 -Saruman,Wormtongue,18157 -Gimli,Legolas,18177 -Pippin,Merry,18196 -Treebeard,Merry,18213 -Treebeard,Gandalf,18230 -Gimli,Legolas,18252 -Gimli,Aragorn,18252 -Legolas,Aragorn,18252 -Gimli,Merry,18299 -Pippin,Aragorn,18431 -Gimli,Gandalf,18452 -Pippin,Saruman,18465 -Treebeard,Saruman,18528 -Gandalf,Galadriel,18554 -Gandalf,Elrond,18554 -Galadriel,Elrond,18554 -Treebeard,Merry,18634 -Treebeard,Gandalf,18654 -Treebeard,Gandalf,18664 -Treebeard,Gandalf,18669 -Sam,Frodo,18689 -Sam,Boromir,18689 -Frodo,Boromir,18689 -Saruman,Aragorn,18825 -Treebeard,Merry,18829 -Pippin,Merry,18936 -Saruman,Wormtongue,18954 -Gandalf,Saruman,18988 -Gandalf,Saruman,18993 -Gandalf,Saruman,19202 -Gandalf,Saruman,19207 -Saruman,Aragorn,19218 -Gandalf,Saruman,19271 -Gimli,Legolas,19275 -Gimli,Aragorn,19275 -Legolas,Aragorn,19275 -Treebeard,Gandalf,19276 -Gimli,Legolas,19292 -Gimli,Legolas,19300 -Pippin,Merry,19326 -Pippin,Gandalf,19362 -Pippin,Merry,19362 -Pippin,Aragorn,19362 -Gandalf,Merry,19362 -Gandalf,Aragorn,19362 -Merry,Aragorn,19362 -Pippin,Merry,19366 -Pippin,Merry,19437 -Pippin,Merry,19453 -Gandalf,Saruman,19471 -Pippin,Gandalf,19475 -Pippin,Sam,19489 -Pippin,Gandalf,19519 -Gandalf,Aragorn,19687 -Pippin,Gandalf,19756 -Pippin,Gandalf,19762 -Pippin,Aragorn,19762 -Gandalf,Aragorn,19762 -Merry,Aragorn,19771 -Shadowfax,Gandalf,19793 -Elendil,Elrond,19848 -Wormtongue,Aragorn,19918 -Peregrin,Shadowfax,19921 -Pippin,Gandalf,19957 -Sam,Frodo,20041 -Sam,Frodo,20047 -Sam,Frodo,20230 -Sam,Frodo,20267 -Frodo,Bilbo,20483 -Frodo,Gollum,20519 -Sam,Frodo,20598 -Sam,Frodo,20607 -Sam,Gollum,20627 -Frodo,Gollum,20653 -Sam,Frodo,20655 -Sam,Gollum,20686 -Frodo,Gollum,20688 -Sam,Frodo,20735 -Sam,Gollum,20769 -Sam,Frodo,20777 -Sam,Frodo,20792 -Sam,Gollum,20838 -Sam,Frodo,20844 -Sam,Gollum,20844 -Frodo,Gollum,20844 -Sam,Frodo,20849 -Sam,Gollum,20859 -Sam,Gollum,20861 -Sam,Frodo,20888 -Frodo,Gollum,20903 -Sam,Samwise,20916 -Sam,Frodo,20985 -Sam,Gollum,20985 -Frodo,Gollum,20985 -Sam,Frodo,21041 -Sam,Frodo,21066 -Sam,Frodo,21137 -Sam,Frodo,21147 -Sam,Frodo,21153 -Sam,Gollum,21167 -Sam,Frodo,21168 -Sam,Gollum,21172 -Sam,Frodo,21214 -Sam,Frodo,21229 -Frodo,Gollum,21299 -Sam,Gollum,21301 -Frodo,Gollum,21369 -Sam,Gollum,21455 -Frodo,Gollum,21466 -Frodo,Gollum,21490 -Sam,Frodo,21492 -Sam,Gollum,21492 -Frodo,Gollum,21492 -Frodo,Gollum,21501 -Frodo,Gandalf,21551 -Frodo,Bilbo,21551 -Gandalf,Bilbo,21551 -Frodo,Gollum,21558 -Isildur,Elendil,21588 -Isildur,Frodo,21588 -Elendil,Frodo,21588 -Sam,Frodo,21626 -Sam,Gollum,21626 -Frodo,Gollum,21626 -Gollum,Aragorn,21673 -Frodo,Gollum,21680 -Frodo,Gollum,21683 -Gollum,Aragorn,21687 -Gandalf,Aragorn,21702 -Gandalf,Saruman,21703 -Gandalf,Aragorn,21703 -Saruman,Aragorn,21703 -Frodo,Samwise,21705 -Frodo,Gandalf,21706 -Sam,Frodo,21727 -Frodo,Gollum,21753 -Sam,Frodo,21765 -Frodo,Gollum,21825 -Sam,Frodo,21860 -Sam,Gollum,21860 -Frodo,Gollum,21860 -Sam,Gollum,21887 -Frodo,Elrond,21938 -Sam,Gollum,21945 -Sam,Frodo,21970 -Sam,Gollum,21970 -Frodo,Gollum,21970 -Sam,Frodo,22061 -Sam,Gollum,22081 -Sam,Frodo,22143 -Frodo,Boromir,22150 -Frodo,Hamfast,22183 -Frodo,Samwise,22183 -Hamfast,Samwise,22183 -Boromir,Aragorn,22187 -Denethor,Boromir,22194 -Mablung,Damrod,22233 -Sam,Frodo,22280 -Sam,Faramir,22341 -Frodo,Faramir,22343 -Sam,Frodo,22349 -Sam,Boromir,22349 -Frodo,Boromir,22349 -Elendil,Boromir,22366 -Isildur,Elendil,22367 -Isildur,Aragorn,22367 -Elendil,Aragorn,22367 -Sam,Frodo,22384 -Sam,Faramir,22384 -Frodo,Faramir,22384 -Frodo,Boromir,22391 -Frodo,Denethor,22441 -Frodo,Faramir,22441 -Denethor,Faramir,22441 -Mablung,Damrod,22556 -Samwise,Frodo,22557 -Samwise,Faramir,22557 -Frodo,Faramir,22557 -Sam,Frodo,22568 -Sam,Mablung,22568 -Sam,Damrod,22568 -Sam,Faramir,22568 -Frodo,Mablung,22568 -Frodo,Damrod,22568 -Frodo,Faramir,22568 -Mablung,Damrod,22568 -Mablung,Faramir,22568 -Damrod,Faramir,22568 -Samwise,Faramir,22571 -Isildur,Frodo,22593 -Boromir,Faramir,22687 -Mablung,Damrod,22692 -Mablung,Damrod,22710 -Sam,Frodo,22718 -Mablung,Damrod,22723 -Sam,Frodo,22824 -Sam,Faramir,22824 -Frodo,Faramir,22824 -Sam,Frodo,22832 -Sam,Frodo,22844 -Frodo,Boromir,22845 -Frodo,Aragorn,22849 -Boromir,Faramir,22852 -Sam,Faramir,22890 -Samwise,Faramir,22891 -Bilbo,Aragorn,22908 -Sam,Frodo,22959 -Frodo,Faramir,22986 -Frodo,Faramir,23030 -Sam,Frodo,23064 -Frodo,Faramir,23077 -Frodo,Faramir,23094 -Anborn,Faramir,23143 -Anborn,Frodo,23166 -Frodo,Gandalf,23212 -Frodo,Gollum,23277 -Frodo,Faramir,23277 -Gollum,Faramir,23277 -Anborn,Gollum,23341 -Frodo,Faramir,23375 -Gollum,Faramir,23389 -Frodo,Gollum,23399 -Frodo,Faramir,23420 -Frodo,Gollum,23435 -Frodo,Faramir,23435 -Gollum,Faramir,23435 -Anborn,Gollum,23477 -Sam,Frodo,23545 -Frodo,Elrond,23572 -Frodo,Gollum,23576 -Frodo,Faramir,23576 -Gollum,Faramir,23576 -Samwise,Faramir,23577 -Frodo,Gollum,23578 -Sam,Frodo,23617 -Sam,Gollum,23624 -Sam,Frodo,23678 -Sam,Gollum,23678 -Frodo,Gollum,23678 -Sam,Frodo,23726 -Sam,Gollum,23726 -Frodo,Gollum,23726 -Sam,Gandalf,23787 -Sam,Gollum,23789 -Sam,Frodo,23816 -Sam,Frodo,23837 -Frodo,Gollum,23855 -Sam,Frodo,23864 -Sam,Gollum,23890 -Sam,Frodo,23896 -Sam,Frodo,23910 -Sam,Frodo,23917 -Sam,Frodo,23940 -Gandalf,Galadriel,24026 -Gandalf,Faramir,24026 -Gandalf,Elrond,24026 -Gandalf,Aragorn,24026 -Galadriel,Faramir,24026 -Galadriel,Elrond,24026 -Galadriel,Aragorn,24026 -Faramir,Elrond,24026 -Faramir,Aragorn,24026 -Elrond,Aragorn,24026 -Sam,Frodo,24050 -Sam,Frodo,24058 -Sam,Frodo,24099 -Sam,Frodo,24138 -Sam,Frodo,24219 -Sam,Frodo,24220 -Sam,Frodo,24253 -Sam,Frodo,24284 -Sam,Frodo,24292 -Sam,Frodo,24334 -Frodo,Gollum,24339 -Sam,Frodo,24355 -Gollum,Shelob,24375 -Sam,Frodo,24413 -Sam,Frodo,24436 -Sam,Frodo,24442 -Sam,Frodo,24515 -Gollum,Faramir,24578 -Ungoliant,Shelob,24618 -Sam,Frodo,24664 -Sam,Gollum,24681 -Sam,Gollum,24687 -Sam,Gollum,24690 -Sam,Gollum,24692 -Sam,Gollum,24693 -Sam,Frodo,24711 -Sam,Frodo,24715 -Sam,Samwise,24715 -Frodo,Samwise,24715 -Sam,Shelob,24736 -Sam,Frodo,24742 -Hamfast,Samwise,24753 -Frodo,Galadriel,24792 -Frodo,Faramir,24816 -Frodo,Bilbo,24817 -Frodo,Bilbo,24857 -Sam,Frodo,24873 -Sam,Shagrat,25066 -Shelob,Shagrat,25096 -Gorbag,Shagrat,25308 -Gorbag,Shagrat,25310 -Pippin,Gandalf,25348 -Pippin,Gandalf,25359 -Pippin,Gandalf,25366 -Pippin,Gandalf,25380 -Pippin,Gandalf,25394 -Gandalf,Denethor,25399 -Gandalf,Boromir,25411 -Amroth,Imrahil,25440 -Pippin,Gandalf,25441 -Pippin,Ecthelion,25446 -Peregrin,Gandalf,25447 -Gandalf,Shadowfax,25472 -Pippin,Gandalf,25480 -Pippin,Gandalf,25485 -Denethor,Ecthelion,25521 -Pippin,Boromir,25523 -Pippin,Aragorn,25523 -Boromir,Aragorn,25523 -Gandalf,Faramir,25533 -Pippin,Boromir,25541 -Pippin,Denethor,25550 -Meriadoc,Pippin,25560 -Pippin,Denethor,25561 -Pippin,Gandalf,25572 -Pippin,Denethor,25573 -Pippin,Peregrin,25574 -Pippin,Denethor,25574 -Pippin,Ecthelion,25574 -Peregrin,Denethor,25574 -Peregrin,Ecthelion,25574 -Denethor,Ecthelion,25574 -Pippin,Gandalf,25578 -Gandalf,Denethor,25579 -Théoden,Saruman,25583 -Pippin,Gandalf,25584 -Gandalf,Denethor,25585 -Pippin,Gandalf,25586 -Treebeard,Gandalf,25589 -Gandalf,Denethor,25592 -Pippin,Gandalf,25599 -Pippin,Denethor,25600 -Pippin,Gandalf,25621 -Pippin,Gandalf,25629 -Isildur,Denethor,25635 -Isildur,Boromir,25635 -Denethor,Boromir,25635 -Pippin,Boromir,25636 -Pippin,Faramir,25636 -Boromir,Faramir,25636 -Denethor,Faramir,25654 -Beregond,Aragorn,25692 -Pippin,Peregrin,25697 -Gandalf,Shadowfax,25715 -Beregond,Shadowfax,25716 -Pippin,Shadowfax,25723 -Pippin,Shadowfax,25726 -Beregond,Pippin,25741 -Beregond,Pippin,25749 -Beregond,Peregrin,25753 -Beregond,Pippin,25780 -Pippin,Peregrin,25817 -Pippin,Peregrin,25867 -Beregond,Gandalf,25868 -Beregond,Pippin,25868 -Gandalf,Pippin,25868 -Pippin,Beregond,25869 -Pippin,Denethor,25872 -Pippin,Boromir,25872 -Denethor,Boromir,25872 -Beregond,Bergil,25917 -Pippin,Merry,25928 -Pippin,Bergil,25928 -Merry,Bergil,25928 -Pippin,Bergil,25930 -Pippin,Bergil,25935 -Pippin,Forlong,25939 -Forlong,Bergil,25941 -Amroth,Imrahil,25961 -Pippin,Gandalf,25987 -Pippin,Faramir,26001 -Gandalf,Merry,26010 -Gandalf,Shadowfax,26010 -Gandalf,Aragorn,26010 -Merry,Shadowfax,26010 -Merry,Aragorn,26010 -Shadowfax,Aragorn,26010 -Gimli,Legolas,26013 -Merry,Aragorn,26031 -Gimli,Legolas,26034 -Gimli,Merry,26034 -Gimli,Aragorn,26034 -Legolas,Merry,26034 -Legolas,Aragorn,26034 -Merry,Aragorn,26034 -Merry,Aragorn,26042 -Théoden,Saruman,26073 -Halbarad,Aragorn,26089 -Gimli,Legolas,26099 -Gimli,Merry,26099 -Legolas,Merry,26099 -Gimli,Legolas,26102 -Gimli,Halbarad,26115 -Legolas,Aragorn,26116 -Legolas,Gandalf,26125 -Legolas,Galadriel,26125 -Gandalf,Galadriel,26125 -Gimli,Gandalf,26126 -Meriadoc,Théoden,26151 -Théoden,Merry,26160 -Théoden,Aragorn,26160 -Merry,Aragorn,26160 -Éomer,Elrond,26169 -Éomer,Halbarad,26169 -Éomer,Aragorn,26169 -Elrond,Halbarad,26169 -Elrond,Aragorn,26169 -Halbarad,Aragorn,26169 -Gimli,Legolas,26170 -Merry,Aragorn,26171 -Éomer,Théoden,26176 -Éomer,Merry,26186 -Éomer,Aragorn,26186 -Merry,Aragorn,26186 -Éomer,Aragorn,26188 -Éomer,Théoden,26190 -Éomer,Aragorn,26190 -Théoden,Aragorn,26190 -Gimli,Legolas,26197 -Gimli,Merry,26197 -Legolas,Merry,26197 -Éomer,Théoden,26202 -Gimli,Legolas,26212 -Gandalf,Aragorn,26219 -Isildur,Elendil,26234 -Isildur,Théoden,26234 -Isildur,Sauron,26234 -Elendil,Théoden,26234 -Elendil,Sauron,26234 -Théoden,Sauron,26234 -Gimli,Aragorn,26236 -Legolas,Elrond,26249 -Legolas,Aragorn,26249 -Elrond,Aragorn,26249 -Arvedui,Aragorn,26250 -Gimli,Aragorn,26257 -Gimli,Legolas,26260 -Gimli,Aragorn,26260 -Legolas,Aragorn,26260 -Isildur,Sauron,26263 -Isildur,Sauron,26266 -Gimli,Legolas,26272 -Gimli,Aragorn,26272 -Legolas,Aragorn,26272 -Gimli,Legolas,26274 -Helm,Halbarad,26276 -Éowyn,Elrond,26278 -Éowyn,Aragorn,26278 -Elrond,Aragorn,26278 -Helm,Théoden,26279 -Éowyn,Aragorn,26283 -Gimli,Legolas,26295 -Gimli,Aragorn,26295 -Legolas,Aragorn,26295 -Gimli,Legolas,26330 -Gimli,Glóin,26361 -Gimli,Aragorn,26362 -Gimli,Glóin,26371 -Gimli,Legolas,26398 -Gimli,Legolas,26400 -Gimli,Amroth,26407 -Isildur,Elessar,26428 -Isildur,Halbarad,26428 -Elessar,Halbarad,26428 -Gimli,Legolas,26433 -Pippin,Amroth,26438 -Gimli,Pippin,26465 -Gimli,Legolas,26465 -Gimli,Aragorn,26465 -Pippin,Legolas,26465 -Pippin,Aragorn,26465 -Legolas,Aragorn,26465 -Sam,Frodo,26466 -Éomer,Théoden,26476 -Gandalf,Shadowfax,26497 -Dúnhere,Gandalf,26500 -Éowyn,Théoden,26544 -Éowyn,Merry,26544 -Théoden,Merry,26544 -Éomer,Aragorn,26549 -Pippin,Gandalf,26571 -Pippin,Frodo,26571 -Pippin,Gimli,26571 -Pippin,Sam,26571 -Pippin,Legolas,26571 -Gandalf,Frodo,26571 -Gandalf,Gimli,26571 -Gandalf,Sam,26571 -Gandalf,Legolas,26571 -Frodo,Gimli,26571 -Frodo,Sam,26571 -Frodo,Legolas,26571 -Gimli,Sam,26571 -Gimli,Legolas,26571 -Sam,Legolas,26571 -Éowyn,Dúnhere,26576 -Éowyn,Éomer,26576 -Éowyn,Théoden,26576 -Dúnhere,Éomer,26576 -Dúnhere,Théoden,26576 -Éomer,Théoden,26576 -Éomer,Aragorn,26584 -Brego,Baldor,26592 -Merry,Aragorn,26598 -Éowyn,Éomer,26599 -Brego,Baldor,26603 -Éomer,Aragorn,26614 -Merry,Boromir,26622 -Hirgon,Denethor,26628 -Théoden,Denethor,26630 -Hirgon,Denethor,26634 -Hirgon,Gandalf,26638 -Hirgon,Denethor,26638 -Gandalf,Denethor,26638 -Éomer,Théoden,26705 -Éomer,Merry,26706 -Éowyn,Merry,26712 -Théoden,Merry,26714 -Éowyn,Aragorn,26721 -Gimli,Merry,26725 -Meriadoc,Merry,26759 -Pippin,Gandalf,26788 -Pippin,Denethor,26800 -Pippin,Gandalf,26801 -Éomer,Gandalf,26819 -Éomer,Denethor,26819 -Gandalf,Denethor,26819 -Pippin,Denethor,26820 -Pippin,Denethor,26821 -Pippin,Gandalf,26862 -Beregond,Pippin,26868 -Beregond,Faramir,26868 -Pippin,Faramir,26868 -Beregond,Faramir,26909 -Beregond,Pippin,26910 -Gandalf,Shadowfax,26929 -Pippin,Faramir,26930 -Pippin,Boromir,26932 -Pippin,Denethor,26949 -Pippin,Faramir,26953 -Pippin,Gandalf,26955 -Pippin,Gandalf,26957 -Pippin,Faramir,26957 -Gandalf,Faramir,26957 -Pippin,Frodo,26958 -Pippin,Gandalf,26958 -Frodo,Gandalf,26958 -Pippin,Gandalf,26959 -Boromir,Denethor,26991 -Boromir,Faramir,26991 -Denethor,Faramir,26991 -Boromir,Denethor,26998 -Boromir,Faramir,26998 -Denethor,Faramir,26998 -Pippin,Gandalf,27035 -Pippin,Denethor,27035 -Gandalf,Denethor,27035 -Pippin,Gandalf,27056 -Pippin,Gandalf,27058 -Pippin,Frodo,27060 -Pippin,Gandalf,27060 -Frodo,Gandalf,27060 -Pippin,Gollum,27090 -Frodo,Gollum,27094 -Boromir,Faramir,27114 -Denethor,Boromir,27125 -Gandalf,Faramir,27128 -Beregond,Faramir,27161 -Nazgûl,Denethor,27184 -Nazgûl,Sauron,27184 -Denethor,Sauron,27184 -Pippin,Gandalf,27187 -Shadowfax,Amroth,27234 -Shadowfax,Faramir,27234 -Amroth,Faramir,27234 -Denethor,Faramir,27255 -Nazgûl,Faramir,27259 -Imrahil,Faramir,27261 -Denethor,Faramir,27265 -Pippin,Gandalf,27269 -Gandalf,Denethor,27322 -Denethor,Faramir,27350 -Denethor,Faramir,27362 -Denethor,Faramir,27368 -Denethor,Faramir,27387 -Pippin,Denethor,27389 -Denethor,Faramir,27404 -Pippin,Denethor,27422 -Beregond,Pippin,27428 -Beregond,Faramir,27428 -Pippin,Faramir,27428 -Beregond,Pippin,27435 -Beregond,Faramir,27435 -Pippin,Faramir,27435 -Gandalf,Shadowfax,27488 -Pippin,Merry,27536 -Éomer,Merry,27539 -Éomer,Théoden,27567 -Théoden,Ghân-buri-Ghân,27665 -Éomer,Denethor,27690 -Éomer,Théoden,27771 -Théoden,Thengel,27796 -Éowyn,Merry,27873 -Éowyn,Meriadoc,27881 -Éomer,Amroth,27905 -Merry,Gandalf,27906 -Éowyn,Éomer,27908 -Éomer,Théoden,27918 -Éowyn,Théoden,27951 -Imrahil,Amroth,27953 -Éowyn,Éomer,27962 -Éomer,Imrahil,27972 -Éomer,Húrin,27972 -Imrahil,Húrin,27972 -Arwen,Elrond,28013 -Isildur,Elessar,28014 -Isildur,Aragorn,28014 -Elessar,Aragorn,28014 -Gimli,Legolas,28019 -Gimli,Halbarad,28019 -Legolas,Halbarad,28019 -Elendil,Aragorn,28020 -Éomer,Aragorn,28021 -Éomer,Imrahil,28034 -Éomer,Aragorn,28034 -Imrahil,Aragorn,28034 -Grimbold,Halbarad,28038 -Grimbold,Dúnhere,28044 -Grimbold,Fastred,28044 -Grimbold,Horn,28044 -Grimbold,Harding,28044 -Dúnhere,Fastred,28044 -Dúnhere,Horn,28044 -Dúnhere,Harding,28044 -Fastred,Horn,28044 -Fastred,Harding,28044 -Horn,Harding,28044 -Gandalf,Denethor,28049 -Gandalf,Shadowfax,28053 -Pippin,Shadowfax,28055 -Pippin,Faramir,28067 -Pippin,Denethor,28070 -Pippin,Faramir,28070 -Denethor,Faramir,28070 -Pippin,Gandalf,28073 -Gandalf,Faramir,28074 -Pippin,Shadowfax,28077 -Pippin,Beregond,28089 -Beregond,Denethor,28102 -Pippin,Gandalf,28104 -Pippin,Denethor,28104 -Gandalf,Denethor,28104 -Gandalf,Denethor,28109 -Denethor,Faramir,28116 -Beregond,Gandalf,28122 -Beregond,Pippin,28122 -Beregond,Denethor,28122 -Gandalf,Pippin,28122 -Gandalf,Denethor,28122 -Pippin,Denethor,28122 -Gandalf,Denethor,28174 -Beregond,Faramir,28177 -Beregond,Gandalf,28191 -Beregond,Denethor,28191 -Beregond,Ecthelion,28191 -Gandalf,Denethor,28191 -Gandalf,Ecthelion,28191 -Denethor,Ecthelion,28191 -Beregond,Faramir,28195 -Beregond,Gandalf,28197 -Beregond,Pippin,28197 -Beregond,Faramir,28197 -Gandalf,Pippin,28197 -Gandalf,Faramir,28197 -Pippin,Faramir,28197 -Gandalf,Amroth,28204 -Beregond,Gandalf,28210 -Beregond,Pippin,28210 -Beregond,Faramir,28210 -Gandalf,Pippin,28210 -Gandalf,Faramir,28210 -Pippin,Faramir,28210 -Pippin,Beregond,28212 -Denethor,Sauron,28225 -Beregond,Faramir,28229 -Éowyn,Théoden,28249 -Pippin,Merry,28276 -Beregond,Bergil,28285 -Éowyn,Meriadoc,28301 -Éowyn,Faramir,28301 -Meriadoc,Faramir,28301 -Éomer,Imrahil,28323 -Éomer,Aragorn,28323 -Imrahil,Aragorn,28323 -Éomer,Elendil,28327 -Éomer,Imrahil,28334 -Éowyn,Éomer,28338 -Éowyn,Imrahil,28339 -Éowyn,Gandalf,28345 -Denethor,Faramir,28346 -Galadriel,Aragorn,28350 -Amroth,Faramir,28352 -Éomer,Imrahil,28365 -Elessar,Aragorn,28366 -Éowyn,Meriadoc,28368 -Éowyn,Gandalf,28368 -Meriadoc,Gandalf,28368 -Éowyn,Merry,28370 -Éowyn,Faramir,28370 -Éowyn,Aragorn,28370 -Merry,Faramir,28370 -Merry,Aragorn,28370 -Faramir,Aragorn,28370 -Éomer,Faramir,28373 -Éomer,Elrond,28373 -Éomer,Aragorn,28373 -Faramir,Elrond,28373 -Faramir,Aragorn,28373 -Elrond,Aragorn,28373 -Gandalf,Faramir,28386 -Shadowfax,Aragorn,28387 -Gandalf,Aragorn,28390 -Aragorn,Faramir,28419 -Faramir,Aragorn,28421 -Faramir,Aragorn,28432 -Faramir,Aragorn,28436 -Beregond,Gandalf,28443 -Beregond,Imrahil,28443 -Gandalf,Imrahil,28443 -Pippin,Gandalf,28444 -Éowyn,Aragorn,28448 -Théoden,Wormtongue,28464 -Saruman,Wormtongue,28468 -Éomer,Aragorn,28472 -Éowyn,Aragorn,28483 -Éowyn,Aragorn,28484 -Éomer,Éowyn,28488 -Éowyn,Éomer,28499 -Éomer,Gandalf,28503 -Pippin,Gandalf,28510 -Pippin,Merry,28510 -Pippin,Aragorn,28510 -Gandalf,Merry,28510 -Gandalf,Aragorn,28510 -Merry,Aragorn,28510 -Pippin,Merry,28511 -Merry,Aragorn,28516 -Pippin,Gandalf,28518 -Pippin,Aragorn,28528 -Meriadoc,Saruman,28532 -Meriadoc,Aragorn,28532 -Saruman,Aragorn,28532 -Merry,Gandalf,28543 -Éowyn,Gandalf,28565 -Éowyn,Faramir,28565 -Éowyn,Aragorn,28565 -Gandalf,Faramir,28565 -Gandalf,Aragorn,28565 -Faramir,Aragorn,28565 -Éowyn,Gandalf,28566 -Éowyn,Faramir,28566 -Éowyn,Aragorn,28566 -Gandalf,Faramir,28566 -Gandalf,Aragorn,28566 -Faramir,Aragorn,28566 -Beregond,Meriadoc,28568 -Elrond,Aragorn,28573 -Gimli,Pippin,28578 -Gimli,Legolas,28578 -Gimli,Merry,28578 -Pippin,Legolas,28578 -Pippin,Merry,28578 -Legolas,Merry,28578 -Gimli,Legolas,28579 -Legolas,Aragorn,28581 -Legolas,Imrahil,28583 -Legolas,Aragorn,28583 -Imrahil,Aragorn,28583 -Legolas,Aragorn,28587 -Meriadoc,Peregrin,28589 -Meriadoc,Imrahil,28589 -Peregrin,Imrahil,28589 -Legolas,Merry,28603 -Legolas,Merry,28619 -Gimli,Glóin,28632 -Éowyn,Merry,28635 -Pippin,Legolas,28639 -Pippin,Merry,28639 -Legolas,Merry,28639 -Angbor,Aragorn,28652 -Lúthien,Sauron,28683 -Isildur,Aragorn,28690 -Gandalf,Aragorn,28715 -Théoden,Imrahil,28719 -Théoden,Gimli,28719 -Théoden,Aragorn,28719 -Théoden,Éomer,28719 -Théoden,Legolas,28719 -Imrahil,Gimli,28719 -Imrahil,Aragorn,28719 -Imrahil,Éomer,28719 -Imrahil,Legolas,28719 -Gimli,Aragorn,28719 -Gimli,Éomer,28719 -Gimli,Legolas,28719 -Aragorn,Éomer,28719 -Aragorn,Legolas,28719 -Éomer,Legolas,28719 -Gandalf,Elrond,28720 -Gandalf,Aragorn,28720 -Elrond,Aragorn,28720 -Amroth,Imrahil,28730 -Gandalf,Sauron,28793 -Imrahil,Aragorn,28799 -Imrahil,Gimli,28818 -Imrahil,Glóin,28818 -Imrahil,Aragorn,28818 -Gimli,Glóin,28818 -Gimli,Aragorn,28818 -Glóin,Aragorn,28818 -Éomer,Elrond,28820 -Éomer,Amroth,28820 -Éomer,Imrahil,28820 -Éomer,Aragorn,28820 -Elrond,Amroth,28820 -Elrond,Imrahil,28820 -Elrond,Aragorn,28820 -Amroth,Imrahil,28820 -Amroth,Aragorn,28820 -Imrahil,Aragorn,28820 -Gimli,Legolas,28836 -Gimli,Gandalf,28836 -Gimli,Elrond,28836 -Gimli,Aragorn,28836 -Legolas,Gandalf,28836 -Legolas,Elrond,28836 -Legolas,Aragorn,28836 -Gandalf,Elrond,28836 -Gandalf,Aragorn,28836 -Elrond,Aragorn,28836 -Gandalf,Imrahil,28872 -Gandalf,Faramir,28872 -Imrahil,Faramir,28872 -Gandalf,Aragorn,28876 -Nazgûl,Aragorn,28894 -Gandalf,Imrahil,28926 -Gandalf,Gimli,28926 -Gandalf,Peregrin,28926 -Gandalf,Elrond,28926 -Gandalf,Aragorn,28926 -Gandalf,Éomer,28926 -Gandalf,Legolas,28926 -Imrahil,Gimli,28926 -Imrahil,Peregrin,28926 -Imrahil,Elrond,28926 -Imrahil,Aragorn,28926 -Imrahil,Éomer,28926 -Imrahil,Legolas,28926 -Gimli,Peregrin,28926 -Gimli,Elrond,28926 -Gimli,Aragorn,28926 -Gimli,Éomer,28926 -Gimli,Legolas,28926 -Peregrin,Elrond,28926 -Peregrin,Aragorn,28926 -Peregrin,Éomer,28926 -Peregrin,Legolas,28926 -Elrond,Aragorn,28926 -Elrond,Éomer,28926 -Elrond,Legolas,28926 -Aragorn,Éomer,28926 -Aragorn,Legolas,28926 -Éomer,Legolas,28926 -Gandalf,Sauron,28956 -Sam,Frodo,28958 -Pippin,Imrahil,28960 -Saruman,Sauron,28982 -Elrond,Amroth,29020 -Elrond,Imrahil,29020 -Amroth,Imrahil,29020 -Beregond,Frodo,29024 -Beregond,Gandalf,29024 -Beregond,Pippin,29024 -Beregond,Imrahil,29024 -Frodo,Gandalf,29024 -Frodo,Pippin,29024 -Frodo,Imrahil,29024 -Gandalf,Pippin,29024 -Gandalf,Imrahil,29024 -Pippin,Imrahil,29024 -Beregond,Pippin,29040 -Pippin,Merry,29069 -Pippin,Denethor,29069 -Pippin,Aragorn,29069 -Merry,Denethor,29069 -Merry,Aragorn,29069 -Denethor,Aragorn,29069 -Sam,Frodo,29070 -Hamfast,Samwise,29072 -Frodo,Shelob,29084 -Frodo,Shagrat,29084 -Shelob,Shagrat,29084 -Gorbag,Shagrat,29101 -Gorbag,Shagrat,29163 -Gorbag,Shagrat,29164 -Gorbag,Sam,29169 -Sam,Frodo,29207 -Gorbag,Shagrat,29246 -Gorbag,Shagrat,29288 -Nazgûl,Shagrat,29309 -Sam,Shagrat,29317 -Sam,Shagrat,29323 -Sam,Shagrat,29344 -Sam,Shagrat,29350 -Sam,Frodo,29427 -Sam,Frodo,29440 -Sam,Frodo,29449 -Sam,Frodo,29461 -Sam,Frodo,29480 -Sam,Frodo,29499 -Sam,Frodo,29503 -Sam,Frodo,29511 -Sam,Frodo,29521 -Sam,Frodo,29532 -Sam,Frodo,29545 -Gorbag,Sam,29578 -Sam,Frodo,29583 -Sam,Frodo,29584 -Sam,Frodo,29593 -Sam,Faramir,29619 -Sam,Frodo,29623 -Sam,Galadriel,29649 -Sam,Frodo,29654 -Sam,Frodo,29679 -Sam,Frodo,29684 -Sam,Frodo,29689 -Sam,Frodo,29694 -Sam,Frodo,29709 -Sam,Shagrat,29717 -Sam,Frodo,29739 -Sam,Frodo,29751 -Sam,Frodo,29764 -Sam,Frodo,29780 -Sam,Frodo,29788 -Gorbag,Shagrat,29792 -Sam,Frodo,29795 -Sam,Frodo,29807 -Sam,Frodo,29836 -Sam,Frodo,29851 -Sam,Faramir,29855 -Sam,Frodo,29863 -Sam,Frodo,29874 -Sam,Frodo,29888 -Sam,Frodo,29972 -Sam,Frodo,29987 -Sam,Shelob,29987 -Sam,Gollum,29987 -Frodo,Shelob,29987 -Frodo,Gollum,29987 -Shelob,Gollum,29987 -Sam,Frodo,29988 -Sam,Frodo,30000 -Sam,Frodo,30011 -Sam,Frodo,30029 -Sam,Frodo,30031 -Sam,Frodo,30035 -Sam,Frodo,30042 -Sam,Frodo,30043 -Frodo,Gollum,30061 -Sam,Frodo,30064 -Sam,Frodo,30085 -Sam,Frodo,30118 -Sam,Frodo,30141 -Sam,Frodo,30153 -Sam,Frodo,30170 -Frodo,Gandalf,30173 -Sam,Frodo,30187 -Sam,Frodo,30276 -Sam,Faramir,30276 -Frodo,Faramir,30276 -Sam,Frodo,30316 -Sam,Frodo,30351 -Sam,Frodo,30365 -Sam,Frodo,30368 -Sam,Frodo,30371 -Sam,Frodo,30379 -Sam,Frodo,30413 -Sam,Frodo,30419 -Frodo,Gollum,30437 -Sam,Gollum,30440 -Sam,Frodo,30453 -Sam,Gollum,30453 -Frodo,Gollum,30453 -Sam,Gollum,30488 -Sam,Frodo,30513 -Sam,Gollum,30536 -Sam,Frodo,30547 -Gollum,Gandalf,30571 -Gandalf,Sauron,30601 -Gandalf,Aragorn,30610 -Meneldor,Gandalf,30615 -Sam,Frodo,30618 -Sam,Frodo,30622 -Sam,Frodo,30628 -Sam,Frodo,30630 -Sam,Gandalf,30668 -Gandalf,Sauron,30681 -Sam,Frodo,30694 -Sam,Galadriel,30694 -Frodo,Galadriel,30694 -Frodo,Samwise,30706 -Sam,Frodo,30716 -Sam,Frodo,30721 -Sam,Aragorn,30723 -Sam,Frodo,30725 -Sam,Frodo,30729 -Sam,Frodo,30736 -Frodo,Gandalf,30737 -Sam,Frodo,30741 -Frodo,Gandalf,30744 -Frodo,Bilbo,30744 -Gandalf,Bilbo,30744 -Éomer,Gimli,30745 -Éomer,Legolas,30745 -Éomer,Gandalf,30745 -Éomer,Imrahil,30745 -Gimli,Legolas,30745 -Gimli,Gandalf,30745 -Gimli,Imrahil,30745 -Legolas,Gandalf,30745 -Legolas,Imrahil,30745 -Gandalf,Imrahil,30745 -Peregrin,Merry,30751 -Pippin,Merry,30759 -Pippin,Gandalf,30759 -Pippin,Frodo,30759 -Pippin,Gimli,30759 -Pippin,Sam,30759 -Pippin,Legolas,30759 -Merry,Gandalf,30759 -Merry,Frodo,30759 -Merry,Gimli,30759 -Merry,Sam,30759 -Merry,Legolas,30759 -Gandalf,Frodo,30759 -Gandalf,Gimli,30759 -Gandalf,Sam,30759 -Gandalf,Legolas,30759 -Frodo,Gimli,30759 -Frodo,Sam,30759 -Frodo,Legolas,30759 -Gimli,Sam,30759 -Gimli,Legolas,30759 -Sam,Legolas,30759 -Sam,Frodo,30760 -Pippin,Frodo,30763 -Pippin,Merry,30763 -Frodo,Merry,30763 -Gandalf,Bilbo,30772 -Gimli,Pippin,30775 -Gimli,Frodo,30775 -Gimli,Sam,30775 -Pippin,Frodo,30775 -Pippin,Sam,30775 -Frodo,Sam,30775 -Sam,Frodo,30798 -Éomer,Théoden,30860 -Éomer,Faramir,30860 -Théoden,Faramir,30860 -Éowyn,Merry,30888 -Éowyn,Faramir,30888 -Merry,Faramir,30888 -Merry,Faramir,30889 -Éowyn,Faramir,30896 -Éowyn,Amroth,30903 -Éowyn,Finduilas,30903 -Amroth,Finduilas,30903 -Éowyn,Faramir,30905 -Merry,Faramir,30945 -Éowyn,Faramir,30949 -Elendil,Aragorn,30950 -Éowyn,Faramir,30960 -Éowyn,Faramir,30981 -Éowyn,Húrin,30991 -Éowyn,Faramir,30991 -Húrin,Faramir,30991 -Éomer,Gandalf,30995 -Éomer,Imrahil,30995 -Gandalf,Imrahil,30995 -Húrin,Faramir,31008 -Faramir,Aragorn,31009 -Isildur,Valandil,31013 -Isildur,Elendil,31013 -Isildur,Elessar,31013 -Isildur,Aragorn,31013 -Valandil,Elendil,31013 -Valandil,Elessar,31013 -Valandil,Aragorn,31013 -Elendil,Elessar,31013 -Elendil,Aragorn,31013 -Elessar,Aragorn,31013 -Eärnur,Faramir,31017 -Faramir,Aragorn,31022 -Frodo,Gandalf,31023 -Frodo,Faramir,31023 -Frodo,Aragorn,31023 -Gandalf,Faramir,31023 -Gandalf,Aragorn,31023 -Faramir,Aragorn,31023 -Húrin,Elessar,31025 -Húrin,Faramir,31025 -Elessar,Faramir,31025 -Beregond,Faramir,31038 -Faramir,Aragorn,31039 -Éomer,Aragorn,31040 -Éowyn,Elrond,31045 -Éowyn,Faramir,31045 -Elrond,Faramir,31045 -Gimli,Legolas,31048 -Gimli,Aragorn,31048 -Legolas,Aragorn,31048 -Frodo,Gandalf,31052 -Frodo,Aragorn,31052 -Gandalf,Aragorn,31052 -Gandalf,Bilbo,31053 -Pippin,Frodo,31054 -Pippin,Gandalf,31054 -Frodo,Gandalf,31054 -Gandalf,Aragorn,31057 -Gandalf,Aragorn,31058 -Gandalf,Aragorn,31067 -Gandalf,Nimloth,31086 -Nimloth,Aragorn,31094 -Glorfindel,Arwen,31102 -Glorfindel,Erestor,31102 -Glorfindel,Elrond,31102 -Glorfindel,Galadriel,31102 -Glorfindel,Celeborn,31102 -Arwen,Erestor,31102 -Arwen,Elrond,31102 -Arwen,Galadriel,31102 -Arwen,Celeborn,31102 -Erestor,Elrond,31102 -Erestor,Galadriel,31102 -Erestor,Celeborn,31102 -Elrond,Galadriel,31102 -Elrond,Celeborn,31102 -Galadriel,Celeborn,31102 -Frodo,Gandalf,31103 -Arwen,Elessar,31106 -Arwen,Aragorn,31106 -Elessar,Aragorn,31106 -Arwen,Frodo,31108 -Frodo,Aragorn,31109 -Arwen,Bilbo,31113 -Arwen,Elrond,31113 -Bilbo,Elrond,31113 -Éomer,Théoden,31119 -Éomer,Gimli,31129 -Éomer,Glóin,31129 -Gimli,Glóin,31129 -Éomer,Gimli,31131 -Théoden,Merry,31141 -Pippin,Gandalf,31142 -Pippin,Shadowfax,31142 -Pippin,Frodo,31142 -Pippin,Gimli,31142 -Pippin,Aragorn,31142 -Pippin,Legolas,31142 -Pippin,Samwise,31142 -Gandalf,Shadowfax,31142 -Gandalf,Frodo,31142 -Gandalf,Gimli,31142 -Gandalf,Aragorn,31142 -Gandalf,Legolas,31142 -Gandalf,Samwise,31142 -Shadowfax,Frodo,31142 -Shadowfax,Gimli,31142 -Shadowfax,Aragorn,31142 -Shadowfax,Legolas,31142 -Shadowfax,Samwise,31142 -Frodo,Gimli,31142 -Frodo,Aragorn,31142 -Frodo,Legolas,31142 -Frodo,Samwise,31142 -Gimli,Aragorn,31142 -Gimli,Legolas,31142 -Gimli,Samwise,31142 -Aragorn,Legolas,31142 -Aragorn,Samwise,31142 -Legolas,Samwise,31142 -Elrond,Arwen,31143 -Elrond,Galadriel,31143 -Elrond,Amroth,31143 -Elrond,Celeborn,31143 -Arwen,Galadriel,31143 -Arwen,Amroth,31143 -Arwen,Celeborn,31143 -Galadriel,Amroth,31143 -Galadriel,Celeborn,31143 -Amroth,Celeborn,31143 -Théoden,Thengel,31144 -Elessar,Aragorn,31146 -Théoden,Thengel,31152 -Helm,Théoden,31153 -Théoden,Merry,31156 -Éowyn,Éomer,31161 -Fréa,Fengel,31162 -Fréa,Helm,31162 -Fréa,Théoden,31162 -Fréa,Baldor,31162 -Fréa,Déor,31162 -Fréa,Brego,31162 -Fréa,Goldwine,31162 -Fréa,Folcwine,31162 -Fréa,Aldor,31162 -Fréa,Walda,31162 -Fréa,Folca,31162 -Fréa,Thengel,31162 -Fréa,Gram,31162 -Fengel,Helm,31162 -Fengel,Théoden,31162 -Fengel,Baldor,31162 -Fengel,Déor,31162 -Fengel,Brego,31162 -Fengel,Goldwine,31162 -Fengel,Folcwine,31162 -Fengel,Aldor,31162 -Fengel,Walda,31162 -Fengel,Folca,31162 -Fengel,Thengel,31162 -Fengel,Gram,31162 -Helm,Théoden,31162 -Helm,Baldor,31162 -Helm,Déor,31162 -Helm,Brego,31162 -Helm,Goldwine,31162 -Helm,Folcwine,31162 -Helm,Aldor,31162 -Helm,Walda,31162 -Helm,Folca,31162 -Helm,Thengel,31162 -Helm,Gram,31162 -Théoden,Baldor,31162 -Théoden,Déor,31162 -Théoden,Brego,31162 -Théoden,Goldwine,31162 -Théoden,Folcwine,31162 -Théoden,Aldor,31162 -Théoden,Walda,31162 -Théoden,Folca,31162 -Théoden,Thengel,31162 -Théoden,Gram,31162 -Baldor,Déor,31162 -Baldor,Brego,31162 -Baldor,Goldwine,31162 -Baldor,Folcwine,31162 -Baldor,Aldor,31162 -Baldor,Walda,31162 -Baldor,Folca,31162 -Baldor,Thengel,31162 -Baldor,Gram,31162 -Déor,Brego,31162 -Déor,Goldwine,31162 -Déor,Folcwine,31162 -Déor,Aldor,31162 -Déor,Walda,31162 -Déor,Folca,31162 -Déor,Thengel,31162 -Déor,Gram,31162 -Brego,Goldwine,31162 -Brego,Folcwine,31162 -Brego,Aldor,31162 -Brego,Walda,31162 -Brego,Folca,31162 -Brego,Thengel,31162 -Brego,Gram,31162 -Goldwine,Folcwine,31162 -Goldwine,Aldor,31162 -Goldwine,Walda,31162 -Goldwine,Folca,31162 -Goldwine,Thengel,31162 -Goldwine,Gram,31162 -Folcwine,Aldor,31162 -Folcwine,Walda,31162 -Folcwine,Folca,31162 -Folcwine,Thengel,31162 -Folcwine,Gram,31162 -Aldor,Walda,31162 -Aldor,Folca,31162 -Aldor,Thengel,31162 -Aldor,Gram,31162 -Walda,Folca,31162 -Walda,Thengel,31162 -Walda,Gram,31162 -Folca,Thengel,31162 -Folca,Gram,31162 -Thengel,Gram,31162 -Éomer,Théoden,31163 -Éowyn,Éomer,31164 -Éowyn,Théoden,31164 -Éomer,Théoden,31164 -Éowyn,Faramir,31166 -Éowyn,Faramir,31167 -Éowyn,Éomer,31168 -Éowyn,Aragorn,31168 -Éomer,Aragorn,31168 -Arwen,Faramir,31170 -Arwen,Imrahil,31170 -Arwen,Aragorn,31170 -Faramir,Imrahil,31170 -Faramir,Aragorn,31170 -Imrahil,Aragorn,31170 -Éowyn,Meriadoc,31172 -Éowyn,Éomer,31172 -Éowyn,Merry,31172 -Meriadoc,Éomer,31172 -Meriadoc,Merry,31172 -Éomer,Merry,31172 -Éowyn,Merry,31174 -Éowyn,Merry,31178 -Gimli,Legolas,31180 -Treebeard,Gandalf,31188 -Treebeard,Gandalf,31192 -Treebeard,Aragorn,31197 -Treebeard,Gandalf,31202 -Treebeard,Merry,31202 -Gandalf,Merry,31202 -Galadriel,Celeborn,31245 -Gimli,Legolas,31247 -Treebeard,Galadriel,31255 -Treebeard,Celeborn,31255 -Galadriel,Celeborn,31255 -Galadriel,Celeborn,31259 -Pippin,Merry,31261 -Pippin,Aragorn,31268 -Pippin,Aragorn,31270 -Celeborn,Galadriel,31274 -Celeborn,Aragorn,31274 -Galadriel,Aragorn,31274 -Gandalf,Saruman,31282 -Saruman,Galadriel,31295 -Saruman,Gandalf,31313 -Saruman,Wormtongue,31313 -Gandalf,Wormtongue,31313 -Saruman,Merry,31330 -Saruman,Wormtongue,31330 -Merry,Wormtongue,31330 -Galadriel,Celeborn,31349 -Gandalf,Elrond,31350 -Frodo,Galadriel,31356 -Sam,Elrond,31357 -Sam,Frodo,31389 -Sam,Elrond,31389 -Frodo,Elrond,31389 -Pippin,Merry,31401 -Pippin,Bilbo,31402 -Frodo,Bilbo,31406 -Sam,Aragorn,31411 -Sam,Frodo,31423 -Gandalf,Bilbo,31436 -Gandalf,Elrond,31436 -Bilbo,Elrond,31436 -Frodo,Elrond,31437 -Frodo,Gandalf,31445 -Pippin,Sam,31474 -Pippin,Sam,31475 -Gandalf,Butterbur,31524 -Gandalf,Barliman,31581 -Butterbur,Merry,31614 -Sam,Bill,31626 -Sam,Bill,31629 -Sam,Bill,31651 -Frodo,Barliman,31652 -Pippin,Gandalf,31656 -Pippin,Lotho,31656 -Gandalf,Lotho,31656 -Merry,Gandalf,31658 -Gandalf,Shadowfax,31684 -Frodo,Lotho,31707 -Pippin,Merry,31716 -Pippin,Merry,31717 -Merry,Bill,31723 -Sam,Bill,31731 -Robin,Sam,31816 -Sam,Frodo,31834 -Frodo,Merry,31844 -Pippin,Sam,31845 -Pippin,Merry,31845 -Sam,Merry,31845 -Sam,Frodo,31863 -Sam,Merry,31875 -Sam,Bill,31880 -Pippin,Frodo,31930 -Pippin,Merry,31930 -Pippin,Sam,31930 -Frodo,Merry,31930 -Frodo,Sam,31930 -Merry,Sam,31930 -Frodo,Lotho,31940 -Pippin,Frodo,31943 -Frodo,Lotho,31950 -Pippin,Lotho,31957 -Sam,Frodo,32044 -Tom,Frodo,32058 -Tom,Sam,32058 -Frodo,Sam,32058 -Frodo,Merry,32075 -Peregrin,Lotho,32080 -Frodo,Merry,32095 -Sam,Frodo,32234 -Sam,Frodo,32238 -Sam,Frodo,32239 -Pippin,Merry,32283 -Meriadoc,Peregrin,32293 -Pippin,Sam,32295 -Pippin,Merry,32295 -Sam,Merry,32295 -Frodo,Merry,32296 -Pippin,Frodo,32301 -Pippin,Merry,32301 -Pippin,Sam,32301 -Frodo,Merry,32301 -Frodo,Sam,32301 -Merry,Sam,32301 -Sam,Frodo,32331 -Merry,Lotho,32348 -Merry,Saruman,32355 -Gandalf,Saruman,32372 -Saruman,Wormtongue,32396 -Saruman,Wormtongue,32399 -Frodo,Saruman,32400 -Sam,Frodo,32404 -Frodo,Saruman,32409 -Frodo,Wormtongue,32422 -Saruman,Wormtongue,32432 -Saruman,Wormtongue,32443 -Frodo,Wormtongue,32444 -Pippin,Fredegar,32459 -Frodo,Lotho,32469 -Pippin,Merry,32473 -Sam,Frodo,32499 -Sam,Frodo,32520 -Sam,Frodo,32523 -Pippin,Merry,32524 -Sam,Frodo,32525 -Sam,Frodo,32527 -Sam,Frodo,32538 -Pippin,Merry,32541 -Sam,Frodo,32544 -Sam,Frodo,32547 -Meriadoc,Peregrin,32548 -Meriadoc,Sam,32548 -Peregrin,Sam,32548 -Sam,Frodo,32552 -Sam,Frodo,32557 -Sam,Frodo,32563 -Sam,Frodo,32565 -Sam,Frodo,32568 -Sam,Frodo,32569 -Sam,Bilbo,32570 -Sam,Frodo,32573 -Sam,Frodo,32579 -Sam,Bilbo,32579 -Frodo,Bilbo,32579 -Sam,Frodo,32584 -Frodo,Bilbo,32586 -Frodo,Bilbo,32595 -Sam,Frodo,32597 -Sam,Frodo,32598 -Sam,Frodo,32599 -Sam,Bill,32599 -Frodo,Bill,32599 -Sam,Frodo,32603 -Sam,Frodo,32613 -Elrond,Sam,32614 -Elrond,Galadriel,32614 -Sam,Galadriel,32614 -Elrond,Galadriel,32619 -Sam,Frodo,32631 -Pippin,Merry,32645 -Pippin,Goldilocks,32645 -Merry,Goldilocks,32645 -Galadriel,Elrond,32650 -Sam,Frodo,32651 -Sam,Bilbo,32651 -Frodo,Bilbo,32651 -Frodo,Gandalf,32656 -Pippin,Merry,32659 -Sam,Gandalf,32663 -Pippin,Frodo,32666 -Pippin,Merry,32666 -Pippin,Galadriel,32666 -Pippin,Sam,32666 -Frodo,Merry,32666 -Frodo,Galadriel,32666 -Frodo,Sam,32666 -Merry,Galadriel,32666 -Merry,Sam,32666 -Galadriel,Sam,32666 -Pippin,Merry,32671 -Pippin,Merry,32674 diff --git a/mainproject/src/main/scala/com/raphtory/examples/test/actors/RandomGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/test/actors/RandomGraphBuilder.scala deleted file mode 100644 index d852e6af56..0000000000 --- a/mainproject/src/main/scala/com/raphtory/examples/test/actors/RandomGraphBuilder.scala +++ /dev/null @@ -1,95 +0,0 @@ -package com.raphtory.examples.test.actors - -import com.raphtory.core.actors.Router.GraphBuilder -import com.raphtory.core.model.communication.{GraphUpdate, _} -import spray.json._ - -/** - * The Graph Manager is the top level actor in this system (under the stream) - * which tracks all the graph partitions - passing commands processed by the 'command processor' actors - * to the correct partition - */ -/** - * The Command Processor takes string message from Kafka and translates them into - * the correct case Class which can then be passed to the graph manager - * which will then pass it to the graph partition dealing with the associated vertex - */ -class RandomGraphBuilder extends GraphBuilder[String]{ - - //************* MESSAGE HANDLING BLOCK - - override def parseTuple(tuple:String) = { - val command = tuple.asInstanceOf[String] - val parsedOBJ = command.parseJson.asJsObject //get the json object - val commandKey = parsedOBJ.fields //get the command type - if (commandKey.contains("VertexAdd")) - sendUpdate(vertexAdd(parsedOBJ.getFields("VertexAdd").head.asJsObject)) - //else if(commandKey.contains("VertexUpdateProperties")) vertexUpdateProperties(parsedOBJ.getFields("VertexUpdateProperties").head.asJsObject) - else if (commandKey.contains("VertexRemoval")) - sendUpdate(vertexRemoval(parsedOBJ.getFields("VertexRemoval").head.asJsObject)) - else if (commandKey.contains("EdgeAdd")) - sendUpdate(edgeAdd(parsedOBJ.getFields("EdgeAdd").head.asJsObject)) //if addVertex, parse to handling function - // else if(commandKey.contains("EdgeUpdateProperties")) edgeUpdateProperties(parsedOBJ.getFields("EdgeUpdateProperties").head.asJsObject) - else if (commandKey.contains("EdgeRemoval")) - sendUpdate(edgeRemoval(parsedOBJ.getFields("EdgeRemoval").head.asJsObject)) - } - - def vertexAdd(command: JsObject) = { - val msgTime = command.fields("messageID").toString().toLong - val srcId = command.fields("srcID").toString().toInt //extract the srcID - if (command.fields.contains("properties")) { //if there are properties within the command - var properties = Properties() //create a vertex map - //command.fields("properties").asJsObject.fields.foreach( pair => { //add all of the pairs to the map - // properties = properties updated (pair._1, pair._2.toString()) - // }) - //send the srcID and properties to the graph manager - VertexAddWithProperties(msgTime, srcId, properties) - } else - VertexAdd(msgTime, srcId) - // if there are not any properties, just send the srcID - } - -// def vertexUpdateProperties(command:JsObject):Unit={ -// val msgTime = command.fields("messageID").toString().toLong -// val srcId = command.fields("srcID").toString().toInt //extract the srcID -// var properties = Properties() //create a vertex map -// //command.fields("properties").asJsObject.fields.foreach( pair => {properties = properties updated (pair._1,pair._2.toString())}) -// sendGraphUpdate(VertexUpdateProperties(msgTime,srcId,properties)) //send the srcID and properties to the graph parition -// } - - def vertexRemoval(command: JsObject):GraphUpdate = { - val msgTime = command.fields("messageID").toString().toLong - val srcId = command.fields("srcID").toString().toInt //extract the srcID - VertexDelete(msgTime, srcId) - } - - def edgeAdd(command: JsObject):GraphUpdate = { - val msgTime = command.fields("messageID").toString().toLong - val srcId = command.fields("srcID").toString().toInt //extract the srcID - val dstId = command.fields("dstID").toString().toInt //extract the dstID - if (command.fields.contains("properties")) { //if there are properties within the command - var properties = Properties() //create a vertex map -// command.fields("properties").asJsObject.fields.foreach( pair => { //add all of the pairs to the map -// properties = properties updated (pair._1,pair._2.toString()) -// }) - EdgeAddWithProperties(msgTime, srcId, dstId, properties) - } else EdgeAdd(msgTime, srcId, dstId) - } - -// def edgeUpdateProperties(command:JsObject):Unit={ -// val msgTime = command.fields("messageID").toString().toLong -// val srcId = command.fields("srcID").toString().toInt //extract the srcID -// val dstId = command.fields("dstID").toString().toInt //extract the dstID -// var properties =Properties() //create a vertex map -// //command.fields("properties").asJsObject.fields.foreach( pair => {properties = properties updated (pair._1,pair._2.toString())}) -// sendGraphUpdate(EdgeUpdateProperties(msgTime,srcId,dstId,properties))//send the srcID, dstID and properties to the graph manager -// } - - def edgeRemoval(command: JsObject):GraphUpdate = { - val msgTime = command.fields("messageID").toString().toLong - val srcId = command.fields("srcID").toString().toInt //extract the srcID - val dstId = command.fields("dstID").toString().toInt //extract the dstID - EdgeDelete(msgTime, srcId, dstId) - } - -} diff --git a/mainproject/src/main/scala/com/raphtory/examples/test/actors/RandomSpout.scala b/mainproject/src/main/scala/com/raphtory/examples/test/actors/RandomSpout.scala deleted file mode 100644 index a3217beef0..0000000000 --- a/mainproject/src/main/scala/com/raphtory/examples/test/actors/RandomSpout.scala +++ /dev/null @@ -1,94 +0,0 @@ -package com.raphtory.examples.test.actors - -import com.raphtory.core.actors.Spout.Spout - -import scala.language.postfixOps -import scala.util.Random - -class RandomSpout extends Spout[String] { - - var totalCount = 100 - var freq = 1000 - var pool = System.getenv().getOrDefault("ENTITY_POOL", "1000000").toInt - var msgID = 0 - - override def setupDataSource(): Unit = {} - - override def generateData(): Option[String] = Some(distribution()) - - override def closeDataSource(): Unit = {} - - - def distribution(): String = { - val random = Random.nextFloat() - if (random <= 0.3) genVertexAdd() - else if (random <= 0.7) genEdgeAdd() - else if (random <= 0.8) genVertexRemoval() - else genEdgeRemoval() - } - - def genVertexAdd(): String = - s""" {"VertexAdd":{${getMessageID()}, ${genSrcID()}, ${genProperties(2)}}}""" - - def genVertexAdd(src: Int): String = //overloaded method if you want to specify src ID - s""" {"VertexAdd":{${getMessageID()}, ${genSrcID(src)}, ${genProperties(2)}}}""" - - def genVertexUpdateProperties(): String = - s""" {"VertexUpdateProperties":{${getMessageID()}, ${genSrcID(1)}}}""" - - def genVertexUpdateProperties(src: Int): String = //overloaded to mass src - s""" {"VertexUpdateProperties":{${getMessageID()}, ${genSrcID(src)}, ${genProperties(2)}}}""" - - def genVertexRemoval(): String = - s""" {"VertexRemoval":{${getMessageID()}, ${genSrcID()}}}""" - - def genVertexRemoval(src: Int): String = - s""" {"VertexRemoval":{${getMessageID()}, ${genSrcID(src)}}}""" - - def genEdgeAdd(): String = - s""" {"EdgeAdd":{${getMessageID()}, ${genSrcID()}, ${genDstID()}, ${genProperties(2)}}}""" - - def genEdgeAdd(src: Int, dst: Int): String = - s""" {"EdgeAdd":{${getMessageID()}, ${genSrcID(src)}, ${genDstID(dst)}}}""" - - def genEdgeUpdateProperties(): String = - s""" {"EdgeUpdateProperties":{${getMessageID()}, ${genSrcID()}, ${genDstID()}, ${genProperties(2)}}}""" - - def genEdgeUpdateProperties(src: Int, dst: Int): String = - s""" {"EdgeUpdateProperties":{${getMessageID()}, ${genSrcID(src)}, ${genDstID(dst)}, ${genProperties(2)}}}""" - - def genEdgeRemoval(): String = - s""" {"EdgeRemoval":{${getMessageID()}, ${genSrcID()}, ${genDstID()}}}""" - - def genEdgeRemoval(src: Int, dst: Int): String = - s""" {"EdgeRemoval":{${getMessageID()}, ${genSrcID(src)}, ${genDstID(dst)}}}""" - - def genSetSrcID(): String = s""" "srcID":9 """ - - def genSetDstID(): String = s""" "dstID":10 """ - - def genSrcID(): String = s""" "srcID":${Random.nextInt(pool)} """ - - def genDstID(): String = s""" "dstID":${Random.nextInt(pool)} """ - - def genSrcID(src: Int): String = s""" "srcID":$src """ - - def genDstID(dst: Int): String = s""" "dstID":$dst """ - - def getMessageID(): String = { - msgID += 1 - //s""" "messageID":${System.currentTimeMillis()} """ - s""" "messageID":$msgID """ - } - - def genProperties(numOfProps: Int): String = { - var properties = "\"properties\":{" - for (i <- 1 to numOfProps) { - val propnum = i - if (i < numOfProps) properties = properties + s""" "property$propnum":"${"test"}", """ - else properties = properties + s""" "property$propnum":"${"test"}" }""" - } - properties - } - -} diff --git a/mainproject/src/main/scala/com/raphtory/examples/test/actors/TriangleGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/test/actors/TriangleGraphBuilder.scala deleted file mode 100644 index 5626cec18d..0000000000 --- a/mainproject/src/main/scala/com/raphtory/examples/test/actors/TriangleGraphBuilder.scala +++ /dev/null @@ -1,14 +0,0 @@ -package com.raphtory.examples.test.actors - -import com.raphtory.core.actors.Router.GraphBuilder -import com.raphtory.core.model.communication.EdgeAdd - -class TriangleGraphBuilder extends GraphBuilder[String] { - override def parseTuple(tuple: String)= { - val command = tuple.split(",") - val msgTime = command(0).toLong - val sourceID = command(1).toLong - val destID = command(2).toLong - sendUpdate(EdgeAdd(msgTime, sourceID, destID)) - } -} diff --git a/mainproject/src/main/scala/com/raphtory/examples/test/actors/TriangleSpout.scala b/mainproject/src/main/scala/com/raphtory/examples/test/actors/TriangleSpout.scala deleted file mode 100644 index b8a45a7b5c..0000000000 --- a/mainproject/src/main/scala/com/raphtory/examples/test/actors/TriangleSpout.scala +++ /dev/null @@ -1,23 +0,0 @@ -package com.raphtory.examples.test.actors - -import com.raphtory.core.actors.Spout.Spout - -import scala.collection.mutable.Queue - -class TriangleSpout extends Spout[String] { - - val edges = Queue[String]("3,3,1", "4,3,4", "2,2,3", "5,4,1", "6,1,3", "1,1,2") - - override def setupDataSource(): Unit = {} - - override def generateData(): Option[String] = { - if(edges.isEmpty) { - dataSourceComplete() - None - } - else - Some(edges.dequeue()) - } - - override def closeDataSource(): Unit = {} -} diff --git a/mainproject/src/main/scala/com/raphtory/examples/trackAndTrace/TrackAndTraceGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/trackAndTrace/TrackAndTraceGraphBuilder.scala deleted file mode 100644 index ffa979e7df..0000000000 --- a/mainproject/src/main/scala/com/raphtory/examples/trackAndTrace/TrackAndTraceGraphBuilder.scala +++ /dev/null @@ -1,101 +0,0 @@ -package com.raphtory.examples.trackAndTrace - -import com.raphtory.core.actors.Router.GraphBuilder -import com.raphtory.core.model.communication.{Type, _} - -class TrackAndTraceGraphBuilder extends GraphBuilder[String] { - val EARTH_EQU = 6378137.0 //m - val EARTH_POL = 6356752.3142 //m - val STEPSIZE = System.getenv().getOrDefault("MAP_GRID_SIZE", "100").trim.toDouble //m - - override def parseTuple(tuple: String) = { - val datapoint = lineToDatapoint(tuple.split(",").map(_.trim)) - val eventTime = datapoint.time - val userID = datapoint.userId - val latitude = datapoint.latitude - val longitude = datapoint.longitude - val locationID = locationIDGenerator(latitude, longitude) - - sendUpdate(VertexAdd(eventTime, userID, Type("User"))) - - sendUpdate( - VertexAddWithProperties( - eventTime, - locationID, - Properties(DoubleProperty("latitude", latitude), DoubleProperty("longitude", longitude)), - Type("Location") - ) - ) - - sendUpdate(EdgeAdd(eventTime, userID, locationID, Type("User Visted Location"))) - } - - //converts the line into a case class which has all of the data via the correct name and type - def lineToDatapoint(line: Array[String]): Datapoint = - Datapoint( - line(0).toLong, //User ID - line(1).toLong, //Trip ID - line(2).toLong, //OSM way ID - line(3).toLong, //OSM node ID - line(4).toDouble, //Latitude - line(5).toDouble, //Longitude - line(6), //Node type - line(7), //Road classification - line(8), //Rail road classification - line(9), //Water way classification - line(10), //Aerial way classification - line(11).toLong * 1000, //Location time, in seconds (milli for raphtory) - line(15).toLong, //Location duration - line(16).toDouble, //Location speed - line(17).toDouble, // Location distance - line(18).toDouble, //Location accuracy - line(19), //Location quality - line(20), //Location modality - longCheck(line(21)), //SM relation ID - line(22).toDouble, //O Altitude in m - (line(23).toDouble * 1000).toLong, //Time estimate in decimal seconds (converted to millis) - line(24) // Source - ) - - def longCheck(data: String): Option[Long] = if (data equals "") None else Some(data.toLong) - - def getCartCoord(lat: Double, long: Double): (Double, Double) = { - val e = 1 - (Math.pow(EARTH_EQU, 2) / Math.pow(EARTH_POL, 2)) - val N = EARTH_EQU / (Math.sqrt(1 - (e * Math.pow(Math.sin(lat), 2)))) - val x = N * Math.cos(lat) * Math.cos(long) - val y = N * Math.cos(lat) * Math.sin(long) - (x, y) - } - - def locationIDGenerator(latitude: Double, longitude: Double): Long = { //make a unique id for the location - val (x, y) = getCartCoord(latitude, longitude) // converts lat-long to cartesian coordinates - val ptx = Math.floor(x / STEPSIZE) * STEPSIZE // gets coordinates of grid s.t. (x,y) in grid - val pty = Math.floor(y / STEPSIZE) * STEPSIZE - assignID(ptx.toString + pty.toString) //assigns unique ID to grid - } -} - -case class Datapoint( - userId: Long, //User ID - tripId: Long, //Trip ID - wayId: Long, //OSM way ID - nodeId: Long, //OSM node ID - latitude: Double, //Latitude - longitude: Double, //Longitude - nodeType: String, //Node type - highway: String, //Road classification - railway: String, //Rail road classification - waterway: String, //Water way classification - aerialway: String, //Aerial way classification - time: Long, //Location time, in seconds (milli for raphtory) - duration: Long, //Location duration - speed: Double, //Location speed - distance: Double, // Location distance - deviation: Double, //Location accuracy - quality: String, //Location quality - modality: String, //Location modality from accelerometer; when quality=missing this indicates the modality of the used corridor or filled-in segment in a sensing gap - relation: Option[Long], //SM relation ID //wrapped as an option because data missing - altitude: Double, //O Altitude in m - estimatedTime: Long, //Time estimate in decimal seconds - source: String // Where this location originates from, either map matching (M) or sensing (S) -) diff --git a/mainproject/src/main/scala/com/raphtory/examples/tsvnet/SamplerSpout.scala b/mainproject/src/main/scala/com/raphtory/examples/tsvnet/SamplerSpout.scala deleted file mode 100644 index 4ba3214a5e..0000000000 --- a/mainproject/src/main/scala/com/raphtory/examples/tsvnet/SamplerSpout.scala +++ /dev/null @@ -1,56 +0,0 @@ -//package com.raphtory.examples.tsvnet -// -//import java.time.LocalDateTime -// -//import com.raphtory.core.components.Spout.Spout -//import com.raphtory.core.components.Spout.Spout.BasicDomain -//import com.raphtory.core.components.Spout.Spout.CommonMessage.Next -//import com.raphtory.core.model.communication.StringSpoutGoing -// -//import scala.concurrent.duration.{Duration, NANOSECONDS} -//import scala.util.Random -//import scala.util.control.Breaks._ -// -//class SamplerSpout extends Spout[BasicDomain,StringSpoutGoing] { -// -// val prop = 1.0/10 -// val r = new Random() -// val directory = System.getenv().getOrDefault("TSV_DIRECTORY", "/app").trim -// val file_name = System.getenv().getOrDefault("TSV_FILE_NAME", "sx_reordered.txt").trim -// val fileLines = scala.io.Source.fromFile(directory + "/" + file_name).getLines.drop(1).toArray -// // upstream/master -// var position = 0 -// var linesNumber = fileLines.length -// println("Start: " + LocalDateTime.now()) -// -// override def handleDomainMessage(message: BasicDomain): Unit = message match { -// case Next => -// try { -// if (position < linesNumber) { -// for (i <- 1 to 100) { -// val randFloat = r.nextFloat() -// breakable { -// if (randFloat > prop) { -// position +=1 -// break -// } else { -// val line = fileLines(position) -// sendTuple(StringSpoutGoing(line)) -// position += 1 -// } -// } -// } -// AllocateSpoutTask(Duration(1, NANOSECONDS), "newLine") -// } -// else -// { -// println("Finished ingestion") -// } -// } -// catch {case e:Exception => println("Finished ingestion")} -// case _ => println("message not recognized!") -// } -// -// override def startSpout(): Unit = self ! Next -// -//} diff --git a/mainproject/src/main/scala/com/raphtory/examples/tsvnet/TSVGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/tsvnet/TSVGraphBuilder.scala deleted file mode 100644 index 4fda7b5fef..0000000000 --- a/mainproject/src/main/scala/com/raphtory/examples/tsvnet/TSVGraphBuilder.scala +++ /dev/null @@ -1,28 +0,0 @@ -package com.raphtory.examples.tsvnet - -import com.raphtory.core.actors.Router.GraphBuilder -import com.raphtory.core.model.communication.{EdgeAdd, Type, VertexAdd} - -/** Spout for network datasets of the form SRC_NODE_ID DEST_NODE_ID TIMESTAMP */ -class TSVGraphBuilder extends GraphBuilder[String] { - override def parseTuple(tuple: String) = { - val fileLine = tuple.split(" ").map(_.trim) - //user wise - val sourceNode = fileLine(0).toInt - val targetNode = fileLine(1).toInt - val creationDate = fileLine(2).toLong - - - - //comment wise - // val sourceNode=fileLine(1).toInt - //val targetNode=fileLine(4).toInt - if (targetNode > 0) { - val creationDate = fileLine(2).toLong - sendUpdate(VertexAdd(creationDate, sourceNode, Type("User"))) - sendUpdate(VertexAdd(creationDate, targetNode, Type("User"))) - sendUpdate(EdgeAdd(creationDate, sourceNode, targetNode, Type("User to User"))) - - } - } -} diff --git a/mainproject/src/main/scala/com/raphtory/examples/wordSemantic/graphbuilders/CooccurrenceMatrixGraphBuilder.scala b/mainproject/src/main/scala/com/raphtory/examples/wordSemantic/graphbuilders/CooccurrenceMatrixGraphBuilder.scala deleted file mode 100644 index 247be1a7f4..0000000000 --- a/mainproject/src/main/scala/com/raphtory/examples/wordSemantic/graphbuilders/CooccurrenceMatrixGraphBuilder.scala +++ /dev/null @@ -1,40 +0,0 @@ -package com.raphtory.examples.wordSemantic.graphbuilders - -import com.raphtory.core.actors.Router.GraphBuilder -import com.raphtory.core.model.communication._ - -class CooccurrenceMatrixGraphBuilder extends GraphBuilder[String] { - - override def parseTuple(tuple: String) = { - //println(record) - var dp = tuple.split(" ").map(_.trim) - val occurenceTime = dp.head.toLong//DateFormatting(dp.head) //.slice(4, dp.head.length) - try { - dp = dp.last.split("\t") - val srcClusterId = dp.head.toLong//assignID(dp.head) - val len = dp.length - - for (i <- 1 until len by 2) { - val dstClusterId = dp(i).toLong//assignID(dp(i)) - val coocWeight = dp(i + 1).toLong - - sendUpdate( - EdgeAddWithProperties( - msgTime = occurenceTime, - srcID = srcClusterId, - dstID = dstClusterId, - Properties(LongProperty("Frequency", coocWeight)) - ) - ) - } - - }catch { - case e: Exception => println(e, dp.length, tuple) - } - } - - def DateFormatting(date: String): Long = { - val format = new java.text.SimpleDateFormat("yyyyMM") - format.parse(date).getTime - } -} diff --git a/mainproject/src/main/scala/com/raphtory/examples/wordSemantic/graphbuilders/CooccurrenceMatrixGraphBuilderFiltered.scala b/mainproject/src/main/scala/com/raphtory/examples/wordSemantic/graphbuilders/CooccurrenceMatrixGraphBuilderFiltered.scala deleted file mode 100644 index 3e15adb318..0000000000 --- a/mainproject/src/main/scala/com/raphtory/examples/wordSemantic/graphbuilders/CooccurrenceMatrixGraphBuilderFiltered.scala +++ /dev/null @@ -1,38 +0,0 @@ -package com.raphtory.examples.wordSemantic.graphbuilders - -import com.raphtory.core.actors.Router.GraphBuilder -import com.raphtory.core.model.communication._ - -class CooccurrenceMatrixGraphBuilderFiltered extends GraphBuilder[String] { - val THR = System.getenv().getOrDefault("COOC_FREQ_THRESHOLD ", "0.05").trim.toDouble - - override def parseTuple(tuple: String) = { - //println(record) - var dp =tuple.split(" ").map(_.trim) - val occurenceTime = dp.head.toLong//DateFormatting(dp.head) //.slice(4, dp.head.length) - val scale = dp(1).toDouble - try { - dp = dp.last.split("\t") - val srcClusterId = assignID(dp.head) - val len = dp.length - for (i <- 1 until len by 2) { - if ((dp(i+1).toLong/scale) >= THR) { - val dstClusterId = assignID(dp(i)) - val coocWeight = dp(i + 1).toLong - - sendUpdate( - EdgeAddWithProperties( - msgTime = occurenceTime, - srcID = srcClusterId, - dstID = dstClusterId, - Properties(LongProperty("Frequency", coocWeight), - DoubleProperty("ScaledFreq", coocWeight/scale)) - ) - ) - } - } - }catch { - case e: Exception => println(e, dp.length, tuple.asInstanceOf[String]) - } - } -} diff --git a/mainproject/src/main/scala/com/raphtory/examples/wordSemantic/spouts/CooccurrenceMatrixSpout.scala b/mainproject/src/main/scala/com/raphtory/examples/wordSemantic/spouts/CooccurrenceMatrixSpout.scala deleted file mode 100644 index a28e723c6c..0000000000 --- a/mainproject/src/main/scala/com/raphtory/examples/wordSemantic/spouts/CooccurrenceMatrixSpout.scala +++ /dev/null @@ -1,126 +0,0 @@ -//package com.raphtory.examples.wordSemantic.spouts -// -// -//import java.io.{BufferedReader, File, FileReader} -//import java.time.LocalDateTime -// -//import com.raphtory.core.components.Spout.Spout -//import com.raphtory.core.components.Spout.Spout.{BasicDomain, DomainMessage} -//import com.raphtory.core.model.communication.StringSpoutGoing -//import com.raphtory.examples.wordSemantic.spouts.CooccurrenceMatrixSpout.Message.{CooccuranceDomain, NextFile, NextLineBlock, NextLineSlice} -// -//import scala.concurrent.duration._ -//import scala.language.postfixOps -// -//class CooccurrenceMatrixSpout extends Spout[CooccuranceDomain,StringSpoutGoing] { -// -// println("Start: " + LocalDateTime.now()) -// val directory = System.getenv().getOrDefault("FILE_SPOUT_DIRECTORY", "/app").trim -// val fileName = System.getenv().getOrDefault("FILE_SPOUT_FILENAME", "").trim //gabNetwork500.csv -// val dropHeader = System.getenv().getOrDefault("FILE_SPOUT_DROP_HEADER", "false").trim.toBoolean -// var JUMP = System.getenv().getOrDefault("FILE_SPOUT_BLOCK_SIZE", "100").trim.toInt -// var INCREMENT = System.getenv().getOrDefault("FILE_SPOUT_INCREMENT", "1").trim.toInt -// var TIME = System.getenv().getOrDefault("FILE_SPOUT_TIME", "60").trim.toInt -// -// var directoryPosition = 0 -// -// val filesToRead = if(fileName.isEmpty) -// getListOfFiles(directory) -// else -// Array(directory + "/" + fileName) -// -// var currentFile = fileToArray(directoryPosition) -// val JUMP2 = 20 -// var posSlice = 1 -// var cline = currentFile.readLine() -// var currentLine = cline.split("\t") -// var filename = filesToRead(directoryPosition) //D-200001_merge_occ -// var time = filename.split('/').last.stripPrefix("D-").stripSuffix("_merge_occ").toLong * 1000000000L -// var cnt = time + 1 -// -// -// -// def handleDomainMessage(message: CooccuranceDomain): Unit = message match { -// -// case NextLineSlice => nextLineSlice() -// case NextLineBlock => nextLineBlock() -// case NextFile => nextFile() -// case _ => println("message not recognized!") -//} -// def nextLineSlice() = { -// try { -// if (posSlice <= currentLine.length-1) { -// val head = currentLine(0) -// for (i<- 1 to Set(JUMP, currentLine.length-posSlice/JUMP2).min) { -// val currentSlice = currentLine.slice(posSlice, posSlice + JUMP2) -// sendTuple(StringSpoutGoing(cnt.toString + ' ' + head + "\t" + currentSlice.mkString("\t"))) -// posSlice += JUMP2 -// } -// AllocateSpoutTask(Duration(1, MILLISECONDS), "nextLineSlice") -// } -// else { -// posSlice = 1 -// AllocateSpoutTask(Duration(1, NANOSECONDS), "nextLineBLock") -// } -// }catch { -// case e: Exception => println(e, posSlice) -// } -// } -// -// def nextLineBlock() = { -// try { -// cnt += 1 -// cline = currentFile.readLine() -// currentLine = cline.split("\t") -// AllocateSpoutTask(Duration(1, NANOSECONDS), "nextLineSlice") -// } -// catch { -// case e:Exception => AllocateSpoutTask(Duration(1, NANOSECONDS), "nextFile") -// } -// } -// -// def nextFile() = { -// directoryPosition += 1 -// if (filesToRead.length > directoryPosition) { -// currentFile = fileToArray(directoryPosition) -// filename = filesToRead(directoryPosition) //D-200001_merge_occ -// time = filename.split('/').last.stripPrefix("D-").stripSuffix("_merge_occ").toLong * 1000000000L -// cnt = time + 1 -// AllocateSpoutTask(Duration(1, NANOSECONDS), "nextLineBLock") -// } -// else { -// println("All files read "+ LocalDateTime.now()) -// } -// } -// -// def fileToArray(pos:Int) ={ -// println(s"Now reading ${filesToRead(pos)}") -// if(dropHeader){ -// val br = new BufferedReader(new FileReader(filesToRead(pos))) -// br.readLine() -// br -// } -// else -// new BufferedReader(new FileReader(filesToRead(pos))) -// } -// -// def getListOfFiles(dir: String):Array[String] = { -// val d = new File(dir) -// if (d.exists && d.isDirectory) { -// d.listFiles.filter(f=> f.isFile && !f.isHidden).map(f=> f.getCanonicalPath).sorted -// } else { -// Array[String]() -// } -// } -// -// override def startSpout(): Unit = self ! NextLineSlice -//} -// -//object CooccurrenceMatrixSpout { -// object Message { -// sealed trait CooccuranceDomain extends DomainMessage -// case object NextLineSlice extends CooccuranceDomain -// case object NextLineBlock extends CooccuranceDomain -// case object NextFile extends CooccuranceDomain -// } -//} \ No newline at end of file diff --git a/mainproject/src/main/scala/com/raphtory/examples/wordSemantic/spouts/CooccurrenceMatrixSpoutFiltered.scala b/mainproject/src/main/scala/com/raphtory/examples/wordSemantic/spouts/CooccurrenceMatrixSpoutFiltered.scala deleted file mode 100644 index eb73e266b8..0000000000 --- a/mainproject/src/main/scala/com/raphtory/examples/wordSemantic/spouts/CooccurrenceMatrixSpoutFiltered.scala +++ /dev/null @@ -1,53 +0,0 @@ -//package com.raphtory.examples.wordSemantic.spouts -// -//import java.io.{BufferedReader, File, FileReader} -//import java.time.LocalDateTime -// -//import com.raphtory.core.components.Spout.Spout -//import com.raphtory.core.model.communication.StringSpoutGoing -// -//import scala.concurrent.duration._ -//import scala.language.postfixOps -// -//class CooccurrenceMatrixSpoutFiltered extends CooccurrenceMatrixSpout { -// var freq = currentLine.drop(2).grouped(2).map(_.head.toInt).toArray -// var scale = scalling(freq) -// -// override def nextLineSlice() = { -// try { -// if (posSlice <= currentLine.length-1) { -// val head = currentLine(0) -// for (i<- 1 to Set(JUMP, currentLine.length-posSlice/JUMP2).min) { -// val currentSlice = currentLine.slice(posSlice, posSlice + JUMP2) -// sendTuple(StringSpoutGoing(cnt.toString + ' ' + scale.toString + ' ' + head + "\t" + currentSlice.mkString("\t"))) -// posSlice += JUMP2 -// } -// AllocateSpoutTask(Duration(1, MILLISECONDS), "nextLineSlice") -// } -// else { -// posSlice = 1 -// AllocateSpoutTask(Duration(1, NANOSECONDS), "nextLineBLock") -// } -// }catch { -// case e: Exception => println(e, posSlice) -// } -// } -// -// override def nextLineBlock() = { -// try { -// cnt += 1 -// cline = currentFile.readLine() -// currentLine = cline.split("\t") -// freq = currentLine.drop(2).grouped(2).map(_.head.toInt).toArray -// scale = scalling(freq) -// AllocateSpoutTask(Duration(1, NANOSECONDS), "nextLineSlice") -// } -// catch { -// case e:Exception => AllocateSpoutTask(Duration(1, NANOSECONDS), "nextFile") -// } -// } -// -// def scalling(freq: Array[Int]): Double = { -// math.sqrt(freq.map(math.pow(_, 2)).sum) -// } -//} \ No newline at end of file From a0f49d389ba92cfd60b4648eaf7db1a08de7647f Mon Sep 17 00:00:00 2001 From: Miratepuffin Date: Thu, 3 Dec 2020 01:00:04 +0000 Subject: [PATCH 56/58] Minor changes to PageRank and Connected Components Output --- .../com/raphtory/algorithms/ConnectedComponents.scala | 8 ++++---- .../src/main/scala/com/raphtory/algorithms/PageRank.scala | 5 ++++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/mainproject/src/main/scala/com/raphtory/algorithms/ConnectedComponents.scala b/mainproject/src/main/scala/com/raphtory/algorithms/ConnectedComponents.scala index 13bbde4052..2db3c4ea55 100644 --- a/mainproject/src/main/scala/com/raphtory/algorithms/ConnectedComponents.scala +++ b/mainproject/src/main/scala/com/raphtory/algorithms/ConnectedComponents.scala @@ -36,12 +36,12 @@ class ConnectedComponents(args:Array[String]) extends Analyser(args){ override def processResults(results: ArrayBuffer[Any], timestamp: Long, viewCompleteTime: Long): Unit = { val er = extractData(results) - val text = s"""{"time":$timestamp,"top5":[${er.top5.mkString(",")}],"total":${er.total},"totalIslands":${er.totalIslands},"proportion":${er.proportion},"clustersGT2":${er.totalGT2},"viewTime":$viewCompleteTime}""" + val text = s"""{"time":$timestamp,"top5":[${er.top5.mkString(",")}],"total":${er.total},"totalIslands":${er.totalIslands},"proportion":${er.proportion},"clustersGT2":${er.totalGT2},"viewTime":$viewCompleteTime},""" var output_folder = System.getenv().getOrDefault("OUTPUT_FOLDER", "/app").trim var output_file = output_folder + "/" + System.getenv().getOrDefault("OUTPUT_FILE","ConnectedComponents.json").trim println(text) - writeLines(output_file, text, "{\"views\":[") + writeLines(output_file, text, "[") publishData(text) } @@ -50,8 +50,8 @@ class ConnectedComponents(args:Array[String]) extends Analyser(args){ val er = extractData(results) var output_folder = System.getenv().getOrDefault("OUTPUT_FOLDER", "/app").trim var output_file = output_folder + "/" + System.getenv().getOrDefault("OUTPUT_FILE","ConnectedComponents.json").trim - val text = s"""{"time":$timestamp,"windowsize":$windowSize,"top5":[${er.top5.mkString(",")}],"total":${er.total},"totalIslands":${er.totalIslands},"proportion":${er.proportion},"clustersGT2":${er.totalGT2},"viewTime":$viewCompleteTime}""" - writeLines(output_file, text, "{\"views\":[") + val text = s"""{"time":$timestamp,"windowsize":$windowSize,"top5":[${er.top5.mkString(",")}],"total":${er.total},"totalIslands":${er.totalIslands},"proportion":${er.proportion},"clustersGT2":${er.totalGT2},"viewTime":$viewCompleteTime},""" + writeLines(output_file, text, "[") println(text) //publishData(text) diff --git a/mainproject/src/main/scala/com/raphtory/algorithms/PageRank.scala b/mainproject/src/main/scala/com/raphtory/algorithms/PageRank.scala index 18680ae362..bdd79f2428 100644 --- a/mainproject/src/main/scala/com/raphtory/algorithms/PageRank.scala +++ b/mainproject/src/main/scala/com/raphtory/algorithms/PageRank.scala @@ -69,6 +69,9 @@ class PageRank(args:Array[String]) extends Analyser(args) { .take(10) .map(x => s"""{"id":${x._1},"pagerank":${x._2}}""").mkString("[",",","]") val text = s"""{"time":$timeStamp,"vertices":$totalVert,"bestusers":$bestUsers,"viewTime":$viewCompleteTime}""" + var output_folder = System.getenv().getOrDefault("OUTPUT_FOLDER", "/app").trim + var output_file = output_folder + "/" + System.getenv().getOrDefault("OUTPUT_FILE","WeightedPageRank.json").trim + writeLines(output_file, text, "[") println(text) } @@ -86,7 +89,7 @@ class PageRank(args:Array[String]) extends Analyser(args) { .map(x => s"""{"id":${x._1},"pagerank":${x._2}}""").mkString("[",",","]") val text = s"""{"time":$timestamp,"windowsize":$windowSize,"vertices":$totalVert,"bestusers":$bestUsers,"viewTime":$viewCompleteTime}""" - writeLines(output_file, text, "{\"views\":[") + writeLines(output_file, text, "[") println(text) //publishData(text) } From 06325abc45ad8664b523b23d9e6cf237a4e9cf82 Mon Sep 17 00:00:00 2001 From: Miratepuffin Date: Tue, 8 Dec 2020 20:50:48 +0000 Subject: [PATCH 57/58] Fixed the window ordering and turned off all messages --- .../src/main/resources/application.conf | 2 + .../scala/com/raphtory/RaphtoryServer.scala | 8 +- .../scala/com/raphtory/api/Analyser.scala | 36 +- .../LiveTasks/BWindowedLiveAnalysisTask.scala | 1 + .../BWindowedRangeAnalysisTask.scala | 2 +- .../ViewTasks/BWindowedViewAnalysisTask.scala | 2 +- .../Workers/IngestionWorker.scala | 2 +- .../Workers/ReaderWorker.scala | 6 +- .../raphtory/core/actors/RaphtoryActor.scala | 4 +- .../core/actors/Router/RouterWorker.scala | 4 +- .../src/tests/scala/lotr/LOTRDeployment.scala | 20 + .../tests/scala/lotr/LOTRGraphBuilder.scala | 26 + .../src/tests/scala/lotr/LOTRSpout.scala | 27 + mainproject/src/tests/scala/lotr/lotr.csv | 2649 +++++++++++++++++ 14 files changed, 2760 insertions(+), 29 deletions(-) create mode 100644 mainproject/src/tests/scala/lotr/LOTRDeployment.scala create mode 100644 mainproject/src/tests/scala/lotr/LOTRGraphBuilder.scala create mode 100644 mainproject/src/tests/scala/lotr/LOTRSpout.scala create mode 100644 mainproject/src/tests/scala/lotr/lotr.csv diff --git a/mainproject/src/main/resources/application.conf b/mainproject/src/main/resources/application.conf index 7966b69a1f..38d7fcb282 100644 --- a/mainproject/src/main/resources/application.conf +++ b/mainproject/src/main/resources/application.conf @@ -12,7 +12,9 @@ settings { //ip = 0.0.0.0 http = 8080 port = 1600 + port = ${?HOST_PORT} bport = 1600 + bport = ${?HOST_PORT} } akka.management.cluster.bootstrap.contact-point-discovery { diff --git a/mainproject/src/main/scala/com/raphtory/RaphtoryServer.scala b/mainproject/src/main/scala/com/raphtory/RaphtoryServer.scala index 3d16b5c576..9bbc13057c 100644 --- a/mainproject/src/main/scala/com/raphtory/RaphtoryServer.scala +++ b/mainproject/src/main/scala/com/raphtory/RaphtoryServer.scala @@ -26,8 +26,8 @@ object RaphtoryServer extends App { val clusterSystemName = "Raphtory" val ssn: String = java.util.UUID.randomUUID.toString - val partitionCount = sys.env("PARTITION_MIN").toInt - val routerCount = sys.env("ROUTER_MIN").toInt + val partitionCount = sys.env.getOrElse("PARTITION_MIN","1").toInt + val routerCount = sys.env.getOrElse("ROUTER_MIN","1").toInt val docker = System.getenv().getOrDefault("DOCKER", "false").trim.toBoolean args(0) match { @@ -41,7 +41,7 @@ object RaphtoryServer extends App { } def seedNode() = { - val seedLoc = s"${sys.env("HOST_IP")}:${conf.getInt("settings.bport")}" + val seedLoc = s"${sys.env.getOrElse("HOST_IP","127.0.0.1")}:${conf.getInt("settings.bport")}" println(s"Creating seed node at $seedLoc") implicit val system: ActorSystem = initialiseActorSystem(seeds = List(seedLoc)) system.actorOf(Props(new SeedActor()), "cluster") @@ -99,7 +99,7 @@ object RaphtoryServer extends App { Thread.sleep(3000) } InetAddress.getByName("seedNode").getHostAddress() + ":1600" - } else "127.0.0.1" + } else "127.0.0.1:1600" diff --git a/mainproject/src/main/scala/com/raphtory/api/Analyser.scala b/mainproject/src/main/scala/com/raphtory/api/Analyser.scala index b55fbaf90e..28066370f4 100644 --- a/mainproject/src/main/scala/com/raphtory/api/Analyser.scala +++ b/mainproject/src/main/scala/com/raphtory/api/Analyser.scala @@ -57,22 +57,26 @@ abstract class Analyser(args:Array[String]) extends java.io.Serializable { //TODO THIS IS A CURSED FUNCTION AND SHOULD BE DESTROYED def writeLines(fileName: String, line: String, header: String): Unit = { - val f = new File(fileName) - if (!f.exists()) { - f.createNewFile() - val file = new FileWriter(fileName, true) - var bw = new BufferedWriter(file) - bw.write(header) - bw.newLine() - bw.write(line) - bw.newLine() - bw.flush() - } else { - val file = new FileWriter(fileName, true) - var bw = new BufferedWriter(file) - bw.write(line) - bw.newLine() - bw.flush() + try { + val f = new File(fileName) + if (!f.exists()) { + f.createNewFile() + val file = new FileWriter(fileName, true) + var bw = new BufferedWriter(file) + bw.write(header) + bw.newLine() + bw.write(line) + bw.newLine() + bw.flush() + } else { + val file = new FileWriter(fileName, true) + var bw = new BufferedWriter(file) + bw.write(line) + bw.newLine() + bw.flush() + } + }catch { + case e:Exception => } } diff --git a/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/LiveTasks/BWindowedLiveAnalysisTask.scala b/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/LiveTasks/BWindowedLiveAnalysisTask.scala index 708b4f06c1..9958736a8d 100644 --- a/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/LiveTasks/BWindowedLiveAnalysisTask.scala +++ b/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/LiveTasks/BWindowedLiveAnalysisTask.scala @@ -21,6 +21,7 @@ class BWindowedLiveAnalysisTask(managerCount:Int, jobID: String,args:Array[Strin invertedArray.asInstanceOf[ArrayBuffer[Any]] } else original } + override def windowSet(): Array[Long] = windowset.sortBy(x=>x)(sortOrdering) override protected def analysisType(): AnalysisType.Value = AnalysisType.live override def processResults(time: Long): Unit = { var i = 0 diff --git a/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/RangeTasks/BWindowedRangeAnalysisTask.scala b/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/RangeTasks/BWindowedRangeAnalysisTask.scala index db270a3900..91a73a37ce 100644 --- a/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/RangeTasks/BWindowedRangeAnalysisTask.scala +++ b/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/RangeTasks/BWindowedRangeAnalysisTask.scala @@ -32,7 +32,7 @@ class BWindowedRangeAnalysisTask( } else original } - override def windowSet(): Array[Long] = windows + override def windowSet(): Array[Long] = windows.sortBy(x=>x)(sortOrdering) override protected def analysisType(): AnalysisType.Value = AnalysisType.range override def processResults(time: Long): Unit = { var i = 0 diff --git a/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/ViewTasks/BWindowedViewAnalysisTask.scala b/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/ViewTasks/BWindowedViewAnalysisTask.scala index 37cad86f68..f995ee2696 100644 --- a/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/ViewTasks/BWindowedViewAnalysisTask.scala +++ b/mainproject/src/main/scala/com/raphtory/core/actors/AnalysisManager/Tasks/ViewTasks/BWindowedViewAnalysisTask.scala @@ -22,7 +22,7 @@ class BWindowedViewAnalysisTask(managerCount:Int, jobID: String, args:Array[Stri } else original } -override def windowSet(): Array[Long] = windows +override def windowSet(): Array[Long] = windows.sortBy(x=>x)(sortOrdering) override def processResults(time: Long): Unit = { var i = 0 val vtime = viewCompleteTime diff --git a/mainproject/src/main/scala/com/raphtory/core/actors/PartitionManager/Workers/IngestionWorker.scala b/mainproject/src/main/scala/com/raphtory/core/actors/PartitionManager/Workers/IngestionWorker.scala index c6ea9895a5..4d31699e91 100644 --- a/mainproject/src/main/scala/com/raphtory/core/actors/PartitionManager/Workers/IngestionWorker.scala +++ b/mainproject/src/main/scala/com/raphtory/core/actors/PartitionManager/Workers/IngestionWorker.scala @@ -271,7 +271,7 @@ class IngestionWorker(workerId: Int,partitionID:Int, storage: EntityStorage) ext setSafePoint(queue._1, queue._2) }) val timestamps = queueState.map(q => q.timestamp) - println(s"$increments Writer Worker $partitionID $workerId $timestamps") + //println(s"$increments Writer Worker $partitionID $workerId $timestamps") //TODO FIX //println(s"Writer Worker $partitionID $workerId ${queueState.mkString("[",",","]")} ${storage.vertices.size}") //println(s"$increments Writer Worker $partitionID $workerId ${timestamps.min} ${storage.vertices.size} $updates ${updates-updates2}") diff --git a/mainproject/src/main/scala/com/raphtory/core/actors/PartitionManager/Workers/ReaderWorker.scala b/mainproject/src/main/scala/com/raphtory/core/actors/PartitionManager/Workers/ReaderWorker.scala index eafc0170e4..4fe4755b1f 100644 --- a/mainproject/src/main/scala/com/raphtory/core/actors/PartitionManager/Workers/ReaderWorker.scala +++ b/mainproject/src/main/scala/com/raphtory/core/actors/PartitionManager/Workers/ReaderWorker.scala @@ -64,7 +64,7 @@ class ReaderWorker(managerCountVal: Int, managerID: Int, workerId: Int, storage: .withTag("JobID",req.jobID) .withTag("timestamp",req.timestamp) .withTag("superstep",req.superStep) - try setup(req.analyzer, req.jobID, req.args, req.superStep, req.timestamp, req.analysisType, req.window, req.windowSet) + try setup(req.analyzer, req.jobID, req.args, req.superStep, req.timestamp, req.analysisType, req.window, req.windowSet.sortBy(x=>x)(sortOrdering)) catch { case e: Exception => log.error("Failed to run setup due to [{}].", e.printStackTrace()) } superstepTimer.update(System.currentTimeMillis()-beforeTime) } @@ -100,7 +100,7 @@ class ReaderWorker(managerCountVal: Int, managerID: Int, workerId: Int, storage: .withTag("timestamp",req.timestamp) .withTag("superstep",req.superStep) - try nextStep(req.analyzer, req.jobID, req.args, req.superStep, req.timestamp, req.analysisType, req.window, req.windowSet) + try nextStep(req.analyzer, req.jobID, req.args, req.superStep, req.timestamp, req.analysisType, req.window, req.windowSet.sortBy(x=>x)(sortOrdering)) catch { case e: Exception => log.error("Failed to run nextStep due to [{}].", e) } superstepTimer.update(System.currentTimeMillis()-beforeTime) } @@ -130,7 +130,7 @@ class ReaderWorker(managerCountVal: Int, managerID: Int, workerId: Int, storage: .withTag("timestamp",req.timestamp) .withTag("superstep",req.superStep) - try returnResults(req.analyzer, req.jobID, req.args, req.superStep, req.timestamp, req.analysisType, req.window, req.windowSet) + try returnResults(req.analyzer, req.jobID, req.args, req.superStep, req.timestamp, req.analysisType, req.window, req.windowSet.sortBy(x=>x)(sortOrdering)) catch { case e: Exception => log.error("Failed to run returnResults due to [{}].", e) } superstepTimer.update(System.currentTimeMillis()-beforeTime) } diff --git a/mainproject/src/main/scala/com/raphtory/core/actors/RaphtoryActor.scala b/mainproject/src/main/scala/com/raphtory/core/actors/RaphtoryActor.scala index 94bc2faaa3..781480e36f 100644 --- a/mainproject/src/main/scala/com/raphtory/core/actors/RaphtoryActor.scala +++ b/mainproject/src/main/scala/com/raphtory/core/actors/RaphtoryActor.scala @@ -92,6 +92,8 @@ trait RaphtoryActor extends Actor with ActorLogging with Timers { isCancelled } - + object sortOrdering extends Ordering[Long] { + def compare(key1: Long, key2: Long) = key2.compareTo(key1) + } } diff --git a/mainproject/src/main/scala/com/raphtory/core/actors/Router/RouterWorker.scala b/mainproject/src/main/scala/com/raphtory/core/actors/Router/RouterWorker.scala index dc71383f10..2ce3010a24 100644 --- a/mainproject/src/main/scala/com/raphtory/core/actors/Router/RouterWorker.scala +++ b/mainproject/src/main/scala/com/raphtory/core/actors/Router/RouterWorker.scala @@ -19,7 +19,7 @@ import scala.concurrent.duration._ class RouterWorker[T](val graphBuilder: GraphBuilder[T],val routerId: Int, val workerID: Int, val initialManagerCount: Int,val initialRouterCount:Int) extends RaphtoryActor { implicit val executionContext: ExecutionContext = context.system.dispatcher - println(s"Router $routerId $workerID with $initialManagerCount $initialRouterCount") + //println(s"Router $routerId $workerID with $initialManagerCount $initialRouterCount") private val messageIDs = ParTrieMap[String, Int]() private val routerWorkerUpdates = @@ -88,7 +88,7 @@ class RouterWorker[T](val graphBuilder: GraphBuilder[T],val routerId: Int, val w case DataFinishedSync(time) => { if (time >= newestTime) { - println(s"Router $routerId $workerID ${time}") + //println(s"Router $routerId $workerID ${time}") getAllWriterWorkers(managerCount).foreach { workerPath => mediator ! DistributedPubSubMediator.Send( workerPath, diff --git a/mainproject/src/tests/scala/lotr/LOTRDeployment.scala b/mainproject/src/tests/scala/lotr/LOTRDeployment.scala new file mode 100644 index 0000000000..72e3b495d2 --- /dev/null +++ b/mainproject/src/tests/scala/lotr/LOTRDeployment.scala @@ -0,0 +1,20 @@ +package examples.lotr + +import com.raphtory.RaphtoryGraph +import com.raphtory.algorithms.{ConnectedComponents, DegreeBasic} + +object LOTRDeployment extends App{ + val source = new LOTRSpout() + val builder = new LOTRGraphBuilder() + val rg = RaphtoryGraph[String](source,builder) + val arguments = Array[String]() + + //rg.rangeQuery(ConnectedComponents(),start = 1,end = 32674,increment = 100,arguments) + //rg.rangeQuery(ConnectedComponents(),start = 1,end = 32674,increment = 100,window=100,arguments) + //rg.rangeQuery(ConnectedComponents(),start = 1,end = 32674,increment = 100,windowBatch=Array(10,50,100),arguments) + + //rg.viewQuery(DegreeBasic(),timestamp = 10000,arguments) + // rg.viewQuery(DegreeBasic(),timestamp = 10000,window=100,arguments) + rg.viewQuery(DegreeBasic(),timestamp = 10000,windowBatch=Array(100,50,10),arguments) + rg.viewQuery(DegreeBasic(),timestamp = 10000,windowBatch=Array(10,50,100),arguments) +} diff --git a/mainproject/src/tests/scala/lotr/LOTRGraphBuilder.scala b/mainproject/src/tests/scala/lotr/LOTRGraphBuilder.scala new file mode 100644 index 0000000000..3775bfa265 --- /dev/null +++ b/mainproject/src/tests/scala/lotr/LOTRGraphBuilder.scala @@ -0,0 +1,26 @@ +package examples.lotr + +import com.raphtory.core.actors.Router.GraphBuilder +import com.raphtory.core.model.communication._ + +class LOTRGraphBuilder extends GraphBuilder[String]{ + + override def parseTuple(tuple: String) = { + + val fileLine = tuple.split(",").map(_.trim) + val sourceNode = fileLine(0) + val srcID = assignID(sourceNode) + + val targetNode = fileLine(1) + val tarID = assignID(targetNode) + + val timeStamp = fileLine(2).toLong + + sendUpdate(VertexAddWithProperties(timeStamp, srcID, + Properties(ImmutableProperty("name",sourceNode)),Type("Character"))) + sendUpdate(VertexAddWithProperties(timeStamp, tarID, + Properties(ImmutableProperty("name",targetNode)),Type("Character"))) + + sendUpdate(EdgeAdd(timeStamp,srcID,tarID, Type("Character Co-occurence"))) + } +} diff --git a/mainproject/src/tests/scala/lotr/LOTRSpout.scala b/mainproject/src/tests/scala/lotr/LOTRSpout.scala new file mode 100644 index 0000000000..7e7f8d245d --- /dev/null +++ b/mainproject/src/tests/scala/lotr/LOTRSpout.scala @@ -0,0 +1,27 @@ +package examples.lotr + +import com.raphtory.core.actors.Spout.Spout + +import scala.collection.mutable + + +class LOTRSpout extends Spout[String] { + + val fileQueue = mutable.Queue[String]() + + override def setupDataSource(): Unit = { + fileQueue++= + scala.io.Source.fromFile("src/tests/scala/lotr/lotr.csv") + .getLines + }//no setup + + override def generateData(): Option[String] = { + if(fileQueue isEmpty){ + dataSourceComplete() + None + } + else + Some(fileQueue.dequeue()) + } + override def closeDataSource(): Unit = {}//no file closure already done +} diff --git a/mainproject/src/tests/scala/lotr/lotr.csv b/mainproject/src/tests/scala/lotr/lotr.csv new file mode 100644 index 0000000000..897532fcf5 --- /dev/null +++ b/mainproject/src/tests/scala/lotr/lotr.csv @@ -0,0 +1,2649 @@ +Gandalf,Elrond,33 +Frodo,Bilbo,114 +Blanco,Marcho,146 +Frodo,Bilbo,205 +Thorin,Gandalf,270 +Thorin,Bilbo,270 +Gandalf,Bilbo,270 +Gollum,Bilbo,286 +Gollum,Bilbo,306 +Gollum,Bilbo,308 +Bilbo,Elrond,317 +Frodo,Samwise,319 +Gandalf,Bilbo,320 +Gollum,Bilbo,324 +Frodo,Gandalf,329 +Peregrin,Elessar,356 +Arwen,Aragorn,358 +Barahir,Faramir,359 +Bilbo,Findegil,360 +Meriadoc,Peregrin,363 +Peregrin,Elendil,368 +Galadriel,Celeborn,374 +Frodo,Bilbo,387 +Frodo,Bilbo,388 +Frodo,Bilbo,389 +Frodo,Bilbo,390 +Frodo,Bilbo,393 +Frodo,Bilbo,399 +Hamfast,Bilbo,402 +Gandalf,Bilbo,483 +Gandalf,Bilbo,543 +Frodo,Gandalf,555 +Frodo,Bilbo,555 +Gandalf,Bilbo,555 +Frodo,Bilbo,562 +Gandalf,Bilbo,730 +Frodo,Bilbo,808 +Frodo,Bilbo,815 +Gandalf,Bilbo,843 +Frodo,Gandalf,861 +Frodo,Bilbo,898 +Frodo,Merry,929 +Frodo,Merry,930 +Frodo,Bilbo,942 +Frodo,Bilbo,944 +Odo,Frodo,959 +Frodo,Bilbo,1038 +Meriadoc,Peregrin,1042 +Meriadoc,Merry,1042 +Meriadoc,Pippin,1042 +Meriadoc,Fredegar,1042 +Peregrin,Merry,1042 +Peregrin,Pippin,1042 +Peregrin,Fredegar,1042 +Merry,Pippin,1042 +Merry,Fredegar,1042 +Pippin,Fredegar,1042 +Pippin,Merry,1044 +Pippin,Bilbo,1044 +Merry,Bilbo,1044 +Frodo,Gandalf,1056 +Sam,Halfast,1089 +Frodo,Bilbo,1106 +Frodo,Gandalf,1130 +Gandalf,Bilbo,1135 +Gandalf,Bilbo,1156 +Frodo,Gandalf,1160 +Frodo,Saruman,1185 +Gandalf,Bilbo,1229 +Frodo,Gandalf,1234 +Frodo,Gandalf,1241 +Isildur,Elendil,1309 +Isildur,Gil-galad,1309 +Isildur,Sauron,1309 +Elendil,Gil-galad,1309 +Elendil,Sauron,1309 +Gil-galad,Sauron,1309 +Sméagol,Déagol,1324 +Sméagol,Déagol,1330 +Sméagol,Déagol,1331 +Sméagol,Déagol,1336 +Frodo,Gollum,1356 +Frodo,Gollum,1359 +Frodo,Gandalf,1390 +Frodo,Gollum,1400 +Frodo,Bilbo,1401 +Isildur,Gollum,1407 +Isildur,Déagol,1407 +Gollum,Déagol,1407 +Frodo,Gandalf,1417 +Isildur,Elendil,1420 +Frodo,Gollum,1429 +Gollum,Déagol,1441 +Gollum,Bilbo,1459 +Gandalf,Bilbo,1469 +Frodo,Bilbo,1484 +Frodo,Gollum,1484 +Bilbo,Gollum,1484 +Frodo,Bilbo,1567 +Frodo,Gollum,1567 +Bilbo,Gollum,1567 +Gandalf,Bilbo,1654 +Frodo,Gandalf,1656 +Sam,Gandalf,1677 +Sam,Frodo,1687 +Sam,Frodo,1705 +Sam,Gandalf,1710 +Sam,Gandalf,1711 +Sam,Frodo,1714 +Gandalf,Bilbo,1721 +Frodo,Gandalf,1741 +Gandalf,Bilbo,1748 +Sam,Elrond,1756 +Frodo,Gandalf,1783 +Frodo,Gandalf,1785 +Pippin,Frodo,1789 +Pippin,Merry,1789 +Pippin,Fredegar,1789 +Frodo,Merry,1789 +Frodo,Fredegar,1789 +Merry,Fredegar,1789 +Frodo,Gandalf,1792 +Frodo,Bilbo,1802 +Frodo,Gandalf,1804 +Merry,Fredegar,1806 +Frodo,Gandalf,1809 +Lobelia,Frodo,1816 +Lobelia,Lotho,1816 +Frodo,Lotho,1816 +Pippin,Sam,1825 +Lobelia,Sam,1826 +Lobelia,Frodo,1826 +Sam,Frodo,1826 +Pippin,Sam,1829 +Sam,Frodo,1869 +Sam,Frodo,1871 +Peregrin,Bilbo,1879 +Pippin,Sam,1888 +Pippin,Frodo,1951 +Pippin,Sam,1955 +Pippin,Bilbo,1981 +Frodo,Gandalf,1999 +Pippin,Frodo,2024 +Pippin,Sam,2024 +Frodo,Sam,2024 +Frodo,Gandalf,2056 +Frodo,Bilbo,2083 +Pippin,Frodo,2110 +Pippin,Sam,2110 +Frodo,Sam,2110 +Frodo,Bilbo,2175 +Sam,Frodo,2204 +Pippin,Frodo,2242 +Frodo,Bilbo,2246 +Frodo,Bilbo,2250 +Sam,Frodo,2254 +Sam,Gandalf,2254 +Frodo,Gandalf,2254 +Sam,Frodo,2318 +Sam,Frodo,2358 +Sam,Frodo,2393 +Pippin,Sam,2446 +Pippin,Frodo,2488 +Maggot,Pippin,2490 +Pippin,Frodo,2511 +Pippin,Sam,2511 +Frodo,Sam,2511 +Pippin,Sam,2513 +Maggot,Pippin,2519 +Pippin,Peregrin,2521 +Sam,Frodo,2541 +Maggot,Frodo,2607 +Frodo,Bilbo,2613 +Frodo,Peregrin,2628 +Pippin,Frodo,2633 +Pippin,Sam,2633 +Frodo,Sam,2633 +Pippin,Sam,2659 +Sam,Frodo,2668 +Frodo,Merry,2694 +Pippin,Frodo,2742 +Pippin,Merry,2742 +Pippin,Sam,2742 +Frodo,Merry,2742 +Frodo,Sam,2742 +Merry,Sam,2742 +Maggot,Frodo,2756 +Meriadoc,Frodo,2786 +Pippin,Bilbo,2794 +Maggot,Frodo,2821 +Pippin,Frodo,2829 +Maggot,Merry,2834 +Pippin,Frodo,2835 +Maggot,Frodo,2837 +Maggot,Bilbo,2837 +Frodo,Bilbo,2837 +Maggot,Merry,2838 +Pippin,Merry,2845 +Frodo,Merry,2850 +Pippin,Frodo,2857 +Sam,Frodo,2882 +Sam,Merry,2919 +Sam,Frodo,2922 +Frodo,Gandalf,2925 +Sam,Frodo,2938 +Pippin,Merry,2949 +Gandalf,Merry,2972 +Frodo,Gandalf,2999 +Gandalf,Fredegar,3060 +Pippin,Merry,3239 +Sam,Frodo,3265 +Pippin,Frodo,3271 +Pippin,Sam,3271 +Frodo,Sam,3271 +Sam,Frodo,3278 +Pippin,Sam,3282 +Pippin,Merry,3291 +Sam,Frodo,3316 +Sam,Frodo,3337 +Sam,Frodo,3345 +Sam,Merry,3356 +Frodo,Goldberry,3461 +Tom,Goldberry,3482 +Tom,Goldberry,3499 +Tom,Goldberry,3514 +Tom,Frodo,3526 +Pippin,Merry,3544 +Tom,Goldberry,3616 +Tom,Goldberry,3686 +Tom,Frodo,3702 +Tom,Frodo,3703 +Tom,Gandalf,3703 +Tom,Bilbo,3703 +Frodo,Gandalf,3703 +Frodo,Bilbo,3703 +Gandalf,Bilbo,3703 +Tom,Frodo,3705 +Tom,Frodo,3714 +Tom,Gandalf,3718 +Frodo,Merry,3721 +Tom,Frodo,3733 +Pippin,Sam,3837 +Pippin,Merry,3837 +Sam,Merry,3837 +Frodo,Gandalf,3914 +Frodo,Bilbo,3914 +Gandalf,Bilbo,3914 +Pippin,Sam,3919 +Pippin,Merry,3919 +Sam,Merry,3919 +Pippin,Sam,3938 +Pippin,Merry,3938 +Sam,Merry,3938 +Frodo,Merry,3946 +Tom,Frodo,3960 +Pippin,Sam,3976 +Pippin,Merry,3976 +Sam,Merry,3976 +Tom,Frodo,3988 +Tom,Merry,4037 +Pippin,Sam,4046 +Pippin,Merry,4046 +Sam,Merry,4046 +Tom,Goldberry,4060 +Butterbur,Barliman,4126 +Tom,Goldberry,4131 +Butterbur,Barliman,4261 +Sam,Frodo,4269 +Nob,Butterbur,4332 +Pippin,Frodo,4342 +Pippin,Sam,4342 +Frodo,Sam,4342 +Butterbur,Barliman,4353 +Pippin,Sam,4380 +Frodo,Butterbur,4389 +Frodo,Bilbo,4418 +Butterbur,Bilbo,4421 +Pippin,Sam,4482 +Pippin,Frodo,4565 +Pippin,Sam,4565 +Frodo,Sam,4565 +Sam,Frodo,4620 +Pippin,Frodo,4629 +Pippin,Sam,4629 +Frodo,Sam,4629 +Sam,Frodo,4692 +Sam,Frodo,4697 +Nob,Butterbur,4714 +Pippin,Sam,4734 +Pippin,Butterbur,4736 +Frodo,Butterbur,4749 +Pippin,Frodo,4902 +Pippin,Sam,4902 +Frodo,Sam,4902 +Frodo,Gandalf,4910 +Pippin,Gandalf,4926 +Pippin,Sam,4930 +Nob,Merry,5012 +Frodo,Merry,5089 +Nob,Merry,5090 +Nob,Gandalf,5090 +Merry,Gandalf,5090 +Bob,Bill,5228 +Bob,Bill,5229 +Butterbur,Merry,5242 +Tom,Butterbur,5249 +Nob,Butterbur,5268 +Nob,Bob,5268 +Butterbur,Bob,5268 +Pippin,Frodo,5274 +Pippin,Merry,5274 +Pippin,Sam,5274 +Frodo,Merry,5274 +Frodo,Sam,5274 +Merry,Sam,5274 +Nob,Bob,5276 +Pippin,Frodo,5410 +Elendil,Gil-galad,5434 +Pippin,Sam,5468 +Frodo,Merry,5470 +Merry,Gandalf,5500 +Frodo,Merry,5534 +Sam,Peregrin,5538 +Pippin,Gandalf,5544 +Merry,Gil-galad,5615 +Pippin,Frodo,5619 +Pippin,Merry,5619 +Frodo,Merry,5619 +Frodo,Gandalf,5620 +Lúthien,Beren,5653 +Lúthien,Barahir,5653 +Beren,Barahir,5653 +Beren,Lúthien,5654 +Beren,Thingol,5654 +Lúthien,Thingol,5654 +Beren,Barahir,5657 +Beren,Thingol,5657 +Barahir,Thingol,5657 +Beren,Lúthien,5660 +Beren,Sauron,5660 +Beren,Thingol,5660 +Lúthien,Sauron,5660 +Lúthien,Thingol,5660 +Sauron,Thingol,5660 +Beren,Thingol,5667 +Beren,Elwing,5667 +Beren,Eärendil,5667 +Beren,Lúthien,5667 +Beren,Dior,5667 +Thingol,Elwing,5667 +Thingol,Eärendil,5667 +Thingol,Lúthien,5667 +Thingol,Dior,5667 +Elwing,Eärendil,5667 +Elwing,Lúthien,5667 +Elwing,Dior,5667 +Eärendil,Lúthien,5667 +Eärendil,Dior,5667 +Lúthien,Dior,5667 +Sam,Merry,5682 +Pippin,Frodo,5683 +Pippin,Merry,5713 +Sam,Frodo,5714 +Sam,Frodo,5752 +Pippin,Merry,5758 +Frodo,Gandalf,5775 +Sam,Merry,5999 +Pippin,Merry,6010 +Sam,Frodo,6011 +Pippin,Merry,6012 +Sam,Frodo,6020 +Frodo,Bilbo,6037 +Frodo,Bilbo,6044 +Gandalf,Bilbo,6118 +Frodo,Bilbo,6126 +Frodo,Bilbo,6127 +Glorfindel,Elrond,6161 +Glorfindel,Frodo,6195 +Elrond,Aragorn,6200 +Glorfindel,Frodo,6220 +Pippin,Sam,6225 +Pippin,Merry,6225 +Sam,Merry,6225 +Glorfindel,Frodo,6267 +Frodo,Lúthien,6325 +Frodo,Gandalf,6381 +Butterbur,Bill,6429 +Gandalf,Barliman,6431 +Frodo,Gandalf,6531 +Glorfindel,Aragorn,6549 +Frodo,Gandalf,6578 +Glorfindel,Elrond,6583 +Sam,Frodo,6628 +Glorfindel,Gandalf,6660 +Glorfindel,Elrond,6660 +Gandalf,Elrond,6660 +Glorfindel,Frodo,6661 +Glorfindel,Gandalf,6661 +Glorfindel,Elrond,6661 +Frodo,Gandalf,6661 +Frodo,Elrond,6661 +Gandalf,Elrond,6661 +Frodo,Elrond,6669 +Arwen,Frodo,6673 +Arwen,Lúthien,6673 +Arwen,Elrond,6673 +Frodo,Lúthien,6673 +Frodo,Elrond,6673 +Lúthien,Elrond,6673 +Frodo,Elrond,6676 +Pippin,Frodo,6682 +Pippin,Merry,6682 +Frodo,Merry,6682 +Thorin,Glóin,6693 +Gandalf,Elrond,6701 +Frodo,Glóin,6709 +Frodo,Beorn,6710 +Frodo,Grimbeorn,6710 +Beorn,Grimbeorn,6710 +Bard,Bain,6717 +Bard,Brand,6717 +Bain,Brand,6717 +Dwalin,Bombur,6728 +Dwalin,Glóin,6728 +Dwalin,Bofur,6728 +Bombur,Glóin,6728 +Bombur,Bofur,6728 +Glóin,Bofur,6728 +Ori,Balin,6730 +Ori,Óin,6730 +Balin,Óin,6730 +Frodo,Glóin,6736 +Smaug,Bilbo,6747 +Frodo,Glóin,6748 +Arwen,Elrond,6754 +Frodo,Gandalf,6757 +Sam,Frodo,6810 +Sam,Bilbo,6810 +Frodo,Bilbo,6810 +Gandalf,Elrond,6833 +Gandalf,Bilbo,6835 +Frodo,Bilbo,6855 +Frodo,Bilbo,6935 +Eärendil,Elrond,6969 +Frodo,Bilbo,6987 +Frodo,Aragorn,6994 +Arwen,Frodo,6995 +Sam,Bilbo,7006 +Sam,Bilbo,7009 +Frodo,Elrond,7018 +Gandalf,Bilbo,7024 +Gandalf,Elrond,7037 +Sam,Frodo,7040 +Sam,Bilbo,7040 +Frodo,Bilbo,7040 +Frodo,Gandalf,7041 +Glorfindel,Glóin,7047 +Glorfindel,Frodo,7047 +Glóin,Frodo,7047 +Frodo,Elrond,7048 +Gimli,Glóin,7051 +Elrond,Glorfindel,7052 +Elrond,Galdor,7052 +Elrond,Erestor,7052 +Elrond,Círdan,7052 +Glorfindel,Galdor,7052 +Glorfindel,Erestor,7052 +Glorfindel,Círdan,7052 +Galdor,Erestor,7052 +Galdor,Círdan,7052 +Erestor,Círdan,7052 +Legolas,Thranduil,7053 +Frodo,Bilbo,7058 +Gandalf,Elrond,7059 +Gandalf,Boromir,7059 +Elrond,Boromir,7059 +Frodo,Glóin,7064 +Balin,Óin,7077 +Balin,Ori,7077 +Óin,Ori,7077 +Elrond,Sauron,7127 +Elrond,Sauron,7128 +Isildur,Elendil,7134 +Isildur,Anárion,7134 +Elendil,Anárion,7134 +Elendil,Gil-galad,7135 +Elendil,Sauron,7135 +Gil-galad,Sauron,7135 +Gil-galad,Elrond,7142 +Elwing,Eärendil,7145 +Elwing,Lúthien,7145 +Elwing,Dior,7145 +Eärendil,Lúthien,7145 +Eärendil,Dior,7145 +Lúthien,Dior,7145 +Elendil,Gil-galad,7148 +Isildur,Elendil,7149 +Isildur,Gil-galad,7149 +Isildur,Sauron,7149 +Elendil,Gil-galad,7149 +Elendil,Sauron,7149 +Gil-galad,Sauron,7149 +Gil-galad,Círdan,7162 +Isildur,Valandil,7170 +Isildur,Elendil,7170 +Isildur,Ohtar,7170 +Valandil,Elendil,7170 +Valandil,Ohtar,7170 +Elendil,Ohtar,7170 +Isildur,Elendil,7179 +Isildur,Gil-galad,7179 +Isildur,Anárion,7179 +Elendil,Gil-galad,7179 +Elendil,Anárion,7179 +Gil-galad,Anárion,7179 +Meneldil,Anárion,7192 +Elrond,Aragorn,7235 +Isildur,Elendil,7241 +Isildur,Elrond,7241 +Isildur,Aragorn,7241 +Elendil,Elrond,7241 +Elendil,Aragorn,7241 +Elrond,Aragorn,7241 +Frodo,Bilbo,7270 +Aragorn,Boromir,7279 +Isildur,Elendil,7281 +Isildur,Denethor,7281 +Elendil,Denethor,7281 +Gollum,Bilbo,7330 +Frodo,Bilbo,7336 +Galdor,Elrond,7353 +Frodo,Gandalf,7356 +Frodo,Galdor,7356 +Frodo,Glóin,7356 +Gandalf,Galdor,7356 +Gandalf,Glóin,7356 +Galdor,Glóin,7356 +Gollum,Gandalf,7391 +Isildur,Aragorn,7404 +Gollum,Aragorn,7405 +Isildur,Gandalf,7407 +Isildur,Saruman,7434 +Isildur,Boromir,7434 +Saruman,Boromir,7434 +Isildur,Elendil,7440 +Gil-galad,Sauron,7448 +Celebrimbor,Sauron,7454 +Gollum,Denethor,7455 +Gollum,Aragorn,7455 +Denethor,Aragorn,7455 +Gandalf,Elrond,7484 +Sméagol,Gollum,7509 +Legolas,Glóin,7525 +Saruman,Radagast,7581 +Gandalf,Saruman,7599 +Saruman,Sauron,7714 +Saruman,Radagast,7739 +Sauron,Aragorn,7769 +Glorfindel,Galdor,7904 +Gandalf,Elrond,7911 +Glorfindel,Saruman,7913 +Glorfindel,Erestor,7932 +Frodo,Boromir,7948 +Balin,Thrór,7992 +Glorfindel,Sauron,8028 +Bilbo,Boromir,8060 +Gandalf,Bilbo,8062 +Frodo,Elrond,8097 +Beren,Túrin,8105 +Beren,Hador,8105 +Beren,Húrin,8105 +Túrin,Hador,8105 +Túrin,Húrin,8105 +Hador,Húrin,8105 +Pippin,Frodo,8117 +Pippin,Merry,8117 +Pippin,Sam,8117 +Frodo,Merry,8117 +Frodo,Sam,8117 +Merry,Sam,8117 +Legolas,Gollum,8145 +Sam,Frodo,8150 +Sam,Bilbo,8150 +Frodo,Bilbo,8150 +Elrond,Thranduil,8157 +Elrond,Aragorn,8158 +Frodo,Gandalf,8183 +Frodo,Gandalf,8190 +Frodo,Bilbo,8190 +Gandalf,Bilbo,8190 +Gimli,Legolas,8255 +Gimli,Glóin,8255 +Legolas,Glóin,8255 +Isildur,Aragorn,8257 +Frodo,Boromir,8262 +Pippin,Frodo,8276 +Pippin,Gandalf,8276 +Frodo,Gandalf,8276 +Pippin,Elrond,8286 +Elendil,Aragorn,8292 +Gandalf,Elrond,8295 +Gandalf,Aragorn,8295 +Elrond,Aragorn,8295 +Frodo,Bilbo,8296 +Pippin,Merry,8297 +Pippin,Frodo,8297 +Pippin,Beren,8297 +Pippin,Bilbo,8297 +Pippin,Sam,8297 +Pippin,Lúthien,8297 +Merry,Frodo,8297 +Merry,Beren,8297 +Merry,Bilbo,8297 +Merry,Sam,8297 +Merry,Lúthien,8297 +Frodo,Beren,8297 +Frodo,Bilbo,8297 +Frodo,Sam,8297 +Frodo,Lúthien,8297 +Beren,Bilbo,8297 +Beren,Sam,8297 +Beren,Lúthien,8297 +Bilbo,Sam,8297 +Bilbo,Lúthien,8297 +Sam,Lúthien,8297 +Frodo,Bilbo,8300 +Frodo,Bilbo,8336 +Frodo,Bilbo,8341 +Elrond,Boromir,8373 +Frodo,Bilbo,8379 +Thorin,Gandalf,8380 +Sam,Bill,8384 +Pippin,Sam,8386 +Frodo,Bilbo,8390 +Elrond,Aragorn,8391 +Sam,Frodo,8397 +Gandalf,Elrond,8405 +Gandalf,Aragorn,8438 +Frodo,Gandalf,8459 +Gimli,Legolas,8473 +Sam,Aragorn,8525 +Sam,Aragorn,8534 +Sam,Aragorn,8538 +Sam,Frodo,8563 +Frodo,Gandalf,8598 +Frodo,Aragorn,8598 +Gandalf,Aragorn,8598 +Gandalf,Aragorn,8600 +Gandalf,Aragorn,8614 +Frodo,Aragorn,8619 +Gandalf,Aragorn,8647 +Gimli,Sauron,8686 +Frodo,Merry,8690 +Frodo,Bilbo,8710 +Gandalf,Boromir,8712 +Legolas,Gandalf,8765 +Aragorn,Boromir,8779 +Aragorn,Boromir,8791 +Aragorn,Boromir,8796 +Merry,Aragorn,8822 +Pippin,Legolas,8827 +Pippin,Merry,8827 +Legolas,Merry,8827 +Sam,Boromir,8828 +Gimli,Gandalf,8829 +Gimli,Bill,8829 +Gandalf,Bill,8829 +Frodo,Aragorn,8830 +Pippin,Sam,8866 +Pippin,Merry,8866 +Sam,Merry,8866 +Aragorn,Boromir,8867 +Frodo,Gandalf,8871 +Gandalf,Boromir,8892 +Gandalf,Sauron,8912 +Balin,Fundin,8919 +Sam,Frodo,8952 +Pippin,Sam,8978 +Pippin,Elrond,8978 +Sam,Elrond,8978 +Pippin,Sam,8982 +Gandalf,Aragorn,9006 +Boromir,Aragorn,9018 +Gandalf,Boromir,9059 +Gimli,Frodo,9098 +Gimli,Gandalf,9098 +Frodo,Gandalf,9098 +Sam,Bill,9134 +Gimli,Legolas,9154 +Gandalf,Bill,9162 +Sam,Frodo,9169 +Sam,Bill,9179 +Gandalf,Fëanor,9210 +Gandalf,Aragorn,9240 +Peregrin,Gandalf,9258 +Sam,Bill,9282 +Gimli,Gandalf,9316 +Sam,Frodo,9329 +Sam,Frodo,9334 +Sam,Frodo,9343 +Sam,Gandalf,9343 +Frodo,Gandalf,9343 +Sam,Frodo,9350 +Gimli,Gandalf,9374 +Legolas,Sam,9402 +Legolas,Frodo,9402 +Legolas,Boromir,9402 +Sam,Frodo,9402 +Sam,Boromir,9402 +Frodo,Boromir,9402 +Gimli,Gandalf,9410 +Gimli,Glóin,9413 +Gimli,Gandalf,9417 +Gimli,Legolas,9444 +Gimli,Boromir,9444 +Gimli,Aragorn,9444 +Legolas,Boromir,9444 +Legolas,Aragorn,9444 +Boromir,Aragorn,9444 +Pippin,Gandalf,9468 +Pippin,Merry,9468 +Gandalf,Merry,9468 +Merry,Aragorn,9478 +Thorin,Bilbo,9605 +Gandalf,Daeron,9670 +Balin,Frodo,9676 +Balin,Bilbo,9676 +Frodo,Bilbo,9676 +Gimli,Frodo,9687 +Ori,Gimli,9717 +Balin,Fundin,9755 +Balin,Gimli,9852 +Aragorn,Boromir,9853 +Pippin,Merry,9861 +Frodo,Aragorn,9864 +Aragorn,Boromir,9870 +Pippin,Frodo,9877 +Pippin,Merry,9877 +Pippin,Aragorn,9877 +Frodo,Merry,9877 +Frodo,Aragorn,9877 +Merry,Aragorn,9877 +Balin,Gimli,9878 +Balin,Legolas,9878 +Gimli,Legolas,9878 +Sam,Frodo,9901 +Frodo,Gandalf,9903 +Gandalf,Bilbo,9982 +Gandalf,Boromir,10098 +Gandalf,Aragorn,10098 +Boromir,Aragorn,10098 +Aragorn,Boromir,10144 +Aragorn,Boromir,10152 +Sam,Frodo,10155 +Sam,Frodo,10235 +Pippin,Sam,10251 +Sam,Frodo,10273 +Sam,Aragorn,10273 +Frodo,Aragorn,10273 +Legolas,Aragorn,10279 +Aragorn,Boromir,10280 +Gimli,Frodo,10298 +Gimli,Sam,10298 +Gimli,Aragorn,10298 +Frodo,Sam,10298 +Frodo,Aragorn,10298 +Sam,Aragorn,10298 +Sam,Aragorn,10299 +Merry,Bilbo,10334 +Sam,Frodo,10356 +Sam,Aragorn,10356 +Frodo,Aragorn,10356 +Gimli,Frodo,10360 +Gimli,Aragorn,10478 +Legolas,Sam,10515 +Legolas,Frodo,10515 +Sam,Frodo,10515 +Legolas,Frodo,10518 +Sam,Frodo,10521 +Orophin,Rúmil,10531 +Haldir,Aragorn,10540 +Frodo,Elrond,10550 +Legolas,Aragorn,10555 +Haldir,Pippin,10568 +Haldir,Legolas,10568 +Haldir,Merry,10568 +Pippin,Legolas,10568 +Pippin,Merry,10568 +Legolas,Merry,10568 +Legolas,Frodo,10591 +Haldir,Rúmil,10645 +Haldir,Amroth,10795 +Frodo,Aragorn,10843 +Frodo,Aragorn,10845 +Frodo,Amroth,10850 +Galadriel,Celeborn,10863 +Haldir,Frodo,10869 +Haldir,Galadriel,10879 +Haldir,Celeborn,10879 +Galadriel,Celeborn,10879 +Legolas,Frodo,10883 +Galadriel,Celeborn,10892 +Haldir,Frodo,10896 +Gimli,Glóin,10909 +Haldir,Legolas,10933 +Balin,Celeborn,10938 +Balin,Aragorn,10938 +Celeborn,Aragorn,10938 +Frodo,Galadriel,10964 +Legolas,Aragorn,10978 +Frodo,Boromir,11009 +Sam,Frodo,11054 +Sam,Frodo,11062 +Sam,Frodo,11095 +Sam,Frodo,11140 +Frodo,Gandalf,11189 +Frodo,Gandalf,11192 +Frodo,Saruman,11192 +Gandalf,Saruman,11192 +Frodo,Galadriel,11248 +Frodo,Gandalf,11279 +Legolas,Boromir,11333 +Legolas,Aragorn,11333 +Boromir,Aragorn,11333 +Celeborn,Aragorn,11344 +Legolas,Galadriel,11359 +Frodo,Aragorn,11362 +Gandalf,Boromir,11363 +Elendil,Sauron,11364 +Frodo,Gandalf,11365 +Frodo,Boromir,11365 +Gandalf,Boromir,11365 +Frodo,Boromir,11367 +Frodo,Boromir,11375 +Frodo,Boromir,11380 +Frodo,Aragorn,11380 +Boromir,Aragorn,11380 +Pippin,Sam,11382 +Pippin,Merry,11382 +Sam,Merry,11382 +Pippin,Merry,11471 +Pippin,Frodo,11471 +Pippin,Boromir,11471 +Pippin,Gimli,11471 +Pippin,Sam,11471 +Pippin,Aragorn,11471 +Pippin,Legolas,11471 +Merry,Frodo,11471 +Merry,Boromir,11471 +Merry,Gimli,11471 +Merry,Sam,11471 +Merry,Aragorn,11471 +Merry,Legolas,11471 +Frodo,Boromir,11471 +Frodo,Gimli,11471 +Frodo,Sam,11471 +Frodo,Aragorn,11471 +Frodo,Legolas,11471 +Boromir,Gimli,11471 +Boromir,Sam,11471 +Boromir,Aragorn,11471 +Boromir,Legolas,11471 +Gimli,Sam,11471 +Gimli,Aragorn,11471 +Gimli,Legolas,11471 +Sam,Aragorn,11471 +Sam,Legolas,11471 +Aragorn,Legolas,11471 +Galadriel,Celeborn,11486 +Aragorn,Boromir,11516 +Galadriel,Celeborn,11528 +Celeborn,Galadriel,11539 +Celeborn,Aragorn,11539 +Galadriel,Aragorn,11539 +Elendil,Elessar,11549 +Arwen,Celebrían,11551 +Pippin,Merry,11553 +Pippin,Boromir,11553 +Merry,Boromir,11553 +Gimli,Galadriel,11566 +Gimli,Glóin,11572 +Gimli,Galadriel,11575 +Gimli,Glóin,11589 +Gimli,Glóin,11649 +Gimli,Glóin,11655 +Sam,Frodo,11680 +Gimli,Legolas,11730 +Pippin,Merry,11731 +Pippin,Boromir,11731 +Pippin,Aragorn,11731 +Merry,Boromir,11731 +Merry,Aragorn,11731 +Boromir,Aragorn,11731 +Pippin,Frodo,11732 +Frodo,Aragorn,11736 +Sam,Frodo,11739 +Sam,Frodo,11744 +Sam,Frodo,11771 +Sam,Frodo,11794 +Gollum,Aragorn,11841 +Merry,Aragorn,11895 +Sam,Frodo,11903 +Legolas,Galadriel,11943 +Frodo,Boromir,11956 +Frodo,Boromir,12025 +Frodo,Aragorn,12025 +Boromir,Aragorn,12025 +Legolas,Aragorn,12030 +Legolas,Frodo,12042 +Legolas,Aragorn,12042 +Frodo,Aragorn,12042 +Pippin,Merry,12063 +Aragorn,Boromir,12066 +Sam,Frodo,12117 +Isildur,Anárion,12126 +Isildur,Valandil,12127 +Isildur,Elendil,12127 +Isildur,Elessar,12127 +Valandil,Elendil,12127 +Valandil,Elessar,12127 +Elendil,Elessar,12127 +Frodo,Aragorn,12205 +Sam,Frodo,12226 +Sam,Boromir,12226 +Frodo,Boromir,12226 +Gandalf,Bilbo,12236 +Frodo,Boromir,12260 +Frodo,Boromir,12321 +Pippin,Sam,12469 +Pippin,Merry,12469 +Sam,Merry,12469 +Denethor,Elrond,12500 +Gimli,Sam,12520 +Legolas,Meriadoc,12521 +Legolas,Peregrin,12521 +Legolas,Boromir,12521 +Meriadoc,Peregrin,12521 +Meriadoc,Boromir,12521 +Peregrin,Boromir,12521 +Sam,Aragorn,12563 +Aragorn,Boromir,12601 +Frodo,Boromir,12615 +Pippin,Frodo,12634 +Pippin,Merry,12634 +Frodo,Merry,12634 +Gimli,Legolas,12637 +Sam,Aragorn,12645 +Sam,Frodo,12702 +Sam,Frodo,12730 +Sam,Frodo,12736 +Sam,Frodo,12769 +Samwise,Boromir,12805 +Samwise,Aragorn,12805 +Boromir,Aragorn,12805 +Gimli,Legolas,12876 +Gimli,Legolas,12878 +Legolas,Aragorn,12881 +Pippin,Frodo,12893 +Pippin,Merry,12893 +Pippin,Sam,12893 +Frodo,Merry,12893 +Frodo,Sam,12893 +Merry,Sam,12893 +Gimli,Sauron,12935 +Gandalf,Saruman,12946 +Gimli,Legolas,12961 +Denethor,Boromir,12986 +Frodo,Aragorn,13052 +Gimli,Frodo,13056 +Legolas,Aragorn,13191 +Gimli,Legolas,13238 +Gimli,Legolas,13309 +Gimli,Legolas,13429 +Gimli,Aragorn,13429 +Legolas,Aragorn,13429 +Saruman,Sauron,13474 +Gimli,Gandalf,13478 +Aragorn,Boromir,13479 +Gimli,Legolas,13530 +Sauron,Aragorn,13562 +Isildur,Elendil,13572 +Isildur,Elessar,13572 +Isildur,Aragorn,13572 +Elendil,Elessar,13572 +Elendil,Aragorn,13572 +Elessar,Aragorn,13572 +Gimli,Legolas,13576 +Legolas,Aragorn,13578 +Denethor,Boromir,13585 +Sauron,Thengel,13588 +Denethor,Sauron,13644 +Gandalf,Saruman,13655 +Gandalf,Shadowfax,13660 +Gandalf,Shadowfax,13663 +Gandalf,Aragorn,13663 +Shadowfax,Aragorn,13663 +Gimli,Legolas,13739 +Gimli,Legolas,13794 +Pippin,Merry,13839 +Legolas,Gandalf,13845 +Gimli,Gandalf,13846 +Gandalf,Aragorn,13848 +Legolas,Aragorn,13870 +Celeborn,Aragorn,13882 +Gimli,Saruman,13908 +Legolas,Aragorn,13909 +Gimli,Legolas,13949 +Gimli,Aragorn,13949 +Legolas,Aragorn,13949 +Pippin,Frodo,13952 +Pippin,Merry,13967 +Gandalf,Elrond,13986 +Pippin,Merry,14151 +Pippin,Merry,14177 +Pippin,Merry,14236 +Pippin,Merry,14331 +Frodo,Gandalf,14364 +Gandalf,Shadowfax,14365 +Pippin,Merry,14394 +Pippin,Merry,14436 +Pippin,Merry,14468 +Pippin,Merry,14544 +Pippin,Merry,14621 +Meriadoc,Merry,14799 +Pippin,Peregrin,14800 +Pippin,Merry,14804 +Pippin,Merry,15032 +Tom,Galadriel,15038 +Tom,Elrond,15038 +Galadriel,Elrond,15038 +Treebeard,Gandalf,15049 +Treebeard,Saruman,15049 +Gandalf,Saruman,15049 +Sam,Gandalf,15050 +Gandalf,Galadriel,15059 +Pippin,Merry,15258 +Treebeard,Merry,15265 +Treebeard,Pippin,15265 +Merry,Pippin,15265 +Pippin,Merry,15267 +Pippin,Merry,15292 +Treebeard,Pippin,15294 +Treebeard,Merry,15298 +Treebeard,Pippin,15298 +Merry,Pippin,15298 +Pippin,Merry,15312 +Pippin,Merry,15338 +Sam,Frodo,15374 +Legolas,Aragorn,15600 +Pippin,Merry,15612 +Pippin,Merry,15619 +Pippin,Boromir,15619 +Merry,Boromir,15619 +Gimli,Legolas,15669 +Gimli,Aragorn,15669 +Legolas,Aragorn,15669 +Pippin,Merry,15672 +Aragorn,Boromir,15901 +Pippin,Merry,15958 +Gandalf,Saruman,16001 +Gandalf,Saruman,16002 +Treebeard,Gandalf,16018 +Pippin,Merry,16036 +Gimli,Saruman,16040 +Legolas,Aragorn,16046 +Gandalf,Aragorn,16109 +Elessar,Aragorn,16187 +Gandalf,Shadowfax,16250 +Legolas,Aragorn,16259 +Gimli,Shadowfax,16260 +Gandalf,Shadowfax,16268 +Gimli,Legolas,16293 +Gimli,Gandalf,16293 +Gimli,Aragorn,16293 +Legolas,Gandalf,16293 +Legolas,Aragorn,16293 +Gandalf,Aragorn,16293 +Gimli,Gandalf,16298 +Gimli,Legolas,16410 +Elendil,Thengel,16455 +Elendil,Aragorn,16455 +Thengel,Aragorn,16455 +Denethor,Aragorn,16456 +Gandalf,Wormtongue,16556 +Gandalf,Thengel,16571 +Gimli,Gandalf,16579 +Gandalf,Wormtongue,16663 +Gandalf,Wormtongue,16851 +Gandalf,Saruman,16910 +Legolas,Aragorn,16949 +Gimli,Gandalf,16951 +Gimli,Legolas,17015 +Gandalf,Shadowfax,17026 +Legolas,Aragorn,17034 +Gimli,Galadriel,17041 +Gandalf,Shadowfax,17059 +Legolas,Gandalf,17094 +Helm,Erkenbrand,17122 +Helm,Erkenbrand,17178 +Gandalf,Wormtongue,17194 +Helm,Gandalf,17196 +Helm,Erkenbrand,17196 +Gandalf,Erkenbrand,17196 +Legolas,Aragorn,17207 +Helm,Erkenbrand,17239 +Helm,Gamling,17258 +Legolas,Aragorn,17457 +Gamling,Saruman,17476 +Gandalf,Aragorn,17581 +Elendil,Aragorn,17650 +Legolas,Erkenbrand,17707 +Legolas,Aragorn,17707 +Erkenbrand,Aragorn,17707 +Gimli,Gamling,17710 +Gandalf,Saruman,17770 +Gimli,Legolas,17785 +Gimli,Gandalf,17785 +Gimli,Aragorn,17785 +Legolas,Gandalf,17785 +Legolas,Aragorn,17785 +Gandalf,Aragorn,17785 +Gimli,Legolas,17820 +Gimli,Gandalf,17820 +Legolas,Gandalf,17820 +Legolas,Gandalf,17821 +Gimli,Legolas,17828 +Legolas,Helm,17848 +Legolas,Galadriel,17857 +Gimli,Legolas,17881 +Gandalf,Shadowfax,17989 +Shadowfax,Gandalf,18002 +Grimbold,Erkenbrand,18006 +Helm,Saruman,18010 +Saruman,Wormtongue,18116 +Saruman,Wormtongue,18157 +Gimli,Legolas,18177 +Pippin,Merry,18196 +Treebeard,Merry,18213 +Treebeard,Gandalf,18230 +Gimli,Legolas,18252 +Gimli,Aragorn,18252 +Legolas,Aragorn,18252 +Gimli,Merry,18299 +Pippin,Aragorn,18431 +Gimli,Gandalf,18452 +Pippin,Saruman,18465 +Treebeard,Saruman,18528 +Gandalf,Galadriel,18554 +Gandalf,Elrond,18554 +Galadriel,Elrond,18554 +Treebeard,Merry,18634 +Treebeard,Gandalf,18654 +Treebeard,Gandalf,18664 +Treebeard,Gandalf,18669 +Sam,Frodo,18689 +Sam,Boromir,18689 +Frodo,Boromir,18689 +Saruman,Aragorn,18825 +Treebeard,Merry,18829 +Pippin,Merry,18936 +Saruman,Wormtongue,18954 +Gandalf,Saruman,18988 +Gandalf,Saruman,18993 +Gandalf,Saruman,19202 +Gandalf,Saruman,19207 +Saruman,Aragorn,19218 +Gandalf,Saruman,19271 +Gimli,Legolas,19275 +Gimli,Aragorn,19275 +Legolas,Aragorn,19275 +Treebeard,Gandalf,19276 +Gimli,Legolas,19292 +Gimli,Legolas,19300 +Pippin,Merry,19326 +Pippin,Gandalf,19362 +Pippin,Merry,19362 +Pippin,Aragorn,19362 +Gandalf,Merry,19362 +Gandalf,Aragorn,19362 +Merry,Aragorn,19362 +Pippin,Merry,19366 +Pippin,Merry,19437 +Pippin,Merry,19453 +Gandalf,Saruman,19471 +Pippin,Gandalf,19475 +Pippin,Sam,19489 +Pippin,Gandalf,19519 +Gandalf,Aragorn,19687 +Pippin,Gandalf,19756 +Pippin,Gandalf,19762 +Pippin,Aragorn,19762 +Gandalf,Aragorn,19762 +Merry,Aragorn,19771 +Shadowfax,Gandalf,19793 +Elendil,Elrond,19848 +Wormtongue,Aragorn,19918 +Peregrin,Shadowfax,19921 +Pippin,Gandalf,19957 +Sam,Frodo,20041 +Sam,Frodo,20047 +Sam,Frodo,20230 +Sam,Frodo,20267 +Frodo,Bilbo,20483 +Frodo,Gollum,20519 +Sam,Frodo,20598 +Sam,Frodo,20607 +Sam,Gollum,20627 +Frodo,Gollum,20653 +Sam,Frodo,20655 +Sam,Gollum,20686 +Frodo,Gollum,20688 +Sam,Frodo,20735 +Sam,Gollum,20769 +Sam,Frodo,20777 +Sam,Frodo,20792 +Sam,Gollum,20838 +Sam,Frodo,20844 +Sam,Gollum,20844 +Frodo,Gollum,20844 +Sam,Frodo,20849 +Sam,Gollum,20859 +Sam,Gollum,20861 +Sam,Frodo,20888 +Frodo,Gollum,20903 +Sam,Samwise,20916 +Sam,Frodo,20985 +Sam,Gollum,20985 +Frodo,Gollum,20985 +Sam,Frodo,21041 +Sam,Frodo,21066 +Sam,Frodo,21137 +Sam,Frodo,21147 +Sam,Frodo,21153 +Sam,Gollum,21167 +Sam,Frodo,21168 +Sam,Gollum,21172 +Sam,Frodo,21214 +Sam,Frodo,21229 +Frodo,Gollum,21299 +Sam,Gollum,21301 +Frodo,Gollum,21369 +Sam,Gollum,21455 +Frodo,Gollum,21466 +Frodo,Gollum,21490 +Sam,Frodo,21492 +Sam,Gollum,21492 +Frodo,Gollum,21492 +Frodo,Gollum,21501 +Frodo,Gandalf,21551 +Frodo,Bilbo,21551 +Gandalf,Bilbo,21551 +Frodo,Gollum,21558 +Isildur,Elendil,21588 +Isildur,Frodo,21588 +Elendil,Frodo,21588 +Sam,Frodo,21626 +Sam,Gollum,21626 +Frodo,Gollum,21626 +Gollum,Aragorn,21673 +Frodo,Gollum,21680 +Frodo,Gollum,21683 +Gollum,Aragorn,21687 +Gandalf,Aragorn,21702 +Gandalf,Saruman,21703 +Gandalf,Aragorn,21703 +Saruman,Aragorn,21703 +Frodo,Samwise,21705 +Frodo,Gandalf,21706 +Sam,Frodo,21727 +Frodo,Gollum,21753 +Sam,Frodo,21765 +Frodo,Gollum,21825 +Sam,Frodo,21860 +Sam,Gollum,21860 +Frodo,Gollum,21860 +Sam,Gollum,21887 +Frodo,Elrond,21938 +Sam,Gollum,21945 +Sam,Frodo,21970 +Sam,Gollum,21970 +Frodo,Gollum,21970 +Sam,Frodo,22061 +Sam,Gollum,22081 +Sam,Frodo,22143 +Frodo,Boromir,22150 +Frodo,Hamfast,22183 +Frodo,Samwise,22183 +Hamfast,Samwise,22183 +Boromir,Aragorn,22187 +Denethor,Boromir,22194 +Mablung,Damrod,22233 +Sam,Frodo,22280 +Sam,Faramir,22341 +Frodo,Faramir,22343 +Sam,Frodo,22349 +Sam,Boromir,22349 +Frodo,Boromir,22349 +Elendil,Boromir,22366 +Isildur,Elendil,22367 +Isildur,Aragorn,22367 +Elendil,Aragorn,22367 +Sam,Frodo,22384 +Sam,Faramir,22384 +Frodo,Faramir,22384 +Frodo,Boromir,22391 +Frodo,Denethor,22441 +Frodo,Faramir,22441 +Denethor,Faramir,22441 +Mablung,Damrod,22556 +Samwise,Frodo,22557 +Samwise,Faramir,22557 +Frodo,Faramir,22557 +Sam,Frodo,22568 +Sam,Mablung,22568 +Sam,Damrod,22568 +Sam,Faramir,22568 +Frodo,Mablung,22568 +Frodo,Damrod,22568 +Frodo,Faramir,22568 +Mablung,Damrod,22568 +Mablung,Faramir,22568 +Damrod,Faramir,22568 +Samwise,Faramir,22571 +Isildur,Frodo,22593 +Boromir,Faramir,22687 +Mablung,Damrod,22692 +Mablung,Damrod,22710 +Sam,Frodo,22718 +Mablung,Damrod,22723 +Sam,Frodo,22824 +Sam,Faramir,22824 +Frodo,Faramir,22824 +Sam,Frodo,22832 +Sam,Frodo,22844 +Frodo,Boromir,22845 +Frodo,Aragorn,22849 +Boromir,Faramir,22852 +Sam,Faramir,22890 +Samwise,Faramir,22891 +Bilbo,Aragorn,22908 +Sam,Frodo,22959 +Frodo,Faramir,22986 +Frodo,Faramir,23030 +Sam,Frodo,23064 +Frodo,Faramir,23077 +Frodo,Faramir,23094 +Anborn,Faramir,23143 +Anborn,Frodo,23166 +Frodo,Gandalf,23212 +Frodo,Gollum,23277 +Frodo,Faramir,23277 +Gollum,Faramir,23277 +Anborn,Gollum,23341 +Frodo,Faramir,23375 +Gollum,Faramir,23389 +Frodo,Gollum,23399 +Frodo,Faramir,23420 +Frodo,Gollum,23435 +Frodo,Faramir,23435 +Gollum,Faramir,23435 +Anborn,Gollum,23477 +Sam,Frodo,23545 +Frodo,Elrond,23572 +Frodo,Gollum,23576 +Frodo,Faramir,23576 +Gollum,Faramir,23576 +Samwise,Faramir,23577 +Frodo,Gollum,23578 +Sam,Frodo,23617 +Sam,Gollum,23624 +Sam,Frodo,23678 +Sam,Gollum,23678 +Frodo,Gollum,23678 +Sam,Frodo,23726 +Sam,Gollum,23726 +Frodo,Gollum,23726 +Sam,Gandalf,23787 +Sam,Gollum,23789 +Sam,Frodo,23816 +Sam,Frodo,23837 +Frodo,Gollum,23855 +Sam,Frodo,23864 +Sam,Gollum,23890 +Sam,Frodo,23896 +Sam,Frodo,23910 +Sam,Frodo,23917 +Sam,Frodo,23940 +Gandalf,Galadriel,24026 +Gandalf,Faramir,24026 +Gandalf,Elrond,24026 +Gandalf,Aragorn,24026 +Galadriel,Faramir,24026 +Galadriel,Elrond,24026 +Galadriel,Aragorn,24026 +Faramir,Elrond,24026 +Faramir,Aragorn,24026 +Elrond,Aragorn,24026 +Sam,Frodo,24050 +Sam,Frodo,24058 +Sam,Frodo,24099 +Sam,Frodo,24138 +Sam,Frodo,24219 +Sam,Frodo,24220 +Sam,Frodo,24253 +Sam,Frodo,24284 +Sam,Frodo,24292 +Sam,Frodo,24334 +Frodo,Gollum,24339 +Sam,Frodo,24355 +Gollum,Shelob,24375 +Sam,Frodo,24413 +Sam,Frodo,24436 +Sam,Frodo,24442 +Sam,Frodo,24515 +Gollum,Faramir,24578 +Ungoliant,Shelob,24618 +Sam,Frodo,24664 +Sam,Gollum,24681 +Sam,Gollum,24687 +Sam,Gollum,24690 +Sam,Gollum,24692 +Sam,Gollum,24693 +Sam,Frodo,24711 +Sam,Frodo,24715 +Sam,Samwise,24715 +Frodo,Samwise,24715 +Sam,Shelob,24736 +Sam,Frodo,24742 +Hamfast,Samwise,24753 +Frodo,Galadriel,24792 +Frodo,Faramir,24816 +Frodo,Bilbo,24817 +Frodo,Bilbo,24857 +Sam,Frodo,24873 +Sam,Shagrat,25066 +Shelob,Shagrat,25096 +Gorbag,Shagrat,25308 +Gorbag,Shagrat,25310 +Pippin,Gandalf,25348 +Pippin,Gandalf,25359 +Pippin,Gandalf,25366 +Pippin,Gandalf,25380 +Pippin,Gandalf,25394 +Gandalf,Denethor,25399 +Gandalf,Boromir,25411 +Amroth,Imrahil,25440 +Pippin,Gandalf,25441 +Pippin,Ecthelion,25446 +Peregrin,Gandalf,25447 +Gandalf,Shadowfax,25472 +Pippin,Gandalf,25480 +Pippin,Gandalf,25485 +Denethor,Ecthelion,25521 +Pippin,Boromir,25523 +Pippin,Aragorn,25523 +Boromir,Aragorn,25523 +Gandalf,Faramir,25533 +Pippin,Boromir,25541 +Pippin,Denethor,25550 +Meriadoc,Pippin,25560 +Pippin,Denethor,25561 +Pippin,Gandalf,25572 +Pippin,Denethor,25573 +Pippin,Peregrin,25574 +Pippin,Denethor,25574 +Pippin,Ecthelion,25574 +Peregrin,Denethor,25574 +Peregrin,Ecthelion,25574 +Denethor,Ecthelion,25574 +Pippin,Gandalf,25578 +Gandalf,Denethor,25579 +Théoden,Saruman,25583 +Pippin,Gandalf,25584 +Gandalf,Denethor,25585 +Pippin,Gandalf,25586 +Treebeard,Gandalf,25589 +Gandalf,Denethor,25592 +Pippin,Gandalf,25599 +Pippin,Denethor,25600 +Pippin,Gandalf,25621 +Pippin,Gandalf,25629 +Isildur,Denethor,25635 +Isildur,Boromir,25635 +Denethor,Boromir,25635 +Pippin,Boromir,25636 +Pippin,Faramir,25636 +Boromir,Faramir,25636 +Denethor,Faramir,25654 +Beregond,Aragorn,25692 +Pippin,Peregrin,25697 +Gandalf,Shadowfax,25715 +Beregond,Shadowfax,25716 +Pippin,Shadowfax,25723 +Pippin,Shadowfax,25726 +Beregond,Pippin,25741 +Beregond,Pippin,25749 +Beregond,Peregrin,25753 +Beregond,Pippin,25780 +Pippin,Peregrin,25817 +Pippin,Peregrin,25867 +Beregond,Gandalf,25868 +Beregond,Pippin,25868 +Gandalf,Pippin,25868 +Pippin,Beregond,25869 +Pippin,Denethor,25872 +Pippin,Boromir,25872 +Denethor,Boromir,25872 +Beregond,Bergil,25917 +Pippin,Merry,25928 +Pippin,Bergil,25928 +Merry,Bergil,25928 +Pippin,Bergil,25930 +Pippin,Bergil,25935 +Pippin,Forlong,25939 +Forlong,Bergil,25941 +Amroth,Imrahil,25961 +Pippin,Gandalf,25987 +Pippin,Faramir,26001 +Gandalf,Merry,26010 +Gandalf,Shadowfax,26010 +Gandalf,Aragorn,26010 +Merry,Shadowfax,26010 +Merry,Aragorn,26010 +Shadowfax,Aragorn,26010 +Gimli,Legolas,26013 +Merry,Aragorn,26031 +Gimli,Legolas,26034 +Gimli,Merry,26034 +Gimli,Aragorn,26034 +Legolas,Merry,26034 +Legolas,Aragorn,26034 +Merry,Aragorn,26034 +Merry,Aragorn,26042 +Théoden,Saruman,26073 +Halbarad,Aragorn,26089 +Gimli,Legolas,26099 +Gimli,Merry,26099 +Legolas,Merry,26099 +Gimli,Legolas,26102 +Gimli,Halbarad,26115 +Legolas,Aragorn,26116 +Legolas,Gandalf,26125 +Legolas,Galadriel,26125 +Gandalf,Galadriel,26125 +Gimli,Gandalf,26126 +Meriadoc,Théoden,26151 +Théoden,Merry,26160 +Théoden,Aragorn,26160 +Merry,Aragorn,26160 +Éomer,Elrond,26169 +Éomer,Halbarad,26169 +Éomer,Aragorn,26169 +Elrond,Halbarad,26169 +Elrond,Aragorn,26169 +Halbarad,Aragorn,26169 +Gimli,Legolas,26170 +Merry,Aragorn,26171 +Éomer,Théoden,26176 +Éomer,Merry,26186 +Éomer,Aragorn,26186 +Merry,Aragorn,26186 +Éomer,Aragorn,26188 +Éomer,Théoden,26190 +Éomer,Aragorn,26190 +Théoden,Aragorn,26190 +Gimli,Legolas,26197 +Gimli,Merry,26197 +Legolas,Merry,26197 +Éomer,Théoden,26202 +Gimli,Legolas,26212 +Gandalf,Aragorn,26219 +Isildur,Elendil,26234 +Isildur,Théoden,26234 +Isildur,Sauron,26234 +Elendil,Théoden,26234 +Elendil,Sauron,26234 +Théoden,Sauron,26234 +Gimli,Aragorn,26236 +Legolas,Elrond,26249 +Legolas,Aragorn,26249 +Elrond,Aragorn,26249 +Arvedui,Aragorn,26250 +Gimli,Aragorn,26257 +Gimli,Legolas,26260 +Gimli,Aragorn,26260 +Legolas,Aragorn,26260 +Isildur,Sauron,26263 +Isildur,Sauron,26266 +Gimli,Legolas,26272 +Gimli,Aragorn,26272 +Legolas,Aragorn,26272 +Gimli,Legolas,26274 +Helm,Halbarad,26276 +Éowyn,Elrond,26278 +Éowyn,Aragorn,26278 +Elrond,Aragorn,26278 +Helm,Théoden,26279 +Éowyn,Aragorn,26283 +Gimli,Legolas,26295 +Gimli,Aragorn,26295 +Legolas,Aragorn,26295 +Gimli,Legolas,26330 +Gimli,Glóin,26361 +Gimli,Aragorn,26362 +Gimli,Glóin,26371 +Gimli,Legolas,26398 +Gimli,Legolas,26400 +Gimli,Amroth,26407 +Isildur,Elessar,26428 +Isildur,Halbarad,26428 +Elessar,Halbarad,26428 +Gimli,Legolas,26433 +Pippin,Amroth,26438 +Gimli,Pippin,26465 +Gimli,Legolas,26465 +Gimli,Aragorn,26465 +Pippin,Legolas,26465 +Pippin,Aragorn,26465 +Legolas,Aragorn,26465 +Sam,Frodo,26466 +Éomer,Théoden,26476 +Gandalf,Shadowfax,26497 +Dúnhere,Gandalf,26500 +Éowyn,Théoden,26544 +Éowyn,Merry,26544 +Théoden,Merry,26544 +Éomer,Aragorn,26549 +Pippin,Gandalf,26571 +Pippin,Frodo,26571 +Pippin,Gimli,26571 +Pippin,Sam,26571 +Pippin,Legolas,26571 +Gandalf,Frodo,26571 +Gandalf,Gimli,26571 +Gandalf,Sam,26571 +Gandalf,Legolas,26571 +Frodo,Gimli,26571 +Frodo,Sam,26571 +Frodo,Legolas,26571 +Gimli,Sam,26571 +Gimli,Legolas,26571 +Sam,Legolas,26571 +Éowyn,Dúnhere,26576 +Éowyn,Éomer,26576 +Éowyn,Théoden,26576 +Dúnhere,Éomer,26576 +Dúnhere,Théoden,26576 +Éomer,Théoden,26576 +Éomer,Aragorn,26584 +Brego,Baldor,26592 +Merry,Aragorn,26598 +Éowyn,Éomer,26599 +Brego,Baldor,26603 +Éomer,Aragorn,26614 +Merry,Boromir,26622 +Hirgon,Denethor,26628 +Théoden,Denethor,26630 +Hirgon,Denethor,26634 +Hirgon,Gandalf,26638 +Hirgon,Denethor,26638 +Gandalf,Denethor,26638 +Éomer,Théoden,26705 +Éomer,Merry,26706 +Éowyn,Merry,26712 +Théoden,Merry,26714 +Éowyn,Aragorn,26721 +Gimli,Merry,26725 +Meriadoc,Merry,26759 +Pippin,Gandalf,26788 +Pippin,Denethor,26800 +Pippin,Gandalf,26801 +Éomer,Gandalf,26819 +Éomer,Denethor,26819 +Gandalf,Denethor,26819 +Pippin,Denethor,26820 +Pippin,Denethor,26821 +Pippin,Gandalf,26862 +Beregond,Pippin,26868 +Beregond,Faramir,26868 +Pippin,Faramir,26868 +Beregond,Faramir,26909 +Beregond,Pippin,26910 +Gandalf,Shadowfax,26929 +Pippin,Faramir,26930 +Pippin,Boromir,26932 +Pippin,Denethor,26949 +Pippin,Faramir,26953 +Pippin,Gandalf,26955 +Pippin,Gandalf,26957 +Pippin,Faramir,26957 +Gandalf,Faramir,26957 +Pippin,Frodo,26958 +Pippin,Gandalf,26958 +Frodo,Gandalf,26958 +Pippin,Gandalf,26959 +Boromir,Denethor,26991 +Boromir,Faramir,26991 +Denethor,Faramir,26991 +Boromir,Denethor,26998 +Boromir,Faramir,26998 +Denethor,Faramir,26998 +Pippin,Gandalf,27035 +Pippin,Denethor,27035 +Gandalf,Denethor,27035 +Pippin,Gandalf,27056 +Pippin,Gandalf,27058 +Pippin,Frodo,27060 +Pippin,Gandalf,27060 +Frodo,Gandalf,27060 +Pippin,Gollum,27090 +Frodo,Gollum,27094 +Boromir,Faramir,27114 +Denethor,Boromir,27125 +Gandalf,Faramir,27128 +Beregond,Faramir,27161 +Nazgûl,Denethor,27184 +Nazgûl,Sauron,27184 +Denethor,Sauron,27184 +Pippin,Gandalf,27187 +Shadowfax,Amroth,27234 +Shadowfax,Faramir,27234 +Amroth,Faramir,27234 +Denethor,Faramir,27255 +Nazgûl,Faramir,27259 +Imrahil,Faramir,27261 +Denethor,Faramir,27265 +Pippin,Gandalf,27269 +Gandalf,Denethor,27322 +Denethor,Faramir,27350 +Denethor,Faramir,27362 +Denethor,Faramir,27368 +Denethor,Faramir,27387 +Pippin,Denethor,27389 +Denethor,Faramir,27404 +Pippin,Denethor,27422 +Beregond,Pippin,27428 +Beregond,Faramir,27428 +Pippin,Faramir,27428 +Beregond,Pippin,27435 +Beregond,Faramir,27435 +Pippin,Faramir,27435 +Gandalf,Shadowfax,27488 +Pippin,Merry,27536 +Éomer,Merry,27539 +Éomer,Théoden,27567 +Théoden,Ghân-buri-Ghân,27665 +Éomer,Denethor,27690 +Éomer,Théoden,27771 +Théoden,Thengel,27796 +Éowyn,Merry,27873 +Éowyn,Meriadoc,27881 +Éomer,Amroth,27905 +Merry,Gandalf,27906 +Éowyn,Éomer,27908 +Éomer,Théoden,27918 +Éowyn,Théoden,27951 +Imrahil,Amroth,27953 +Éowyn,Éomer,27962 +Éomer,Imrahil,27972 +Éomer,Húrin,27972 +Imrahil,Húrin,27972 +Arwen,Elrond,28013 +Isildur,Elessar,28014 +Isildur,Aragorn,28014 +Elessar,Aragorn,28014 +Gimli,Legolas,28019 +Gimli,Halbarad,28019 +Legolas,Halbarad,28019 +Elendil,Aragorn,28020 +Éomer,Aragorn,28021 +Éomer,Imrahil,28034 +Éomer,Aragorn,28034 +Imrahil,Aragorn,28034 +Grimbold,Halbarad,28038 +Grimbold,Dúnhere,28044 +Grimbold,Fastred,28044 +Grimbold,Horn,28044 +Grimbold,Harding,28044 +Dúnhere,Fastred,28044 +Dúnhere,Horn,28044 +Dúnhere,Harding,28044 +Fastred,Horn,28044 +Fastred,Harding,28044 +Horn,Harding,28044 +Gandalf,Denethor,28049 +Gandalf,Shadowfax,28053 +Pippin,Shadowfax,28055 +Pippin,Faramir,28067 +Pippin,Denethor,28070 +Pippin,Faramir,28070 +Denethor,Faramir,28070 +Pippin,Gandalf,28073 +Gandalf,Faramir,28074 +Pippin,Shadowfax,28077 +Pippin,Beregond,28089 +Beregond,Denethor,28102 +Pippin,Gandalf,28104 +Pippin,Denethor,28104 +Gandalf,Denethor,28104 +Gandalf,Denethor,28109 +Denethor,Faramir,28116 +Beregond,Gandalf,28122 +Beregond,Pippin,28122 +Beregond,Denethor,28122 +Gandalf,Pippin,28122 +Gandalf,Denethor,28122 +Pippin,Denethor,28122 +Gandalf,Denethor,28174 +Beregond,Faramir,28177 +Beregond,Gandalf,28191 +Beregond,Denethor,28191 +Beregond,Ecthelion,28191 +Gandalf,Denethor,28191 +Gandalf,Ecthelion,28191 +Denethor,Ecthelion,28191 +Beregond,Faramir,28195 +Beregond,Gandalf,28197 +Beregond,Pippin,28197 +Beregond,Faramir,28197 +Gandalf,Pippin,28197 +Gandalf,Faramir,28197 +Pippin,Faramir,28197 +Gandalf,Amroth,28204 +Beregond,Gandalf,28210 +Beregond,Pippin,28210 +Beregond,Faramir,28210 +Gandalf,Pippin,28210 +Gandalf,Faramir,28210 +Pippin,Faramir,28210 +Pippin,Beregond,28212 +Denethor,Sauron,28225 +Beregond,Faramir,28229 +Éowyn,Théoden,28249 +Pippin,Merry,28276 +Beregond,Bergil,28285 +Éowyn,Meriadoc,28301 +Éowyn,Faramir,28301 +Meriadoc,Faramir,28301 +Éomer,Imrahil,28323 +Éomer,Aragorn,28323 +Imrahil,Aragorn,28323 +Éomer,Elendil,28327 +Éomer,Imrahil,28334 +Éowyn,Éomer,28338 +Éowyn,Imrahil,28339 +Éowyn,Gandalf,28345 +Denethor,Faramir,28346 +Galadriel,Aragorn,28350 +Amroth,Faramir,28352 +Éomer,Imrahil,28365 +Elessar,Aragorn,28366 +Éowyn,Meriadoc,28368 +Éowyn,Gandalf,28368 +Meriadoc,Gandalf,28368 +Éowyn,Merry,28370 +Éowyn,Faramir,28370 +Éowyn,Aragorn,28370 +Merry,Faramir,28370 +Merry,Aragorn,28370 +Faramir,Aragorn,28370 +Éomer,Faramir,28373 +Éomer,Elrond,28373 +Éomer,Aragorn,28373 +Faramir,Elrond,28373 +Faramir,Aragorn,28373 +Elrond,Aragorn,28373 +Gandalf,Faramir,28386 +Shadowfax,Aragorn,28387 +Gandalf,Aragorn,28390 +Aragorn,Faramir,28419 +Faramir,Aragorn,28421 +Faramir,Aragorn,28432 +Faramir,Aragorn,28436 +Beregond,Gandalf,28443 +Beregond,Imrahil,28443 +Gandalf,Imrahil,28443 +Pippin,Gandalf,28444 +Éowyn,Aragorn,28448 +Théoden,Wormtongue,28464 +Saruman,Wormtongue,28468 +Éomer,Aragorn,28472 +Éowyn,Aragorn,28483 +Éowyn,Aragorn,28484 +Éomer,Éowyn,28488 +Éowyn,Éomer,28499 +Éomer,Gandalf,28503 +Pippin,Gandalf,28510 +Pippin,Merry,28510 +Pippin,Aragorn,28510 +Gandalf,Merry,28510 +Gandalf,Aragorn,28510 +Merry,Aragorn,28510 +Pippin,Merry,28511 +Merry,Aragorn,28516 +Pippin,Gandalf,28518 +Pippin,Aragorn,28528 +Meriadoc,Saruman,28532 +Meriadoc,Aragorn,28532 +Saruman,Aragorn,28532 +Merry,Gandalf,28543 +Éowyn,Gandalf,28565 +Éowyn,Faramir,28565 +Éowyn,Aragorn,28565 +Gandalf,Faramir,28565 +Gandalf,Aragorn,28565 +Faramir,Aragorn,28565 +Éowyn,Gandalf,28566 +Éowyn,Faramir,28566 +Éowyn,Aragorn,28566 +Gandalf,Faramir,28566 +Gandalf,Aragorn,28566 +Faramir,Aragorn,28566 +Beregond,Meriadoc,28568 +Elrond,Aragorn,28573 +Gimli,Pippin,28578 +Gimli,Legolas,28578 +Gimli,Merry,28578 +Pippin,Legolas,28578 +Pippin,Merry,28578 +Legolas,Merry,28578 +Gimli,Legolas,28579 +Legolas,Aragorn,28581 +Legolas,Imrahil,28583 +Legolas,Aragorn,28583 +Imrahil,Aragorn,28583 +Legolas,Aragorn,28587 +Meriadoc,Peregrin,28589 +Meriadoc,Imrahil,28589 +Peregrin,Imrahil,28589 +Legolas,Merry,28603 +Legolas,Merry,28619 +Gimli,Glóin,28632 +Éowyn,Merry,28635 +Pippin,Legolas,28639 +Pippin,Merry,28639 +Legolas,Merry,28639 +Angbor,Aragorn,28652 +Lúthien,Sauron,28683 +Isildur,Aragorn,28690 +Gandalf,Aragorn,28715 +Théoden,Imrahil,28719 +Théoden,Gimli,28719 +Théoden,Aragorn,28719 +Théoden,Éomer,28719 +Théoden,Legolas,28719 +Imrahil,Gimli,28719 +Imrahil,Aragorn,28719 +Imrahil,Éomer,28719 +Imrahil,Legolas,28719 +Gimli,Aragorn,28719 +Gimli,Éomer,28719 +Gimli,Legolas,28719 +Aragorn,Éomer,28719 +Aragorn,Legolas,28719 +Éomer,Legolas,28719 +Gandalf,Elrond,28720 +Gandalf,Aragorn,28720 +Elrond,Aragorn,28720 +Amroth,Imrahil,28730 +Gandalf,Sauron,28793 +Imrahil,Aragorn,28799 +Imrahil,Gimli,28818 +Imrahil,Glóin,28818 +Imrahil,Aragorn,28818 +Gimli,Glóin,28818 +Gimli,Aragorn,28818 +Glóin,Aragorn,28818 +Éomer,Elrond,28820 +Éomer,Amroth,28820 +Éomer,Imrahil,28820 +Éomer,Aragorn,28820 +Elrond,Amroth,28820 +Elrond,Imrahil,28820 +Elrond,Aragorn,28820 +Amroth,Imrahil,28820 +Amroth,Aragorn,28820 +Imrahil,Aragorn,28820 +Gimli,Legolas,28836 +Gimli,Gandalf,28836 +Gimli,Elrond,28836 +Gimli,Aragorn,28836 +Legolas,Gandalf,28836 +Legolas,Elrond,28836 +Legolas,Aragorn,28836 +Gandalf,Elrond,28836 +Gandalf,Aragorn,28836 +Elrond,Aragorn,28836 +Gandalf,Imrahil,28872 +Gandalf,Faramir,28872 +Imrahil,Faramir,28872 +Gandalf,Aragorn,28876 +Nazgûl,Aragorn,28894 +Gandalf,Imrahil,28926 +Gandalf,Gimli,28926 +Gandalf,Peregrin,28926 +Gandalf,Elrond,28926 +Gandalf,Aragorn,28926 +Gandalf,Éomer,28926 +Gandalf,Legolas,28926 +Imrahil,Gimli,28926 +Imrahil,Peregrin,28926 +Imrahil,Elrond,28926 +Imrahil,Aragorn,28926 +Imrahil,Éomer,28926 +Imrahil,Legolas,28926 +Gimli,Peregrin,28926 +Gimli,Elrond,28926 +Gimli,Aragorn,28926 +Gimli,Éomer,28926 +Gimli,Legolas,28926 +Peregrin,Elrond,28926 +Peregrin,Aragorn,28926 +Peregrin,Éomer,28926 +Peregrin,Legolas,28926 +Elrond,Aragorn,28926 +Elrond,Éomer,28926 +Elrond,Legolas,28926 +Aragorn,Éomer,28926 +Aragorn,Legolas,28926 +Éomer,Legolas,28926 +Gandalf,Sauron,28956 +Sam,Frodo,28958 +Pippin,Imrahil,28960 +Saruman,Sauron,28982 +Elrond,Amroth,29020 +Elrond,Imrahil,29020 +Amroth,Imrahil,29020 +Beregond,Frodo,29024 +Beregond,Gandalf,29024 +Beregond,Pippin,29024 +Beregond,Imrahil,29024 +Frodo,Gandalf,29024 +Frodo,Pippin,29024 +Frodo,Imrahil,29024 +Gandalf,Pippin,29024 +Gandalf,Imrahil,29024 +Pippin,Imrahil,29024 +Beregond,Pippin,29040 +Pippin,Merry,29069 +Pippin,Denethor,29069 +Pippin,Aragorn,29069 +Merry,Denethor,29069 +Merry,Aragorn,29069 +Denethor,Aragorn,29069 +Sam,Frodo,29070 +Hamfast,Samwise,29072 +Frodo,Shelob,29084 +Frodo,Shagrat,29084 +Shelob,Shagrat,29084 +Gorbag,Shagrat,29101 +Gorbag,Shagrat,29163 +Gorbag,Shagrat,29164 +Gorbag,Sam,29169 +Sam,Frodo,29207 +Gorbag,Shagrat,29246 +Gorbag,Shagrat,29288 +Nazgûl,Shagrat,29309 +Sam,Shagrat,29317 +Sam,Shagrat,29323 +Sam,Shagrat,29344 +Sam,Shagrat,29350 +Sam,Frodo,29427 +Sam,Frodo,29440 +Sam,Frodo,29449 +Sam,Frodo,29461 +Sam,Frodo,29480 +Sam,Frodo,29499 +Sam,Frodo,29503 +Sam,Frodo,29511 +Sam,Frodo,29521 +Sam,Frodo,29532 +Sam,Frodo,29545 +Gorbag,Sam,29578 +Sam,Frodo,29583 +Sam,Frodo,29584 +Sam,Frodo,29593 +Sam,Faramir,29619 +Sam,Frodo,29623 +Sam,Galadriel,29649 +Sam,Frodo,29654 +Sam,Frodo,29679 +Sam,Frodo,29684 +Sam,Frodo,29689 +Sam,Frodo,29694 +Sam,Frodo,29709 +Sam,Shagrat,29717 +Sam,Frodo,29739 +Sam,Frodo,29751 +Sam,Frodo,29764 +Sam,Frodo,29780 +Sam,Frodo,29788 +Gorbag,Shagrat,29792 +Sam,Frodo,29795 +Sam,Frodo,29807 +Sam,Frodo,29836 +Sam,Frodo,29851 +Sam,Faramir,29855 +Sam,Frodo,29863 +Sam,Frodo,29874 +Sam,Frodo,29888 +Sam,Frodo,29972 +Sam,Frodo,29987 +Sam,Shelob,29987 +Sam,Gollum,29987 +Frodo,Shelob,29987 +Frodo,Gollum,29987 +Shelob,Gollum,29987 +Sam,Frodo,29988 +Sam,Frodo,30000 +Sam,Frodo,30011 +Sam,Frodo,30029 +Sam,Frodo,30031 +Sam,Frodo,30035 +Sam,Frodo,30042 +Sam,Frodo,30043 +Frodo,Gollum,30061 +Sam,Frodo,30064 +Sam,Frodo,30085 +Sam,Frodo,30118 +Sam,Frodo,30141 +Sam,Frodo,30153 +Sam,Frodo,30170 +Frodo,Gandalf,30173 +Sam,Frodo,30187 +Sam,Frodo,30276 +Sam,Faramir,30276 +Frodo,Faramir,30276 +Sam,Frodo,30316 +Sam,Frodo,30351 +Sam,Frodo,30365 +Sam,Frodo,30368 +Sam,Frodo,30371 +Sam,Frodo,30379 +Sam,Frodo,30413 +Sam,Frodo,30419 +Frodo,Gollum,30437 +Sam,Gollum,30440 +Sam,Frodo,30453 +Sam,Gollum,30453 +Frodo,Gollum,30453 +Sam,Gollum,30488 +Sam,Frodo,30513 +Sam,Gollum,30536 +Sam,Frodo,30547 +Gollum,Gandalf,30571 +Gandalf,Sauron,30601 +Gandalf,Aragorn,30610 +Meneldor,Gandalf,30615 +Sam,Frodo,30618 +Sam,Frodo,30622 +Sam,Frodo,30628 +Sam,Frodo,30630 +Sam,Gandalf,30668 +Gandalf,Sauron,30681 +Sam,Frodo,30694 +Sam,Galadriel,30694 +Frodo,Galadriel,30694 +Frodo,Samwise,30706 +Sam,Frodo,30716 +Sam,Frodo,30721 +Sam,Aragorn,30723 +Sam,Frodo,30725 +Sam,Frodo,30729 +Sam,Frodo,30736 +Frodo,Gandalf,30737 +Sam,Frodo,30741 +Frodo,Gandalf,30744 +Frodo,Bilbo,30744 +Gandalf,Bilbo,30744 +Éomer,Gimli,30745 +Éomer,Legolas,30745 +Éomer,Gandalf,30745 +Éomer,Imrahil,30745 +Gimli,Legolas,30745 +Gimli,Gandalf,30745 +Gimli,Imrahil,30745 +Legolas,Gandalf,30745 +Legolas,Imrahil,30745 +Gandalf,Imrahil,30745 +Peregrin,Merry,30751 +Pippin,Merry,30759 +Pippin,Gandalf,30759 +Pippin,Frodo,30759 +Pippin,Gimli,30759 +Pippin,Sam,30759 +Pippin,Legolas,30759 +Merry,Gandalf,30759 +Merry,Frodo,30759 +Merry,Gimli,30759 +Merry,Sam,30759 +Merry,Legolas,30759 +Gandalf,Frodo,30759 +Gandalf,Gimli,30759 +Gandalf,Sam,30759 +Gandalf,Legolas,30759 +Frodo,Gimli,30759 +Frodo,Sam,30759 +Frodo,Legolas,30759 +Gimli,Sam,30759 +Gimli,Legolas,30759 +Sam,Legolas,30759 +Sam,Frodo,30760 +Pippin,Frodo,30763 +Pippin,Merry,30763 +Frodo,Merry,30763 +Gandalf,Bilbo,30772 +Gimli,Pippin,30775 +Gimli,Frodo,30775 +Gimli,Sam,30775 +Pippin,Frodo,30775 +Pippin,Sam,30775 +Frodo,Sam,30775 +Sam,Frodo,30798 +Éomer,Théoden,30860 +Éomer,Faramir,30860 +Théoden,Faramir,30860 +Éowyn,Merry,30888 +Éowyn,Faramir,30888 +Merry,Faramir,30888 +Merry,Faramir,30889 +Éowyn,Faramir,30896 +Éowyn,Amroth,30903 +Éowyn,Finduilas,30903 +Amroth,Finduilas,30903 +Éowyn,Faramir,30905 +Merry,Faramir,30945 +Éowyn,Faramir,30949 +Elendil,Aragorn,30950 +Éowyn,Faramir,30960 +Éowyn,Faramir,30981 +Éowyn,Húrin,30991 +Éowyn,Faramir,30991 +Húrin,Faramir,30991 +Éomer,Gandalf,30995 +Éomer,Imrahil,30995 +Gandalf,Imrahil,30995 +Húrin,Faramir,31008 +Faramir,Aragorn,31009 +Isildur,Valandil,31013 +Isildur,Elendil,31013 +Isildur,Elessar,31013 +Isildur,Aragorn,31013 +Valandil,Elendil,31013 +Valandil,Elessar,31013 +Valandil,Aragorn,31013 +Elendil,Elessar,31013 +Elendil,Aragorn,31013 +Elessar,Aragorn,31013 +Eärnur,Faramir,31017 +Faramir,Aragorn,31022 +Frodo,Gandalf,31023 +Frodo,Faramir,31023 +Frodo,Aragorn,31023 +Gandalf,Faramir,31023 +Gandalf,Aragorn,31023 +Faramir,Aragorn,31023 +Húrin,Elessar,31025 +Húrin,Faramir,31025 +Elessar,Faramir,31025 +Beregond,Faramir,31038 +Faramir,Aragorn,31039 +Éomer,Aragorn,31040 +Éowyn,Elrond,31045 +Éowyn,Faramir,31045 +Elrond,Faramir,31045 +Gimli,Legolas,31048 +Gimli,Aragorn,31048 +Legolas,Aragorn,31048 +Frodo,Gandalf,31052 +Frodo,Aragorn,31052 +Gandalf,Aragorn,31052 +Gandalf,Bilbo,31053 +Pippin,Frodo,31054 +Pippin,Gandalf,31054 +Frodo,Gandalf,31054 +Gandalf,Aragorn,31057 +Gandalf,Aragorn,31058 +Gandalf,Aragorn,31067 +Gandalf,Nimloth,31086 +Nimloth,Aragorn,31094 +Glorfindel,Arwen,31102 +Glorfindel,Erestor,31102 +Glorfindel,Elrond,31102 +Glorfindel,Galadriel,31102 +Glorfindel,Celeborn,31102 +Arwen,Erestor,31102 +Arwen,Elrond,31102 +Arwen,Galadriel,31102 +Arwen,Celeborn,31102 +Erestor,Elrond,31102 +Erestor,Galadriel,31102 +Erestor,Celeborn,31102 +Elrond,Galadriel,31102 +Elrond,Celeborn,31102 +Galadriel,Celeborn,31102 +Frodo,Gandalf,31103 +Arwen,Elessar,31106 +Arwen,Aragorn,31106 +Elessar,Aragorn,31106 +Arwen,Frodo,31108 +Frodo,Aragorn,31109 +Arwen,Bilbo,31113 +Arwen,Elrond,31113 +Bilbo,Elrond,31113 +Éomer,Théoden,31119 +Éomer,Gimli,31129 +Éomer,Glóin,31129 +Gimli,Glóin,31129 +Éomer,Gimli,31131 +Théoden,Merry,31141 +Pippin,Gandalf,31142 +Pippin,Shadowfax,31142 +Pippin,Frodo,31142 +Pippin,Gimli,31142 +Pippin,Aragorn,31142 +Pippin,Legolas,31142 +Pippin,Samwise,31142 +Gandalf,Shadowfax,31142 +Gandalf,Frodo,31142 +Gandalf,Gimli,31142 +Gandalf,Aragorn,31142 +Gandalf,Legolas,31142 +Gandalf,Samwise,31142 +Shadowfax,Frodo,31142 +Shadowfax,Gimli,31142 +Shadowfax,Aragorn,31142 +Shadowfax,Legolas,31142 +Shadowfax,Samwise,31142 +Frodo,Gimli,31142 +Frodo,Aragorn,31142 +Frodo,Legolas,31142 +Frodo,Samwise,31142 +Gimli,Aragorn,31142 +Gimli,Legolas,31142 +Gimli,Samwise,31142 +Aragorn,Legolas,31142 +Aragorn,Samwise,31142 +Legolas,Samwise,31142 +Elrond,Arwen,31143 +Elrond,Galadriel,31143 +Elrond,Amroth,31143 +Elrond,Celeborn,31143 +Arwen,Galadriel,31143 +Arwen,Amroth,31143 +Arwen,Celeborn,31143 +Galadriel,Amroth,31143 +Galadriel,Celeborn,31143 +Amroth,Celeborn,31143 +Théoden,Thengel,31144 +Elessar,Aragorn,31146 +Théoden,Thengel,31152 +Helm,Théoden,31153 +Théoden,Merry,31156 +Éowyn,Éomer,31161 +Fréa,Fengel,31162 +Fréa,Helm,31162 +Fréa,Théoden,31162 +Fréa,Baldor,31162 +Fréa,Déor,31162 +Fréa,Brego,31162 +Fréa,Goldwine,31162 +Fréa,Folcwine,31162 +Fréa,Aldor,31162 +Fréa,Walda,31162 +Fréa,Folca,31162 +Fréa,Thengel,31162 +Fréa,Gram,31162 +Fengel,Helm,31162 +Fengel,Théoden,31162 +Fengel,Baldor,31162 +Fengel,Déor,31162 +Fengel,Brego,31162 +Fengel,Goldwine,31162 +Fengel,Folcwine,31162 +Fengel,Aldor,31162 +Fengel,Walda,31162 +Fengel,Folca,31162 +Fengel,Thengel,31162 +Fengel,Gram,31162 +Helm,Théoden,31162 +Helm,Baldor,31162 +Helm,Déor,31162 +Helm,Brego,31162 +Helm,Goldwine,31162 +Helm,Folcwine,31162 +Helm,Aldor,31162 +Helm,Walda,31162 +Helm,Folca,31162 +Helm,Thengel,31162 +Helm,Gram,31162 +Théoden,Baldor,31162 +Théoden,Déor,31162 +Théoden,Brego,31162 +Théoden,Goldwine,31162 +Théoden,Folcwine,31162 +Théoden,Aldor,31162 +Théoden,Walda,31162 +Théoden,Folca,31162 +Théoden,Thengel,31162 +Théoden,Gram,31162 +Baldor,Déor,31162 +Baldor,Brego,31162 +Baldor,Goldwine,31162 +Baldor,Folcwine,31162 +Baldor,Aldor,31162 +Baldor,Walda,31162 +Baldor,Folca,31162 +Baldor,Thengel,31162 +Baldor,Gram,31162 +Déor,Brego,31162 +Déor,Goldwine,31162 +Déor,Folcwine,31162 +Déor,Aldor,31162 +Déor,Walda,31162 +Déor,Folca,31162 +Déor,Thengel,31162 +Déor,Gram,31162 +Brego,Goldwine,31162 +Brego,Folcwine,31162 +Brego,Aldor,31162 +Brego,Walda,31162 +Brego,Folca,31162 +Brego,Thengel,31162 +Brego,Gram,31162 +Goldwine,Folcwine,31162 +Goldwine,Aldor,31162 +Goldwine,Walda,31162 +Goldwine,Folca,31162 +Goldwine,Thengel,31162 +Goldwine,Gram,31162 +Folcwine,Aldor,31162 +Folcwine,Walda,31162 +Folcwine,Folca,31162 +Folcwine,Thengel,31162 +Folcwine,Gram,31162 +Aldor,Walda,31162 +Aldor,Folca,31162 +Aldor,Thengel,31162 +Aldor,Gram,31162 +Walda,Folca,31162 +Walda,Thengel,31162 +Walda,Gram,31162 +Folca,Thengel,31162 +Folca,Gram,31162 +Thengel,Gram,31162 +Éomer,Théoden,31163 +Éowyn,Éomer,31164 +Éowyn,Théoden,31164 +Éomer,Théoden,31164 +Éowyn,Faramir,31166 +Éowyn,Faramir,31167 +Éowyn,Éomer,31168 +Éowyn,Aragorn,31168 +Éomer,Aragorn,31168 +Arwen,Faramir,31170 +Arwen,Imrahil,31170 +Arwen,Aragorn,31170 +Faramir,Imrahil,31170 +Faramir,Aragorn,31170 +Imrahil,Aragorn,31170 +Éowyn,Meriadoc,31172 +Éowyn,Éomer,31172 +Éowyn,Merry,31172 +Meriadoc,Éomer,31172 +Meriadoc,Merry,31172 +Éomer,Merry,31172 +Éowyn,Merry,31174 +Éowyn,Merry,31178 +Gimli,Legolas,31180 +Treebeard,Gandalf,31188 +Treebeard,Gandalf,31192 +Treebeard,Aragorn,31197 +Treebeard,Gandalf,31202 +Treebeard,Merry,31202 +Gandalf,Merry,31202 +Galadriel,Celeborn,31245 +Gimli,Legolas,31247 +Treebeard,Galadriel,31255 +Treebeard,Celeborn,31255 +Galadriel,Celeborn,31255 +Galadriel,Celeborn,31259 +Pippin,Merry,31261 +Pippin,Aragorn,31268 +Pippin,Aragorn,31270 +Celeborn,Galadriel,31274 +Celeborn,Aragorn,31274 +Galadriel,Aragorn,31274 +Gandalf,Saruman,31282 +Saruman,Galadriel,31295 +Saruman,Gandalf,31313 +Saruman,Wormtongue,31313 +Gandalf,Wormtongue,31313 +Saruman,Merry,31330 +Saruman,Wormtongue,31330 +Merry,Wormtongue,31330 +Galadriel,Celeborn,31349 +Gandalf,Elrond,31350 +Frodo,Galadriel,31356 +Sam,Elrond,31357 +Sam,Frodo,31389 +Sam,Elrond,31389 +Frodo,Elrond,31389 +Pippin,Merry,31401 +Pippin,Bilbo,31402 +Frodo,Bilbo,31406 +Sam,Aragorn,31411 +Sam,Frodo,31423 +Gandalf,Bilbo,31436 +Gandalf,Elrond,31436 +Bilbo,Elrond,31436 +Frodo,Elrond,31437 +Frodo,Gandalf,31445 +Pippin,Sam,31474 +Pippin,Sam,31475 +Gandalf,Butterbur,31524 +Gandalf,Barliman,31581 +Butterbur,Merry,31614 +Sam,Bill,31626 +Sam,Bill,31629 +Sam,Bill,31651 +Frodo,Barliman,31652 +Pippin,Gandalf,31656 +Pippin,Lotho,31656 +Gandalf,Lotho,31656 +Merry,Gandalf,31658 +Gandalf,Shadowfax,31684 +Frodo,Lotho,31707 +Pippin,Merry,31716 +Pippin,Merry,31717 +Merry,Bill,31723 +Sam,Bill,31731 +Robin,Sam,31816 +Sam,Frodo,31834 +Frodo,Merry,31844 +Pippin,Sam,31845 +Pippin,Merry,31845 +Sam,Merry,31845 +Sam,Frodo,31863 +Sam,Merry,31875 +Sam,Bill,31880 +Pippin,Frodo,31930 +Pippin,Merry,31930 +Pippin,Sam,31930 +Frodo,Merry,31930 +Frodo,Sam,31930 +Merry,Sam,31930 +Frodo,Lotho,31940 +Pippin,Frodo,31943 +Frodo,Lotho,31950 +Pippin,Lotho,31957 +Sam,Frodo,32044 +Tom,Frodo,32058 +Tom,Sam,32058 +Frodo,Sam,32058 +Frodo,Merry,32075 +Peregrin,Lotho,32080 +Frodo,Merry,32095 +Sam,Frodo,32234 +Sam,Frodo,32238 +Sam,Frodo,32239 +Pippin,Merry,32283 +Meriadoc,Peregrin,32293 +Pippin,Sam,32295 +Pippin,Merry,32295 +Sam,Merry,32295 +Frodo,Merry,32296 +Pippin,Frodo,32301 +Pippin,Merry,32301 +Pippin,Sam,32301 +Frodo,Merry,32301 +Frodo,Sam,32301 +Merry,Sam,32301 +Sam,Frodo,32331 +Merry,Lotho,32348 +Merry,Saruman,32355 +Gandalf,Saruman,32372 +Saruman,Wormtongue,32396 +Saruman,Wormtongue,32399 +Frodo,Saruman,32400 +Sam,Frodo,32404 +Frodo,Saruman,32409 +Frodo,Wormtongue,32422 +Saruman,Wormtongue,32432 +Saruman,Wormtongue,32443 +Frodo,Wormtongue,32444 +Pippin,Fredegar,32459 +Frodo,Lotho,32469 +Pippin,Merry,32473 +Sam,Frodo,32499 +Sam,Frodo,32520 +Sam,Frodo,32523 +Pippin,Merry,32524 +Sam,Frodo,32525 +Sam,Frodo,32527 +Sam,Frodo,32538 +Pippin,Merry,32541 +Sam,Frodo,32544 +Sam,Frodo,32547 +Meriadoc,Peregrin,32548 +Meriadoc,Sam,32548 +Peregrin,Sam,32548 +Sam,Frodo,32552 +Sam,Frodo,32557 +Sam,Frodo,32563 +Sam,Frodo,32565 +Sam,Frodo,32568 +Sam,Frodo,32569 +Sam,Bilbo,32570 +Sam,Frodo,32573 +Sam,Frodo,32579 +Sam,Bilbo,32579 +Frodo,Bilbo,32579 +Sam,Frodo,32584 +Frodo,Bilbo,32586 +Frodo,Bilbo,32595 +Sam,Frodo,32597 +Sam,Frodo,32598 +Sam,Frodo,32599 +Sam,Bill,32599 +Frodo,Bill,32599 +Sam,Frodo,32603 +Sam,Frodo,32613 +Elrond,Sam,32614 +Elrond,Galadriel,32614 +Sam,Galadriel,32614 +Elrond,Galadriel,32619 +Sam,Frodo,32631 +Pippin,Merry,32645 +Pippin,Goldilocks,32645 +Merry,Goldilocks,32645 +Galadriel,Elrond,32650 +Sam,Frodo,32651 +Sam,Bilbo,32651 +Frodo,Bilbo,32651 +Frodo,Gandalf,32656 +Pippin,Merry,32659 +Sam,Gandalf,32663 +Pippin,Frodo,32666 +Pippin,Merry,32666 +Pippin,Galadriel,32666 +Pippin,Sam,32666 +Frodo,Merry,32666 +Frodo,Galadriel,32666 +Frodo,Sam,32666 +Merry,Galadriel,32666 +Merry,Sam,32666 +Galadriel,Sam,32666 +Pippin,Merry,32671 +Pippin,Merry,32674 From c5544f56a08578b636a62448d41e5654c1de598a Mon Sep 17 00:00:00 2001 From: Miratepuffin Date: Tue, 8 Dec 2020 21:32:20 +0000 Subject: [PATCH 58/58] Shrank the Jar. Moved Bitcoin to Examples project --- mainproject/build.sbt | 36 +++---- .../scala/com/raphtory/api/S3Serialiser.scala | 35 ------ .../Workers/IngestionWorker.scala | 6 +- .../spouts/blockchain/BitcoinNodeSpout.scala | 79 -------------- .../spouts/blockchain/EthereumGethSpout.scala | 100 ------------------ .../blockchain/EthereumPostgresSpout.scala | 57 ---------- .../src/tests/scala/lotr/LOTRDeployment.scala | 3 +- .../tests/scala/lotr/LOTRGraphBuilder.scala | 2 +- .../src/tests/scala/lotr/LOTRSpout.scala | 2 +- 9 files changed, 24 insertions(+), 296 deletions(-) delete mode 100644 mainproject/src/main/scala/com/raphtory/api/S3Serialiser.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/spouts/blockchain/BitcoinNodeSpout.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/spouts/blockchain/EthereumGethSpout.scala delete mode 100644 mainproject/src/main/scala/com/raphtory/spouts/blockchain/EthereumPostgresSpout.scala diff --git a/mainproject/build.sbt b/mainproject/build.sbt index b5c07d8646..f1d43d863d 100644 --- a/mainproject/build.sbt +++ b/mainproject/build.sbt @@ -55,23 +55,23 @@ val kamon_prometheus = "io.kamon" %% "kamon-prometheus" % "2.1.0" val kamon_akka = "io.kamon" %% "kamon-akka" % "2.1.0" val kamon_system = "io.kamon" %% "kamon-system-metrics" % "2.1.0" val kamon_netty = "io.kamon" %% "kamon-netty" % "1.0.0" -val monix = "io.monix" %% "monix" % "3.0.0-RC1" +//val monix = "io.monix" %% "monix" % "3.0.0-RC1" val mongo = "org.mongodb" % "mongo-java-driver" % "3.12.4" val casbah = "org.mongodb" %% "casbah-core" % "3.1.1" -val doobie = "org.tpolecat" %% "doobie-core" % "0.8.4" -val doobiepostgres = - "org.tpolecat" %% "doobie-postgres" % "0.8.4" // Postgres driver 42.2.8 + type mappings. +//val doobie = "org.tpolecat" %% "doobie-core" % "0.8.4" +//val doobiepostgres = +// "org.tpolecat" %% "doobie-postgres" % "0.8.4" // Postgres driver 42.2.8 + type mappings. val lift = "net.liftweb" %% "lift-json" % "3.3.0" -val bitcoin = "org.scalaj" %% "scalaj-http" % "2.3.0" -val twitter_eval = "com.twitter" %% "util-eval" % "6.43.0" +//val bitcoin = "org.scalaj" %% "scalaj-http" % "2.3.0" +//val twitter_eval = "com.twitter" %% "util-eval" % "6.43.0" val hadoop = "org.apache.hadoop" % "hadoop-client" % "3.3.0" // https://mvnrepository.com/artifact/com.amazonaws/aws-java-sdk -val aws = "com.amazonaws" % "aws-java-sdk" % "1.11.897" +//val aws = "com.amazonaws" % "aws-java-sdk" % "1.11.897" val parquet = "com.github.mjakubowski84" %% "parquet4s-core" % "1.6.0" -val h3 = "com.uber" % "h3" % "3.6.4" +//val h3 = "com.uber" % "h3" % "3.6.4" val IP = java.net.InetAddress.getLocalHost.getHostAddress @@ -129,9 +129,9 @@ lazy val mergeStrategy: String => MergeStrategy = { case _ => MergeStrategy.first } -lazy val root = Project(id = "raphtory", base = file(".")) aggregate (cluster) +lazy val root = Project(id = "raphtory", base = file(".")) aggregate (raphtory) -lazy val cluster = project +lazy val raphtory = project .in(file(".")) .enablePlugins(JavaAppPackaging) .enablePlugins(AshScriptPlugin) @@ -172,22 +172,22 @@ lazy val cluster = project kamon_akka, kamon_prometheus, kamon_system, - monix, - bitcoin, - twitter_eval, + //monix, + //bitcoin, + //twitter_eval, lift, apacheLang, kafka, kafkac, - doobie, - doobiepostgres, + //doobie, + //doobiepostgres, joda, casbah, mongo, - aws, + //aws, parquet, - hadoop, - h3 + hadoop + // h3 ) ) .settings( diff --git a/mainproject/src/main/scala/com/raphtory/api/S3Serialiser.scala b/mainproject/src/main/scala/com/raphtory/api/S3Serialiser.scala deleted file mode 100644 index b03c4a5c70..0000000000 --- a/mainproject/src/main/scala/com/raphtory/api/S3Serialiser.scala +++ /dev/null @@ -1,35 +0,0 @@ -package com.raphtory.api - -import java.io.{BufferedWriter, File, FileWriter} - -import com.amazonaws.auth.{AWSStaticCredentialsProvider, BasicAWSCredentials} -import com.amazonaws.services.s3.{AmazonS3, AmazonS3ClientBuilder} - -abstract class S3Serialiser extends Serialiser { - val AWS_ACCESS_KEY = s"${sys.env.getOrElse("AWS_ACCESS_KEY", "")}" - val AWS_SECRET_KEY = s"${sys.env.getOrElse("AWS_SECRET_KEY", "")}" - val bucketName = s"${sys.env.getOrElse("AWS_BUCKET_NAME", "")}" - val yourAWSCredentials = new BasicAWSCredentials(AWS_ACCESS_KEY, AWS_SECRET_KEY) - //val amazonS3Client = new AmazonS3Client(yourAWSCredentials) - - - val s3Client: AmazonS3 = AmazonS3ClientBuilder.standard.withCredentials(new AWSStaticCredentialsProvider(yourAWSCredentials)).build - //s3Client.setRegion(com.amazonaws.regions.Region.getRegion(Regions.EU_WEST_2)) - try{s3Client.createBucket(bucketName)}catch {case e:Exception =>} - - override def write(serialisedResults:(Array[String],Array[String]),file:File) = { - println("vertices & edges", serialisedResults._1.length, serialisedResults._2.length) - val bw = new BufferedWriter(new FileWriter(file)) - bw.write(startOfFile()) - bw.write(serialisedResults._1.mkString(rowDelimeter())) - bw.write(middleOfFile()) - bw.write(serialisedResults._2.mkString(rowDelimeter())) - bw.write(endOfFile()) - bw.newLine() - bw.close() - new Thread(() => { - s3Client.putObject(bucketName, file.getName,file) - file.delete() - }).start() - } -} diff --git a/mainproject/src/main/scala/com/raphtory/core/actors/PartitionManager/Workers/IngestionWorker.scala b/mainproject/src/main/scala/com/raphtory/core/actors/PartitionManager/Workers/IngestionWorker.scala index 4d31699e91..64cd3c56fc 100644 --- a/mainproject/src/main/scala/com/raphtory/core/actors/PartitionManager/Workers/IngestionWorker.scala +++ b/mainproject/src/main/scala/com/raphtory/core/actors/PartitionManager/Workers/IngestionWorker.scala @@ -300,11 +300,11 @@ class IngestionWorker(workerId: Int,partitionID:Int, storage: EntityStorage) ext currentSafePoint = messageQueue.dequeue() else if (messageQueue.head.routerEpoch == currentSafePoint.routerEpoch) currentSafePoint = messageQueue.dequeue() - else { - currentSafePoint = messageQueue.dequeue() + //else { + // currentSafePoint = messageQueue.dequeue() //println(s"$increments Writer Worker $partitionID $workerId --- ${safeMessageMap.get(routerName).get} $currentSafePoint ${messageQueue.head.routerEpoch}") //break - } + //} } safeMessageMap put(routerName, currentSafePoint) currentSafePoint diff --git a/mainproject/src/main/scala/com/raphtory/spouts/blockchain/BitcoinNodeSpout.scala b/mainproject/src/main/scala/com/raphtory/spouts/blockchain/BitcoinNodeSpout.scala deleted file mode 100644 index c1eb4a9f6f..0000000000 --- a/mainproject/src/main/scala/com/raphtory/spouts/blockchain/BitcoinNodeSpout.scala +++ /dev/null @@ -1,79 +0,0 @@ -package com.raphtory.spouts.blockchain -// -import java.io.{File, PrintWriter} - -import com.raphtory.core.actors.Spout.Spout -import scalaj.http.{Http, HttpRequest} -import spray.json._ - -import scala.collection.mutable -import scala.language.postfixOps -import scala.sys.process._ -// -case class BitcoinTransaction(time: JsValue, block: Int, blockID: JsValue, transaction: JsValue) - -class BitcoinNodeSpout extends Spout[BitcoinTransaction]{ - - var blockcount = 1 - val rpcuser = System.getenv().getOrDefault("BITCOIN_USERNAME", "").trim - val rpcpassword = System.getenv().getOrDefault("BITCOIN_PASSWORD", "").trim - val serverAddress = System.getenv().getOrDefault("BITCOIN_NODE", "").trim - val id = "scala-jsonrpc" - val baseRequest = Http(serverAddress).auth(rpcuser, rpcpassword).header("content-type", "text/plain") - - override def setupDataSource(): Unit = {} - override def closeDataSource(): Unit = {} - - val queue = mutable.Queue[Option[BitcoinTransaction]]() - - override def generateData(): Option[BitcoinTransaction] = { - if(queue.isEmpty) - getTransactions() - queue.dequeue() - } - - def getTransactions(): Unit = { - try { - - val re = request("getblockhash", blockcount.toString).execute().body.toString.parseJson.asJsObject - val blockID = re.fields("result") - val blockData = request("getblock", s"$blockID,2").execute().body.toString.parseJson.asJsObject - val result = blockData.fields("result") - val time = result.asJsObject.fields("time") - for (transaction <- result.asJsObject().fields("tx").asInstanceOf[JsArray].elements) - queue += Some(BitcoinTransaction(time, blockcount, blockID, transaction)) - //val time = transaction.asJsObject.fields("time") - blockcount += 1 - } catch { - case e: java.net.SocketTimeoutException => queue += None - } - } - - //************* MESSAGE HANDLING BLOCK - def handleDomainMessage(): Unit = { - - } - - def outputScript() = { - val pw = new PrintWriter(new File("bitcoin.sh")) - pw.write("""curl --user $1:$2 --data-binary $3 -H 'content-type: text/plain;' $4""") - pw.close - "chmod 777 bitcoin.sh" ! - } - - def curlRequest(command: String, params: String): String = { - //val data = """{"jsonrpc":"1.0","id":"scala-jsonrpc","method":"getblockhash","params":[2]}""" - val data = s"""{"jsonrpc":"1.0","id":"$id","method":"$command","params":[$params]}""" - s"bash bitcoin.sh $rpcuser $rpcpassword $data $serverAddress" !! - } - - def request(command: String, params: String = ""): HttpRequest = - baseRequest.postData(s"""{"jsonrpc": "1.0", "id":"$id", "method": "$command", "params": [$params] }""") - - - - -} - - -//def request(command: String, params: String = ""): HttpRequest = baseRequest.postData(s"""{"jsonrpc": "1.0", "id":"$id", "method": "$command", "params": [$params] }""") diff --git a/mainproject/src/main/scala/com/raphtory/spouts/blockchain/EthereumGethSpout.scala b/mainproject/src/main/scala/com/raphtory/spouts/blockchain/EthereumGethSpout.scala deleted file mode 100644 index afd0cd63f0..0000000000 --- a/mainproject/src/main/scala/com/raphtory/spouts/blockchain/EthereumGethSpout.scala +++ /dev/null @@ -1,100 +0,0 @@ -package com.raphtory.spouts.blockchain - -import java.net.InetAddress -import java.util.NoSuchElementException - -import com.raphtory.core.actors.Spout.Spout -import scalaj.http.{Http, HttpRequest} -import spray.json.DefaultJsonProtocol._ -import spray.json._ - -import scala.collection.mutable -import scala.language.postfixOps - - - -class EthereumGethSpout extends Spout[String] { - - var currentBlock = System.getenv().getOrDefault("SPOUT_ETHEREUM_START_BLOCK_INDEX", "9014194").trim.toInt - var highestBlock = System.getenv().getOrDefault("SPOUT_ETHEREUM_MAXIMUM_BLOCK_INDEX", "10026447").trim.toInt - val nodeIP = System.getenv().getOrDefault("SPOUT_ETHEREUM_IP_ADDRESS", "127.0.0.1").trim - val nodePort = System.getenv().getOrDefault("SPOUT_ETHEREUM_PORT", "8545").trim -// val nodePort = System.getenv().getOrDefault("SPOUT_ETHEREUM_PORT", "30303").trim -def IPRegex = - "\\b(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\\.(25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9]?[0-9])\\b" - print(currentBlock) - - val queue = mutable.Queue[Option[String]]() - - val baseRequest = requestBuilder() - - override def setupDataSource(): Unit = {} - override def closeDataSource(): Unit = {} - override def generateData(): Option[String] = { - if(queue.isEmpty) - pullNextBlock() - queue.dequeue() - } - - - - - - implicit val EthFormat = jsonFormat14(EthResult) - implicit val EthTransactionFormat = jsonFormat3(EthTransaction) - if (nodeIP.matches(IPRegex)) - println(s"Connecting to Ethereum RPC \n Address:$nodeIP \n Port:$nodePort") - else - println(s"Connecting to Ethereum RPC \n Address:${hostname2Ip(nodeIP)} \n Port:$nodePort") - - - def pullNextBlock(): Unit = { - if (currentBlock > highestBlock) - return - try { - val transactionCountHex = executeRequest("eth_getBlockTransactionCountByNumber", "\"0x" + currentBlock.toHexString + "\""); - val transactionCount = Integer.parseInt(transactionCountHex.fields("result").toString().drop(3).dropRight(1), 16) - if(transactionCount>0){ - var transactions = "[" - for (i <- 0 until transactionCount) - transactions = transactions + batchRequestBuilder("eth_getTransactionByBlockNumberAndIndex",s""""0x${currentBlock.toHexString}","0x${i.toHexString}"""")+"," - val trasnactionBlock = executeBatchRequest(transactions.dropRight(1)+"]") - val transList = trasnactionBlock.parseJson.convertTo[List[EthTransaction]] - transList.foreach(t => { //try needed to ignore contracts //todo include them - try{queue +=(Some(s"${t.result.blockNumber.get},${t.result.from.get},${t.result.to.get},${t.result.value.get}"))} - catch {case e:NoSuchElementException =>} - - }) - - } - currentBlock += 1 - } catch { - case e: NumberFormatException => - case e: Exception => e.printStackTrace(); - } - } - - - def batchRequestBuilder(command:String,params:String):String = s"""{"jsonrpc": "2.0", "id":"100", "method": "$command", "params": [$params]}""" - def executeBatchRequest(data: String) = requestBatch(data).execute().body.toString - def requestBatch(data: String): HttpRequest = baseRequest.postData(data) - def requestBuilder() = - if (nodeIP.matches(IPRegex)) - Http("http://" + nodeIP + ":" + nodePort).header("content-type", "application/json") - else - Http("http://" + hostname2Ip(nodeIP) + ":" + nodePort).header("content-type", "application/json") - def request(command: String, params: String = ""): HttpRequest = - baseRequest.postData(s"""{"jsonrpc": "2.0", "id":"100", "method": "$command", "params": [$params]}""") - - def executeRequest(command: String, params: String = "") = { - request(command, params).execute().body.toString.parseJson.asJsObject - } - - def hostname2Ip(hostname: String): String = InetAddress.getByName(hostname).getHostAddress() - - -} - -case class EthResult(blockHash:Option[String],blockNumber:Option[String],from:Option[String],gas:Option[String],gasPrice:Option[String],hash:Option[String],input:Option[String],nonce:Option[String],r:Option[String],s:Option[String],to:Option[String],transactionIndex:Option[String],v:Option[String],value:Option[String]) -case class EthTransaction(id:Option[String],jsonrpc:Option[String],result:EthResult) - diff --git a/mainproject/src/main/scala/com/raphtory/spouts/blockchain/EthereumPostgresSpout.scala b/mainproject/src/main/scala/com/raphtory/spouts/blockchain/EthereumPostgresSpout.scala deleted file mode 100644 index 01f92d7382..0000000000 --- a/mainproject/src/main/scala/com/raphtory/spouts/blockchain/EthereumPostgresSpout.scala +++ /dev/null @@ -1,57 +0,0 @@ -package com.raphtory.spouts.blockchain - -import cats.effect.{Blocker, IO} -import com.raphtory.core.actors.Spout.Spout -import doobie.implicits._ -import doobie.util.ExecutionContexts -import doobie.util.transactor.Transactor - -import scala.collection.mutable - -class EthereumPostgresSpout extends Spout[String]{ - var startBlock = System.getenv().getOrDefault("STARTING_BLOCK", "46147").trim.toInt //first block to have a transaction by default - val batchSize = System.getenv().getOrDefault("BLOCK_BATCH_SIZE", "100").trim.toInt //number of blocks to pull each query - val maxblock = System.getenv().getOrDefault("MAX_BLOCK", "8828337").trim.toInt //Maximum block in database to stop querying once this is reached - - val dbURL = System.getenv().getOrDefault("DB_URL", "jdbc:postgresql:ether").trim //db connection string, default is for local with db called ether - val dbUSER = System.getenv().getOrDefault("DB_USER", "postgres").trim //db user defaults to postgres - val dbPASSWORD = System.getenv().getOrDefault("DB_PASSWORD", "").trim //default no password - - // querying done with doobie wrapper for JDBC (https://tpolecat.github.io/doobie/) - implicit val cs = IO.contextShift(ExecutionContexts.synchronous) - val dbconnector = Transactor.fromDriverManager[IO]( - "org.postgresql.Driver", - dbURL, - dbUSER, - dbPASSWORD, - Blocker.liftExecutionContext(ExecutionContexts.synchronous) - ) - val queue = mutable.Queue[Option[String]]() - - override def generateData(): Option[String] = { - if(queue isEmpty) - pullBlocks() - queue.dequeue() - } - - protected def pullBlocks(): Unit = { - sql"select from_address, to_address, value,block_timestamp from transactions where block_number >= $startBlock AND block_number < ${startBlock + batchSize} " - .query[ - (String, String, String, String) - ] //get the to,from,value and time for transactions within the set block batch - .to[List] // ConnectionIO[List[String]] - .transact(dbconnector) // IO[List[String]] - .unsafeRunSync // List[String] - .foreach(x => queue+=(Some(x.toString()))) //send each transaction to the routers - - startBlock += batchSize//increment batch for the next query - if (startBlock <= maxblock) - dataSourceComplete()//if we have reached the max block we stop querying the database - - - - } - - override def setupDataSource(): Unit = {} - override def closeDataSource(): Unit = {} -} diff --git a/mainproject/src/tests/scala/lotr/LOTRDeployment.scala b/mainproject/src/tests/scala/lotr/LOTRDeployment.scala index 72e3b495d2..e83ec4d173 100644 --- a/mainproject/src/tests/scala/lotr/LOTRDeployment.scala +++ b/mainproject/src/tests/scala/lotr/LOTRDeployment.scala @@ -1,5 +1,4 @@ -package examples.lotr - +package lotr import com.raphtory.RaphtoryGraph import com.raphtory.algorithms.{ConnectedComponents, DegreeBasic} diff --git a/mainproject/src/tests/scala/lotr/LOTRGraphBuilder.scala b/mainproject/src/tests/scala/lotr/LOTRGraphBuilder.scala index 3775bfa265..9e8b82de7b 100644 --- a/mainproject/src/tests/scala/lotr/LOTRGraphBuilder.scala +++ b/mainproject/src/tests/scala/lotr/LOTRGraphBuilder.scala @@ -1,4 +1,4 @@ -package examples.lotr +package lotr import com.raphtory.core.actors.Router.GraphBuilder import com.raphtory.core.model.communication._ diff --git a/mainproject/src/tests/scala/lotr/LOTRSpout.scala b/mainproject/src/tests/scala/lotr/LOTRSpout.scala index 7e7f8d245d..3e0bd59de1 100644 --- a/mainproject/src/tests/scala/lotr/LOTRSpout.scala +++ b/mainproject/src/tests/scala/lotr/LOTRSpout.scala @@ -1,4 +1,4 @@ -package examples.lotr +package lotr import com.raphtory.core.actors.Spout.Spout