From 244f928f3c514706a3418ae71498b35375aee237 Mon Sep 17 00:00:00 2001 From: Minjae Gwon Date: Sun, 26 Nov 2023 16:29:45 +0900 Subject: [PATCH 01/39] dependency: setup log4j --- build.sbt | 9 ++++----- core/src/main/resources/log4j2.properties | 17 +++++++++++++++++ master/src/main/resources/log4j2.properties | 17 +++++++++++++++++ rpc/src/main/resources/log4j2.properties | 17 +++++++++++++++++ utils/src/main/resources/log4j2.properties | 17 +++++++++++++++++ worker/src/main/resources/log4j2.properties | 17 +++++++++++++++++ 6 files changed, 89 insertions(+), 5 deletions(-) create mode 100644 core/src/main/resources/log4j2.properties create mode 100644 master/src/main/resources/log4j2.properties create mode 100644 rpc/src/main/resources/log4j2.properties create mode 100644 utils/src/main/resources/log4j2.properties create mode 100644 worker/src/main/resources/log4j2.properties diff --git a/build.sbt b/build.sbt index 5cddebe..43a4e89 100644 --- a/build.sbt +++ b/build.sbt @@ -18,7 +18,10 @@ lazy val commonSettings = Seq( "org.scalactic" %% "scalactic" % "3.2.17", "org.scalatest" %% "scalatest" % "3.2.17" % "test", "org.scalatest" %% "scalatest-flatspec" % "3.2.17" % "test", - "io.reactivex.rxjava3" % "rxjava" % "3.0.4" + "io.grpc" % "grpc-netty" % scalapb.compiler.Version.grpcJavaVersion, + "com.thesamet.scalapb" %% "scalapb-runtime-grpc" % scalapb.compiler.Version.scalapbVersion, + "org.apache.logging.log4j" %% "log4j-api-scala" % "13.0.0", + "org.apache.logging.log4j" % "log4j-core" % "2.22.0" % Runtime ) ) @@ -42,10 +45,6 @@ lazy val rpc = (project in file("rpc")) .settings( commonSettings, idePackagePrefix := Some("kr.ac.postech.paranode.rpc"), - libraryDependencies ++= Seq( - "io.grpc" % "grpc-netty" % scalapb.compiler.Version.grpcJavaVersion, - "com.thesamet.scalapb" %% "scalapb-runtime-grpc" % scalapb.compiler.Version.scalapbVersion - ), Compile / PB.targets := Seq( scalapb.gen() -> (Compile / sourceManaged).value / "scalapb" ) diff --git a/core/src/main/resources/log4j2.properties b/core/src/main/resources/log4j2.properties new file mode 100644 index 0000000..59b1671 --- /dev/null +++ b/core/src/main/resources/log4j2.properties @@ -0,0 +1,17 @@ +# Set to debug or trace if log4j initialization is failing +status = warn + +# Name of the configuration +name = ConsoleLogConfigDemo + +# Console appender configuration +appender.console.type = Console +appender.console.name = consoleLogger +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n + +# Root logger level +rootLogger.level = debug + +# Root logger referring to console appender +rootLogger.appenderRef.stdout.ref = consoleLogger diff --git a/master/src/main/resources/log4j2.properties b/master/src/main/resources/log4j2.properties new file mode 100644 index 0000000..59b1671 --- /dev/null +++ b/master/src/main/resources/log4j2.properties @@ -0,0 +1,17 @@ +# Set to debug or trace if log4j initialization is failing +status = warn + +# Name of the configuration +name = ConsoleLogConfigDemo + +# Console appender configuration +appender.console.type = Console +appender.console.name = consoleLogger +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n + +# Root logger level +rootLogger.level = debug + +# Root logger referring to console appender +rootLogger.appenderRef.stdout.ref = consoleLogger diff --git a/rpc/src/main/resources/log4j2.properties b/rpc/src/main/resources/log4j2.properties new file mode 100644 index 0000000..59b1671 --- /dev/null +++ b/rpc/src/main/resources/log4j2.properties @@ -0,0 +1,17 @@ +# Set to debug or trace if log4j initialization is failing +status = warn + +# Name of the configuration +name = ConsoleLogConfigDemo + +# Console appender configuration +appender.console.type = Console +appender.console.name = consoleLogger +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n + +# Root logger level +rootLogger.level = debug + +# Root logger referring to console appender +rootLogger.appenderRef.stdout.ref = consoleLogger diff --git a/utils/src/main/resources/log4j2.properties b/utils/src/main/resources/log4j2.properties new file mode 100644 index 0000000..59b1671 --- /dev/null +++ b/utils/src/main/resources/log4j2.properties @@ -0,0 +1,17 @@ +# Set to debug or trace if log4j initialization is failing +status = warn + +# Name of the configuration +name = ConsoleLogConfigDemo + +# Console appender configuration +appender.console.type = Console +appender.console.name = consoleLogger +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n + +# Root logger level +rootLogger.level = debug + +# Root logger referring to console appender +rootLogger.appenderRef.stdout.ref = consoleLogger diff --git a/worker/src/main/resources/log4j2.properties b/worker/src/main/resources/log4j2.properties new file mode 100644 index 0000000..59b1671 --- /dev/null +++ b/worker/src/main/resources/log4j2.properties @@ -0,0 +1,17 @@ +# Set to debug or trace if log4j initialization is failing +status = warn + +# Name of the configuration +name = ConsoleLogConfigDemo + +# Console appender configuration +appender.console.type = Console +appender.console.name = consoleLogger +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n + +# Root logger level +rootLogger.level = debug + +# Root logger referring to console appender +rootLogger.appenderRef.stdout.ref = consoleLogger From acc17cbccaf33267601e139a3e02c06d94c3aba7 Mon Sep 17 00:00:00 2001 From: Minjae Gwon Date: Sun, 26 Nov 2023 16:30:31 +0900 Subject: [PATCH 02/39] refactor: using non-blocking stub --- rpc/src/main/scala/WorkerClient.scala | 29 +++++++++++++++------------ 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/rpc/src/main/scala/WorkerClient.scala b/rpc/src/main/scala/WorkerClient.scala index 4dcded4..2333097 100644 --- a/rpc/src/main/scala/WorkerClient.scala +++ b/rpc/src/main/scala/WorkerClient.scala @@ -8,29 +8,30 @@ import kr.ac.postech.paranode.core.WorkerMetadata import java.util.concurrent.TimeUnit import java.util.logging.Logger - import worker._ -import worker.WorkerGrpc.WorkerBlockingStub +import worker.WorkerGrpc.WorkerStub import common.{ Node, KeyRange => RpcKeyRange, WorkerMetadata => RpcWorkerMetadata } +import scala.concurrent.Future + object WorkerClient { def apply(host: String, port: Int): WorkerClient = { val channel = ManagedChannelBuilder .forAddress(host, port) .usePlaintext() .build - val blockingStub = WorkerGrpc.blockingStub(channel) - new WorkerClient(channel, blockingStub) + val stub = WorkerGrpc.stub(channel) + new WorkerClient(channel, stub) } } class WorkerClient private ( private val channel: ManagedChannel, - private val blockingStub: WorkerBlockingStub + private val stub: WorkerStub ) { Logger.getLogger(classOf[WorkerClient].getName) @@ -38,16 +39,16 @@ class WorkerClient private ( channel.shutdown.awaitTermination(5, TimeUnit.SECONDS) } - def sample(numberOfKeys: Int): SampleReply = { + def sample(numberOfKeys: Int): Future[SampleReply] = { val request = SampleRequest(numberOfKeys) - val response = blockingStub.sample(request) + val response = stub.sample(request) response } def partition( workers: List[(WorkerMetadata, KeyRange)] - ): PartitionReply = { + ): Future[PartitionReply] = { val request = PartitionRequest(workers.map({ case (worker, keyRange) => RpcWorkerMetadata( Some(Node(worker.host, worker.port)), @@ -60,10 +61,12 @@ class WorkerClient private ( ) })) - blockingStub.partition(request) + stub.partition(request) } - def exchange(workers: List[(WorkerMetadata, KeyRange)]): ExchangeReply = { + def exchange( + workers: List[(WorkerMetadata, KeyRange)] + ): Future[ExchangeReply] = { val request = ExchangeRequest(workers.map({ case (worker, keyRange) => RpcWorkerMetadata( Some(Node(worker.host, worker.port)), @@ -76,11 +79,11 @@ class WorkerClient private ( ) })) - blockingStub.exchange(request) + stub.exchange(request) } - def merge(): MergeReply = { + def merge(): Future[MergeReply] = { val request = MergeRequest() - blockingStub.merge(request) + stub.merge(request) } } From a7061db6c754d3656557256444c3029d67bf02da Mon Sep 17 00:00:00 2001 From: Minjae Gwon Date: Sun, 26 Nov 2023 16:31:26 +0900 Subject: [PATCH 03/39] refactor: sample with first n records --- core/src/main/scala/Block.scala | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/core/src/main/scala/Block.scala b/core/src/main/scala/Block.scala index 3f6d308..b1f99e7 100644 --- a/core/src/main/scala/Block.scala +++ b/core/src/main/scala/Block.scala @@ -1,12 +1,14 @@ package kr.ac.postech.paranode.core +import org.apache.logging.log4j.scala.Logging + import java.io.BufferedOutputStream import java.io.File import java.io.FileOutputStream import scala.io.Source import scala.reflect.io.Path -object Block { +object Block extends Logging { def fromBytes( bytes: LazyList[Byte], keyLength: Int = 10, @@ -28,8 +30,11 @@ object Block { path: Path, keyLength: Int = 10, valueLength: Int = 90 - ): Block = + ): Block = { + logger.debug(s"Reading block from $path") + Block.fromSource(Source.fromURI(path.toURI), keyLength, valueLength) + } } @@ -60,7 +65,7 @@ class Block(val records: LazyList[Record]) extends AnyVal { def sort(): Block = new Block(records.sortBy(_.key)) - def sample(): LazyList[Key] = - Record.sampleWithInterval(records) + def sample(number: Int = 64): LazyList[Key] = + Record.sample(records, number) } From cf4ae827d28cc0989bbb32576db9ad0e683f01ab Mon Sep 17 00:00:00 2001 From: Minjae Gwon Date: Sun, 26 Nov 2023 16:31:50 +0900 Subject: [PATCH 04/39] feat: make key constructible from bytestring --- core/src/main/scala/Key.scala | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/core/src/main/scala/Key.scala b/core/src/main/scala/Key.scala index 0e16ca3..01051c2 100644 --- a/core/src/main/scala/Key.scala +++ b/core/src/main/scala/Key.scala @@ -1,7 +1,13 @@ package kr.ac.postech.paranode.core +import com.google.protobuf.ByteString + object Key { def fromString(string: String): Key = new Key(string.getBytes()) + + def fromByteString(byteString: ByteString): Key = new Key( + byteString.toByteArray + ) } class Key(val underlying: Array[Byte]) extends AnyVal with Ordered[Key] { From 9c92c7f0e0c586dbd0365595a04f5fc683bc68be Mon Sep 17 00:00:00 2001 From: Minjae Gwon Date: Sun, 26 Nov 2023 16:32:05 +0900 Subject: [PATCH 05/39] refactor: sample with first n records --- core/src/main/scala/Record.scala | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/core/src/main/scala/Record.scala b/core/src/main/scala/Record.scala index bbe80be..e14d4cf 100644 --- a/core/src/main/scala/Record.scala +++ b/core/src/main/scala/Record.scala @@ -1,6 +1,8 @@ package kr.ac.postech.paranode.core -object Record { +import org.apache.logging.log4j.scala.Logging + +object Record extends Logging { def fromString(string: String, keyLength: Int = 10): Record = Record.fromBytes(string.getBytes(), keyLength) @@ -25,18 +27,10 @@ object Record { ) } - def sampleWithInterval( + def sample( records: LazyList[Record], - interval: Int = 10 - ): LazyList[Key] = { - if (records.isEmpty) - LazyList.empty[Key] - else { - val (current, rest) = records.splitAt(interval) - val head = current.head.key - head #:: sampleWithInterval(rest, interval) - } - } + number: Int = 64 + ): LazyList[Key] = records.take(number).map(_.key) } class Record(val key: Key, val value: Array[Byte]) extends Ordered[Record] { From a37a359490b17a7a55d042ee91d7e976ffdb5880 Mon Sep 17 00:00:00 2001 From: Minjae Gwon Date: Sun, 26 Nov 2023 16:32:36 +0900 Subject: [PATCH 06/39] refactor: make client to return reply --- rpc/src/main/scala/MasterClient.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rpc/src/main/scala/MasterClient.scala b/rpc/src/main/scala/MasterClient.scala index 872017c..66a3cf6 100644 --- a/rpc/src/main/scala/MasterClient.scala +++ b/rpc/src/main/scala/MasterClient.scala @@ -6,10 +6,11 @@ import kr.ac.postech.paranode.core.WorkerMetadata import java.util.concurrent.TimeUnit import java.util.logging.Logger - import common.Node import master.MasterGrpc.MasterStub -import master.{MasterGrpc, RegisterRequest} +import master.{MasterGrpc, RegisterReply, RegisterRequest} + +import scala.concurrent.Future object MasterClient { def apply(host: String, port: Int): MasterClient = { @@ -41,10 +42,9 @@ class MasterClient private ( channel.shutdown.awaitTermination(5, TimeUnit.SECONDS) } - /** Say hello to server. */ def register( workerMetadata: WorkerMetadata - ): Unit = { + ): Future[RegisterReply] = { val request = RegisterRequest( Some(Node(workerMetadata.host, workerMetadata.port)) ) From 7ec1596741cf938b43852ade4feeca961d4a92a1 Mon Sep 17 00:00:00 2001 From: Minjae Gwon Date: Sun, 26 Nov 2023 16:33:07 +0900 Subject: [PATCH 07/39] refactor: make servers write logs --- rpc/src/main/scala/MasterServer.scala | 23 +++++------- rpc/src/main/scala/WorkerServer.scala | 53 ++++++++++++--------------- 2 files changed, 33 insertions(+), 43 deletions(-) diff --git a/rpc/src/main/scala/MasterServer.scala b/rpc/src/main/scala/MasterServer.scala index 2728a55..0de2a24 100644 --- a/rpc/src/main/scala/MasterServer.scala +++ b/rpc/src/main/scala/MasterServer.scala @@ -11,22 +11,15 @@ import scala.collection.mutable.WrappedArray import scala.concurrent.ExecutionContext import scala.concurrent.Future import scala.concurrent.Promise - import master.{MasterGrpc, RegisterReply, RegisterRequest} -object MasterServer { - private val logger = Logger.getLogger(classOf[MasterServer].getName) - - def main(args: Array[String]): Unit = { - val server = new MasterServer(ExecutionContext.global) - server.start() - server.blockUntilShutdown() - } +import org.apache.logging.log4j.scala.Logging +object MasterServer { private val port = 50051 } -class MasterServer(executionContext: ExecutionContext) { self => +class MasterServer(executionContext: ExecutionContext) extends Logging { self => private[this] val server: Server = ServerBuilder .forPort(MasterServer.port) .addService(MasterGrpc.bindService(new MasterImpl, executionContext)) @@ -45,16 +38,16 @@ class MasterServer(executionContext: ExecutionContext) { self => private def start(): Unit = { server.start() - MasterServer.logger.info( - "Server started, listening on " + MasterServer.port + logger.info( + s"MasterServer listening on port $port" ) sys.addShutdownHook { - System.err.println( + logger.error( "*** shutting down gRPC server since JVM is shutting down" ) self.stop() - System.err.println("*** server shut down") + logger.error("*** server shut down") } } @@ -78,6 +71,8 @@ class MasterServer(executionContext: ExecutionContext) { self => val promise = Promise[RegisterReply] Future { + logger.debug(s"Register request: $request") + val workerMetadata = WorkerMetadata(request.worker.get.host, request.worker.get.port, None) addWorkerInfo(workerMetadata) diff --git a/rpc/src/main/scala/WorkerServer.scala b/rpc/src/main/scala/WorkerServer.scala index 309bc75..fd5b9d1 100644 --- a/rpc/src/main/scala/WorkerServer.scala +++ b/rpc/src/main/scala/WorkerServer.scala @@ -11,50 +11,43 @@ import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.Future import scala.concurrent.Promise import scala.reflect.io.Path - import worker._ -object WorkerServer { - private val logger = Logger.getLogger(classOf[WorkerServer].getName) - - def main(args: Array[String]): Unit = { - val server = new WorkerServer(ExecutionContext.global) - server.start() - server.blockUntilShutdown() - } - - private val port = 30040 -} +import org.apache.logging.log4j.scala.Logging -class WorkerServer(executionContext: ExecutionContext) { self => +class WorkerServer( + executionContext: ExecutionContext, + port: Int, + inputDirectories: Array[Path], + outputDirectory: Path +) extends Logging { self => private[this] val server: Server = ServerBuilder - .forPort(WorkerServer.port) + .forPort(port) .addService(WorkerGrpc.bindService(new WorkerImpl, executionContext)) .build() - private def start(): Unit = { + def start(): Unit = { server.start() - WorkerServer.logger.info( - "Server started, listening on " + WorkerServer.port + logger.debug( + s"WorkerServer listening on port $port with inputDirectories: ${inputDirectories + .mkString(", ")} and outputDirectory: $outputDirectory" ) sys.addShutdownHook { - System.err.println( - "*** shutting down gRPC server since JVM is shutting down" - ) + logger.error("*** shutting down gRPC server since JVM is shutting down") self.stop() - System.err.println("*** server shut down") + logger.error("*** server shut down") } } - private def stop(): Unit = { + def stop(): Unit = { if (server != null) { server.shutdown() } } - private def blockUntilShutdown(): Unit = { + def blockUntilShutdown(): Unit = { if (server != null) { server.awaitTermination() } @@ -65,15 +58,17 @@ class WorkerServer(executionContext: ExecutionContext) { self => val promise = Promise[SampleReply] Future { + logger.debug(s"Sample request: $request") + try { - val sortedBlock = Block.fromPath(Path("data/block"), 10, 90).sort() - val sampledKeys = sortedBlock - .sample() + val sampledKeys = inputDirectories + .map(_.toDirectory) + .flatMap(_.list) + .map(f => Block.fromPath(f.path)) + .flatMap(_.sample(request.numberOfKeys)) .map(key => ByteString.copyFrom(key.underlying)) - .toList - val reply = SampleReply(sampledKeys) - promise.success(reply) + promise.success(SampleReply(sampledKeys)) } catch { case e: Exception => println(e) From c99307068626468ba4cadc346fd4bdbbe20745af Mon Sep 17 00:00:00 2001 From: Minjae Gwon Date: Sun, 26 Nov 2023 16:33:28 +0900 Subject: [PATCH 08/39] refactor: main logics --- .../main/scala/{Main.scala => Master.scala} | 23 +++++- worker/src/main/scala/Main.scala | 46 ------------ worker/src/main/scala/Worker.scala | 73 +++++++++++++++++++ 3 files changed, 92 insertions(+), 50 deletions(-) rename master/src/main/scala/{Main.scala => Master.scala} (54%) delete mode 100644 worker/src/main/scala/Main.scala create mode 100644 worker/src/main/scala/Worker.scala diff --git a/master/src/main/scala/Main.scala b/master/src/main/scala/Master.scala similarity index 54% rename from master/src/main/scala/Main.scala rename to master/src/main/scala/Master.scala index 08537a9..f1dc950 100644 --- a/master/src/main/scala/Main.scala +++ b/master/src/main/scala/Master.scala @@ -1,12 +1,15 @@ package kr.ac.postech.paranode.master -import kr.ac.postech.paranode.core.WorkerMetadata -import kr.ac.postech.paranode.rpc.MasterServer +import kr.ac.postech.paranode.core.{Key, WorkerMetadata} +import kr.ac.postech.paranode.rpc.{MasterServer, WorkerClient} +import org.apache.logging.log4j.scala.Logging import java.net._ +import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.{Await, Future} import scala.util.Try -object Main { +object Master extends Logging { def main(args: Array[String]): Unit = { val numberOfWorker = Try(args(0).toInt).getOrElse { println("Invalid command") @@ -32,8 +35,20 @@ object Main { } catch { case e: Exception => e.printStackTrace() } - // TODO: save workerInfo and start WorkerClient + val clients = workerInfo.map { worker => + WorkerClient(worker.host, worker.port) + } + + val sampledKeys = Await + .result( + Future.sequence(clients.map(_.sample(64))), + scala.concurrent.duration.Duration.Inf + ) + .flatMap(_.sampledKeys) + .map(Key.fromByteString) + + logger.debug(s"Sampled keys: $sampledKeys") } } diff --git a/worker/src/main/scala/Main.scala b/worker/src/main/scala/Main.scala deleted file mode 100644 index 2e87062..0000000 --- a/worker/src/main/scala/Main.scala +++ /dev/null @@ -1,46 +0,0 @@ -package kr.ac.postech.paranode.worker - -import kr.ac.postech.paranode.core.WorkerMetadata -import kr.ac.postech.paranode.rpc.MasterClient - -import java.net.InetAddress -import scala.util.Try - -object Main { - def main(args: Array[String]): Unit = { - if (args.length < 5) { - println("Usage: worker -I -O ") - return - } - - val Array(ip, portStr) = args(0).split(":") - val port = Try(portStr.toInt).getOrElse { - println("Invalid port number.") - return - } - - val inputDirIndex = args.indexOf("-I") + 1 - val outputDirIndex = args.indexOf("-O") + 1 - - if (inputDirIndex <= 0 || outputDirIndex <= 0) { - println("Input or Output directories not specified correctly.") - return - } - - args.slice(inputDirIndex, outputDirIndex - 1) - args(outputDirIndex) - - // Open MasterClient and request register - val client = MasterClient(ip, port) - try { - val ipAddress = InetAddress.getLocalHost.getHostAddress - val workerMetadata = WorkerMetadata(ipAddress, -1, None) - client.register(workerMetadata) - - } finally { - client.shutdown() - } - - } - -} diff --git a/worker/src/main/scala/Worker.scala b/worker/src/main/scala/Worker.scala new file mode 100644 index 0000000..5cc80cd --- /dev/null +++ b/worker/src/main/scala/Worker.scala @@ -0,0 +1,73 @@ +package kr.ac.postech.paranode.worker + +import kr.ac.postech.paranode.core.WorkerMetadata +import kr.ac.postech.paranode.rpc.{MasterClient, WorkerServer} + +import java.net.{InetAddress, ServerSocket} +import scala.util.Using +import scala.reflect.io.Path +import org.apache.logging.log4j.scala.Logging + +import scala.concurrent.Await + +object Worker extends Logging { + private class WorkerArguments(args: Array[String]) { + def masterIp: String = args(0).split(":")(0) + + def masterPort: Int = args(0).split(":")(1).toInt + + def inputDirectories(): Array[Path] = + args + .slice(inputDirectoriesIndex, outputDirectoryIndex - 1) + .map(Path.string2path) + + def outputDirectory(): Path = Path.string2path(args(outputDirectoryIndex)) + + private def inputDirectoriesIndex = args.indexOf("-I") + 1 + + private def outputDirectoryIndex = args.indexOf("-O") + 1 + } + + def main(args: Array[String]): Unit = { + val workerArguments = new WorkerArguments(args) + + logger.debug( + s"Worker arguments: \n" + + s"masterIp: ${workerArguments.masterIp}\n" + + s"masterPort: ${workerArguments.masterPort}\n" + + s"inputDirectories: ${workerArguments.inputDirectories().mkString(", ")}\n" + + s"outputDirectory: ${workerArguments.outputDirectory()}\n" + ) + + val workerPort = Using(new ServerSocket(0))(_.getLocalPort).get + + val server = new WorkerServer( + scala.concurrent.ExecutionContext.global, + workerPort, + workerArguments.inputDirectories(), + workerArguments.outputDirectory() + ) + + server.start() + + val client = + MasterClient(workerArguments.masterIp, workerArguments.masterPort) + + try { + val ipAddress = InetAddress.getLocalHost.getHostAddress + val workerMetadata = WorkerMetadata(ipAddress, workerPort, None) + + Await.result( + client.register(workerMetadata), + scala.concurrent.duration.Duration.Inf + ) + + } finally { + client.shutdown() + } + + + server.blockUntilShutdown() + } + +} From 125ccb83660d33b6d3d368df0307a9bd9cb739d7 Mon Sep 17 00:00:00 2001 From: Minjae Gwon Date: Sun, 26 Nov 2023 16:33:48 +0900 Subject: [PATCH 09/39] feat: dockerize --- .dockerignore | 7 ++ docker-compose.yml | 14 +++ docker/master/Dockerfile | 13 ++ docker/worker/Dockerfile | 23 ++++ docker/worker/data/0/0 | 256 +++++++++++++++++++++++++++++++++++++++ docker/worker/data/0/1 | 256 +++++++++++++++++++++++++++++++++++++++ docker/worker/data/1/2 | 256 +++++++++++++++++++++++++++++++++++++++ docker/worker/data/1/3 | 256 +++++++++++++++++++++++++++++++++++++++ docker/worker/data/2/4 | 256 +++++++++++++++++++++++++++++++++++++++ docker/worker/data/2/5 | 256 +++++++++++++++++++++++++++++++++++++++ 10 files changed, 1593 insertions(+) create mode 100644 .dockerignore create mode 100644 docker-compose.yml create mode 100644 docker/master/Dockerfile create mode 100644 docker/worker/Dockerfile create mode 100644 docker/worker/data/0/0 create mode 100644 docker/worker/data/0/1 create mode 100644 docker/worker/data/1/2 create mode 100644 docker/worker/data/1/3 create mode 100644 docker/worker/data/2/4 create mode 100644 docker/worker/data/2/5 diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..6a047fe --- /dev/null +++ b/.dockerignore @@ -0,0 +1,7 @@ +.bsp +.idea +docs +**/.bloop +target +**/target +**/Dockerfile diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..13e8811 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,14 @@ +services: + master: + build: + context: . + dockerfile: docker/master/Dockerfile + worker: + build: + context: . + dockerfile: docker/worker/Dockerfile + args: + - MASTER_HOST=master + - MASTER_PORT=50051 + depends_on: + - master diff --git a/docker/master/Dockerfile b/docker/master/Dockerfile new file mode 100644 index 0000000..5405e99 --- /dev/null +++ b/docker/master/Dockerfile @@ -0,0 +1,13 @@ +FROM sbtscala/scala-sbt:eclipse-temurin-jammy-20.0.2_9_1.9.6_2.13.12 + +RUN mkdir -p /app + +WORKDIR /app + +COPY project/build.properties project/plugins.sbt project/scalapb.sbt /app/project/ + +COPY . . + +RUN sbt --batch compile + +ENTRYPOINT sbt --batch -v "master/run 1" diff --git a/docker/worker/Dockerfile b/docker/worker/Dockerfile new file mode 100644 index 0000000..ccc48a6 --- /dev/null +++ b/docker/worker/Dockerfile @@ -0,0 +1,23 @@ +FROM sbtscala/scala-sbt:eclipse-temurin-jammy-20.0.2_9_1.9.6_2.13.12 + +ARG MASTER_HOST +ARG MASTER_PORT + +ENV MASTER_HOST=${MASTER_HOST} +ENV MASTER_PORT=${MASTER_PORT} + +ENV SBT_OPTS="-Xmx2G -Xss2M" + +RUN mkdir -p /app /data /output + +COPY docker/worker/data /data + +WORKDIR /app + +COPY project/build.properties project/plugins.sbt project/scalapb.sbt /app/project/ + +COPY . . + +RUN sbt --batch compile + +ENTRYPOINT sbt --batch -v "worker/run ${MASTER_HOST}:${MASTER_PORT} -I /data/0 data/1 data/2 -O /output/" diff --git a/docker/worker/data/0/0 b/docker/worker/data/0/0 new file mode 100644 index 0000000..f70be53 --- /dev/null +++ b/docker/worker/data/0/0 @@ -0,0 +1,256 @@ +AsfAGHM5om 00000000000000000000000000000000 0000222200002222000022220000222200002222000000001111 +~sHd0jDv6X 00000000000000000000000000000001 77779999444488885555CCCC777755555555BBBB666644446666 +uI^EYm8s=| 00000000000000000000000000000002 CCCCFFFF777799995555FFFF11112222999988884444DDDDFFFF +Q)JN)R9z-L 00000000000000000000000000000003 FFFF111100000000000066668888BBBB33333333AAAA1111CCCC +o4FoBkqERn 00000000000000000000000000000004 7777AAAABBBBBBBB22224444444499995555BBBB11118888DDDD +*}-Wz1;TD- 00000000000000000000000000000005 AAAA88883333BBBB888888884444777722227777999900002222 +0fssx}~[oB 00000000000000000000000000000006 FFFF999977774444AAAA7777EEEEDDDDAAAAAAAA99998888BBBB +mz4VCN@a#" 00000000000000000000000000000007 DDDDBBBB1111FFFF2222DDDDFFFFBBBBFFFF6666444477778888 +my+=5r7(N| 00000000000000000000000000000008 22226666CCCC66662222FFFF0000EEEE11118888444455559999 +5HA\z%qt{% 00000000000000000000000000000009 0000AAAA8888FFFF0000888800000000222255551111FFFFEEEE +`PkXQ<&+cc 0000000000000000000000000000000A 66667777FFFFFFFF2222FFFF3333FFFF22224444DDDD77777777 +GLSnlm0*P* 0000000000000000000000000000000B 5555EEEE1111BBBB55555555AAAABBBB33335555BBBB11114444 +swBzQ#' K< 0000000000000000000000000000000C DDDDEEEE777777777777EEEEBBBBEEEECCCCEEEE444466665555 +9SCzyMKz% 00000000000000000000000000000014 99997777777722222222999911116666DDDDBBBBCCCC3333DDDD +kXrG^ech1/ 00000000000000000000000000000015 FFFF2222AAAA66660000AAAA9999333322227777AAAA77772222 +v%^RG^ci[X 00000000000000000000000000000016 7777BBBBFFFF999922229999DDDD3333CCCCAAAACCCCBBBBBBBB +vi@>8jMaYX 00000000000000000000000000000017 CCCCFFFFFFFF6666AAAA999999997777AAAA6666000066668888 +}dA7*<6[lx 00000000000000000000000000000018 6666FFFF77775555CCCC88881111AAAA00000000CCCC00009999 +>Y$Kc)9DOf 00000000000000000000000000000019 AAAAEEEEEEEE222266660000AAAA00000000000044446666EEEE +uF?T;*:x0+ 0000000000000000000000000000001A 444466665555BBBB8888BBBB00000000BBBB00005555AAAA7777 +~%.@@,3>7+ 0000000000000000000000000000001B FFFFDDDD333399999999666644449999BBBB2222000000004444 +JPrAKc{W>j 0000000000000000000000000000001C 222211111111333355554444DDDDEEEE33339999999911115555 +*i@lF30|aO 0000000000000000000000000000001D CCCC88880000FFFFCCCCDDDDBBBB4444EEEE99990000AAAAAAAA +9UH>%FZbUK 0000000000000000000000000000001E DDDD00005555EEEE111199998888CCCC888877779999DDDD3333 +-^1~=aYZC1 0000000000000000000000000000001F 11112222555599996666111144445555DDDDBBBB7777EEEE0000 +.\KqT&Z4i4 00000000000000000000000000000020 3333DDDDCCCC888888885555222299993333EEEEFFFF66661111 +9RtsPa gIN 00000000000000000000000000000021 555544440000AAAACCCC8888AAAA55552222DDDDBBBB22226666 +Fqy~]OATYw 00000000000000000000000000000022 9999CCCC0000888888884444333355553333BBBB55553333FFFF +Wj-%?e*2y\ 00000000000000000000000000000023 AAAA444477776666FFFF444455550000EEEE88883333FFFFCCCC +Rv]QHo9f6+ 00000000000000000000000000000024 99996666BBBB7777444466660000AAAADDDDDDDDAAAAEEEEDDDD +1 e!og=!hi 00000000000000000000000000000025 3333EEEE88887777CCCCEEEE777733337777DDDDFFFFEEEE2222 +N,gk>37c ( 00000000000000000000000000000026 DDDD11116666DDDD5555EEEE99996666AAAAAAAA3333EEEEBBBB +s.72Y.uzRN 00000000000000000000000000000027 0000CCCCFFFFAAAAAAAA11117777AAAA66662222000055558888 +j*0`gczW{u 00000000000000000000000000000028 9999DDDDCCCC8888AAAA999911111111BBBB66667777BBBB9999 +R+Sh=Kgd.6 00000000000000000000000000000029 DDDD0000BBBB77770000222200009999AAAADDDDAAAADDDDEEEE +x]w]=Dp6\s 0000000000000000000000000000002A 6666FFFFEEEEBBBBCCCC000033331111666677771111DDDD7777 +maU!Uo]He& 0000000000000000000000000000002B AAAA9999BBBB55557777CCCC6666EEEE555566668888FFFF4444 +smN$i pU-~ 0000000000000000000000000000002C AAAADDDDEEEE333388885555DDDD3333FFFFEEEE1111CCCC5555 +T}# `9Cseu 0000000000000000000000000000002D 4444FFFF777700001111EEEE999944447777444488881111AAAA +G2oMVHVK`! 0000000000000000000000000000002E 4444888844448888CCCC1111AAAA555577778888BBBB00003333 +(tv1'>|,s[ 0000000000000000000000000000002F 0000CCCC222277778888FFFFCCCCBBBB111177779999DDDD0000 +RE%Z{o)9E: 00000000000000000000000000000030 00002222AAAAEEEE444400008888CCCC3333DDDD555511111111 +'%:YY?@b+M 00000000000000000000000000000031 9999777777770000AAAA0000444433332222EEEE333399996666 +N(JuG%.`7c 00000000000000000000000000000032 66665555AAAA5555BBBBAAAAEEEE111111119999BBBB6666FFFF +`E$,xR{?Kc 00000000000000000000000000000033 7777AAAA222200008888CCCC3333AAAAFFFFAAAAEEEEEEEECCCC +jzu$Fn3ZN+ 00000000000000000000000000000034 55552222EEEE5555BBBBAAAA8888AAAA11110000DDDD9999DDDD +|xaPO\~!V* 00000000000000000000000000000035 EEEE2222EEEEFFFFEEEEBBBB2222BBBB9999AAAA999955552222 +GCU4gfl|&! 00000000000000000000000000000036 1111DDDD8888AAAA999999990000333377778888FFFF1111BBBB +JU9s03v&}o 00000000000000000000000000000037 7777555500001111DDDD3333111122221111AAAA444444448888 +D@v9""8"!` 00000000000000000000000000000038 2222BBBB55552222CCCC555533336666EEEE9999777766669999 +OffB8(#k@N 00000000000000000000000000000039 CCCC555555556666DDDDCCCC111111119999DDDD55554444EEEE +-_) R'4T<" 0000000000000000000000000000003A BBBBCCCCEEEECCCCDDDD6666BBBB333377779999222200007777 +'z4P|n>\to 0000000000000000000000000000003B DDDD5555AAAA3333CCCC222233330000111133335555EEEE4444 +[!Nbx&X=+V 0000000000000000000000000000003C 77778888AAAA7777888844446666EEEECCCCBBBBEEEE77775555 +p?"\\6:W[' 0000000000000000000000000000003D 8888EEEE6666555599992222DDDD2222FFFFEEEE33338888AAAA ++2A^xP5dA- 0000000000000000000000000000003E CCCCBBBBFFFFFFFF99995555AAAAAAAA77771111000033333333 +-cuJvy7;=h 0000000000000000000000000000003F 99993333CCCC8888EEEEAAAAEEEE5555CCCC7777FFFFCCCC0000 +ix^TQ-ma5o 00000000000000000000000000000047 88885555CCCC88889999888811110000BBBBEEEECCCC33338888 +E(g$5-WpBC 00000000000000000000000000000048 BBBB7777EEEE4444888866661111FFFF44449999BBBB11119999 +~\ptsn1L{/ 00000000000000000000000000000049 CCCC0000FFFFCCCC7777BBBB333300003333FFFF3333BBBBEEEE +YhlPk`q6SN 0000000000000000000000000000004A 00001111AAAABBBBAAAA66669999DDDD11116666666633337777 +WBzul,K** 0000000000000000000000000000004B 4444AAAADDDDEEEE33337777AAAA9999DDDD88886666DDDD4444 +q)epJsSLXU 0000000000000000000000000000004C 88887777999955558888AAAA9999111155552222FFFF22225555 +`WVkW&( /[ 0000000000000000000000000000004D FFFF999922221111DDDD9999AAAA9999FFFF66662222FFFFAAAA +HXg#Z^7@7/ 0000000000000000000000000000004E 777799999999DDDD44441111BBBB6666AAAA0000999966663333 +-&8$lcXS$; 0000000000000000000000000000004F 333300004444AAAA3333666600009999DDDDCCCC9999BBBB0000 +^SpqwoVET; 00000000000000000000000000000050 BBBB77775555CCCC1111DDDD2222BBBB66669999CCCC77771111 +P-6Ybw3=a| 00000000000000000000000000000051 2222FFFFDDDDAAAACCCC3333EEEEBBBB8888EEEE000077776666 +a`:;ydJv)Y 00000000000000000000000000000052 55554444AAAAAAAA6666DDDD7777AAAAEEEE00003333CCCCFFFF +dO]G~-YGFJ 00000000000000000000000000000053 8888555533333333EEEE11113333AAAA444411110000CCCCCCCC +WM^;rvO/u] 00000000000000000000000000000054 BBBB00007777999977772222AAAA0000CCCCAAAAEEEEFFFFDDDD +-WwU4BF*P< 00000000000000000000000000000055 BBBB8888EEEEEEEE55553333AAAA4444EEEE6666888833332222 +y'w/PZ!x"> 00000000000000000000000000000056 DDDD55554444AAAAAAAA444455553333AAAA444411117777BBBB +#jA`l-f@*f 00000000000000000000000000000057 4444AAAA2222AAAAEEEE55556666CCCC3333FFFF888822228888 +:!W.=v,f"z 00000000000000000000000000000058 444422223333666688886666BBBB4444888877772222CCCC9999 +"!r4ugwZcz 00000000000000000000000000000059 22226666CCCCEEEE6666FFFF888822220000333366662222EEEE +O?(|_dXq:X 0000000000000000000000000000005C 2222FFFFBBBB55552222444433332222444433333333DDDD5555 +(cb621Uv~( 0000000000000000000000000000005D 3333BBBB11110000FFFF9999BBBB6666CCCCCCCC66666666AAAA +hg5x s23L3 0000000000000000000000000000005E 22221111AAAA000011117777DDDD999933337777666699993333 +w5nn`wa'[p 0000000000000000000000000000005F 000099997777CCCC666677776666FFFF444455557777AAAA0000 +qeQNoB 2XG 00000000000000000000000000000060 0000EEEE66668888FFFF2222FFFFAAAAFFFF7777EEEE22221111 +6n>iMbi=gw 00000000000000000000000000000061 4444DDDDFFFF9999DDDD9999EEEEEEEECCCCDDDD4444EEEE6666 +uGz7S=#fh: 00000000000000000000000000000062 77771111DDDD222200000000BBBBDDDD222288885555FFFFFFFF +rd)x<>@@H| 00000000000000000000000000000063 99996666AAAABBBB7777CCCC4444DDDD555544447777BBBBCCCC +xEwa^:84K] 00000000000000000000000000000064 000099995555CCCC00000000EEEE5555BBBB1111DDDDAAAADDDD +':]$}b:Z$a 00000000000000000000000000000065 77775555DDDD4444DDDDAAAA66660000FFFF5555DDDDAAAA2222 +vZl"Ji^AO@ 00000000000000000000000000000066 3333777744444444EEEEEEEE11115555666600008888AAAABBBB +o%,!fSdRBa 00000000000000000000000000000067 11119999DDDDAAAA8888AAAA5555EEEE8888CCCC888811118888 +*+o@+t%u&; 00000000000000000000000000000068 22228888999922228888EEEE3333444455551111EEEE77779999 +(CoKUhfB9{ 00000000000000000000000000000069 1111BBBB9999CCCCFFFF3333CCCC777755559999CCCC9999EEEE +I>:9JQb_5T 0000000000000000000000000000006A AAAA66662222CCCCBBBB666600003333BBBB2222AAAA99997777 +eSzU!X,[%/ 0000000000000000000000000000006B AAAACCCC9999FFFFFFFFAAAA444455553333BBBB4444BBBB4444 +[QZ{a&TaCS 0000000000000000000000000000006C AAAA8888CCCC2222FFFFAAAA5555CCCC4444CCCCCCCC88885555 +?:Yp4%.K9n 0000000000000000000000000000006D BBBBCCCC5555111100007777FFFFAAAAFFFF0000DDDDDDDDAAAA +'_>/ 4v}8I 0000000000000000000000000000006E 1111EEEE000088886666333311115555666655557777CCCC3333 +KrNW.!5MP; 0000000000000000000000000000006F 2222AAAAEEEE33330000777711111111FFFF2222999999990000 +j17a3a#bH? 00000000000000000000000000000070 8888CCCCBBBBCCCC88881111EEEE33336666BBBB3333DDDD1111 +)o-QLHy(UO 00000000000000000000000000000071 3333BBBB444400002222EEEEFFFF333355556666DDDD55556666 +al;:OEKSH2 00000000000000000000000000000072 DDDDEEEE3333DDDDBBBB7777BBBB1111FFFF3333CCCC2222FFFF +I@L*`HMPfN 00000000000000000000000000000073 DDDD666699993333999977779999DDDD444488882222AAAACCCC +$kBiG 00000000000000000000000000000090 3333DDDD000044445555444444441111CCCC1111BBBB33331111 +V*W% x\&'q 00000000000000000000000000000091 8888DDDDBBBBAAAABBBB99997777EEEE00008888AAAA33336666 +0Jw^F-}sZr 00000000000000000000000000000092 4444BBBBDDDD2222FFFF888811117777DDDD444444448888FFFF +OLf_u|1d}9 00000000000000000000000000000093 DDDD1111AAAA9999FFFFDDDD9999BBBB9999000044448888CCCC +%V*"P*(/Ez 00000000000000000000000000000094 11110000CCCC666600002222CCCCEEEECCCCEEEE1111BBBBDDDD +jGdC5%yw;# 00000000000000000000000000000095 44449999FFFF00004444222222228888000099995555FFFF2222 +,uSW`H/pXh 00000000000000000000000000000096 8888BBBB111188885555CCCC7777BBBB7777111166663333BBBB +n],.INMkcA 00000000000000000000000000000097 DDDD77777777DDDD99993333FFFFDDDD9999CCCCFFFFEEEE8888 +(P=jd[R]&[ 00000000000000000000000000000098 BBBBAAAA99995555CCCC666688887777EEEE1111999988889999 +*%pAD!6FlS 00000000000000000000000000000099 AAAA777799994444DDDDEEEE6666DDDD2222AAAA7777EEEEEEEE +;u0nwOR{|e 0000000000000000000000000000009A CCCC5555FFFF1111000066667777EEEEEEEE1111777722227777 +X*)4\|>`b. 0000000000000000000000000000009B EEEE33339999CCCC2222999999999999FFFFDDDD777788884444 +#Yh\5HjDLp 0000000000000000000000000000009C 33338888EEEEFFFFDDDDDDDDAAAA7777EEEE0000EEEE99995555 +5Yh?d9@6(_ 0000000000000000000000000000009D 66667777FFFF333377774444FFFFDDDD99993333CCCC2222AAAA +W;mKiM[qVV 0000000000000000000000000000009E 111133334444AAAA55556666CCCCFFFF5555BBBB333355553333 +287P2iA6U: 0000000000000000000000000000009F 88880000FFFF000022220000BBBBCCCCFFFF3333777766660000 +Bj`e'4&VbD 000000000000000000000000000000A0 DDDD888811110000BBBB2222EEEEBBBB00004444CCCCEEEE1111 +Q[7 0VqTg~ 000000000000000000000000000000A1 8888666622222222222222221111DDDD11110000EEEEAAAA6666 +4Y8,(.X0:" 000000000000000000000000000000A2 DDDD7777DDDD5555AAAAEEEE7777DDDD444499996666BBBBFFFF +w)98*ZrTi* 000000000000000000000000000000A3 CCCC4444FFFF0000BBBBAAAA66667777CCCC4444BBBB7777CCCC +pHH7F#q@Tf 000000000000000000000000000000A4 0000FFFFEEEE9999DDDD9999AAAAFFFF9999AAAA00006666DDDD +89_ &UU^-f 000000000000000000000000000000A5 BBBB5555AAAAAAAAAAAA4444FFFF6666DDDD1111BBBB66662222 +IjYZ+5}$cH 000000000000000000000000000000A6 EEEE555577779999666611110000FFFF0000AAAADDDD6666BBBB +HP5;m7ydP, 000000000000000000000000000000A7 DDDDDDDDEEEE222255559999DDDDEEEE7777AAAAFFFFDDDD8888 +(\+CmQF;sh 000000000000000000000000000000A8 7777BBBB9999333377779999AAAAAAAACCCC1111555533339999 +`27}+u%$Z' 000000000000000000000000000000A9 22228888EEEE555511117777DDDD555522229999EEEE5555EEEE +D/XZpkvB{U 000000000000000000000000000000AA 8888BBBB77776666EEEEEEEE00007777BBBB2222333355557777 +6uxeaoldr{ 000000000000000000000000000000AB BBBB11110000CCCC8888222277779999AAAA4444000077774444 +3n glBr={a 000000000000000000000000000000AC EEEE99992222666644446666FFFF11112222FFFF777744445555 +M3Bn!0ePh9 000000000000000000000000000000AD 666611114444999933332222FFFFDDDD5555111133339999AAAA +^:W0]D07,s 000000000000000000000000000000AE 99995555EEEEEEEE33330000AAAA1111CCCC6666444488883333 +R%]??oXCf( 000000000000000000000000000000AF CCCCFFFF00004444222299997777CCCC11111111999955550000 +?jh!g'(/dI 000000000000000000000000000000B0 99994444FFFFBBBBFFFF777711113333EEEEDDDD222299991111 +,qy-T(O~$} 000000000000000000000000000000B1 44446666888800007777DDDDEEEEFFFF22223333777711116666 +g~]6jo( dQ 000000000000000000000000000000B2 DDDD888822224444EEEE88887777EEEE00001111CCCCEEEEFFFF +\1]of=Ah$q 000000000000000000000000000000B3 EEEEAAAACCCCCCCC77779999AAAACCCC9999999966666666CCCC +5,vYEm^\di 000000000000000000000000000000B4 1111EEEE3333EEEEBBBB8888111111111111777733331111DDDD +}uX` 000000000000000000000000000000D1 7777DDDD77778888BBBB8888EEEEAAAA222277773333FFFF6666 +3pU8_.#1JU 000000000000000000000000000000D2 0000777799995555FFFFDDDDFFFF5555FFFFCCCC55554444FFFF +IW,e"{Tnvp 000000000000000000000000000000D3 99995555777755556666AAAA88887777EEEE333388884444CCCC +:=XWS2sImq 000000000000000000000000000000D4 AAAACCCCBBBBAAAA0000EEEE4444CCCCEEEE555544447777DDDD +o8Gcpop:! 000000000000000000000000000000D5 AAAAFFFFDDDD11115555000077778888999900003333BBBB2222 +W-qOhMjDFd 000000000000000000000000000000D6 0000222266660000BBBB55551111777733332222AAAAFFFFBBBB +u?;I>\L_vn 000000000000000000000000000000D7 99991111FFFFFFFF5555EEEE44447777BBBBEEEE7777AAAA8888 +Nyp X=m$h9 000000000000000000000000000000D8 0000EEEE444433333333BBBB8888000011110000000044449999 +g.FN&efgUs 000000000000000000000000000000D9 7777444488885555FFFF3333FFFFDDDD777755559999AAAAEEEE +rhC::yPt=z 000000000000000000000000000000DA 3333666633336666BBBBCCCCEEEE000000007777FFFFEEEE7777 +De9lt=P!{. 000000000000000000000000000000DB DDDD2222BBBB33335555888866663333EEEE9999333344444444 +XG(8.GC,t\ 000000000000000000000000000000DC 7777CCCCEEEE7777222200006666BBBB11112222999955555555 +U;azR 1t!\wV 000000000000000000000000000000E0 2222EEEE00000000FFFFBBBB6666555566665555BBBBAAAA1111 +'-zF>A2f5c 000000000000000000000000000000E1 CCCC999977773333111144444444BBBBFFFF8888888866666666 +^qaslc^dJX 000000000000000000000000000000E2 444411118888AAAA7777CCCC000011119999EEEE77777777FFFF +@gU`)^H:z5 000000000000000000000000000000E3 AAAA33331111FFFF888822225555CCCC33338888FFFF3333CCCC + l;w?EBj0 000000000000000000000000000000E4 BBBBDDDD111188888888AAAA111144449999666633332222DDDD +Iw&)MuUaM, 000000000000000000000000000000E5 5555FFFF00009999888811119999111111111111999922222222 +aGam)y ~t- 000000000000000000000000000000E6 EEEE22228888BBBB999977774444FFFFAAAA999922222222BBBB ++/Yg#6Nf/r 000000000000000000000000000000E7 FFFFBBBB6666CCCC66669999FFFF66662222DDDD777799998888 +SD ,=Yy%.w 000000000000000000000000000000E8 666644446666DDDDDDDD44446666111100004444BBBBFFFF9999 +/e)%&bgl}E 000000000000000000000000000000E9 AAAA6666333388887777DDDDCCCCFFFF1111EEEE00001111EEEE +8mdOTG)j6O 000000000000000000000000000000EA BBBB7777AAAABBBB888855554444999966665555CCCC11117777 +Yp=]xMdlAGG 000000000000000000000000000000F6 1111DDDD555500008888CCCC222288884444EEEEDDDD5555BBBB +X\!l_zyMKz% 00000000000000000000000000000014 99997777777722222222999911116666DDDDBBBBCCCC3333DDDD +kXrG^ech1/ 00000000000000000000000000000015 FFFF2222AAAA66660000AAAA9999333322227777AAAA77772222 +v%^RG^ci[X 00000000000000000000000000000016 7777BBBBFFFF999922229999DDDD3333CCCCAAAACCCCBBBBBBBB +vi@>8jMaYX 00000000000000000000000000000017 CCCCFFFFFFFF6666AAAA999999997777AAAA6666000066668888 +}dA7*<6[lx 00000000000000000000000000000018 6666FFFF77775555CCCC88881111AAAA00000000CCCC00009999 +>Y$Kc)9DOf 00000000000000000000000000000019 AAAAEEEEEEEE222266660000AAAA00000000000044446666EEEE +uF?T;*:x0+ 0000000000000000000000000000001A 444466665555BBBB8888BBBB00000000BBBB00005555AAAA7777 +~%.@@,3>7+ 0000000000000000000000000000001B FFFFDDDD333399999999666644449999BBBB2222000000004444 +JPrAKc{W>j 0000000000000000000000000000001C 222211111111333355554444DDDDEEEE33339999999911115555 +*i@lF30|aO 0000000000000000000000000000001D CCCC88880000FFFFCCCCDDDDBBBB4444EEEE99990000AAAAAAAA +9UH>%FZbUK 0000000000000000000000000000001E DDDD00005555EEEE111199998888CCCC888877779999DDDD3333 +-^1~=aYZC1 0000000000000000000000000000001F 11112222555599996666111144445555DDDDBBBB7777EEEE0000 +.\KqT&Z4i4 00000000000000000000000000000020 3333DDDDCCCC888888885555222299993333EEEEFFFF66661111 +9RtsPa gIN 00000000000000000000000000000021 555544440000AAAACCCC8888AAAA55552222DDDDBBBB22226666 +Fqy~]OATYw 00000000000000000000000000000022 9999CCCC0000888888884444333355553333BBBB55553333FFFF +Wj-%?e*2y\ 00000000000000000000000000000023 AAAA444477776666FFFF444455550000EEEE88883333FFFFCCCC +Rv]QHo9f6+ 00000000000000000000000000000024 99996666BBBB7777444466660000AAAADDDDDDDDAAAAEEEEDDDD +1 e!og=!hi 00000000000000000000000000000025 3333EEEE88887777CCCCEEEE777733337777DDDDFFFFEEEE2222 +N,gk>37c ( 00000000000000000000000000000026 DDDD11116666DDDD5555EEEE99996666AAAAAAAA3333EEEEBBBB +s.72Y.uzRN 00000000000000000000000000000027 0000CCCCFFFFAAAAAAAA11117777AAAA66662222000055558888 +j*0`gczW{u 00000000000000000000000000000028 9999DDDDCCCC8888AAAA999911111111BBBB66667777BBBB9999 +R+Sh=Kgd.6 00000000000000000000000000000029 DDDD0000BBBB77770000222200009999AAAADDDDAAAADDDDEEEE +x]w]=Dp6\s 0000000000000000000000000000002A 6666FFFFEEEEBBBBCCCC000033331111666677771111DDDD7777 +maU!Uo]He& 0000000000000000000000000000002B AAAA9999BBBB55557777CCCC6666EEEE555566668888FFFF4444 +smN$i pU-~ 0000000000000000000000000000002C AAAADDDDEEEE333388885555DDDD3333FFFFEEEE1111CCCC5555 +T}# `9Cseu 0000000000000000000000000000002D 4444FFFF777700001111EEEE999944447777444488881111AAAA +G2oMVHVK`! 0000000000000000000000000000002E 4444888844448888CCCC1111AAAA555577778888BBBB00003333 +(tv1'>|,s[ 0000000000000000000000000000002F 0000CCCC222277778888FFFFCCCCBBBB111177779999DDDD0000 +RE%Z{o)9E: 00000000000000000000000000000030 00002222AAAAEEEE444400008888CCCC3333DDDD555511111111 +'%:YY?@b+M 00000000000000000000000000000031 9999777777770000AAAA0000444433332222EEEE333399996666 +N(JuG%.`7c 00000000000000000000000000000032 66665555AAAA5555BBBBAAAAEEEE111111119999BBBB6666FFFF +`E$,xR{?Kc 00000000000000000000000000000033 7777AAAA222200008888CCCC3333AAAAFFFFAAAAEEEEEEEECCCC +jzu$Fn3ZN+ 00000000000000000000000000000034 55552222EEEE5555BBBBAAAA8888AAAA11110000DDDD9999DDDD +|xaPO\~!V* 00000000000000000000000000000035 EEEE2222EEEEFFFFEEEEBBBB2222BBBB9999AAAA999955552222 +GCU4gfl|&! 00000000000000000000000000000036 1111DDDD8888AAAA999999990000333377778888FFFF1111BBBB +JU9s03v&}o 00000000000000000000000000000037 7777555500001111DDDD3333111122221111AAAA444444448888 +D@v9""8"!` 00000000000000000000000000000038 2222BBBB55552222CCCC555533336666EEEE9999777766669999 +OffB8(#k@N 00000000000000000000000000000039 CCCC555555556666DDDDCCCC111111119999DDDD55554444EEEE +-_) R'4T<" 0000000000000000000000000000003A BBBBCCCCEEEECCCCDDDD6666BBBB333377779999222200007777 +'z4P|n>\to 0000000000000000000000000000003B DDDD5555AAAA3333CCCC222233330000111133335555EEEE4444 +[!Nbx&X=+V 0000000000000000000000000000003C 77778888AAAA7777888844446666EEEECCCCBBBBEEEE77775555 +p?"\\6:W[' 0000000000000000000000000000003D 8888EEEE6666555599992222DDDD2222FFFFEEEE33338888AAAA ++2A^xP5dA- 0000000000000000000000000000003E CCCCBBBBFFFFFFFF99995555AAAAAAAA77771111000033333333 +-cuJvy7;=h 0000000000000000000000000000003F 99993333CCCC8888EEEEAAAAEEEE5555CCCC7777FFFFCCCC0000 +ix^TQ-ma5o 00000000000000000000000000000047 88885555CCCC88889999888811110000BBBBEEEECCCC33338888 +E(g$5-WpBC 00000000000000000000000000000048 BBBB7777EEEE4444888866661111FFFF44449999BBBB11119999 +~\ptsn1L{/ 00000000000000000000000000000049 CCCC0000FFFFCCCC7777BBBB333300003333FFFF3333BBBBEEEE +YhlPk`q6SN 0000000000000000000000000000004A 00001111AAAABBBBAAAA66669999DDDD11116666666633337777 +WBzul,K** 0000000000000000000000000000004B 4444AAAADDDDEEEE33337777AAAA9999DDDD88886666DDDD4444 +q)epJsSLXU 0000000000000000000000000000004C 88887777999955558888AAAA9999111155552222FFFF22225555 +`WVkW&( /[ 0000000000000000000000000000004D FFFF999922221111DDDD9999AAAA9999FFFF66662222FFFFAAAA +HXg#Z^7@7/ 0000000000000000000000000000004E 777799999999DDDD44441111BBBB6666AAAA0000999966663333 +-&8$lcXS$; 0000000000000000000000000000004F 333300004444AAAA3333666600009999DDDDCCCC9999BBBB0000 +^SpqwoVET; 00000000000000000000000000000050 BBBB77775555CCCC1111DDDD2222BBBB66669999CCCC77771111 +P-6Ybw3=a| 00000000000000000000000000000051 2222FFFFDDDDAAAACCCC3333EEEEBBBB8888EEEE000077776666 +a`:;ydJv)Y 00000000000000000000000000000052 55554444AAAAAAAA6666DDDD7777AAAAEEEE00003333CCCCFFFF +dO]G~-YGFJ 00000000000000000000000000000053 8888555533333333EEEE11113333AAAA444411110000CCCCCCCC +WM^;rvO/u] 00000000000000000000000000000054 BBBB00007777999977772222AAAA0000CCCCAAAAEEEEFFFFDDDD +-WwU4BF*P< 00000000000000000000000000000055 BBBB8888EEEEEEEE55553333AAAA4444EEEE6666888833332222 +y'w/PZ!x"> 00000000000000000000000000000056 DDDD55554444AAAAAAAA444455553333AAAA444411117777BBBB +#jA`l-f@*f 00000000000000000000000000000057 4444AAAA2222AAAAEEEE55556666CCCC3333FFFF888822228888 +:!W.=v,f"z 00000000000000000000000000000058 444422223333666688886666BBBB4444888877772222CCCC9999 +"!r4ugwZcz 00000000000000000000000000000059 22226666CCCCEEEE6666FFFF888822220000333366662222EEEE +O?(|_dXq:X 0000000000000000000000000000005C 2222FFFFBBBB55552222444433332222444433333333DDDD5555 +(cb621Uv~( 0000000000000000000000000000005D 3333BBBB11110000FFFF9999BBBB6666CCCCCCCC66666666AAAA +hg5x s23L3 0000000000000000000000000000005E 22221111AAAA000011117777DDDD999933337777666699993333 +w5nn`wa'[p 0000000000000000000000000000005F 000099997777CCCC666677776666FFFF444455557777AAAA0000 +qeQNoB 2XG 00000000000000000000000000000060 0000EEEE66668888FFFF2222FFFFAAAAFFFF7777EEEE22221111 +6n>iMbi=gw 00000000000000000000000000000061 4444DDDDFFFF9999DDDD9999EEEEEEEECCCCDDDD4444EEEE6666 +uGz7S=#fh: 00000000000000000000000000000062 77771111DDDD222200000000BBBBDDDD222288885555FFFFFFFF +rd)x<>@@H| 00000000000000000000000000000063 99996666AAAABBBB7777CCCC4444DDDD555544447777BBBBCCCC +xEwa^:84K] 00000000000000000000000000000064 000099995555CCCC00000000EEEE5555BBBB1111DDDDAAAADDDD +':]$}b:Z$a 00000000000000000000000000000065 77775555DDDD4444DDDDAAAA66660000FFFF5555DDDDAAAA2222 +vZl"Ji^AO@ 00000000000000000000000000000066 3333777744444444EEEEEEEE11115555666600008888AAAABBBB +o%,!fSdRBa 00000000000000000000000000000067 11119999DDDDAAAA8888AAAA5555EEEE8888CCCC888811118888 +*+o@+t%u&; 00000000000000000000000000000068 22228888999922228888EEEE3333444455551111EEEE77779999 +(CoKUhfB9{ 00000000000000000000000000000069 1111BBBB9999CCCCFFFF3333CCCC777755559999CCCC9999EEEE +I>:9JQb_5T 0000000000000000000000000000006A AAAA66662222CCCCBBBB666600003333BBBB2222AAAA99997777 +eSzU!X,[%/ 0000000000000000000000000000006B AAAACCCC9999FFFFFFFFAAAA444455553333BBBB4444BBBB4444 +[QZ{a&TaCS 0000000000000000000000000000006C AAAA8888CCCC2222FFFFAAAA5555CCCC4444CCCCCCCC88885555 +?:Yp4%.K9n 0000000000000000000000000000006D BBBBCCCC5555111100007777FFFFAAAAFFFF0000DDDDDDDDAAAA +'_>/ 4v}8I 0000000000000000000000000000006E 1111EEEE000088886666333311115555666655557777CCCC3333 +KrNW.!5MP; 0000000000000000000000000000006F 2222AAAAEEEE33330000777711111111FFFF2222999999990000 +j17a3a#bH? 00000000000000000000000000000070 8888CCCCBBBBCCCC88881111EEEE33336666BBBB3333DDDD1111 +)o-QLHy(UO 00000000000000000000000000000071 3333BBBB444400002222EEEEFFFF333355556666DDDD55556666 +al;:OEKSH2 00000000000000000000000000000072 DDDDEEEE3333DDDDBBBB7777BBBB1111FFFF3333CCCC2222FFFF +I@L*`HMPfN 00000000000000000000000000000073 DDDD666699993333999977779999DDDD444488882222AAAACCCC +$kBiG 00000000000000000000000000000090 3333DDDD000044445555444444441111CCCC1111BBBB33331111 +V*W% x\&'q 00000000000000000000000000000091 8888DDDDBBBBAAAABBBB99997777EEEE00008888AAAA33336666 +0Jw^F-}sZr 00000000000000000000000000000092 4444BBBBDDDD2222FFFF888811117777DDDD444444448888FFFF +OLf_u|1d}9 00000000000000000000000000000093 DDDD1111AAAA9999FFFFDDDD9999BBBB9999000044448888CCCC +%V*"P*(/Ez 00000000000000000000000000000094 11110000CCCC666600002222CCCCEEEECCCCEEEE1111BBBBDDDD +jGdC5%yw;# 00000000000000000000000000000095 44449999FFFF00004444222222228888000099995555FFFF2222 +,uSW`H/pXh 00000000000000000000000000000096 8888BBBB111188885555CCCC7777BBBB7777111166663333BBBB +n],.INMkcA 00000000000000000000000000000097 DDDD77777777DDDD99993333FFFFDDDD9999CCCCFFFFEEEE8888 +(P=jd[R]&[ 00000000000000000000000000000098 BBBBAAAA99995555CCCC666688887777EEEE1111999988889999 +*%pAD!6FlS 00000000000000000000000000000099 AAAA777799994444DDDDEEEE6666DDDD2222AAAA7777EEEEEEEE +;u0nwOR{|e 0000000000000000000000000000009A CCCC5555FFFF1111000066667777EEEEEEEE1111777722227777 +X*)4\|>`b. 0000000000000000000000000000009B EEEE33339999CCCC2222999999999999FFFFDDDD777788884444 +#Yh\5HjDLp 0000000000000000000000000000009C 33338888EEEEFFFFDDDDDDDDAAAA7777EEEE0000EEEE99995555 +5Yh?d9@6(_ 0000000000000000000000000000009D 66667777FFFF333377774444FFFFDDDD99993333CCCC2222AAAA +W;mKiM[qVV 0000000000000000000000000000009E 111133334444AAAA55556666CCCCFFFF5555BBBB333355553333 +287P2iA6U: 0000000000000000000000000000009F 88880000FFFF000022220000BBBBCCCCFFFF3333777766660000 +Bj`e'4&VbD 000000000000000000000000000000A0 DDDD888811110000BBBB2222EEEEBBBB00004444CCCCEEEE1111 +Q[7 0VqTg~ 000000000000000000000000000000A1 8888666622222222222222221111DDDD11110000EEEEAAAA6666 +4Y8,(.X0:" 000000000000000000000000000000A2 DDDD7777DDDD5555AAAAEEEE7777DDDD444499996666BBBBFFFF +w)98*ZrTi* 000000000000000000000000000000A3 CCCC4444FFFF0000BBBBAAAA66667777CCCC4444BBBB7777CCCC +pHH7F#q@Tf 000000000000000000000000000000A4 0000FFFFEEEE9999DDDD9999AAAAFFFF9999AAAA00006666DDDD +89_ &UU^-f 000000000000000000000000000000A5 BBBB5555AAAAAAAAAAAA4444FFFF6666DDDD1111BBBB66662222 +IjYZ+5}$cH 000000000000000000000000000000A6 EEEE555577779999666611110000FFFF0000AAAADDDD6666BBBB +HP5;m7ydP, 000000000000000000000000000000A7 DDDDDDDDEEEE222255559999DDDDEEEE7777AAAAFFFFDDDD8888 +(\+CmQF;sh 000000000000000000000000000000A8 7777BBBB9999333377779999AAAAAAAACCCC1111555533339999 +`27}+u%$Z' 000000000000000000000000000000A9 22228888EEEE555511117777DDDD555522229999EEEE5555EEEE +D/XZpkvB{U 000000000000000000000000000000AA 8888BBBB77776666EEEEEEEE00007777BBBB2222333355557777 +6uxeaoldr{ 000000000000000000000000000000AB BBBB11110000CCCC8888222277779999AAAA4444000077774444 +3n glBr={a 000000000000000000000000000000AC EEEE99992222666644446666FFFF11112222FFFF777744445555 +M3Bn!0ePh9 000000000000000000000000000000AD 666611114444999933332222FFFFDDDD5555111133339999AAAA +^:W0]D07,s 000000000000000000000000000000AE 99995555EEEEEEEE33330000AAAA1111CCCC6666444488883333 +R%]??oXCf( 000000000000000000000000000000AF CCCCFFFF00004444222299997777CCCC11111111999955550000 +?jh!g'(/dI 000000000000000000000000000000B0 99994444FFFFBBBBFFFF777711113333EEEEDDDD222299991111 +,qy-T(O~$} 000000000000000000000000000000B1 44446666888800007777DDDDEEEEFFFF22223333777711116666 +g~]6jo( dQ 000000000000000000000000000000B2 DDDD888822224444EEEE88887777EEEE00001111CCCCEEEEFFFF +\1]of=Ah$q 000000000000000000000000000000B3 EEEEAAAACCCCCCCC77779999AAAACCCC9999999966666666CCCC +5,vYEm^\di 000000000000000000000000000000B4 1111EEEE3333EEEEBBBB8888111111111111777733331111DDDD +}uX` 000000000000000000000000000000D1 7777DDDD77778888BBBB8888EEEEAAAA222277773333FFFF6666 +3pU8_.#1JU 000000000000000000000000000000D2 0000777799995555FFFFDDDDFFFF5555FFFFCCCC55554444FFFF +IW,e"{Tnvp 000000000000000000000000000000D3 99995555777755556666AAAA88887777EEEE333388884444CCCC +:=XWS2sImq 000000000000000000000000000000D4 AAAACCCCBBBBAAAA0000EEEE4444CCCCEEEE555544447777DDDD +o8Gcpop:! 000000000000000000000000000000D5 AAAAFFFFDDDD11115555000077778888999900003333BBBB2222 +W-qOhMjDFd 000000000000000000000000000000D6 0000222266660000BBBB55551111777733332222AAAAFFFFBBBB +u?;I>\L_vn 000000000000000000000000000000D7 99991111FFFFFFFF5555EEEE44447777BBBBEEEE7777AAAA8888 +Nyp X=m$h9 000000000000000000000000000000D8 0000EEEE444433333333BBBB8888000011110000000044449999 +g.FN&efgUs 000000000000000000000000000000D9 7777444488885555FFFF3333FFFFDDDD777755559999AAAAEEEE +rhC::yPt=z 000000000000000000000000000000DA 3333666633336666BBBBCCCCEEEE000000007777FFFFEEEE7777 +De9lt=P!{. 000000000000000000000000000000DB DDDD2222BBBB33335555888866663333EEEE9999333344444444 +XG(8.GC,t\ 000000000000000000000000000000DC 7777CCCCEEEE7777222200006666BBBB11112222999955555555 +U;azR 1t!\wV 000000000000000000000000000000E0 2222EEEE00000000FFFFBBBB6666555566665555BBBBAAAA1111 +'-zF>A2f5c 000000000000000000000000000000E1 CCCC999977773333111144444444BBBBFFFF8888888866666666 +^qaslc^dJX 000000000000000000000000000000E2 444411118888AAAA7777CCCC000011119999EEEE77777777FFFF +@gU`)^H:z5 000000000000000000000000000000E3 AAAA33331111FFFF888822225555CCCC33338888FFFF3333CCCC + l;w?EBj0 000000000000000000000000000000E4 BBBBDDDD111188888888AAAA111144449999666633332222DDDD +Iw&)MuUaM, 000000000000000000000000000000E5 5555FFFF00009999888811119999111111111111999922222222 +aGam)y ~t- 000000000000000000000000000000E6 EEEE22228888BBBB999977774444FFFFAAAA999922222222BBBB ++/Yg#6Nf/r 000000000000000000000000000000E7 FFFFBBBB6666CCCC66669999FFFF66662222DDDD777799998888 +SD ,=Yy%.w 000000000000000000000000000000E8 666644446666DDDDDDDD44446666111100004444BBBBFFFF9999 +/e)%&bgl}E 000000000000000000000000000000E9 AAAA6666333388887777DDDDCCCCFFFF1111EEEE00001111EEEE +8mdOTG)j6O 000000000000000000000000000000EA BBBB7777AAAABBBB888855554444999966665555CCCC11117777 +Yp=]xMdlAGG 000000000000000000000000000000F6 1111DDDD555500008888CCCC222288884444EEEEDDDD5555BBBB +X\!l_zyMKz% 00000000000000000000000000000014 99997777777722222222999911116666DDDDBBBBCCCC3333DDDD +kXrG^ech1/ 00000000000000000000000000000015 FFFF2222AAAA66660000AAAA9999333322227777AAAA77772222 +v%^RG^ci[X 00000000000000000000000000000016 7777BBBBFFFF999922229999DDDD3333CCCCAAAACCCCBBBBBBBB +vi@>8jMaYX 00000000000000000000000000000017 CCCCFFFFFFFF6666AAAA999999997777AAAA6666000066668888 +}dA7*<6[lx 00000000000000000000000000000018 6666FFFF77775555CCCC88881111AAAA00000000CCCC00009999 +>Y$Kc)9DOf 00000000000000000000000000000019 AAAAEEEEEEEE222266660000AAAA00000000000044446666EEEE +uF?T;*:x0+ 0000000000000000000000000000001A 444466665555BBBB8888BBBB00000000BBBB00005555AAAA7777 +~%.@@,3>7+ 0000000000000000000000000000001B FFFFDDDD333399999999666644449999BBBB2222000000004444 +JPrAKc{W>j 0000000000000000000000000000001C 222211111111333355554444DDDDEEEE33339999999911115555 +*i@lF30|aO 0000000000000000000000000000001D CCCC88880000FFFFCCCCDDDDBBBB4444EEEE99990000AAAAAAAA +9UH>%FZbUK 0000000000000000000000000000001E DDDD00005555EEEE111199998888CCCC888877779999DDDD3333 +-^1~=aYZC1 0000000000000000000000000000001F 11112222555599996666111144445555DDDDBBBB7777EEEE0000 +.\KqT&Z4i4 00000000000000000000000000000020 3333DDDDCCCC888888885555222299993333EEEEFFFF66661111 +9RtsPa gIN 00000000000000000000000000000021 555544440000AAAACCCC8888AAAA55552222DDDDBBBB22226666 +Fqy~]OATYw 00000000000000000000000000000022 9999CCCC0000888888884444333355553333BBBB55553333FFFF +Wj-%?e*2y\ 00000000000000000000000000000023 AAAA444477776666FFFF444455550000EEEE88883333FFFFCCCC +Rv]QHo9f6+ 00000000000000000000000000000024 99996666BBBB7777444466660000AAAADDDDDDDDAAAAEEEEDDDD +1 e!og=!hi 00000000000000000000000000000025 3333EEEE88887777CCCCEEEE777733337777DDDDFFFFEEEE2222 +N,gk>37c ( 00000000000000000000000000000026 DDDD11116666DDDD5555EEEE99996666AAAAAAAA3333EEEEBBBB +s.72Y.uzRN 00000000000000000000000000000027 0000CCCCFFFFAAAAAAAA11117777AAAA66662222000055558888 +j*0`gczW{u 00000000000000000000000000000028 9999DDDDCCCC8888AAAA999911111111BBBB66667777BBBB9999 +R+Sh=Kgd.6 00000000000000000000000000000029 DDDD0000BBBB77770000222200009999AAAADDDDAAAADDDDEEEE +x]w]=Dp6\s 0000000000000000000000000000002A 6666FFFFEEEEBBBBCCCC000033331111666677771111DDDD7777 +maU!Uo]He& 0000000000000000000000000000002B AAAA9999BBBB55557777CCCC6666EEEE555566668888FFFF4444 +smN$i pU-~ 0000000000000000000000000000002C AAAADDDDEEEE333388885555DDDD3333FFFFEEEE1111CCCC5555 +T}# `9Cseu 0000000000000000000000000000002D 4444FFFF777700001111EEEE999944447777444488881111AAAA +G2oMVHVK`! 0000000000000000000000000000002E 4444888844448888CCCC1111AAAA555577778888BBBB00003333 +(tv1'>|,s[ 0000000000000000000000000000002F 0000CCCC222277778888FFFFCCCCBBBB111177779999DDDD0000 +RE%Z{o)9E: 00000000000000000000000000000030 00002222AAAAEEEE444400008888CCCC3333DDDD555511111111 +'%:YY?@b+M 00000000000000000000000000000031 9999777777770000AAAA0000444433332222EEEE333399996666 +N(JuG%.`7c 00000000000000000000000000000032 66665555AAAA5555BBBBAAAAEEEE111111119999BBBB6666FFFF +`E$,xR{?Kc 00000000000000000000000000000033 7777AAAA222200008888CCCC3333AAAAFFFFAAAAEEEEEEEECCCC +jzu$Fn3ZN+ 00000000000000000000000000000034 55552222EEEE5555BBBBAAAA8888AAAA11110000DDDD9999DDDD +|xaPO\~!V* 00000000000000000000000000000035 EEEE2222EEEEFFFFEEEEBBBB2222BBBB9999AAAA999955552222 +GCU4gfl|&! 00000000000000000000000000000036 1111DDDD8888AAAA999999990000333377778888FFFF1111BBBB +JU9s03v&}o 00000000000000000000000000000037 7777555500001111DDDD3333111122221111AAAA444444448888 +D@v9""8"!` 00000000000000000000000000000038 2222BBBB55552222CCCC555533336666EEEE9999777766669999 +OffB8(#k@N 00000000000000000000000000000039 CCCC555555556666DDDDCCCC111111119999DDDD55554444EEEE +-_) R'4T<" 0000000000000000000000000000003A BBBBCCCCEEEECCCCDDDD6666BBBB333377779999222200007777 +'z4P|n>\to 0000000000000000000000000000003B DDDD5555AAAA3333CCCC222233330000111133335555EEEE4444 +[!Nbx&X=+V 0000000000000000000000000000003C 77778888AAAA7777888844446666EEEECCCCBBBBEEEE77775555 +p?"\\6:W[' 0000000000000000000000000000003D 8888EEEE6666555599992222DDDD2222FFFFEEEE33338888AAAA ++2A^xP5dA- 0000000000000000000000000000003E CCCCBBBBFFFFFFFF99995555AAAAAAAA77771111000033333333 +-cuJvy7;=h 0000000000000000000000000000003F 99993333CCCC8888EEEEAAAAEEEE5555CCCC7777FFFFCCCC0000 +ix^TQ-ma5o 00000000000000000000000000000047 88885555CCCC88889999888811110000BBBBEEEECCCC33338888 +E(g$5-WpBC 00000000000000000000000000000048 BBBB7777EEEE4444888866661111FFFF44449999BBBB11119999 +~\ptsn1L{/ 00000000000000000000000000000049 CCCC0000FFFFCCCC7777BBBB333300003333FFFF3333BBBBEEEE +YhlPk`q6SN 0000000000000000000000000000004A 00001111AAAABBBBAAAA66669999DDDD11116666666633337777 +WBzul,K** 0000000000000000000000000000004B 4444AAAADDDDEEEE33337777AAAA9999DDDD88886666DDDD4444 +q)epJsSLXU 0000000000000000000000000000004C 88887777999955558888AAAA9999111155552222FFFF22225555 +`WVkW&( /[ 0000000000000000000000000000004D FFFF999922221111DDDD9999AAAA9999FFFF66662222FFFFAAAA +HXg#Z^7@7/ 0000000000000000000000000000004E 777799999999DDDD44441111BBBB6666AAAA0000999966663333 +-&8$lcXS$; 0000000000000000000000000000004F 333300004444AAAA3333666600009999DDDDCCCC9999BBBB0000 +^SpqwoVET; 00000000000000000000000000000050 BBBB77775555CCCC1111DDDD2222BBBB66669999CCCC77771111 +P-6Ybw3=a| 00000000000000000000000000000051 2222FFFFDDDDAAAACCCC3333EEEEBBBB8888EEEE000077776666 +a`:;ydJv)Y 00000000000000000000000000000052 55554444AAAAAAAA6666DDDD7777AAAAEEEE00003333CCCCFFFF +dO]G~-YGFJ 00000000000000000000000000000053 8888555533333333EEEE11113333AAAA444411110000CCCCCCCC +WM^;rvO/u] 00000000000000000000000000000054 BBBB00007777999977772222AAAA0000CCCCAAAAEEEEFFFFDDDD +-WwU4BF*P< 00000000000000000000000000000055 BBBB8888EEEEEEEE55553333AAAA4444EEEE6666888833332222 +y'w/PZ!x"> 00000000000000000000000000000056 DDDD55554444AAAAAAAA444455553333AAAA444411117777BBBB +#jA`l-f@*f 00000000000000000000000000000057 4444AAAA2222AAAAEEEE55556666CCCC3333FFFF888822228888 +:!W.=v,f"z 00000000000000000000000000000058 444422223333666688886666BBBB4444888877772222CCCC9999 +"!r4ugwZcz 00000000000000000000000000000059 22226666CCCCEEEE6666FFFF888822220000333366662222EEEE +O?(|_dXq:X 0000000000000000000000000000005C 2222FFFFBBBB55552222444433332222444433333333DDDD5555 +(cb621Uv~( 0000000000000000000000000000005D 3333BBBB11110000FFFF9999BBBB6666CCCCCCCC66666666AAAA +hg5x s23L3 0000000000000000000000000000005E 22221111AAAA000011117777DDDD999933337777666699993333 +w5nn`wa'[p 0000000000000000000000000000005F 000099997777CCCC666677776666FFFF444455557777AAAA0000 +qeQNoB 2XG 00000000000000000000000000000060 0000EEEE66668888FFFF2222FFFFAAAAFFFF7777EEEE22221111 +6n>iMbi=gw 00000000000000000000000000000061 4444DDDDFFFF9999DDDD9999EEEEEEEECCCCDDDD4444EEEE6666 +uGz7S=#fh: 00000000000000000000000000000062 77771111DDDD222200000000BBBBDDDD222288885555FFFFFFFF +rd)x<>@@H| 00000000000000000000000000000063 99996666AAAABBBB7777CCCC4444DDDD555544447777BBBBCCCC +xEwa^:84K] 00000000000000000000000000000064 000099995555CCCC00000000EEEE5555BBBB1111DDDDAAAADDDD +':]$}b:Z$a 00000000000000000000000000000065 77775555DDDD4444DDDDAAAA66660000FFFF5555DDDDAAAA2222 +vZl"Ji^AO@ 00000000000000000000000000000066 3333777744444444EEEEEEEE11115555666600008888AAAABBBB +o%,!fSdRBa 00000000000000000000000000000067 11119999DDDDAAAA8888AAAA5555EEEE8888CCCC888811118888 +*+o@+t%u&; 00000000000000000000000000000068 22228888999922228888EEEE3333444455551111EEEE77779999 +(CoKUhfB9{ 00000000000000000000000000000069 1111BBBB9999CCCCFFFF3333CCCC777755559999CCCC9999EEEE +I>:9JQb_5T 0000000000000000000000000000006A AAAA66662222CCCCBBBB666600003333BBBB2222AAAA99997777 +eSzU!X,[%/ 0000000000000000000000000000006B AAAACCCC9999FFFFFFFFAAAA444455553333BBBB4444BBBB4444 +[QZ{a&TaCS 0000000000000000000000000000006C AAAA8888CCCC2222FFFFAAAA5555CCCC4444CCCCCCCC88885555 +?:Yp4%.K9n 0000000000000000000000000000006D BBBBCCCC5555111100007777FFFFAAAAFFFF0000DDDDDDDDAAAA +'_>/ 4v}8I 0000000000000000000000000000006E 1111EEEE000088886666333311115555666655557777CCCC3333 +KrNW.!5MP; 0000000000000000000000000000006F 2222AAAAEEEE33330000777711111111FFFF2222999999990000 +j17a3a#bH? 00000000000000000000000000000070 8888CCCCBBBBCCCC88881111EEEE33336666BBBB3333DDDD1111 +)o-QLHy(UO 00000000000000000000000000000071 3333BBBB444400002222EEEEFFFF333355556666DDDD55556666 +al;:OEKSH2 00000000000000000000000000000072 DDDDEEEE3333DDDDBBBB7777BBBB1111FFFF3333CCCC2222FFFF +I@L*`HMPfN 00000000000000000000000000000073 DDDD666699993333999977779999DDDD444488882222AAAACCCC +$kBiG 00000000000000000000000000000090 3333DDDD000044445555444444441111CCCC1111BBBB33331111 +V*W% x\&'q 00000000000000000000000000000091 8888DDDDBBBBAAAABBBB99997777EEEE00008888AAAA33336666 +0Jw^F-}sZr 00000000000000000000000000000092 4444BBBBDDDD2222FFFF888811117777DDDD444444448888FFFF +OLf_u|1d}9 00000000000000000000000000000093 DDDD1111AAAA9999FFFFDDDD9999BBBB9999000044448888CCCC +%V*"P*(/Ez 00000000000000000000000000000094 11110000CCCC666600002222CCCCEEEECCCCEEEE1111BBBBDDDD +jGdC5%yw;# 00000000000000000000000000000095 44449999FFFF00004444222222228888000099995555FFFF2222 +,uSW`H/pXh 00000000000000000000000000000096 8888BBBB111188885555CCCC7777BBBB7777111166663333BBBB +n],.INMkcA 00000000000000000000000000000097 DDDD77777777DDDD99993333FFFFDDDD9999CCCCFFFFEEEE8888 +(P=jd[R]&[ 00000000000000000000000000000098 BBBBAAAA99995555CCCC666688887777EEEE1111999988889999 +*%pAD!6FlS 00000000000000000000000000000099 AAAA777799994444DDDDEEEE6666DDDD2222AAAA7777EEEEEEEE +;u0nwOR{|e 0000000000000000000000000000009A CCCC5555FFFF1111000066667777EEEEEEEE1111777722227777 +X*)4\|>`b. 0000000000000000000000000000009B EEEE33339999CCCC2222999999999999FFFFDDDD777788884444 +#Yh\5HjDLp 0000000000000000000000000000009C 33338888EEEEFFFFDDDDDDDDAAAA7777EEEE0000EEEE99995555 +5Yh?d9@6(_ 0000000000000000000000000000009D 66667777FFFF333377774444FFFFDDDD99993333CCCC2222AAAA +W;mKiM[qVV 0000000000000000000000000000009E 111133334444AAAA55556666CCCCFFFF5555BBBB333355553333 +287P2iA6U: 0000000000000000000000000000009F 88880000FFFF000022220000BBBBCCCCFFFF3333777766660000 +Bj`e'4&VbD 000000000000000000000000000000A0 DDDD888811110000BBBB2222EEEEBBBB00004444CCCCEEEE1111 +Q[7 0VqTg~ 000000000000000000000000000000A1 8888666622222222222222221111DDDD11110000EEEEAAAA6666 +4Y8,(.X0:" 000000000000000000000000000000A2 DDDD7777DDDD5555AAAAEEEE7777DDDD444499996666BBBBFFFF +w)98*ZrTi* 000000000000000000000000000000A3 CCCC4444FFFF0000BBBBAAAA66667777CCCC4444BBBB7777CCCC +pHH7F#q@Tf 000000000000000000000000000000A4 0000FFFFEEEE9999DDDD9999AAAAFFFF9999AAAA00006666DDDD +89_ &UU^-f 000000000000000000000000000000A5 BBBB5555AAAAAAAAAAAA4444FFFF6666DDDD1111BBBB66662222 +IjYZ+5}$cH 000000000000000000000000000000A6 EEEE555577779999666611110000FFFF0000AAAADDDD6666BBBB +HP5;m7ydP, 000000000000000000000000000000A7 DDDDDDDDEEEE222255559999DDDDEEEE7777AAAAFFFFDDDD8888 +(\+CmQF;sh 000000000000000000000000000000A8 7777BBBB9999333377779999AAAAAAAACCCC1111555533339999 +`27}+u%$Z' 000000000000000000000000000000A9 22228888EEEE555511117777DDDD555522229999EEEE5555EEEE +D/XZpkvB{U 000000000000000000000000000000AA 8888BBBB77776666EEEEEEEE00007777BBBB2222333355557777 +6uxeaoldr{ 000000000000000000000000000000AB BBBB11110000CCCC8888222277779999AAAA4444000077774444 +3n glBr={a 000000000000000000000000000000AC EEEE99992222666644446666FFFF11112222FFFF777744445555 +M3Bn!0ePh9 000000000000000000000000000000AD 666611114444999933332222FFFFDDDD5555111133339999AAAA +^:W0]D07,s 000000000000000000000000000000AE 99995555EEEEEEEE33330000AAAA1111CCCC6666444488883333 +R%]??oXCf( 000000000000000000000000000000AF CCCCFFFF00004444222299997777CCCC11111111999955550000 +?jh!g'(/dI 000000000000000000000000000000B0 99994444FFFFBBBBFFFF777711113333EEEEDDDD222299991111 +,qy-T(O~$} 000000000000000000000000000000B1 44446666888800007777DDDDEEEEFFFF22223333777711116666 +g~]6jo( dQ 000000000000000000000000000000B2 DDDD888822224444EEEE88887777EEEE00001111CCCCEEEEFFFF +\1]of=Ah$q 000000000000000000000000000000B3 EEEEAAAACCCCCCCC77779999AAAACCCC9999999966666666CCCC +5,vYEm^\di 000000000000000000000000000000B4 1111EEEE3333EEEEBBBB8888111111111111777733331111DDDD +}uX` 000000000000000000000000000000D1 7777DDDD77778888BBBB8888EEEEAAAA222277773333FFFF6666 +3pU8_.#1JU 000000000000000000000000000000D2 0000777799995555FFFFDDDDFFFF5555FFFFCCCC55554444FFFF +IW,e"{Tnvp 000000000000000000000000000000D3 99995555777755556666AAAA88887777EEEE333388884444CCCC +:=XWS2sImq 000000000000000000000000000000D4 AAAACCCCBBBBAAAA0000EEEE4444CCCCEEEE555544447777DDDD +o8Gcpop:! 000000000000000000000000000000D5 AAAAFFFFDDDD11115555000077778888999900003333BBBB2222 +W-qOhMjDFd 000000000000000000000000000000D6 0000222266660000BBBB55551111777733332222AAAAFFFFBBBB +u?;I>\L_vn 000000000000000000000000000000D7 99991111FFFFFFFF5555EEEE44447777BBBBEEEE7777AAAA8888 +Nyp X=m$h9 000000000000000000000000000000D8 0000EEEE444433333333BBBB8888000011110000000044449999 +g.FN&efgUs 000000000000000000000000000000D9 7777444488885555FFFF3333FFFFDDDD777755559999AAAAEEEE +rhC::yPt=z 000000000000000000000000000000DA 3333666633336666BBBBCCCCEEEE000000007777FFFFEEEE7777 +De9lt=P!{. 000000000000000000000000000000DB DDDD2222BBBB33335555888866663333EEEE9999333344444444 +XG(8.GC,t\ 000000000000000000000000000000DC 7777CCCCEEEE7777222200006666BBBB11112222999955555555 +U;azR 1t!\wV 000000000000000000000000000000E0 2222EEEE00000000FFFFBBBB6666555566665555BBBBAAAA1111 +'-zF>A2f5c 000000000000000000000000000000E1 CCCC999977773333111144444444BBBBFFFF8888888866666666 +^qaslc^dJX 000000000000000000000000000000E2 444411118888AAAA7777CCCC000011119999EEEE77777777FFFF +@gU`)^H:z5 000000000000000000000000000000E3 AAAA33331111FFFF888822225555CCCC33338888FFFF3333CCCC + l;w?EBj0 000000000000000000000000000000E4 BBBBDDDD111188888888AAAA111144449999666633332222DDDD +Iw&)MuUaM, 000000000000000000000000000000E5 5555FFFF00009999888811119999111111111111999922222222 +aGam)y ~t- 000000000000000000000000000000E6 EEEE22228888BBBB999977774444FFFFAAAA999922222222BBBB ++/Yg#6Nf/r 000000000000000000000000000000E7 FFFFBBBB6666CCCC66669999FFFF66662222DDDD777799998888 +SD ,=Yy%.w 000000000000000000000000000000E8 666644446666DDDDDDDD44446666111100004444BBBBFFFF9999 +/e)%&bgl}E 000000000000000000000000000000E9 AAAA6666333388887777DDDDCCCCFFFF1111EEEE00001111EEEE +8mdOTG)j6O 000000000000000000000000000000EA BBBB7777AAAABBBB888855554444999966665555CCCC11117777 +Yp=]xMdlAGG 000000000000000000000000000000F6 1111DDDD555500008888CCCC222288884444EEEEDDDD5555BBBB +X\!l_zyMKz% 00000000000000000000000000000014 99997777777722222222999911116666DDDDBBBBCCCC3333DDDD +kXrG^ech1/ 00000000000000000000000000000015 FFFF2222AAAA66660000AAAA9999333322227777AAAA77772222 +v%^RG^ci[X 00000000000000000000000000000016 7777BBBBFFFF999922229999DDDD3333CCCCAAAACCCCBBBBBBBB +vi@>8jMaYX 00000000000000000000000000000017 CCCCFFFFFFFF6666AAAA999999997777AAAA6666000066668888 +}dA7*<6[lx 00000000000000000000000000000018 6666FFFF77775555CCCC88881111AAAA00000000CCCC00009999 +>Y$Kc)9DOf 00000000000000000000000000000019 AAAAEEEEEEEE222266660000AAAA00000000000044446666EEEE +uF?T;*:x0+ 0000000000000000000000000000001A 444466665555BBBB8888BBBB00000000BBBB00005555AAAA7777 +~%.@@,3>7+ 0000000000000000000000000000001B FFFFDDDD333399999999666644449999BBBB2222000000004444 +JPrAKc{W>j 0000000000000000000000000000001C 222211111111333355554444DDDDEEEE33339999999911115555 +*i@lF30|aO 0000000000000000000000000000001D CCCC88880000FFFFCCCCDDDDBBBB4444EEEE99990000AAAAAAAA +9UH>%FZbUK 0000000000000000000000000000001E DDDD00005555EEEE111199998888CCCC888877779999DDDD3333 +-^1~=aYZC1 0000000000000000000000000000001F 11112222555599996666111144445555DDDDBBBB7777EEEE0000 +.\KqT&Z4i4 00000000000000000000000000000020 3333DDDDCCCC888888885555222299993333EEEEFFFF66661111 +9RtsPa gIN 00000000000000000000000000000021 555544440000AAAACCCC8888AAAA55552222DDDDBBBB22226666 +Fqy~]OATYw 00000000000000000000000000000022 9999CCCC0000888888884444333355553333BBBB55553333FFFF +Wj-%?e*2y\ 00000000000000000000000000000023 AAAA444477776666FFFF444455550000EEEE88883333FFFFCCCC +Rv]QHo9f6+ 00000000000000000000000000000024 99996666BBBB7777444466660000AAAADDDDDDDDAAAAEEEEDDDD +1 e!og=!hi 00000000000000000000000000000025 3333EEEE88887777CCCCEEEE777733337777DDDDFFFFEEEE2222 +N,gk>37c ( 00000000000000000000000000000026 DDDD11116666DDDD5555EEEE99996666AAAAAAAA3333EEEEBBBB +s.72Y.uzRN 00000000000000000000000000000027 0000CCCCFFFFAAAAAAAA11117777AAAA66662222000055558888 +j*0`gczW{u 00000000000000000000000000000028 9999DDDDCCCC8888AAAA999911111111BBBB66667777BBBB9999 +R+Sh=Kgd.6 00000000000000000000000000000029 DDDD0000BBBB77770000222200009999AAAADDDDAAAADDDDEEEE +x]w]=Dp6\s 0000000000000000000000000000002A 6666FFFFEEEEBBBBCCCC000033331111666677771111DDDD7777 +maU!Uo]He& 0000000000000000000000000000002B AAAA9999BBBB55557777CCCC6666EEEE555566668888FFFF4444 +smN$i pU-~ 0000000000000000000000000000002C AAAADDDDEEEE333388885555DDDD3333FFFFEEEE1111CCCC5555 +T}# `9Cseu 0000000000000000000000000000002D 4444FFFF777700001111EEEE999944447777444488881111AAAA +G2oMVHVK`! 0000000000000000000000000000002E 4444888844448888CCCC1111AAAA555577778888BBBB00003333 +(tv1'>|,s[ 0000000000000000000000000000002F 0000CCCC222277778888FFFFCCCCBBBB111177779999DDDD0000 +RE%Z{o)9E: 00000000000000000000000000000030 00002222AAAAEEEE444400008888CCCC3333DDDD555511111111 +'%:YY?@b+M 00000000000000000000000000000031 9999777777770000AAAA0000444433332222EEEE333399996666 +N(JuG%.`7c 00000000000000000000000000000032 66665555AAAA5555BBBBAAAAEEEE111111119999BBBB6666FFFF +`E$,xR{?Kc 00000000000000000000000000000033 7777AAAA222200008888CCCC3333AAAAFFFFAAAAEEEEEEEECCCC +jzu$Fn3ZN+ 00000000000000000000000000000034 55552222EEEE5555BBBBAAAA8888AAAA11110000DDDD9999DDDD +|xaPO\~!V* 00000000000000000000000000000035 EEEE2222EEEEFFFFEEEEBBBB2222BBBB9999AAAA999955552222 +GCU4gfl|&! 00000000000000000000000000000036 1111DDDD8888AAAA999999990000333377778888FFFF1111BBBB +JU9s03v&}o 00000000000000000000000000000037 7777555500001111DDDD3333111122221111AAAA444444448888 +D@v9""8"!` 00000000000000000000000000000038 2222BBBB55552222CCCC555533336666EEEE9999777766669999 +OffB8(#k@N 00000000000000000000000000000039 CCCC555555556666DDDDCCCC111111119999DDDD55554444EEEE +-_) R'4T<" 0000000000000000000000000000003A BBBBCCCCEEEECCCCDDDD6666BBBB333377779999222200007777 +'z4P|n>\to 0000000000000000000000000000003B DDDD5555AAAA3333CCCC222233330000111133335555EEEE4444 +[!Nbx&X=+V 0000000000000000000000000000003C 77778888AAAA7777888844446666EEEECCCCBBBBEEEE77775555 +p?"\\6:W[' 0000000000000000000000000000003D 8888EEEE6666555599992222DDDD2222FFFFEEEE33338888AAAA ++2A^xP5dA- 0000000000000000000000000000003E CCCCBBBBFFFFFFFF99995555AAAAAAAA77771111000033333333 +-cuJvy7;=h 0000000000000000000000000000003F 99993333CCCC8888EEEEAAAAEEEE5555CCCC7777FFFFCCCC0000 +ix^TQ-ma5o 00000000000000000000000000000047 88885555CCCC88889999888811110000BBBBEEEECCCC33338888 +E(g$5-WpBC 00000000000000000000000000000048 BBBB7777EEEE4444888866661111FFFF44449999BBBB11119999 +~\ptsn1L{/ 00000000000000000000000000000049 CCCC0000FFFFCCCC7777BBBB333300003333FFFF3333BBBBEEEE +YhlPk`q6SN 0000000000000000000000000000004A 00001111AAAABBBBAAAA66669999DDDD11116666666633337777 +WBzul,K** 0000000000000000000000000000004B 4444AAAADDDDEEEE33337777AAAA9999DDDD88886666DDDD4444 +q)epJsSLXU 0000000000000000000000000000004C 88887777999955558888AAAA9999111155552222FFFF22225555 +`WVkW&( /[ 0000000000000000000000000000004D FFFF999922221111DDDD9999AAAA9999FFFF66662222FFFFAAAA +HXg#Z^7@7/ 0000000000000000000000000000004E 777799999999DDDD44441111BBBB6666AAAA0000999966663333 +-&8$lcXS$; 0000000000000000000000000000004F 333300004444AAAA3333666600009999DDDDCCCC9999BBBB0000 +^SpqwoVET; 00000000000000000000000000000050 BBBB77775555CCCC1111DDDD2222BBBB66669999CCCC77771111 +P-6Ybw3=a| 00000000000000000000000000000051 2222FFFFDDDDAAAACCCC3333EEEEBBBB8888EEEE000077776666 +a`:;ydJv)Y 00000000000000000000000000000052 55554444AAAAAAAA6666DDDD7777AAAAEEEE00003333CCCCFFFF +dO]G~-YGFJ 00000000000000000000000000000053 8888555533333333EEEE11113333AAAA444411110000CCCCCCCC +WM^;rvO/u] 00000000000000000000000000000054 BBBB00007777999977772222AAAA0000CCCCAAAAEEEEFFFFDDDD +-WwU4BF*P< 00000000000000000000000000000055 BBBB8888EEEEEEEE55553333AAAA4444EEEE6666888833332222 +y'w/PZ!x"> 00000000000000000000000000000056 DDDD55554444AAAAAAAA444455553333AAAA444411117777BBBB +#jA`l-f@*f 00000000000000000000000000000057 4444AAAA2222AAAAEEEE55556666CCCC3333FFFF888822228888 +:!W.=v,f"z 00000000000000000000000000000058 444422223333666688886666BBBB4444888877772222CCCC9999 +"!r4ugwZcz 00000000000000000000000000000059 22226666CCCCEEEE6666FFFF888822220000333366662222EEEE +O?(|_dXq:X 0000000000000000000000000000005C 2222FFFFBBBB55552222444433332222444433333333DDDD5555 +(cb621Uv~( 0000000000000000000000000000005D 3333BBBB11110000FFFF9999BBBB6666CCCCCCCC66666666AAAA +hg5x s23L3 0000000000000000000000000000005E 22221111AAAA000011117777DDDD999933337777666699993333 +w5nn`wa'[p 0000000000000000000000000000005F 000099997777CCCC666677776666FFFF444455557777AAAA0000 +qeQNoB 2XG 00000000000000000000000000000060 0000EEEE66668888FFFF2222FFFFAAAAFFFF7777EEEE22221111 +6n>iMbi=gw 00000000000000000000000000000061 4444DDDDFFFF9999DDDD9999EEEEEEEECCCCDDDD4444EEEE6666 +uGz7S=#fh: 00000000000000000000000000000062 77771111DDDD222200000000BBBBDDDD222288885555FFFFFFFF +rd)x<>@@H| 00000000000000000000000000000063 99996666AAAABBBB7777CCCC4444DDDD555544447777BBBBCCCC +xEwa^:84K] 00000000000000000000000000000064 000099995555CCCC00000000EEEE5555BBBB1111DDDDAAAADDDD +':]$}b:Z$a 00000000000000000000000000000065 77775555DDDD4444DDDDAAAA66660000FFFF5555DDDDAAAA2222 +vZl"Ji^AO@ 00000000000000000000000000000066 3333777744444444EEEEEEEE11115555666600008888AAAABBBB +o%,!fSdRBa 00000000000000000000000000000067 11119999DDDDAAAA8888AAAA5555EEEE8888CCCC888811118888 +*+o@+t%u&; 00000000000000000000000000000068 22228888999922228888EEEE3333444455551111EEEE77779999 +(CoKUhfB9{ 00000000000000000000000000000069 1111BBBB9999CCCCFFFF3333CCCC777755559999CCCC9999EEEE +I>:9JQb_5T 0000000000000000000000000000006A AAAA66662222CCCCBBBB666600003333BBBB2222AAAA99997777 +eSzU!X,[%/ 0000000000000000000000000000006B AAAACCCC9999FFFFFFFFAAAA444455553333BBBB4444BBBB4444 +[QZ{a&TaCS 0000000000000000000000000000006C AAAA8888CCCC2222FFFFAAAA5555CCCC4444CCCCCCCC88885555 +?:Yp4%.K9n 0000000000000000000000000000006D BBBBCCCC5555111100007777FFFFAAAAFFFF0000DDDDDDDDAAAA +'_>/ 4v}8I 0000000000000000000000000000006E 1111EEEE000088886666333311115555666655557777CCCC3333 +KrNW.!5MP; 0000000000000000000000000000006F 2222AAAAEEEE33330000777711111111FFFF2222999999990000 +j17a3a#bH? 00000000000000000000000000000070 8888CCCCBBBBCCCC88881111EEEE33336666BBBB3333DDDD1111 +)o-QLHy(UO 00000000000000000000000000000071 3333BBBB444400002222EEEEFFFF333355556666DDDD55556666 +al;:OEKSH2 00000000000000000000000000000072 DDDDEEEE3333DDDDBBBB7777BBBB1111FFFF3333CCCC2222FFFF +I@L*`HMPfN 00000000000000000000000000000073 DDDD666699993333999977779999DDDD444488882222AAAACCCC +$kBiG 00000000000000000000000000000090 3333DDDD000044445555444444441111CCCC1111BBBB33331111 +V*W% x\&'q 00000000000000000000000000000091 8888DDDDBBBBAAAABBBB99997777EEEE00008888AAAA33336666 +0Jw^F-}sZr 00000000000000000000000000000092 4444BBBBDDDD2222FFFF888811117777DDDD444444448888FFFF +OLf_u|1d}9 00000000000000000000000000000093 DDDD1111AAAA9999FFFFDDDD9999BBBB9999000044448888CCCC +%V*"P*(/Ez 00000000000000000000000000000094 11110000CCCC666600002222CCCCEEEECCCCEEEE1111BBBBDDDD +jGdC5%yw;# 00000000000000000000000000000095 44449999FFFF00004444222222228888000099995555FFFF2222 +,uSW`H/pXh 00000000000000000000000000000096 8888BBBB111188885555CCCC7777BBBB7777111166663333BBBB +n],.INMkcA 00000000000000000000000000000097 DDDD77777777DDDD99993333FFFFDDDD9999CCCCFFFFEEEE8888 +(P=jd[R]&[ 00000000000000000000000000000098 BBBBAAAA99995555CCCC666688887777EEEE1111999988889999 +*%pAD!6FlS 00000000000000000000000000000099 AAAA777799994444DDDDEEEE6666DDDD2222AAAA7777EEEEEEEE +;u0nwOR{|e 0000000000000000000000000000009A CCCC5555FFFF1111000066667777EEEEEEEE1111777722227777 +X*)4\|>`b. 0000000000000000000000000000009B EEEE33339999CCCC2222999999999999FFFFDDDD777788884444 +#Yh\5HjDLp 0000000000000000000000000000009C 33338888EEEEFFFFDDDDDDDDAAAA7777EEEE0000EEEE99995555 +5Yh?d9@6(_ 0000000000000000000000000000009D 66667777FFFF333377774444FFFFDDDD99993333CCCC2222AAAA +W;mKiM[qVV 0000000000000000000000000000009E 111133334444AAAA55556666CCCCFFFF5555BBBB333355553333 +287P2iA6U: 0000000000000000000000000000009F 88880000FFFF000022220000BBBBCCCCFFFF3333777766660000 +Bj`e'4&VbD 000000000000000000000000000000A0 DDDD888811110000BBBB2222EEEEBBBB00004444CCCCEEEE1111 +Q[7 0VqTg~ 000000000000000000000000000000A1 8888666622222222222222221111DDDD11110000EEEEAAAA6666 +4Y8,(.X0:" 000000000000000000000000000000A2 DDDD7777DDDD5555AAAAEEEE7777DDDD444499996666BBBBFFFF +w)98*ZrTi* 000000000000000000000000000000A3 CCCC4444FFFF0000BBBBAAAA66667777CCCC4444BBBB7777CCCC +pHH7F#q@Tf 000000000000000000000000000000A4 0000FFFFEEEE9999DDDD9999AAAAFFFF9999AAAA00006666DDDD +89_ &UU^-f 000000000000000000000000000000A5 BBBB5555AAAAAAAAAAAA4444FFFF6666DDDD1111BBBB66662222 +IjYZ+5}$cH 000000000000000000000000000000A6 EEEE555577779999666611110000FFFF0000AAAADDDD6666BBBB +HP5;m7ydP, 000000000000000000000000000000A7 DDDDDDDDEEEE222255559999DDDDEEEE7777AAAAFFFFDDDD8888 +(\+CmQF;sh 000000000000000000000000000000A8 7777BBBB9999333377779999AAAAAAAACCCC1111555533339999 +`27}+u%$Z' 000000000000000000000000000000A9 22228888EEEE555511117777DDDD555522229999EEEE5555EEEE +D/XZpkvB{U 000000000000000000000000000000AA 8888BBBB77776666EEEEEEEE00007777BBBB2222333355557777 +6uxeaoldr{ 000000000000000000000000000000AB BBBB11110000CCCC8888222277779999AAAA4444000077774444 +3n glBr={a 000000000000000000000000000000AC EEEE99992222666644446666FFFF11112222FFFF777744445555 +M3Bn!0ePh9 000000000000000000000000000000AD 666611114444999933332222FFFFDDDD5555111133339999AAAA +^:W0]D07,s 000000000000000000000000000000AE 99995555EEEEEEEE33330000AAAA1111CCCC6666444488883333 +R%]??oXCf( 000000000000000000000000000000AF CCCCFFFF00004444222299997777CCCC11111111999955550000 +?jh!g'(/dI 000000000000000000000000000000B0 99994444FFFFBBBBFFFF777711113333EEEEDDDD222299991111 +,qy-T(O~$} 000000000000000000000000000000B1 44446666888800007777DDDDEEEEFFFF22223333777711116666 +g~]6jo( dQ 000000000000000000000000000000B2 DDDD888822224444EEEE88887777EEEE00001111CCCCEEEEFFFF +\1]of=Ah$q 000000000000000000000000000000B3 EEEEAAAACCCCCCCC77779999AAAACCCC9999999966666666CCCC +5,vYEm^\di 000000000000000000000000000000B4 1111EEEE3333EEEEBBBB8888111111111111777733331111DDDD +}uX` 000000000000000000000000000000D1 7777DDDD77778888BBBB8888EEEEAAAA222277773333FFFF6666 +3pU8_.#1JU 000000000000000000000000000000D2 0000777799995555FFFFDDDDFFFF5555FFFFCCCC55554444FFFF +IW,e"{Tnvp 000000000000000000000000000000D3 99995555777755556666AAAA88887777EEEE333388884444CCCC +:=XWS2sImq 000000000000000000000000000000D4 AAAACCCCBBBBAAAA0000EEEE4444CCCCEEEE555544447777DDDD +o8Gcpop:! 000000000000000000000000000000D5 AAAAFFFFDDDD11115555000077778888999900003333BBBB2222 +W-qOhMjDFd 000000000000000000000000000000D6 0000222266660000BBBB55551111777733332222AAAAFFFFBBBB +u?;I>\L_vn 000000000000000000000000000000D7 99991111FFFFFFFF5555EEEE44447777BBBBEEEE7777AAAA8888 +Nyp X=m$h9 000000000000000000000000000000D8 0000EEEE444433333333BBBB8888000011110000000044449999 +g.FN&efgUs 000000000000000000000000000000D9 7777444488885555FFFF3333FFFFDDDD777755559999AAAAEEEE +rhC::yPt=z 000000000000000000000000000000DA 3333666633336666BBBBCCCCEEEE000000007777FFFFEEEE7777 +De9lt=P!{. 000000000000000000000000000000DB DDDD2222BBBB33335555888866663333EEEE9999333344444444 +XG(8.GC,t\ 000000000000000000000000000000DC 7777CCCCEEEE7777222200006666BBBB11112222999955555555 +U;azR 1t!\wV 000000000000000000000000000000E0 2222EEEE00000000FFFFBBBB6666555566665555BBBBAAAA1111 +'-zF>A2f5c 000000000000000000000000000000E1 CCCC999977773333111144444444BBBBFFFF8888888866666666 +^qaslc^dJX 000000000000000000000000000000E2 444411118888AAAA7777CCCC000011119999EEEE77777777FFFF +@gU`)^H:z5 000000000000000000000000000000E3 AAAA33331111FFFF888822225555CCCC33338888FFFF3333CCCC + l;w?EBj0 000000000000000000000000000000E4 BBBBDDDD111188888888AAAA111144449999666633332222DDDD +Iw&)MuUaM, 000000000000000000000000000000E5 5555FFFF00009999888811119999111111111111999922222222 +aGam)y ~t- 000000000000000000000000000000E6 EEEE22228888BBBB999977774444FFFFAAAA999922222222BBBB ++/Yg#6Nf/r 000000000000000000000000000000E7 FFFFBBBB6666CCCC66669999FFFF66662222DDDD777799998888 +SD ,=Yy%.w 000000000000000000000000000000E8 666644446666DDDDDDDD44446666111100004444BBBBFFFF9999 +/e)%&bgl}E 000000000000000000000000000000E9 AAAA6666333388887777DDDDCCCCFFFF1111EEEE00001111EEEE +8mdOTG)j6O 000000000000000000000000000000EA BBBB7777AAAABBBB888855554444999966665555CCCC11117777 +Yp=]xMdlAGG 000000000000000000000000000000F6 1111DDDD555500008888CCCC222288884444EEEEDDDD5555BBBB +X\!l_zyMKz% 00000000000000000000000000000014 99997777777722222222999911116666DDDDBBBBCCCC3333DDDD +kXrG^ech1/ 00000000000000000000000000000015 FFFF2222AAAA66660000AAAA9999333322227777AAAA77772222 +v%^RG^ci[X 00000000000000000000000000000016 7777BBBBFFFF999922229999DDDD3333CCCCAAAACCCCBBBBBBBB +vi@>8jMaYX 00000000000000000000000000000017 CCCCFFFFFFFF6666AAAA999999997777AAAA6666000066668888 +}dA7*<6[lx 00000000000000000000000000000018 6666FFFF77775555CCCC88881111AAAA00000000CCCC00009999 +>Y$Kc)9DOf 00000000000000000000000000000019 AAAAEEEEEEEE222266660000AAAA00000000000044446666EEEE +uF?T;*:x0+ 0000000000000000000000000000001A 444466665555BBBB8888BBBB00000000BBBB00005555AAAA7777 +~%.@@,3>7+ 0000000000000000000000000000001B FFFFDDDD333399999999666644449999BBBB2222000000004444 +JPrAKc{W>j 0000000000000000000000000000001C 222211111111333355554444DDDDEEEE33339999999911115555 +*i@lF30|aO 0000000000000000000000000000001D CCCC88880000FFFFCCCCDDDDBBBB4444EEEE99990000AAAAAAAA +9UH>%FZbUK 0000000000000000000000000000001E DDDD00005555EEEE111199998888CCCC888877779999DDDD3333 +-^1~=aYZC1 0000000000000000000000000000001F 11112222555599996666111144445555DDDDBBBB7777EEEE0000 +.\KqT&Z4i4 00000000000000000000000000000020 3333DDDDCCCC888888885555222299993333EEEEFFFF66661111 +9RtsPa gIN 00000000000000000000000000000021 555544440000AAAACCCC8888AAAA55552222DDDDBBBB22226666 +Fqy~]OATYw 00000000000000000000000000000022 9999CCCC0000888888884444333355553333BBBB55553333FFFF +Wj-%?e*2y\ 00000000000000000000000000000023 AAAA444477776666FFFF444455550000EEEE88883333FFFFCCCC +Rv]QHo9f6+ 00000000000000000000000000000024 99996666BBBB7777444466660000AAAADDDDDDDDAAAAEEEEDDDD +1 e!og=!hi 00000000000000000000000000000025 3333EEEE88887777CCCCEEEE777733337777DDDDFFFFEEEE2222 +N,gk>37c ( 00000000000000000000000000000026 DDDD11116666DDDD5555EEEE99996666AAAAAAAA3333EEEEBBBB +s.72Y.uzRN 00000000000000000000000000000027 0000CCCCFFFFAAAAAAAA11117777AAAA66662222000055558888 +j*0`gczW{u 00000000000000000000000000000028 9999DDDDCCCC8888AAAA999911111111BBBB66667777BBBB9999 +R+Sh=Kgd.6 00000000000000000000000000000029 DDDD0000BBBB77770000222200009999AAAADDDDAAAADDDDEEEE +x]w]=Dp6\s 0000000000000000000000000000002A 6666FFFFEEEEBBBBCCCC000033331111666677771111DDDD7777 +maU!Uo]He& 0000000000000000000000000000002B AAAA9999BBBB55557777CCCC6666EEEE555566668888FFFF4444 +smN$i pU-~ 0000000000000000000000000000002C AAAADDDDEEEE333388885555DDDD3333FFFFEEEE1111CCCC5555 +T}# `9Cseu 0000000000000000000000000000002D 4444FFFF777700001111EEEE999944447777444488881111AAAA +G2oMVHVK`! 0000000000000000000000000000002E 4444888844448888CCCC1111AAAA555577778888BBBB00003333 +(tv1'>|,s[ 0000000000000000000000000000002F 0000CCCC222277778888FFFFCCCCBBBB111177779999DDDD0000 +RE%Z{o)9E: 00000000000000000000000000000030 00002222AAAAEEEE444400008888CCCC3333DDDD555511111111 +'%:YY?@b+M 00000000000000000000000000000031 9999777777770000AAAA0000444433332222EEEE333399996666 +N(JuG%.`7c 00000000000000000000000000000032 66665555AAAA5555BBBBAAAAEEEE111111119999BBBB6666FFFF +`E$,xR{?Kc 00000000000000000000000000000033 7777AAAA222200008888CCCC3333AAAAFFFFAAAAEEEEEEEECCCC +jzu$Fn3ZN+ 00000000000000000000000000000034 55552222EEEE5555BBBBAAAA8888AAAA11110000DDDD9999DDDD +|xaPO\~!V* 00000000000000000000000000000035 EEEE2222EEEEFFFFEEEEBBBB2222BBBB9999AAAA999955552222 +GCU4gfl|&! 00000000000000000000000000000036 1111DDDD8888AAAA999999990000333377778888FFFF1111BBBB +JU9s03v&}o 00000000000000000000000000000037 7777555500001111DDDD3333111122221111AAAA444444448888 +D@v9""8"!` 00000000000000000000000000000038 2222BBBB55552222CCCC555533336666EEEE9999777766669999 +OffB8(#k@N 00000000000000000000000000000039 CCCC555555556666DDDDCCCC111111119999DDDD55554444EEEE +-_) R'4T<" 0000000000000000000000000000003A BBBBCCCCEEEECCCCDDDD6666BBBB333377779999222200007777 +'z4P|n>\to 0000000000000000000000000000003B DDDD5555AAAA3333CCCC222233330000111133335555EEEE4444 +[!Nbx&X=+V 0000000000000000000000000000003C 77778888AAAA7777888844446666EEEECCCCBBBBEEEE77775555 +p?"\\6:W[' 0000000000000000000000000000003D 8888EEEE6666555599992222DDDD2222FFFFEEEE33338888AAAA ++2A^xP5dA- 0000000000000000000000000000003E CCCCBBBBFFFFFFFF99995555AAAAAAAA77771111000033333333 +-cuJvy7;=h 0000000000000000000000000000003F 99993333CCCC8888EEEEAAAAEEEE5555CCCC7777FFFFCCCC0000 +ix^TQ-ma5o 00000000000000000000000000000047 88885555CCCC88889999888811110000BBBBEEEECCCC33338888 +E(g$5-WpBC 00000000000000000000000000000048 BBBB7777EEEE4444888866661111FFFF44449999BBBB11119999 +~\ptsn1L{/ 00000000000000000000000000000049 CCCC0000FFFFCCCC7777BBBB333300003333FFFF3333BBBBEEEE +YhlPk`q6SN 0000000000000000000000000000004A 00001111AAAABBBBAAAA66669999DDDD11116666666633337777 +WBzul,K** 0000000000000000000000000000004B 4444AAAADDDDEEEE33337777AAAA9999DDDD88886666DDDD4444 +q)epJsSLXU 0000000000000000000000000000004C 88887777999955558888AAAA9999111155552222FFFF22225555 +`WVkW&( /[ 0000000000000000000000000000004D FFFF999922221111DDDD9999AAAA9999FFFF66662222FFFFAAAA +HXg#Z^7@7/ 0000000000000000000000000000004E 777799999999DDDD44441111BBBB6666AAAA0000999966663333 +-&8$lcXS$; 0000000000000000000000000000004F 333300004444AAAA3333666600009999DDDDCCCC9999BBBB0000 +^SpqwoVET; 00000000000000000000000000000050 BBBB77775555CCCC1111DDDD2222BBBB66669999CCCC77771111 +P-6Ybw3=a| 00000000000000000000000000000051 2222FFFFDDDDAAAACCCC3333EEEEBBBB8888EEEE000077776666 +a`:;ydJv)Y 00000000000000000000000000000052 55554444AAAAAAAA6666DDDD7777AAAAEEEE00003333CCCCFFFF +dO]G~-YGFJ 00000000000000000000000000000053 8888555533333333EEEE11113333AAAA444411110000CCCCCCCC +WM^;rvO/u] 00000000000000000000000000000054 BBBB00007777999977772222AAAA0000CCCCAAAAEEEEFFFFDDDD +-WwU4BF*P< 00000000000000000000000000000055 BBBB8888EEEEEEEE55553333AAAA4444EEEE6666888833332222 +y'w/PZ!x"> 00000000000000000000000000000056 DDDD55554444AAAAAAAA444455553333AAAA444411117777BBBB +#jA`l-f@*f 00000000000000000000000000000057 4444AAAA2222AAAAEEEE55556666CCCC3333FFFF888822228888 +:!W.=v,f"z 00000000000000000000000000000058 444422223333666688886666BBBB4444888877772222CCCC9999 +"!r4ugwZcz 00000000000000000000000000000059 22226666CCCCEEEE6666FFFF888822220000333366662222EEEE +O?(|_dXq:X 0000000000000000000000000000005C 2222FFFFBBBB55552222444433332222444433333333DDDD5555 +(cb621Uv~( 0000000000000000000000000000005D 3333BBBB11110000FFFF9999BBBB6666CCCCCCCC66666666AAAA +hg5x s23L3 0000000000000000000000000000005E 22221111AAAA000011117777DDDD999933337777666699993333 +w5nn`wa'[p 0000000000000000000000000000005F 000099997777CCCC666677776666FFFF444455557777AAAA0000 +qeQNoB 2XG 00000000000000000000000000000060 0000EEEE66668888FFFF2222FFFFAAAAFFFF7777EEEE22221111 +6n>iMbi=gw 00000000000000000000000000000061 4444DDDDFFFF9999DDDD9999EEEEEEEECCCCDDDD4444EEEE6666 +uGz7S=#fh: 00000000000000000000000000000062 77771111DDDD222200000000BBBBDDDD222288885555FFFFFFFF +rd)x<>@@H| 00000000000000000000000000000063 99996666AAAABBBB7777CCCC4444DDDD555544447777BBBBCCCC +xEwa^:84K] 00000000000000000000000000000064 000099995555CCCC00000000EEEE5555BBBB1111DDDDAAAADDDD +':]$}b:Z$a 00000000000000000000000000000065 77775555DDDD4444DDDDAAAA66660000FFFF5555DDDDAAAA2222 +vZl"Ji^AO@ 00000000000000000000000000000066 3333777744444444EEEEEEEE11115555666600008888AAAABBBB +o%,!fSdRBa 00000000000000000000000000000067 11119999DDDDAAAA8888AAAA5555EEEE8888CCCC888811118888 +*+o@+t%u&; 00000000000000000000000000000068 22228888999922228888EEEE3333444455551111EEEE77779999 +(CoKUhfB9{ 00000000000000000000000000000069 1111BBBB9999CCCCFFFF3333CCCC777755559999CCCC9999EEEE +I>:9JQb_5T 0000000000000000000000000000006A AAAA66662222CCCCBBBB666600003333BBBB2222AAAA99997777 +eSzU!X,[%/ 0000000000000000000000000000006B AAAACCCC9999FFFFFFFFAAAA444455553333BBBB4444BBBB4444 +[QZ{a&TaCS 0000000000000000000000000000006C AAAA8888CCCC2222FFFFAAAA5555CCCC4444CCCCCCCC88885555 +?:Yp4%.K9n 0000000000000000000000000000006D BBBBCCCC5555111100007777FFFFAAAAFFFF0000DDDDDDDDAAAA +'_>/ 4v}8I 0000000000000000000000000000006E 1111EEEE000088886666333311115555666655557777CCCC3333 +KrNW.!5MP; 0000000000000000000000000000006F 2222AAAAEEEE33330000777711111111FFFF2222999999990000 +j17a3a#bH? 00000000000000000000000000000070 8888CCCCBBBBCCCC88881111EEEE33336666BBBB3333DDDD1111 +)o-QLHy(UO 00000000000000000000000000000071 3333BBBB444400002222EEEEFFFF333355556666DDDD55556666 +al;:OEKSH2 00000000000000000000000000000072 DDDDEEEE3333DDDDBBBB7777BBBB1111FFFF3333CCCC2222FFFF +I@L*`HMPfN 00000000000000000000000000000073 DDDD666699993333999977779999DDDD444488882222AAAACCCC +$kBiG 00000000000000000000000000000090 3333DDDD000044445555444444441111CCCC1111BBBB33331111 +V*W% x\&'q 00000000000000000000000000000091 8888DDDDBBBBAAAABBBB99997777EEEE00008888AAAA33336666 +0Jw^F-}sZr 00000000000000000000000000000092 4444BBBBDDDD2222FFFF888811117777DDDD444444448888FFFF +OLf_u|1d}9 00000000000000000000000000000093 DDDD1111AAAA9999FFFFDDDD9999BBBB9999000044448888CCCC +%V*"P*(/Ez 00000000000000000000000000000094 11110000CCCC666600002222CCCCEEEECCCCEEEE1111BBBBDDDD +jGdC5%yw;# 00000000000000000000000000000095 44449999FFFF00004444222222228888000099995555FFFF2222 +,uSW`H/pXh 00000000000000000000000000000096 8888BBBB111188885555CCCC7777BBBB7777111166663333BBBB +n],.INMkcA 00000000000000000000000000000097 DDDD77777777DDDD99993333FFFFDDDD9999CCCCFFFFEEEE8888 +(P=jd[R]&[ 00000000000000000000000000000098 BBBBAAAA99995555CCCC666688887777EEEE1111999988889999 +*%pAD!6FlS 00000000000000000000000000000099 AAAA777799994444DDDDEEEE6666DDDD2222AAAA7777EEEEEEEE +;u0nwOR{|e 0000000000000000000000000000009A CCCC5555FFFF1111000066667777EEEEEEEE1111777722227777 +X*)4\|>`b. 0000000000000000000000000000009B EEEE33339999CCCC2222999999999999FFFFDDDD777788884444 +#Yh\5HjDLp 0000000000000000000000000000009C 33338888EEEEFFFFDDDDDDDDAAAA7777EEEE0000EEEE99995555 +5Yh?d9@6(_ 0000000000000000000000000000009D 66667777FFFF333377774444FFFFDDDD99993333CCCC2222AAAA +W;mKiM[qVV 0000000000000000000000000000009E 111133334444AAAA55556666CCCCFFFF5555BBBB333355553333 +287P2iA6U: 0000000000000000000000000000009F 88880000FFFF000022220000BBBBCCCCFFFF3333777766660000 +Bj`e'4&VbD 000000000000000000000000000000A0 DDDD888811110000BBBB2222EEEEBBBB00004444CCCCEEEE1111 +Q[7 0VqTg~ 000000000000000000000000000000A1 8888666622222222222222221111DDDD11110000EEEEAAAA6666 +4Y8,(.X0:" 000000000000000000000000000000A2 DDDD7777DDDD5555AAAAEEEE7777DDDD444499996666BBBBFFFF +w)98*ZrTi* 000000000000000000000000000000A3 CCCC4444FFFF0000BBBBAAAA66667777CCCC4444BBBB7777CCCC +pHH7F#q@Tf 000000000000000000000000000000A4 0000FFFFEEEE9999DDDD9999AAAAFFFF9999AAAA00006666DDDD +89_ &UU^-f 000000000000000000000000000000A5 BBBB5555AAAAAAAAAAAA4444FFFF6666DDDD1111BBBB66662222 +IjYZ+5}$cH 000000000000000000000000000000A6 EEEE555577779999666611110000FFFF0000AAAADDDD6666BBBB +HP5;m7ydP, 000000000000000000000000000000A7 DDDDDDDDEEEE222255559999DDDDEEEE7777AAAAFFFFDDDD8888 +(\+CmQF;sh 000000000000000000000000000000A8 7777BBBB9999333377779999AAAAAAAACCCC1111555533339999 +`27}+u%$Z' 000000000000000000000000000000A9 22228888EEEE555511117777DDDD555522229999EEEE5555EEEE +D/XZpkvB{U 000000000000000000000000000000AA 8888BBBB77776666EEEEEEEE00007777BBBB2222333355557777 +6uxeaoldr{ 000000000000000000000000000000AB BBBB11110000CCCC8888222277779999AAAA4444000077774444 +3n glBr={a 000000000000000000000000000000AC EEEE99992222666644446666FFFF11112222FFFF777744445555 +M3Bn!0ePh9 000000000000000000000000000000AD 666611114444999933332222FFFFDDDD5555111133339999AAAA +^:W0]D07,s 000000000000000000000000000000AE 99995555EEEEEEEE33330000AAAA1111CCCC6666444488883333 +R%]??oXCf( 000000000000000000000000000000AF CCCCFFFF00004444222299997777CCCC11111111999955550000 +?jh!g'(/dI 000000000000000000000000000000B0 99994444FFFFBBBBFFFF777711113333EEEEDDDD222299991111 +,qy-T(O~$} 000000000000000000000000000000B1 44446666888800007777DDDDEEEEFFFF22223333777711116666 +g~]6jo( dQ 000000000000000000000000000000B2 DDDD888822224444EEEE88887777EEEE00001111CCCCEEEEFFFF +\1]of=Ah$q 000000000000000000000000000000B3 EEEEAAAACCCCCCCC77779999AAAACCCC9999999966666666CCCC +5,vYEm^\di 000000000000000000000000000000B4 1111EEEE3333EEEEBBBB8888111111111111777733331111DDDD +}uX` 000000000000000000000000000000D1 7777DDDD77778888BBBB8888EEEEAAAA222277773333FFFF6666 +3pU8_.#1JU 000000000000000000000000000000D2 0000777799995555FFFFDDDDFFFF5555FFFFCCCC55554444FFFF +IW,e"{Tnvp 000000000000000000000000000000D3 99995555777755556666AAAA88887777EEEE333388884444CCCC +:=XWS2sImq 000000000000000000000000000000D4 AAAACCCCBBBBAAAA0000EEEE4444CCCCEEEE555544447777DDDD +o8Gcpop:! 000000000000000000000000000000D5 AAAAFFFFDDDD11115555000077778888999900003333BBBB2222 +W-qOhMjDFd 000000000000000000000000000000D6 0000222266660000BBBB55551111777733332222AAAAFFFFBBBB +u?;I>\L_vn 000000000000000000000000000000D7 99991111FFFFFFFF5555EEEE44447777BBBBEEEE7777AAAA8888 +Nyp X=m$h9 000000000000000000000000000000D8 0000EEEE444433333333BBBB8888000011110000000044449999 +g.FN&efgUs 000000000000000000000000000000D9 7777444488885555FFFF3333FFFFDDDD777755559999AAAAEEEE +rhC::yPt=z 000000000000000000000000000000DA 3333666633336666BBBBCCCCEEEE000000007777FFFFEEEE7777 +De9lt=P!{. 000000000000000000000000000000DB DDDD2222BBBB33335555888866663333EEEE9999333344444444 +XG(8.GC,t\ 000000000000000000000000000000DC 7777CCCCEEEE7777222200006666BBBB11112222999955555555 +U;azR 1t!\wV 000000000000000000000000000000E0 2222EEEE00000000FFFFBBBB6666555566665555BBBBAAAA1111 +'-zF>A2f5c 000000000000000000000000000000E1 CCCC999977773333111144444444BBBBFFFF8888888866666666 +^qaslc^dJX 000000000000000000000000000000E2 444411118888AAAA7777CCCC000011119999EEEE77777777FFFF +@gU`)^H:z5 000000000000000000000000000000E3 AAAA33331111FFFF888822225555CCCC33338888FFFF3333CCCC + l;w?EBj0 000000000000000000000000000000E4 BBBBDDDD111188888888AAAA111144449999666633332222DDDD +Iw&)MuUaM, 000000000000000000000000000000E5 5555FFFF00009999888811119999111111111111999922222222 +aGam)y ~t- 000000000000000000000000000000E6 EEEE22228888BBBB999977774444FFFFAAAA999922222222BBBB ++/Yg#6Nf/r 000000000000000000000000000000E7 FFFFBBBB6666CCCC66669999FFFF66662222DDDD777799998888 +SD ,=Yy%.w 000000000000000000000000000000E8 666644446666DDDDDDDD44446666111100004444BBBBFFFF9999 +/e)%&bgl}E 000000000000000000000000000000E9 AAAA6666333388887777DDDDCCCCFFFF1111EEEE00001111EEEE +8mdOTG)j6O 000000000000000000000000000000EA BBBB7777AAAABBBB888855554444999966665555CCCC11117777 +Yp=]xMdlAGG 000000000000000000000000000000F6 1111DDDD555500008888CCCC222288884444EEEEDDDD5555BBBB +X\!l_zyMKz% 00000000000000000000000000000014 99997777777722222222999911116666DDDDBBBBCCCC3333DDDD +kXrG^ech1/ 00000000000000000000000000000015 FFFF2222AAAA66660000AAAA9999333322227777AAAA77772222 +v%^RG^ci[X 00000000000000000000000000000016 7777BBBBFFFF999922229999DDDD3333CCCCAAAACCCCBBBBBBBB +vi@>8jMaYX 00000000000000000000000000000017 CCCCFFFFFFFF6666AAAA999999997777AAAA6666000066668888 +}dA7*<6[lx 00000000000000000000000000000018 6666FFFF77775555CCCC88881111AAAA00000000CCCC00009999 +>Y$Kc)9DOf 00000000000000000000000000000019 AAAAEEEEEEEE222266660000AAAA00000000000044446666EEEE +uF?T;*:x0+ 0000000000000000000000000000001A 444466665555BBBB8888BBBB00000000BBBB00005555AAAA7777 +~%.@@,3>7+ 0000000000000000000000000000001B FFFFDDDD333399999999666644449999BBBB2222000000004444 +JPrAKc{W>j 0000000000000000000000000000001C 222211111111333355554444DDDDEEEE33339999999911115555 +*i@lF30|aO 0000000000000000000000000000001D CCCC88880000FFFFCCCCDDDDBBBB4444EEEE99990000AAAAAAAA +9UH>%FZbUK 0000000000000000000000000000001E DDDD00005555EEEE111199998888CCCC888877779999DDDD3333 +-^1~=aYZC1 0000000000000000000000000000001F 11112222555599996666111144445555DDDDBBBB7777EEEE0000 +.\KqT&Z4i4 00000000000000000000000000000020 3333DDDDCCCC888888885555222299993333EEEEFFFF66661111 +9RtsPa gIN 00000000000000000000000000000021 555544440000AAAACCCC8888AAAA55552222DDDDBBBB22226666 +Fqy~]OATYw 00000000000000000000000000000022 9999CCCC0000888888884444333355553333BBBB55553333FFFF +Wj-%?e*2y\ 00000000000000000000000000000023 AAAA444477776666FFFF444455550000EEEE88883333FFFFCCCC +Rv]QHo9f6+ 00000000000000000000000000000024 99996666BBBB7777444466660000AAAADDDDDDDDAAAAEEEEDDDD +1 e!og=!hi 00000000000000000000000000000025 3333EEEE88887777CCCCEEEE777733337777DDDDFFFFEEEE2222 +N,gk>37c ( 00000000000000000000000000000026 DDDD11116666DDDD5555EEEE99996666AAAAAAAA3333EEEEBBBB +s.72Y.uzRN 00000000000000000000000000000027 0000CCCCFFFFAAAAAAAA11117777AAAA66662222000055558888 +j*0`gczW{u 00000000000000000000000000000028 9999DDDDCCCC8888AAAA999911111111BBBB66667777BBBB9999 +R+Sh=Kgd.6 00000000000000000000000000000029 DDDD0000BBBB77770000222200009999AAAADDDDAAAADDDDEEEE +x]w]=Dp6\s 0000000000000000000000000000002A 6666FFFFEEEEBBBBCCCC000033331111666677771111DDDD7777 +maU!Uo]He& 0000000000000000000000000000002B AAAA9999BBBB55557777CCCC6666EEEE555566668888FFFF4444 +smN$i pU-~ 0000000000000000000000000000002C AAAADDDDEEEE333388885555DDDD3333FFFFEEEE1111CCCC5555 +T}# `9Cseu 0000000000000000000000000000002D 4444FFFF777700001111EEEE999944447777444488881111AAAA +G2oMVHVK`! 0000000000000000000000000000002E 4444888844448888CCCC1111AAAA555577778888BBBB00003333 +(tv1'>|,s[ 0000000000000000000000000000002F 0000CCCC222277778888FFFFCCCCBBBB111177779999DDDD0000 +RE%Z{o)9E: 00000000000000000000000000000030 00002222AAAAEEEE444400008888CCCC3333DDDD555511111111 +'%:YY?@b+M 00000000000000000000000000000031 9999777777770000AAAA0000444433332222EEEE333399996666 +N(JuG%.`7c 00000000000000000000000000000032 66665555AAAA5555BBBBAAAAEEEE111111119999BBBB6666FFFF +`E$,xR{?Kc 00000000000000000000000000000033 7777AAAA222200008888CCCC3333AAAAFFFFAAAAEEEEEEEECCCC +jzu$Fn3ZN+ 00000000000000000000000000000034 55552222EEEE5555BBBBAAAA8888AAAA11110000DDDD9999DDDD +|xaPO\~!V* 00000000000000000000000000000035 EEEE2222EEEEFFFFEEEEBBBB2222BBBB9999AAAA999955552222 +GCU4gfl|&! 00000000000000000000000000000036 1111DDDD8888AAAA999999990000333377778888FFFF1111BBBB +JU9s03v&}o 00000000000000000000000000000037 7777555500001111DDDD3333111122221111AAAA444444448888 +D@v9""8"!` 00000000000000000000000000000038 2222BBBB55552222CCCC555533336666EEEE9999777766669999 +OffB8(#k@N 00000000000000000000000000000039 CCCC555555556666DDDDCCCC111111119999DDDD55554444EEEE +-_) R'4T<" 0000000000000000000000000000003A BBBBCCCCEEEECCCCDDDD6666BBBB333377779999222200007777 +'z4P|n>\to 0000000000000000000000000000003B DDDD5555AAAA3333CCCC222233330000111133335555EEEE4444 +[!Nbx&X=+V 0000000000000000000000000000003C 77778888AAAA7777888844446666EEEECCCCBBBBEEEE77775555 +p?"\\6:W[' 0000000000000000000000000000003D 8888EEEE6666555599992222DDDD2222FFFFEEEE33338888AAAA ++2A^xP5dA- 0000000000000000000000000000003E CCCCBBBBFFFFFFFF99995555AAAAAAAA77771111000033333333 +-cuJvy7;=h 0000000000000000000000000000003F 99993333CCCC8888EEEEAAAAEEEE5555CCCC7777FFFFCCCC0000 +ix^TQ-ma5o 00000000000000000000000000000047 88885555CCCC88889999888811110000BBBBEEEECCCC33338888 +E(g$5-WpBC 00000000000000000000000000000048 BBBB7777EEEE4444888866661111FFFF44449999BBBB11119999 +~\ptsn1L{/ 00000000000000000000000000000049 CCCC0000FFFFCCCC7777BBBB333300003333FFFF3333BBBBEEEE +YhlPk`q6SN 0000000000000000000000000000004A 00001111AAAABBBBAAAA66669999DDDD11116666666633337777 +WBzul,K** 0000000000000000000000000000004B 4444AAAADDDDEEEE33337777AAAA9999DDDD88886666DDDD4444 +q)epJsSLXU 0000000000000000000000000000004C 88887777999955558888AAAA9999111155552222FFFF22225555 +`WVkW&( /[ 0000000000000000000000000000004D FFFF999922221111DDDD9999AAAA9999FFFF66662222FFFFAAAA +HXg#Z^7@7/ 0000000000000000000000000000004E 777799999999DDDD44441111BBBB6666AAAA0000999966663333 +-&8$lcXS$; 0000000000000000000000000000004F 333300004444AAAA3333666600009999DDDDCCCC9999BBBB0000 +^SpqwoVET; 00000000000000000000000000000050 BBBB77775555CCCC1111DDDD2222BBBB66669999CCCC77771111 +P-6Ybw3=a| 00000000000000000000000000000051 2222FFFFDDDDAAAACCCC3333EEEEBBBB8888EEEE000077776666 +a`:;ydJv)Y 00000000000000000000000000000052 55554444AAAAAAAA6666DDDD7777AAAAEEEE00003333CCCCFFFF +dO]G~-YGFJ 00000000000000000000000000000053 8888555533333333EEEE11113333AAAA444411110000CCCCCCCC +WM^;rvO/u] 00000000000000000000000000000054 BBBB00007777999977772222AAAA0000CCCCAAAAEEEEFFFFDDDD +-WwU4BF*P< 00000000000000000000000000000055 BBBB8888EEEEEEEE55553333AAAA4444EEEE6666888833332222 +y'w/PZ!x"> 00000000000000000000000000000056 DDDD55554444AAAAAAAA444455553333AAAA444411117777BBBB +#jA`l-f@*f 00000000000000000000000000000057 4444AAAA2222AAAAEEEE55556666CCCC3333FFFF888822228888 +:!W.=v,f"z 00000000000000000000000000000058 444422223333666688886666BBBB4444888877772222CCCC9999 +"!r4ugwZcz 00000000000000000000000000000059 22226666CCCCEEEE6666FFFF888822220000333366662222EEEE +O?(|_dXq:X 0000000000000000000000000000005C 2222FFFFBBBB55552222444433332222444433333333DDDD5555 +(cb621Uv~( 0000000000000000000000000000005D 3333BBBB11110000FFFF9999BBBB6666CCCCCCCC66666666AAAA +hg5x s23L3 0000000000000000000000000000005E 22221111AAAA000011117777DDDD999933337777666699993333 +w5nn`wa'[p 0000000000000000000000000000005F 000099997777CCCC666677776666FFFF444455557777AAAA0000 +qeQNoB 2XG 00000000000000000000000000000060 0000EEEE66668888FFFF2222FFFFAAAAFFFF7777EEEE22221111 +6n>iMbi=gw 00000000000000000000000000000061 4444DDDDFFFF9999DDDD9999EEEEEEEECCCCDDDD4444EEEE6666 +uGz7S=#fh: 00000000000000000000000000000062 77771111DDDD222200000000BBBBDDDD222288885555FFFFFFFF +rd)x<>@@H| 00000000000000000000000000000063 99996666AAAABBBB7777CCCC4444DDDD555544447777BBBBCCCC +xEwa^:84K] 00000000000000000000000000000064 000099995555CCCC00000000EEEE5555BBBB1111DDDDAAAADDDD +':]$}b:Z$a 00000000000000000000000000000065 77775555DDDD4444DDDDAAAA66660000FFFF5555DDDDAAAA2222 +vZl"Ji^AO@ 00000000000000000000000000000066 3333777744444444EEEEEEEE11115555666600008888AAAABBBB +o%,!fSdRBa 00000000000000000000000000000067 11119999DDDDAAAA8888AAAA5555EEEE8888CCCC888811118888 +*+o@+t%u&; 00000000000000000000000000000068 22228888999922228888EEEE3333444455551111EEEE77779999 +(CoKUhfB9{ 00000000000000000000000000000069 1111BBBB9999CCCCFFFF3333CCCC777755559999CCCC9999EEEE +I>:9JQb_5T 0000000000000000000000000000006A AAAA66662222CCCCBBBB666600003333BBBB2222AAAA99997777 +eSzU!X,[%/ 0000000000000000000000000000006B AAAACCCC9999FFFFFFFFAAAA444455553333BBBB4444BBBB4444 +[QZ{a&TaCS 0000000000000000000000000000006C AAAA8888CCCC2222FFFFAAAA5555CCCC4444CCCCCCCC88885555 +?:Yp4%.K9n 0000000000000000000000000000006D BBBBCCCC5555111100007777FFFFAAAAFFFF0000DDDDDDDDAAAA +'_>/ 4v}8I 0000000000000000000000000000006E 1111EEEE000088886666333311115555666655557777CCCC3333 +KrNW.!5MP; 0000000000000000000000000000006F 2222AAAAEEEE33330000777711111111FFFF2222999999990000 +j17a3a#bH? 00000000000000000000000000000070 8888CCCCBBBBCCCC88881111EEEE33336666BBBB3333DDDD1111 +)o-QLHy(UO 00000000000000000000000000000071 3333BBBB444400002222EEEEFFFF333355556666DDDD55556666 +al;:OEKSH2 00000000000000000000000000000072 DDDDEEEE3333DDDDBBBB7777BBBB1111FFFF3333CCCC2222FFFF +I@L*`HMPfN 00000000000000000000000000000073 DDDD666699993333999977779999DDDD444488882222AAAACCCC +$kBiG 00000000000000000000000000000090 3333DDDD000044445555444444441111CCCC1111BBBB33331111 +V*W% x\&'q 00000000000000000000000000000091 8888DDDDBBBBAAAABBBB99997777EEEE00008888AAAA33336666 +0Jw^F-}sZr 00000000000000000000000000000092 4444BBBBDDDD2222FFFF888811117777DDDD444444448888FFFF +OLf_u|1d}9 00000000000000000000000000000093 DDDD1111AAAA9999FFFFDDDD9999BBBB9999000044448888CCCC +%V*"P*(/Ez 00000000000000000000000000000094 11110000CCCC666600002222CCCCEEEECCCCEEEE1111BBBBDDDD +jGdC5%yw;# 00000000000000000000000000000095 44449999FFFF00004444222222228888000099995555FFFF2222 +,uSW`H/pXh 00000000000000000000000000000096 8888BBBB111188885555CCCC7777BBBB7777111166663333BBBB +n],.INMkcA 00000000000000000000000000000097 DDDD77777777DDDD99993333FFFFDDDD9999CCCCFFFFEEEE8888 +(P=jd[R]&[ 00000000000000000000000000000098 BBBBAAAA99995555CCCC666688887777EEEE1111999988889999 +*%pAD!6FlS 00000000000000000000000000000099 AAAA777799994444DDDDEEEE6666DDDD2222AAAA7777EEEEEEEE +;u0nwOR{|e 0000000000000000000000000000009A CCCC5555FFFF1111000066667777EEEEEEEE1111777722227777 +X*)4\|>`b. 0000000000000000000000000000009B EEEE33339999CCCC2222999999999999FFFFDDDD777788884444 +#Yh\5HjDLp 0000000000000000000000000000009C 33338888EEEEFFFFDDDDDDDDAAAA7777EEEE0000EEEE99995555 +5Yh?d9@6(_ 0000000000000000000000000000009D 66667777FFFF333377774444FFFFDDDD99993333CCCC2222AAAA +W;mKiM[qVV 0000000000000000000000000000009E 111133334444AAAA55556666CCCCFFFF5555BBBB333355553333 +287P2iA6U: 0000000000000000000000000000009F 88880000FFFF000022220000BBBBCCCCFFFF3333777766660000 +Bj`e'4&VbD 000000000000000000000000000000A0 DDDD888811110000BBBB2222EEEEBBBB00004444CCCCEEEE1111 +Q[7 0VqTg~ 000000000000000000000000000000A1 8888666622222222222222221111DDDD11110000EEEEAAAA6666 +4Y8,(.X0:" 000000000000000000000000000000A2 DDDD7777DDDD5555AAAAEEEE7777DDDD444499996666BBBBFFFF +w)98*ZrTi* 000000000000000000000000000000A3 CCCC4444FFFF0000BBBBAAAA66667777CCCC4444BBBB7777CCCC +pHH7F#q@Tf 000000000000000000000000000000A4 0000FFFFEEEE9999DDDD9999AAAAFFFF9999AAAA00006666DDDD +89_ &UU^-f 000000000000000000000000000000A5 BBBB5555AAAAAAAAAAAA4444FFFF6666DDDD1111BBBB66662222 +IjYZ+5}$cH 000000000000000000000000000000A6 EEEE555577779999666611110000FFFF0000AAAADDDD6666BBBB +HP5;m7ydP, 000000000000000000000000000000A7 DDDDDDDDEEEE222255559999DDDDEEEE7777AAAAFFFFDDDD8888 +(\+CmQF;sh 000000000000000000000000000000A8 7777BBBB9999333377779999AAAAAAAACCCC1111555533339999 +`27}+u%$Z' 000000000000000000000000000000A9 22228888EEEE555511117777DDDD555522229999EEEE5555EEEE +D/XZpkvB{U 000000000000000000000000000000AA 8888BBBB77776666EEEEEEEE00007777BBBB2222333355557777 +6uxeaoldr{ 000000000000000000000000000000AB BBBB11110000CCCC8888222277779999AAAA4444000077774444 +3n glBr={a 000000000000000000000000000000AC EEEE99992222666644446666FFFF11112222FFFF777744445555 +M3Bn!0ePh9 000000000000000000000000000000AD 666611114444999933332222FFFFDDDD5555111133339999AAAA +^:W0]D07,s 000000000000000000000000000000AE 99995555EEEEEEEE33330000AAAA1111CCCC6666444488883333 +R%]??oXCf( 000000000000000000000000000000AF CCCCFFFF00004444222299997777CCCC11111111999955550000 +?jh!g'(/dI 000000000000000000000000000000B0 99994444FFFFBBBBFFFF777711113333EEEEDDDD222299991111 +,qy-T(O~$} 000000000000000000000000000000B1 44446666888800007777DDDDEEEEFFFF22223333777711116666 +g~]6jo( dQ 000000000000000000000000000000B2 DDDD888822224444EEEE88887777EEEE00001111CCCCEEEEFFFF +\1]of=Ah$q 000000000000000000000000000000B3 EEEEAAAACCCCCCCC77779999AAAACCCC9999999966666666CCCC +5,vYEm^\di 000000000000000000000000000000B4 1111EEEE3333EEEEBBBB8888111111111111777733331111DDDD +}uX` 000000000000000000000000000000D1 7777DDDD77778888BBBB8888EEEEAAAA222277773333FFFF6666 +3pU8_.#1JU 000000000000000000000000000000D2 0000777799995555FFFFDDDDFFFF5555FFFFCCCC55554444FFFF +IW,e"{Tnvp 000000000000000000000000000000D3 99995555777755556666AAAA88887777EEEE333388884444CCCC +:=XWS2sImq 000000000000000000000000000000D4 AAAACCCCBBBBAAAA0000EEEE4444CCCCEEEE555544447777DDDD +o8Gcpop:! 000000000000000000000000000000D5 AAAAFFFFDDDD11115555000077778888999900003333BBBB2222 +W-qOhMjDFd 000000000000000000000000000000D6 0000222266660000BBBB55551111777733332222AAAAFFFFBBBB +u?;I>\L_vn 000000000000000000000000000000D7 99991111FFFFFFFF5555EEEE44447777BBBBEEEE7777AAAA8888 +Nyp X=m$h9 000000000000000000000000000000D8 0000EEEE444433333333BBBB8888000011110000000044449999 +g.FN&efgUs 000000000000000000000000000000D9 7777444488885555FFFF3333FFFFDDDD777755559999AAAAEEEE +rhC::yPt=z 000000000000000000000000000000DA 3333666633336666BBBBCCCCEEEE000000007777FFFFEEEE7777 +De9lt=P!{. 000000000000000000000000000000DB DDDD2222BBBB33335555888866663333EEEE9999333344444444 +XG(8.GC,t\ 000000000000000000000000000000DC 7777CCCCEEEE7777222200006666BBBB11112222999955555555 +U;azR 1t!\wV 000000000000000000000000000000E0 2222EEEE00000000FFFFBBBB6666555566665555BBBBAAAA1111 +'-zF>A2f5c 000000000000000000000000000000E1 CCCC999977773333111144444444BBBBFFFF8888888866666666 +^qaslc^dJX 000000000000000000000000000000E2 444411118888AAAA7777CCCC000011119999EEEE77777777FFFF +@gU`)^H:z5 000000000000000000000000000000E3 AAAA33331111FFFF888822225555CCCC33338888FFFF3333CCCC + l;w?EBj0 000000000000000000000000000000E4 BBBBDDDD111188888888AAAA111144449999666633332222DDDD +Iw&)MuUaM, 000000000000000000000000000000E5 5555FFFF00009999888811119999111111111111999922222222 +aGam)y ~t- 000000000000000000000000000000E6 EEEE22228888BBBB999977774444FFFFAAAA999922222222BBBB ++/Yg#6Nf/r 000000000000000000000000000000E7 FFFFBBBB6666CCCC66669999FFFF66662222DDDD777799998888 +SD ,=Yy%.w 000000000000000000000000000000E8 666644446666DDDDDDDD44446666111100004444BBBBFFFF9999 +/e)%&bgl}E 000000000000000000000000000000E9 AAAA6666333388887777DDDDCCCCFFFF1111EEEE00001111EEEE +8mdOTG)j6O 000000000000000000000000000000EA BBBB7777AAAABBBB888855554444999966665555CCCC11117777 +Yp=]xMdlAGG 000000000000000000000000000000F6 1111DDDD555500008888CCCC222288884444EEEEDDDD5555BBBB +X\!l_ Date: Sun, 26 Nov 2023 16:34:38 +0900 Subject: [PATCH 10/39] chore: lint --- master/src/main/scala/Master.scala | 9 ++++++--- rpc/src/main/scala/MasterClient.scala | 4 ++-- rpc/src/main/scala/MasterServer.scala | 11 ++--------- rpc/src/main/scala/WorkerClient.scala | 4 ++-- rpc/src/main/scala/WorkerServer.scala | 5 ++--- worker/src/main/scala/Worker.scala | 14 +++++++------- 6 files changed, 21 insertions(+), 26 deletions(-) diff --git a/master/src/main/scala/Master.scala b/master/src/main/scala/Master.scala index f1dc950..575636b 100644 --- a/master/src/main/scala/Master.scala +++ b/master/src/main/scala/Master.scala @@ -1,12 +1,15 @@ package kr.ac.postech.paranode.master -import kr.ac.postech.paranode.core.{Key, WorkerMetadata} -import kr.ac.postech.paranode.rpc.{MasterServer, WorkerClient} +import kr.ac.postech.paranode.core.Key +import kr.ac.postech.paranode.core.WorkerMetadata +import kr.ac.postech.paranode.rpc.MasterServer +import kr.ac.postech.paranode.rpc.WorkerClient import org.apache.logging.log4j.scala.Logging import java.net._ +import scala.concurrent.Await import scala.concurrent.ExecutionContext.Implicits.global -import scala.concurrent.{Await, Future} +import scala.concurrent.Future import scala.util.Try object Master extends Logging { diff --git a/rpc/src/main/scala/MasterClient.scala b/rpc/src/main/scala/MasterClient.scala index 66a3cf6..2ef7e5e 100644 --- a/rpc/src/main/scala/MasterClient.scala +++ b/rpc/src/main/scala/MasterClient.scala @@ -6,12 +6,12 @@ import kr.ac.postech.paranode.core.WorkerMetadata import java.util.concurrent.TimeUnit import java.util.logging.Logger +import scala.concurrent.Future + import common.Node import master.MasterGrpc.MasterStub import master.{MasterGrpc, RegisterReply, RegisterRequest} -import scala.concurrent.Future - object MasterClient { def apply(host: String, port: Int): MasterClient = { val channel = diff --git a/rpc/src/main/scala/MasterServer.scala b/rpc/src/main/scala/MasterServer.scala index 0de2a24..84c2981 100644 --- a/rpc/src/main/scala/MasterServer.scala +++ b/rpc/src/main/scala/MasterServer.scala @@ -4,16 +4,15 @@ import io.grpc.Server import io.grpc.ServerBuilder import kr.ac.postech.paranode.core.WorkerMetadata import kr.ac.postech.paranode.rpc.MasterServer.port +import org.apache.logging.log4j.scala.Logging -import java.util.logging.Logger import scala.collection.mutable.ListBuffer import scala.collection.mutable.WrappedArray import scala.concurrent.ExecutionContext import scala.concurrent.Future import scala.concurrent.Promise -import master.{MasterGrpc, RegisterReply, RegisterRequest} -import org.apache.logging.log4j.scala.Logging +import master.{MasterGrpc, RegisterReply, RegisterRequest} object MasterServer { private val port = 50051 @@ -60,12 +59,6 @@ class MasterServer(executionContext: ExecutionContext) extends Logging { self => } } - private def blockUntilShutdown(): Unit = { - if (server != null) { - server.awaitTermination() - } - } - private class MasterImpl extends MasterGrpc.Master { override def register(request: RegisterRequest): Future[RegisterReply] = { val promise = Promise[RegisterReply] diff --git a/rpc/src/main/scala/WorkerClient.scala b/rpc/src/main/scala/WorkerClient.scala index 2333097..fbf7d48 100644 --- a/rpc/src/main/scala/WorkerClient.scala +++ b/rpc/src/main/scala/WorkerClient.scala @@ -8,6 +8,8 @@ import kr.ac.postech.paranode.core.WorkerMetadata import java.util.concurrent.TimeUnit import java.util.logging.Logger +import scala.concurrent.Future + import worker._ import worker.WorkerGrpc.WorkerStub import common.{ @@ -16,8 +18,6 @@ import common.{ WorkerMetadata => RpcWorkerMetadata } -import scala.concurrent.Future - object WorkerClient { def apply(host: String, port: Int): WorkerClient = { val channel = ManagedChannelBuilder diff --git a/rpc/src/main/scala/WorkerServer.scala b/rpc/src/main/scala/WorkerServer.scala index fd5b9d1..577baa9 100644 --- a/rpc/src/main/scala/WorkerServer.scala +++ b/rpc/src/main/scala/WorkerServer.scala @@ -4,16 +4,15 @@ import com.google.protobuf.ByteString import io.grpc.Server import io.grpc.ServerBuilder import kr.ac.postech.paranode.core._ +import org.apache.logging.log4j.scala.Logging -import java.util.logging.Logger import scala.concurrent.ExecutionContext import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.Future import scala.concurrent.Promise import scala.reflect.io.Path -import worker._ -import org.apache.logging.log4j.scala.Logging +import worker._ class WorkerServer( executionContext: ExecutionContext, diff --git a/worker/src/main/scala/Worker.scala b/worker/src/main/scala/Worker.scala index 5cc80cd..036976c 100644 --- a/worker/src/main/scala/Worker.scala +++ b/worker/src/main/scala/Worker.scala @@ -1,14 +1,15 @@ package kr.ac.postech.paranode.worker import kr.ac.postech.paranode.core.WorkerMetadata -import kr.ac.postech.paranode.rpc.{MasterClient, WorkerServer} - -import java.net.{InetAddress, ServerSocket} -import scala.util.Using -import scala.reflect.io.Path +import kr.ac.postech.paranode.rpc.MasterClient +import kr.ac.postech.paranode.rpc.WorkerServer import org.apache.logging.log4j.scala.Logging +import java.net.InetAddress +import java.net.ServerSocket import scala.concurrent.Await +import scala.reflect.io.Path +import scala.util.Using object Worker extends Logging { private class WorkerArguments(args: Array[String]) { @@ -32,7 +33,7 @@ object Worker extends Logging { val workerArguments = new WorkerArguments(args) logger.debug( - s"Worker arguments: \n" + + "Worker arguments: \n" + s"masterIp: ${workerArguments.masterIp}\n" + s"masterPort: ${workerArguments.masterPort}\n" + s"inputDirectories: ${workerArguments.inputDirectories().mkString(", ")}\n" + @@ -66,7 +67,6 @@ object Worker extends Logging { client.shutdown() } - server.blockUntilShutdown() } From b98f22f06a428b79d0363b8dfc461d38e05473a9 Mon Sep 17 00:00:00 2001 From: Minjae Gwon Date: Sun, 26 Nov 2023 19:18:59 +0900 Subject: [PATCH 11/39] refactor: rename sort into sorted --- core/src/main/scala/Block.scala | 2 +- core/src/test/scala/BlockSpec.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/src/main/scala/Block.scala b/core/src/main/scala/Block.scala index b1f99e7..cbe1730 100644 --- a/core/src/main/scala/Block.scala +++ b/core/src/main/scala/Block.scala @@ -62,7 +62,7 @@ class Block(val records: LazyList[Record]) extends AnyVal { def partition(keyRanges: List[KeyRange]): List[Partition] = keyRanges.map(partition) - def sort(): Block = + def sorted: Block = new Block(records.sortBy(_.key)) def sample(number: Int = 64): LazyList[Key] = diff --git a/core/src/test/scala/BlockSpec.scala b/core/src/test/scala/BlockSpec.scala index 15b9287..d1d6fab 100644 --- a/core/src/test/scala/BlockSpec.scala +++ b/core/src/test/scala/BlockSpec.scala @@ -168,7 +168,7 @@ class BlockSpec extends AnyFlatSpec { ) ) - val sortedBlock = block.sort() + val sortedBlock = block.sorted val expectedBlock = new Block( From 44b630d934e9ab6fd46d439f86bd55dbbca95f4f Mon Sep 17 00:00:00 2001 From: Minjae Gwon Date: Sun, 26 Nov 2023 19:19:58 +0900 Subject: [PATCH 12/39] fix: add base case for recursive function --- core/src/main/scala/Record.scala | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/core/src/main/scala/Record.scala b/core/src/main/scala/Record.scala index e14d4cf..66a0316 100644 --- a/core/src/main/scala/Record.scala +++ b/core/src/main/scala/Record.scala @@ -16,15 +16,19 @@ object Record extends Logging { keyLength: Int = 10, valueLength: Int = 90 ): LazyList[Record] = { - val recordLength = keyLength + valueLength - val (head, tail) = bytes.splitAt(recordLength) - - Record.fromBytes(head.toArray, keyLength) #:: Record - .fromBytesToRecords( - tail, - keyLength, - valueLength - ) + if (bytes.isEmpty) { + LazyList.empty + } else { + val recordLength = keyLength + valueLength + val (head, tail) = bytes.splitAt(recordLength) + + Record.fromBytes(head.toArray, keyLength) #:: Record + .fromBytesToRecords( + tail, + keyLength, + valueLength + ) + } } def sample( From 58bd252e10624ac53c55120c4565f014c63ac5cb Mon Sep 17 00:00:00 2001 From: Minjae Gwon Date: Sun, 26 Nov 2023 20:01:28 +0900 Subject: [PATCH 13/39] feat: make key able to be printed as hex --- core/src/main/scala/Key.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/src/main/scala/Key.scala b/core/src/main/scala/Key.scala index 01051c2..cfa5665 100644 --- a/core/src/main/scala/Key.scala +++ b/core/src/main/scala/Key.scala @@ -13,6 +13,8 @@ object Key { class Key(val underlying: Array[Byte]) extends AnyVal with Ordered[Key] { def is(that: Key): Boolean = underlying sameElements that.underlying + def hex = underlying.map("%02x" format _).mkString + override def compare(that: Key): Int = underlying .zip(that.underlying) .map { case (a, b) => From 41bf128adf93f5cb6bafe3b20e90a8475bd1045e Mon Sep 17 00:00:00 2001 From: Minjae Gwon Date: Sun, 26 Nov 2023 20:06:32 +0900 Subject: [PATCH 14/39] refactor: make master server similar with worker --- rpc/src/main/scala/MasterServer.scala | 37 ++++++++++++--------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/rpc/src/main/scala/MasterServer.scala b/rpc/src/main/scala/MasterServer.scala index 84c2981..af3f9f9 100644 --- a/rpc/src/main/scala/MasterServer.scala +++ b/rpc/src/main/scala/MasterServer.scala @@ -3,24 +3,19 @@ package kr.ac.postech.paranode.rpc import io.grpc.Server import io.grpc.ServerBuilder import kr.ac.postech.paranode.core.WorkerMetadata -import kr.ac.postech.paranode.rpc.MasterServer.port import org.apache.logging.log4j.scala.Logging import scala.collection.mutable.ListBuffer -import scala.collection.mutable.WrappedArray import scala.concurrent.ExecutionContext import scala.concurrent.Future import scala.concurrent.Promise import master.{MasterGrpc, RegisterReply, RegisterRequest} -object MasterServer { - private val port = 50051 -} - -class MasterServer(executionContext: ExecutionContext) extends Logging { self => +class MasterServer(executionContext: ExecutionContext, val port: Int = 50051) + extends Logging { self => private[this] val server: Server = ServerBuilder - .forPort(MasterServer.port) + .forPort(port) .addService(MasterGrpc.bindService(new MasterImpl, executionContext)) .build() @@ -32,39 +27,41 @@ class MasterServer(executionContext: ExecutionContext) extends Logging { self => def getWorkerDetails: List[WorkerMetadata] = workerDetails.toList - def getPort: String = port.toString - - private def start(): Unit = { + def start(): Unit = { server.start() - logger.info( - s"MasterServer listening on port $port" + logger.debug( + "[MasterServer] \n" + + s"port: $port\n" ) sys.addShutdownHook { logger.error( - "*** shutting down gRPC server since JVM is shutting down" + "[MasterServer] shutting down gRPC server since JVM is shutting down" ) self.stop() - logger.error("*** server shut down") + logger.error("[MasterServer] server shut down") } } - def startServer(): Unit = this.start() - def stopServer(): Unit = this.stop() - - private def stop(): Unit = { + def stop(): Unit = { if (server != null) { server.shutdown() } } + def blockUntilShutdown(): Unit = { + if (server != null) { + server.awaitTermination() + } + } + private class MasterImpl extends MasterGrpc.Master { override def register(request: RegisterRequest): Future[RegisterReply] = { val promise = Promise[RegisterReply] Future { - logger.debug(s"Register request: $request") + logger.debug(s"[MasterServer] Register ($request)") val workerMetadata = WorkerMetadata(request.worker.get.host, request.worker.get.port, None) From 599fc2de3fda659a2e18e216f3471dea955970c7 Mon Sep 17 00:00:00 2001 From: Minjae Gwon Date: Sun, 26 Nov 2023 20:07:04 +0900 Subject: [PATCH 15/39] chore: annotate type --- core/src/main/scala/Key.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/Key.scala b/core/src/main/scala/Key.scala index cfa5665..0c11567 100644 --- a/core/src/main/scala/Key.scala +++ b/core/src/main/scala/Key.scala @@ -13,7 +13,7 @@ object Key { class Key(val underlying: Array[Byte]) extends AnyVal with Ordered[Key] { def is(that: Key): Boolean = underlying sameElements that.underlying - def hex = underlying.map("%02x" format _).mkString + def hex: String = underlying.map("%02x" format _).mkString override def compare(that: Key): Int = underlying .zip(that.underlying) From d0ec1232521c0bb6a024bffedf0c13f20d3082a6 Mon Sep 17 00:00:00 2001 From: Minjae Gwon Date: Sun, 26 Nov 2023 20:07:18 +0900 Subject: [PATCH 16/39] chore: fix typo --- docker/worker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/worker/Dockerfile b/docker/worker/Dockerfile index ccc48a6..903c031 100644 --- a/docker/worker/Dockerfile +++ b/docker/worker/Dockerfile @@ -20,4 +20,4 @@ COPY . . RUN sbt --batch compile -ENTRYPOINT sbt --batch -v "worker/run ${MASTER_HOST}:${MASTER_PORT} -I /data/0 data/1 data/2 -O /output/" +ENTRYPOINT sbt --batch -v "worker/run ${MASTER_HOST}:${MASTER_PORT} -I /data/0 /data/1 /data/2 -O /output/" From 743a6dc5c5d5c1de78de00e0b0b3409271932e3e Mon Sep 17 00:00:00 2001 From: Minjae Gwon Date: Sun, 26 Nov 2023 20:07:59 +0900 Subject: [PATCH 17/39] chore: separate worker arguments --- worker/src/main/scala/Worker.scala | 56 +++++++-------------- worker/src/main/scala/WorkerArguments.scala | 23 +++++++++ 2 files changed, 41 insertions(+), 38 deletions(-) create mode 100644 worker/src/main/scala/WorkerArguments.scala diff --git a/worker/src/main/scala/Worker.scala b/worker/src/main/scala/Worker.scala index 036976c..449b4c9 100644 --- a/worker/src/main/scala/Worker.scala +++ b/worker/src/main/scala/Worker.scala @@ -8,64 +8,44 @@ import org.apache.logging.log4j.scala.Logging import java.net.InetAddress import java.net.ServerSocket import scala.concurrent.Await -import scala.reflect.io.Path import scala.util.Using object Worker extends Logging { - private class WorkerArguments(args: Array[String]) { - def masterIp: String = args(0).split(":")(0) - - def masterPort: Int = args(0).split(":")(1).toInt - - def inputDirectories(): Array[Path] = - args - .slice(inputDirectoriesIndex, outputDirectoryIndex - 1) - .map(Path.string2path) - - def outputDirectory(): Path = Path.string2path(args(outputDirectoryIndex)) - - private def inputDirectoriesIndex = args.indexOf("-I") + 1 - - private def outputDirectoryIndex = args.indexOf("-O") + 1 - } def main(args: Array[String]): Unit = { val workerArguments = new WorkerArguments(args) + val workerHost = InetAddress.getLocalHost.getHostAddress + val workerPort = Using(new ServerSocket(0))(_.getLocalPort).get + val workerMetadata = WorkerMetadata(workerHost, workerPort, None) logger.debug( - "Worker arguments: \n" + - s"masterIp: ${workerArguments.masterIp}\n" + + "[Worker] Arguments: \n" + + s"workerHost: $workerHost\n" + + s"workerPort: $workerPort\n" + + s"masterIp: ${workerArguments.masterHost}\n" + s"masterPort: ${workerArguments.masterPort}\n" + - s"inputDirectories: ${workerArguments.inputDirectories().mkString(", ")}\n" + - s"outputDirectory: ${workerArguments.outputDirectory()}\n" + s"inputDirectories: ${workerArguments.inputDirectories.mkString(", ")}\n" + + s"outputDirectory: ${workerArguments.outputDirectory}\n" ) - val workerPort = Using(new ServerSocket(0))(_.getLocalPort).get - val server = new WorkerServer( scala.concurrent.ExecutionContext.global, workerPort, - workerArguments.inputDirectories(), - workerArguments.outputDirectory() + workerArguments.inputDirectories, + workerArguments.outputDirectory ) - server.start() - val client = - MasterClient(workerArguments.masterIp, workerArguments.masterPort) + MasterClient(workerArguments.masterHost, workerArguments.masterPort) - try { - val ipAddress = InetAddress.getLocalHost.getHostAddress - val workerMetadata = WorkerMetadata(ipAddress, workerPort, None) + server.start() - Await.result( - client.register(workerMetadata), - scala.concurrent.duration.Duration.Inf - ) + Await.result( + client.register(workerMetadata), + scala.concurrent.duration.Duration.Inf + ) - } finally { - client.shutdown() - } + client.shutdown() server.blockUntilShutdown() } diff --git a/worker/src/main/scala/WorkerArguments.scala b/worker/src/main/scala/WorkerArguments.scala new file mode 100644 index 0000000..ef38699 --- /dev/null +++ b/worker/src/main/scala/WorkerArguments.scala @@ -0,0 +1,23 @@ +package kr.ac.postech.paranode.worker + +import scala.reflect.io.Directory +import scala.reflect.io.Path + +class WorkerArguments(args: Array[String]) { + def masterHost: String = args(0).split(":")(0) + + def masterPort: Int = args(0).split(":")(1).toInt + + def inputDirectories: Array[Directory] = + args + .slice(inputDirectoriesIndex, outputDirectoryIndex - 1) + .map(Path.string2path) + .map(_.toDirectory) + + def outputDirectory: Directory = + Path.string2path(args(outputDirectoryIndex)).toDirectory + + private def inputDirectoriesIndex = args.indexOf("-I") + 1 + + private def outputDirectoryIndex = args.indexOf("-O") + 1 +} From ef43ff348dc72e2ad0f9e2aefffe69c288292237 Mon Sep 17 00:00:00 2001 From: Minjae Gwon Date: Sun, 26 Nov 2023 20:08:24 +0900 Subject: [PATCH 18/39] feat: make sort endpoint --- rpc/src/main/protobuf/worker.proto | 5 + rpc/src/main/scala/WorkerClient.scala | 7 ++ rpc/src/main/scala/WorkerServer.scala | 163 ++++++++++++++++---------- 3 files changed, 110 insertions(+), 65 deletions(-) diff --git a/rpc/src/main/protobuf/worker.proto b/rpc/src/main/protobuf/worker.proto index d6b3e76..b9935cd 100644 --- a/rpc/src/main/protobuf/worker.proto +++ b/rpc/src/main/protobuf/worker.proto @@ -6,6 +6,7 @@ import "common.proto"; service Worker { rpc Sample (SampleRequest) returns (SampleReply) {} + rpc Sort (SortRequest) returns (SortReply) {} rpc Partition (PartitionRequest) returns (PartitionReply) {} rpc Exchange (ExchangeRequest) returns (ExchangeReply) {} rpc Merge (MergeRequest) returns (MergeReply) {} @@ -19,6 +20,10 @@ message SampleReply { repeated bytes sampledKeys = 1; } +message SortRequest {} + +message SortReply {} + message PartitionRequest { repeated WorkerMetadata workers = 1; } diff --git a/rpc/src/main/scala/WorkerClient.scala b/rpc/src/main/scala/WorkerClient.scala index fbf7d48..bad97cc 100644 --- a/rpc/src/main/scala/WorkerClient.scala +++ b/rpc/src/main/scala/WorkerClient.scala @@ -46,6 +46,13 @@ class WorkerClient private ( response } + def sort(): Future[SortReply] = { + val request = SortRequest() + val response = stub.sort(request) + + response + } + def partition( workers: List[(WorkerMetadata, KeyRange)] ): Future[PartitionReply] = { diff --git a/rpc/src/main/scala/WorkerServer.scala b/rpc/src/main/scala/WorkerServer.scala index 577baa9..1a20596 100644 --- a/rpc/src/main/scala/WorkerServer.scala +++ b/rpc/src/main/scala/WorkerServer.scala @@ -10,33 +10,46 @@ import scala.concurrent.ExecutionContext import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.Future import scala.concurrent.Promise +import scala.reflect.io.Directory import scala.reflect.io.Path +import common.{WorkerMetadata => RpcWorkerMetadata} import worker._ class WorkerServer( executionContext: ExecutionContext, port: Int, - inputDirectories: Array[Path], - outputDirectory: Path + inputDirectories: Array[Directory], + outputDirectory: Directory ) extends Logging { self => private[this] val server: Server = ServerBuilder .forPort(port) .addService(WorkerGrpc.bindService(new WorkerImpl, executionContext)) .build() + private def inputFiles = inputDirectories.flatMap(dir => { + logger.debug(s"Input directory: $dir") + logger.debug(s"Input directory files: ${dir.files.mkString(", ")}") + dir.files + }) + def start(): Unit = { server.start() logger.debug( - s"WorkerServer listening on port $port with inputDirectories: ${inputDirectories - .mkString(", ")} and outputDirectory: $outputDirectory" + "[WorkerServer] \n" + + s"port: $port\n" + + s"inputDirectories: ${inputDirectories.mkString(", ")}\n" + + s"inputFiles: ${inputFiles.mkString(", ")}\n" + + s"outputDirectory: $outputDirectory\n" ) sys.addShutdownHook { - logger.error("*** shutting down gRPC server since JVM is shutting down") + logger.error( + "[WorkerServer] shutting down gRPC server since JVM is shutting down" + ) self.stop() - logger.error("*** server shut down") + logger.error("[WorkerServer] server shut down") } } @@ -53,86 +66,105 @@ class WorkerServer( } private class WorkerImpl extends WorkerGrpc.Worker { + private def toWorkerMetadata(workers: Seq[RpcWorkerMetadata]) = + workers.map(worker => + WorkerMetadata( + worker.node.get.host, + worker.node.get.port, + worker.keyRange.map(keyRange => + KeyRange( + Key.fromByteString(keyRange.from), + Key.fromByteString(keyRange.to) + ) + ) + ) + ) + override def sample(request: SampleRequest): Future[SampleReply] = { val promise = Promise[SampleReply] Future { - logger.debug(s"Sample request: $request") + logger.debug(s"[WorkerServer] Sample ($request)") - try { - val sampledKeys = inputDirectories - .map(_.toDirectory) - .flatMap(_.list) - .map(f => Block.fromPath(f.path)) - .flatMap(_.sample(request.numberOfKeys)) - .map(key => ByteString.copyFrom(key.underlying)) - - promise.success(SampleReply(sampledKeys)) - } catch { - case e: Exception => - println(e) - promise.failure(e) - } + val sampledKeys = inputFiles + .map(f => Block.fromPath(f.path)) + .flatMap(_.sample(request.numberOfKeys)) + .map(key => ByteString.copyFrom(key.underlying)) + + promise.success(SampleReply(sampledKeys)) }(executionContext) promise.future } + override def sort(request: SortRequest): Future[SortReply] = { + val promise = Promise[SortReply] + + Future { + logger.debug(s"[WorkerServer] Sort ($request)") + + inputFiles + .foreach(path => { + val block = Block.fromPath(path) + + val sortedBlock = block.sorted + + logger.debug(s"[WorkerServer] Writing sorted block to $path") + + sortedBlock.writeTo(path) + + logger.debug(s"[WorkerServer] Wrote sorted block to $path") + }) + + promise.success(new SortReply()) + } + + promise.future + } + override def partition( request: PartitionRequest ): Future[PartitionReply] = { val promise = Promise[PartitionReply] Future { - try { - val block = Block.fromPath(Path("data/block"), 10, 90) - request.workers - .map(workerMetadata => { - val keyRange = KeyRange( - Key.fromString(workerMetadata.keyRange.get.from.toStringUtf8), - Key.fromString(workerMetadata.keyRange.get.to.toStringUtf8) - ) - val partition = block.partition(keyRange) - val partitionPath = Path( - s"data/partition/${workerMetadata.node.get.host}:${workerMetadata.node.get.port}" - ) - partition._2.writeTo(partitionPath) - }) - - promise.success(new PartitionReply()) - } catch { - case e: Exception => - println(e) - promise.failure(e) - } + logger.debug(s"[WorkerServer] Partition ($request)") + + val workers = toWorkerMetadata(request.workers) + + inputFiles + .map(path => { + val block = Block.fromPath(path) + workers + .map(_.keyRange.get) + .map(block.partition) + .map({ case (keyRange, partition) => + val partitionPath = Path( + s"$path.${keyRange.from.hex}-${keyRange.to.hex}" + ) + + logger.debug( + s"[WorkerServer] Writing partition to $partitionPath" + ) + + partition.writeTo(partitionPath) + + logger.debug( + s"[WorkerServer] Wrote partition to $partitionPath" + ) + }) + + }) + + promise.success(new PartitionReply()) }(executionContext) promise.future } override def exchange(request: ExchangeRequest): Future[ExchangeReply] = { - val futures = request.workers.map(workerMetadata => - Future { - val host = workerMetadata.node.get.host - val port = workerMetadata.node.get.port - val partitionPath = Path(s"data/partition/${host}:${port}") - - try { - if (partitionPath.exists) { - val partition = Block.fromPath(partitionPath, 10, 90) - val exchangeClient = ExchangeClient.apply(host, port) - val reply = exchangeClient.saveRecords(partition.records) - Some(reply) - } else { - None - } - } finally { - if (partitionPath.exists) { - partitionPath.delete() - } - } - }(executionContext) - ) + val futures = + request.workers.map(_ => Future {}(executionContext)) Future.sequence(futures).map(_ => new ExchangeReply()) } @@ -145,7 +177,7 @@ class WorkerServer( val host = Path("data/host") val port = Path("data/port") val blockPath = Path(s"data/partition/${host}:${port}") - val mergedBlock = Block.fromPath(blockPath, 10, 90).sort() + val mergedBlock = Block.fromPath(blockPath, 10, 90).sorted mergedBlock.writeTo(blockPath) promise.success(new MergeReply()) @@ -158,6 +190,7 @@ class WorkerServer( promise.future } + } } From e71e75915a621d65864ed60778e0b00b48a0df24 Mon Sep 17 00:00:00 2001 From: Minjae Gwon Date: Sun, 26 Nov 2023 20:08:36 +0900 Subject: [PATCH 19/39] feat: sort and partition --- master/src/main/scala/Master.scala | 37 ++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/master/src/main/scala/Master.scala b/master/src/main/scala/Master.scala index 575636b..b21ad79 100644 --- a/master/src/main/scala/Master.scala +++ b/master/src/main/scala/Master.scala @@ -1,6 +1,7 @@ package kr.ac.postech.paranode.master import kr.ac.postech.paranode.core.Key +import kr.ac.postech.paranode.core.KeyRange import kr.ac.postech.paranode.core.WorkerMetadata import kr.ac.postech.paranode.rpc.MasterServer import kr.ac.postech.paranode.rpc.WorkerClient @@ -20,7 +21,7 @@ object Master extends Logging { } val server = new MasterServer(scala.concurrent.ExecutionContext.global) - server.startServer() + server.start() while (server.getWorkerDetails.size < numberOfWorker) { Thread.sleep(1000) @@ -33,7 +34,7 @@ object Master extends Logging { try { val ipAddress = InetAddress.getLocalHost.getHostAddress - println(ipAddress + ":" + server.getPort) + println(ipAddress + ":" + server.port) println(workerInfo.map(_.host).mkString(", ")) } catch { case e: Exception => e.printStackTrace() @@ -52,6 +53,38 @@ object Master extends Logging { .map(Key.fromByteString) logger.debug(s"Sampled keys: $sampledKeys") + + val sortedSampledKeys = sampledKeys.sorted + + logger.debug(s"Sorted Sampled keys: $sortedSampledKeys") + + val keyRanges = sortedSampledKeys + .sliding( + sortedSampledKeys.size / numberOfWorker, + sortedSampledKeys.size / numberOfWorker + ) + .toList + .map(keys => (keys.head, keys.last)) + + val keyRangesWithWorker = workerInfo.zip(keyRanges.map(KeyRange.tupled)) + + logger.debug(s"Key ranges with worker: $keyRangesWithWorker") + + Await.result( + Future.sequence( + clients.map(_.sort()) + ), + scala.concurrent.duration.Duration.Inf + ) + + Await.result( + Future.sequence( + clients.map(_.partition(keyRangesWithWorker)) + ), + scala.concurrent.duration.Duration.Inf + ) + + server.blockUntilShutdown() } } From 71f82082eae5e58ceeafce653e5012eedc55cd0b Mon Sep 17 00:00:00 2001 From: Minjae Gwon Date: Sun, 26 Nov 2023 20:18:28 +0900 Subject: [PATCH 20/39] feat: parse args and logging --- master/src/main/scala/Master.scala | 52 ++++++++++++--------- master/src/main/scala/MasterArguments.scala | 6 +++ 2 files changed, 36 insertions(+), 22 deletions(-) create mode 100644 master/src/main/scala/MasterArguments.scala diff --git a/master/src/main/scala/Master.scala b/master/src/main/scala/Master.scala index b21ad79..c3d55d9 100644 --- a/master/src/main/scala/Master.scala +++ b/master/src/main/scala/Master.scala @@ -11,34 +11,34 @@ import java.net._ import scala.concurrent.Await import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.Future -import scala.util.Try object Master extends Logging { def main(args: Array[String]): Unit = { - val numberOfWorker = Try(args(0).toInt).getOrElse { - println("Invalid command") - return - } + val masterArguments = new MasterArguments(args) + val masterHost = InetAddress.getLocalHost.getHostAddress + val masterPort = sys.env.getOrElse("MASTER_PORT", "50051").toInt + + logger.debug( + "[Master] Arguments: \n" + + s"masterHost: $masterHost\n" + + s"masterPort: $masterPort\n" + + s"numberOfWorkers: ${masterArguments.numberOfWorkers}\n" + ) + + val server = + new MasterServer(scala.concurrent.ExecutionContext.global, masterPort) - val server = new MasterServer(scala.concurrent.ExecutionContext.global) server.start() - while (server.getWorkerDetails.size < numberOfWorker) { + println(masterHost + ":" + server.port) + + while (server.getWorkerDetails.size < masterArguments.numberOfWorkers) { Thread.sleep(1000) } val workerInfo: List[WorkerMetadata] = server.getWorkerDetails - assert(workerInfo.size == numberOfWorker) - - try { - val ipAddress = InetAddress.getLocalHost.getHostAddress - - println(ipAddress + ":" + server.port) - println(workerInfo.map(_.host).mkString(", ")) - } catch { - case e: Exception => e.printStackTrace() - } + println(workerInfo.map(_.host).mkString(", ")) val clients = workerInfo.map { worker => WorkerClient(worker.host, worker.port) @@ -52,23 +52,25 @@ object Master extends Logging { .flatMap(_.sampledKeys) .map(Key.fromByteString) - logger.debug(s"Sampled keys: $sampledKeys") + logger.debug(s"[Master] Sampled keys: $sampledKeys") val sortedSampledKeys = sampledKeys.sorted - logger.debug(s"Sorted Sampled keys: $sortedSampledKeys") + logger.debug(s"[Master] Sorted Sampled keys: $sortedSampledKeys") val keyRanges = sortedSampledKeys .sliding( - sortedSampledKeys.size / numberOfWorker, - sortedSampledKeys.size / numberOfWorker + sortedSampledKeys.size / masterArguments.numberOfWorkers, + sortedSampledKeys.size / masterArguments.numberOfWorkers ) .toList .map(keys => (keys.head, keys.last)) val keyRangesWithWorker = workerInfo.zip(keyRanges.map(KeyRange.tupled)) - logger.debug(s"Key ranges with worker: $keyRangesWithWorker") + logger.debug(s"[Master] Key ranges with worker: $keyRangesWithWorker") + + logger.debug("[Master] Sort started") Await.result( Future.sequence( @@ -77,6 +79,10 @@ object Master extends Logging { scala.concurrent.duration.Duration.Inf ) + logger.debug("[Master] Sort finished") + + logger.debug("[Master] Partition started") + Await.result( Future.sequence( clients.map(_.partition(keyRangesWithWorker)) @@ -84,6 +90,8 @@ object Master extends Logging { scala.concurrent.duration.Duration.Inf ) + logger.debug("[Master] Partition finished") + server.blockUntilShutdown() } diff --git a/master/src/main/scala/MasterArguments.scala b/master/src/main/scala/MasterArguments.scala new file mode 100644 index 0000000..d260f49 --- /dev/null +++ b/master/src/main/scala/MasterArguments.scala @@ -0,0 +1,6 @@ +package kr.ac.postech.paranode.master + +class MasterArguments(args: Array[String]) { + def numberOfWorkers: Int = args(0).toInt + +} From f40747fd62eb4c5403d040b2dd8c6297c4575fb4 Mon Sep 17 00:00:00 2001 From: Minjae Gwon Date: Sun, 26 Nov 2023 20:35:22 +0900 Subject: [PATCH 21/39] chore: remove verbose logs --- rpc/src/main/scala/WorkerServer.scala | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/rpc/src/main/scala/WorkerServer.scala b/rpc/src/main/scala/WorkerServer.scala index 1a20596..4d19c1a 100644 --- a/rpc/src/main/scala/WorkerServer.scala +++ b/rpc/src/main/scala/WorkerServer.scala @@ -27,11 +27,7 @@ class WorkerServer( .addService(WorkerGrpc.bindService(new WorkerImpl, executionContext)) .build() - private def inputFiles = inputDirectories.flatMap(dir => { - logger.debug(s"Input directory: $dir") - logger.debug(s"Input directory files: ${dir.files.mkString(", ")}") - dir.files - }) + private def inputFiles = inputDirectories.flatMap(_.files) def start(): Unit = { server.start() From b5573eb3413f3cd12ab7d781eb343ce1295cbc2d Mon Sep 17 00:00:00 2001 From: Minjae Gwon Date: Sun, 26 Nov 2023 20:35:33 +0900 Subject: [PATCH 22/39] fix: docker caching --- docker-compose.yml | 4 ++++ docker/master/Dockerfile | 17 +++++++++++++++-- docker/worker/Dockerfile | 11 ++++++++++- 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 13e8811..1022b9c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,6 +3,8 @@ services: build: context: . dockerfile: docker/master/Dockerfile + args: + - NUMBER_OF_WORKERS=2 worker: build: context: . @@ -10,5 +12,7 @@ services: args: - MASTER_HOST=master - MASTER_PORT=50051 + deploy: + replicas: 2 depends_on: - master diff --git a/docker/master/Dockerfile b/docker/master/Dockerfile index 5405e99..1ec0e61 100644 --- a/docker/master/Dockerfile +++ b/docker/master/Dockerfile @@ -1,13 +1,26 @@ FROM sbtscala/scala-sbt:eclipse-temurin-jammy-20.0.2_9_1.9.6_2.13.12 +ARG NUMBER_OF_WORKERS + +ENV NUMBER_OF_WORKERS=${NUMBER_OF_WORKERS} + RUN mkdir -p /app WORKDIR /app -COPY project/build.properties project/plugins.sbt project/scalapb.sbt /app/project/ +COPY build.sbt ./ +COPY core/build.sbt ./core/ +COPY master/build.sbt ./master/ +COPY rpc/build.sbt ./rpc/ +COPY utils/build.sbt ./utils/ +COPY worker/build.sbt ./worker/ +COPY rpc/src/main/protobuf ./rpc/src/main/protobuf +COPY project/build.properties project/plugins.sbt project/scalapb.sbt ./project/ + +RUN sbt --batch compile COPY . . RUN sbt --batch compile -ENTRYPOINT sbt --batch -v "master/run 1" +ENTRYPOINT sbt --batch -v "master/run ${NUMBER_OF_WORKERS}" diff --git a/docker/worker/Dockerfile b/docker/worker/Dockerfile index 903c031..134a69b 100644 --- a/docker/worker/Dockerfile +++ b/docker/worker/Dockerfile @@ -14,7 +14,16 @@ COPY docker/worker/data /data WORKDIR /app -COPY project/build.properties project/plugins.sbt project/scalapb.sbt /app/project/ +COPY build.sbt ./ +COPY core/build.sbt ./core/ +COPY master/build.sbt ./master/ +COPY rpc/build.sbt ./rpc/ +COPY utils/build.sbt ./utils/ +COPY worker/build.sbt ./worker/ +COPY rpc/src/main/protobuf ./rpc/src/main/protobuf +COPY project/build.properties project/plugins.sbt project/scalapb.sbt ./project/ + +RUN sbt --batch compile COPY . . From dfcff33e00eef609cf39a71949092cfd323e2bca Mon Sep 17 00:00:00 2001 From: Minjae Gwon Date: Sun, 26 Nov 2023 20:36:22 +0900 Subject: [PATCH 23/39] chore: fix typo --- core/src/main/scala/Block.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/main/scala/Block.scala b/core/src/main/scala/Block.scala index cbe1730..d187207 100644 --- a/core/src/main/scala/Block.scala +++ b/core/src/main/scala/Block.scala @@ -31,7 +31,7 @@ object Block extends Logging { keyLength: Int = 10, valueLength: Int = 90 ): Block = { - logger.debug(s"Reading block from $path") + logger.debug(s"[Block] Reading block from $path") Block.fromSource(Source.fromURI(path.toURI), keyLength, valueLength) } From 988008c7701466f7271a9b7acd058860ec15cdf0 Mon Sep 17 00:00:00 2001 From: Minjae Gwon Date: Sun, 26 Nov 2023 20:48:57 +0900 Subject: [PATCH 24/39] refactor: merge exchange service --- rpc/src/main/protobuf/exchange.proto | 13 ----- rpc/src/main/protobuf/worker.proto | 7 +++ rpc/src/main/scala/ExchangeClient.scala | 43 -------------- rpc/src/main/scala/ExchangeServer.scala | 74 ------------------------- rpc/src/main/scala/WorkerClient.scala | 14 ++++- rpc/src/main/scala/WorkerServer.scala | 28 +++++++++- 6 files changed, 45 insertions(+), 134 deletions(-) delete mode 100644 rpc/src/main/protobuf/exchange.proto delete mode 100644 rpc/src/main/scala/ExchangeClient.scala delete mode 100644 rpc/src/main/scala/ExchangeServer.scala diff --git a/rpc/src/main/protobuf/exchange.proto b/rpc/src/main/protobuf/exchange.proto deleted file mode 100644 index 248b039..0000000 --- a/rpc/src/main/protobuf/exchange.proto +++ /dev/null @@ -1,13 +0,0 @@ -syntax = "proto3"; - -package kr.ac.postech.paranode.rpc; - -service Exchange { - rpc SaveRecords (SaveRecordsRequest) returns (SaveRecordsReply) {} -} - -message SaveRecordsRequest { - repeated bytes records = 1; -} - -message SaveRecordsReply {} diff --git a/rpc/src/main/protobuf/worker.proto b/rpc/src/main/protobuf/worker.proto index b9935cd..91766ba 100644 --- a/rpc/src/main/protobuf/worker.proto +++ b/rpc/src/main/protobuf/worker.proto @@ -9,6 +9,7 @@ service Worker { rpc Sort (SortRequest) returns (SortReply) {} rpc Partition (PartitionRequest) returns (PartitionReply) {} rpc Exchange (ExchangeRequest) returns (ExchangeReply) {} + rpc SaveBlock (SaveBlockRequest) returns (SaveBlockReply) {} rpc Merge (MergeRequest) returns (MergeReply) {} } @@ -36,6 +37,12 @@ message ExchangeRequest { message ExchangeReply {} +message SaveBlockRequest { + bytes block = 1; +} + +message SaveBlockReply {} + message MergeRequest {} message MergeReply {} diff --git a/rpc/src/main/scala/ExchangeClient.scala b/rpc/src/main/scala/ExchangeClient.scala deleted file mode 100644 index 7281e9a..0000000 --- a/rpc/src/main/scala/ExchangeClient.scala +++ /dev/null @@ -1,43 +0,0 @@ -package kr.ac.postech.paranode.rpc - -import com.google.protobuf.ByteString -import io.grpc.ManagedChannel -import io.grpc.ManagedChannelBuilder -import kr.ac.postech.paranode.core.Record - -import java.util.concurrent.TimeUnit - -import exchange.ExchangeGrpc.ExchangeBlockingStub -import exchange.{ExchangeGrpc, SaveRecordsReply, SaveRecordsRequest} - -object ExchangeClient { - def apply(host: String, port: Int): ExchangeClient = { - val channel = ManagedChannelBuilder - .forAddress(host, port) - .usePlaintext() - .build() - - val blockingStub = ExchangeGrpc.blockingStub(channel) - new ExchangeClient(channel, blockingStub) - } -} - -class ExchangeClient private ( - private val channel: ManagedChannel, - private val blockingStub: ExchangeBlockingStub -) { - def shutdown(): Unit = { - channel.shutdown.awaitTermination(5, TimeUnit.SECONDS) - } - - def saveRecords(records: LazyList[Record]): SaveRecordsReply = { - val request = - SaveRecordsRequest( - records.map(x => ByteString.copyFrom(x.toChars.map(_.toByte))) - ) - - // TODO - - blockingStub.saveRecords(request) - } -} diff --git a/rpc/src/main/scala/ExchangeServer.scala b/rpc/src/main/scala/ExchangeServer.scala deleted file mode 100644 index 81694e9..0000000 --- a/rpc/src/main/scala/ExchangeServer.scala +++ /dev/null @@ -1,74 +0,0 @@ -package kr.ac.postech.paranode.rpc - -import io.grpc.Server -import io.grpc.ServerBuilder - -import java.util.logging.Logger -import scala.concurrent.ExecutionContext -import scala.concurrent.Future -import scala.concurrent.Promise - -import exchange.{ExchangeGrpc, SaveRecordsReply, SaveRecordsRequest} - -object ExchangeServer { - private val logger = Logger.getLogger(classOf[ExchangeServer].getName) - - def main(args: Array[String]): Unit = { - val server = new ExchangeServer(ExecutionContext.global) - server.start() - server.blockUntilShutdown() - } - - private val port = 30050 -} - -class ExchangeServer(executionContext: ExecutionContext) { self => - private[this] val server: Server = ServerBuilder - .forPort(ExchangeServer.port) - .addService(ExchangeGrpc.bindService(new ExchangeImpl, executionContext)) - .build() - - private def start(): Unit = { - server.start() - - ExchangeServer.logger.info( - "Server started, listening on " + ExchangeServer.port - ) - - sys.addShutdownHook { - System.err.println( - "*** shutting down gRPC server since JVM is shutting down" - ) - self.stop() - System.err.println("*** server shut down") - } - } - - private def stop(): Unit = { - if (server != null) { - server.shutdown() - } - } - - private def blockUntilShutdown(): Unit = { - if (server != null) { - server.awaitTermination() - } - } - - private class ExchangeImpl extends ExchangeGrpc.Exchange { - override def saveRecords( - request: SaveRecordsRequest - ): Future[SaveRecordsReply] = { - val promise = Promise[SaveRecordsReply] - - Future { - // TODO: Logic - promise.success(new SaveRecordsReply()) - }(executionContext) - - promise.future - } - } - -} diff --git a/rpc/src/main/scala/WorkerClient.scala b/rpc/src/main/scala/WorkerClient.scala index bad97cc..368bad5 100644 --- a/rpc/src/main/scala/WorkerClient.scala +++ b/rpc/src/main/scala/WorkerClient.scala @@ -3,13 +3,11 @@ package kr.ac.postech.paranode.rpc import com.google.protobuf.ByteString import io.grpc.ManagedChannel import io.grpc.ManagedChannelBuilder -import kr.ac.postech.paranode.core.KeyRange -import kr.ac.postech.paranode.core.WorkerMetadata +import kr.ac.postech.paranode.core.{Block, KeyRange, WorkerMetadata} import java.util.concurrent.TimeUnit import java.util.logging.Logger import scala.concurrent.Future - import worker._ import worker.WorkerGrpc.WorkerStub import common.{ @@ -89,6 +87,16 @@ class WorkerClient private ( stub.exchange(request) } + def saveBlock( + block: Block + ): Future[SaveBlockReply] = { + val request = SaveBlockRequest( + ByteString.copyFrom(block.toChars.map(_.toByte).toArray) + ) + + stub.saveBlock(request) + } + def merge(): Future[MergeReply] = { val request = MergeRequest() stub.merge(request) diff --git a/rpc/src/main/scala/WorkerServer.scala b/rpc/src/main/scala/WorkerServer.scala index 4d19c1a..810a5d5 100644 --- a/rpc/src/main/scala/WorkerServer.scala +++ b/rpc/src/main/scala/WorkerServer.scala @@ -12,9 +12,9 @@ import scala.concurrent.Future import scala.concurrent.Promise import scala.reflect.io.Directory import scala.reflect.io.Path - import common.{WorkerMetadata => RpcWorkerMetadata} import worker._ +import java.util.UUID class WorkerServer( executionContext: ExecutionContext, @@ -165,6 +165,32 @@ class WorkerServer( Future.sequence(futures).map(_ => new ExchangeReply()) } + override def saveBlock( + request: SaveBlockRequest + ): Future[SaveBlockReply] = { + val promise = Promise[SaveBlockReply] + + Future { + logger.debug(s"[WorkerServer] SaveBlock ($request)") + + val block = Block.fromBytes( + LazyList.from(request.block.toByteArray) + ) + + val path = outputDirectory / UUID.randomUUID().toString + + logger.debug(s"[WorkerServer] Writing block to $path") + + block.writeTo(path) + + logger.debug(s"[WorkerServer] Wrote block to $path") + + promise.success(new SaveBlockReply()) + } + + promise.future + } + override def merge(request: MergeRequest): Future[MergeReply] = { val promise = Promise[MergeReply] From d5223237e44736e36d4fc2d2f82fab02a34e0713 Mon Sep 17 00:00:00 2001 From: Minjae Gwon Date: Sun, 26 Nov 2023 20:51:04 +0900 Subject: [PATCH 25/39] feat: remove partitioned block --- rpc/src/main/scala/WorkerServer.scala | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/rpc/src/main/scala/WorkerServer.scala b/rpc/src/main/scala/WorkerServer.scala index 810a5d5..fe0125e 100644 --- a/rpc/src/main/scala/WorkerServer.scala +++ b/rpc/src/main/scala/WorkerServer.scala @@ -148,6 +148,14 @@ class WorkerServer( logger.debug( s"[WorkerServer] Wrote partition to $partitionPath" ) + + if (path.exists && path.isFile) { + val result = path.delete() + + logger.debug( + s"[WorkerServer] Deleted $path: $result" + ) + } }) }) From 7b4ad23ce0a127e891d0978f7ebd626d686ef25d Mon Sep 17 00:00:00 2001 From: Minjae Gwon Date: Sun, 26 Nov 2023 21:06:19 +0900 Subject: [PATCH 26/39] fix: docker caching --- docker/master/Dockerfile | 5 ++++- docker/worker/Dockerfile | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docker/master/Dockerfile b/docker/master/Dockerfile index 1ec0e61..d35f6e8 100644 --- a/docker/master/Dockerfile +++ b/docker/master/Dockerfile @@ -14,11 +14,14 @@ COPY master/build.sbt ./master/ COPY rpc/build.sbt ./rpc/ COPY utils/build.sbt ./utils/ COPY worker/build.sbt ./worker/ -COPY rpc/src/main/protobuf ./rpc/src/main/protobuf COPY project/build.properties project/plugins.sbt project/scalapb.sbt ./project/ RUN sbt --batch compile +COPY rpc/src/main/protobuf ./rpc/src/main/protobuf + +RUN sbt --batch compile + COPY . . RUN sbt --batch compile diff --git a/docker/worker/Dockerfile b/docker/worker/Dockerfile index 134a69b..af28466 100644 --- a/docker/worker/Dockerfile +++ b/docker/worker/Dockerfile @@ -20,11 +20,14 @@ COPY master/build.sbt ./master/ COPY rpc/build.sbt ./rpc/ COPY utils/build.sbt ./utils/ COPY worker/build.sbt ./worker/ -COPY rpc/src/main/protobuf ./rpc/src/main/protobuf COPY project/build.properties project/plugins.sbt project/scalapb.sbt ./project/ RUN sbt --batch compile +COPY rpc/src/main/protobuf ./rpc/src/main/protobuf + +RUN sbt --batch compile + COPY . . RUN sbt --batch compile From 5791381795aa6c3490ab312ad760ffe9e09c8712 Mon Sep 17 00:00:00 2001 From: Minjae Gwon Date: Sun, 26 Nov 2023 21:07:00 +0900 Subject: [PATCH 27/39] feat: exchange --- master/src/main/scala/Master.scala | 11 ++++++ rpc/src/main/scala/WorkerClient.scala | 5 ++- rpc/src/main/scala/WorkerServer.scala | 50 ++++++++++++++++++++------- 3 files changed, 53 insertions(+), 13 deletions(-) diff --git a/master/src/main/scala/Master.scala b/master/src/main/scala/Master.scala index c3d55d9..9c7da97 100644 --- a/master/src/main/scala/Master.scala +++ b/master/src/main/scala/Master.scala @@ -92,6 +92,17 @@ object Master extends Logging { logger.debug("[Master] Partition finished") + logger.debug("[Master] Exchange started") + + Await.result( + Future.sequence( + clients.map(_.exchange(keyRangesWithWorker)) + ), + scala.concurrent.duration.Duration.Inf + ) + + logger.debug("[Master] Exchange finished") + server.blockUntilShutdown() } diff --git a/rpc/src/main/scala/WorkerClient.scala b/rpc/src/main/scala/WorkerClient.scala index 368bad5..4b71752 100644 --- a/rpc/src/main/scala/WorkerClient.scala +++ b/rpc/src/main/scala/WorkerClient.scala @@ -3,11 +3,14 @@ package kr.ac.postech.paranode.rpc import com.google.protobuf.ByteString import io.grpc.ManagedChannel import io.grpc.ManagedChannelBuilder -import kr.ac.postech.paranode.core.{Block, KeyRange, WorkerMetadata} +import kr.ac.postech.paranode.core.Block +import kr.ac.postech.paranode.core.KeyRange +import kr.ac.postech.paranode.core.WorkerMetadata import java.util.concurrent.TimeUnit import java.util.logging.Logger import scala.concurrent.Future + import worker._ import worker.WorkerGrpc.WorkerStub import common.{ diff --git a/rpc/src/main/scala/WorkerServer.scala b/rpc/src/main/scala/WorkerServer.scala index fe0125e..f35b45c 100644 --- a/rpc/src/main/scala/WorkerServer.scala +++ b/rpc/src/main/scala/WorkerServer.scala @@ -6,15 +6,17 @@ import io.grpc.ServerBuilder import kr.ac.postech.paranode.core._ import org.apache.logging.log4j.scala.Logging +import java.util.UUID +import scala.concurrent.Await import scala.concurrent.ExecutionContext import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.Future import scala.concurrent.Promise import scala.reflect.io.Directory import scala.reflect.io.Path + import common.{WorkerMetadata => RpcWorkerMetadata} import worker._ -import java.util.UUID class WorkerServer( executionContext: ExecutionContext, @@ -148,14 +150,6 @@ class WorkerServer( logger.debug( s"[WorkerServer] Wrote partition to $partitionPath" ) - - if (path.exists && path.isFile) { - val result = path.delete() - - logger.debug( - s"[WorkerServer] Deleted $path: $result" - ) - } }) }) @@ -167,10 +161,42 @@ class WorkerServer( } override def exchange(request: ExchangeRequest): Future[ExchangeReply] = { - val futures = - request.workers.map(_ => Future {}(executionContext)) + val promise = Promise[ExchangeReply] + + Future { + logger.debug(s"[WorkerServer] Exchange ($request)") + + val workers = toWorkerMetadata(request.workers) + + inputFiles.foreach(path => { + val block = Block.fromPath(path) + val targetWorkers = workers + .filter(_.keyRange.get.includes(block.records.head.key)) - Future.sequence(futures).map(_ => new ExchangeReply()) + logger.debug(s"[WorkerServer] Sending $block to $targetWorkers") + + Await.result( + Future.sequence( + targetWorkers + .map(worker => WorkerClient(worker.host, worker.port)) + .map(_.saveBlock(block)) + ), + scala.concurrent.duration.Duration.Inf + ) + + if (path.exists && path.isFile) { + val result = path.delete() + + logger.debug( + s"[WorkerServer] Deleted $path: $result" + ) + } + }) + + promise.success(new ExchangeReply()) + } + + promise.future } override def saveBlock( From 05f2813787ba2b0e8f28470c8a50687ecb602c66 Mon Sep 17 00:00:00 2001 From: Minjae Gwon Date: Sun, 26 Nov 2023 22:30:41 +0900 Subject: [PATCH 28/39] feat: merge --- core/src/main/scala/Block.scala | 5 +++ core/src/main/scala/Record.scala | 21 +++++++++++ master/src/main/scala/Master.scala | 11 ++++++ rpc/src/main/scala/WorkerServer.scala | 52 ++++++++++++++++----------- 4 files changed, 68 insertions(+), 21 deletions(-) diff --git a/core/src/main/scala/Block.scala b/core/src/main/scala/Block.scala index d187207..2735202 100644 --- a/core/src/main/scala/Block.scala +++ b/core/src/main/scala/Block.scala @@ -9,6 +9,11 @@ import scala.io.Source import scala.reflect.io.Path object Block extends Logging { + + implicit class Blocks(blocks: List[Block]) { + def merged: Block = new Block(Record.merged(blocks.map(_.records))) + } + def fromBytes( bytes: LazyList[Byte], keyLength: Int = 10, diff --git a/core/src/main/scala/Record.scala b/core/src/main/scala/Record.scala index 66a0316..1728f57 100644 --- a/core/src/main/scala/Record.scala +++ b/core/src/main/scala/Record.scala @@ -35,6 +35,27 @@ object Record extends Logging { records: LazyList[Record], number: Int = 64 ): LazyList[Key] = records.take(number).map(_.key) + + def merged( + listOfRecords: List[LazyList[Record]] + ): LazyList[Record] = { + if (listOfRecords.isEmpty) { + LazyList.empty + } else { + val sortedListOfRecords = + listOfRecords.sorted(Ordering.by((_: LazyList[Record]).head.key)) + + logger.debug( + s"[Record] Sorted list of records: ${sortedListOfRecords.map(_.head.key.hex).mkString(", ")}" + ) + + sortedListOfRecords.head.head #:: merged( + (sortedListOfRecords.head.tail :: sortedListOfRecords.tail).filter( + _.nonEmpty + ) + ) + } + } } class Record(val key: Key, val value: Array[Byte]) extends Ordered[Record] { diff --git a/master/src/main/scala/Master.scala b/master/src/main/scala/Master.scala index 9c7da97..ffdb17f 100644 --- a/master/src/main/scala/Master.scala +++ b/master/src/main/scala/Master.scala @@ -103,6 +103,17 @@ object Master extends Logging { logger.debug("[Master] Exchange finished") + logger.debug("[Master] Merge started") + + Await.result( + Future.sequence( + clients.map(_.merge()) + ), + scala.concurrent.duration.Duration.Inf + ) + + logger.debug("[Master] Merge finished") + server.blockUntilShutdown() } diff --git a/rpc/src/main/scala/WorkerServer.scala b/rpc/src/main/scala/WorkerServer.scala index f35b45c..ce39dde 100644 --- a/rpc/src/main/scala/WorkerServer.scala +++ b/rpc/src/main/scala/WorkerServer.scala @@ -31,6 +31,8 @@ class WorkerServer( private def inputFiles = inputDirectories.flatMap(_.files) + private def outputFiles = outputDirectory.files.toList + def start(): Unit = { server.start() @@ -150,6 +152,11 @@ class WorkerServer( logger.debug( s"[WorkerServer] Wrote partition to $partitionPath" ) + + if (path.exists && path.isFile) { + val result = path.delete() + logger.debug(s"[WorkerServer] Deleted $path: $result") + } }) }) @@ -183,14 +190,6 @@ class WorkerServer( ), scala.concurrent.duration.Duration.Inf ) - - if (path.exists && path.isFile) { - val result = path.delete() - - logger.debug( - s"[WorkerServer] Deleted $path: $result" - ) - } }) promise.success(new ExchangeReply()) @@ -229,19 +228,30 @@ class WorkerServer( val promise = Promise[MergeReply] Future { - try { - val host = Path("data/host") - val port = Path("data/port") - val blockPath = Path(s"data/partition/${host}:${port}") - val mergedBlock = Block.fromPath(blockPath, 10, 90).sorted - mergedBlock.writeTo(blockPath) - - promise.success(new MergeReply()) - } catch { - case e: Exception => - println(e) - promise.failure(e) - } + logger.debug(s"[WorkerServer] Merge ($request)") + val targetFiles = outputFiles + + val blocks = targetFiles.map(path => Block.fromPath(path)) + + logger.debug("[WorkerServer] Merging blocks") + + val mergedBlock = blocks.merged + + val results = mergedBlock.writeTo(outputDirectory / "result") + + targetFiles.foreach(file => { + val result = file.delete() + + logger.debug( + s"[WorkerServer] Deleted $file: $result" + ) + }) + + logger.debug( + s"[WorkerServer] Merged blocks: $results" + ) + + promise.success(new MergeReply()) }(executionContext) promise.future From 2a999487c37c7fcea2fb82495441e9019d7346f9 Mon Sep 17 00:00:00 2001 From: Minjae Gwon Date: Mon, 27 Nov 2023 00:24:49 +0900 Subject: [PATCH 29/39] chore: remove the unused --- rpc/src/main/scala/Main.scala | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 rpc/src/main/scala/Main.scala diff --git a/rpc/src/main/scala/Main.scala b/rpc/src/main/scala/Main.scala deleted file mode 100644 index a0590d4..0000000 --- a/rpc/src/main/scala/Main.scala +++ /dev/null @@ -1,7 +0,0 @@ -package kr.ac.postech.paranode.rpc - -object Main { - def main(args: Array[String]): Unit = { - println("Hello world!") - } -} From e288cef7fc8b3ed5c1747918a1de70a138e83d74 Mon Sep 17 00:00:00 2001 From: Minjae Gwon Date: Mon, 27 Nov 2023 00:39:35 +0900 Subject: [PATCH 30/39] refactor: implicit conversion --- rpc/src/main/scala/Implicit.scala | 46 +++++++++++++++++++++++++++ rpc/src/main/scala/MasterServer.scala | 5 +-- rpc/src/main/scala/WorkerServer.scala | 23 +++----------- 3 files changed, 53 insertions(+), 21 deletions(-) create mode 100644 rpc/src/main/scala/Implicit.scala diff --git a/rpc/src/main/scala/Implicit.scala b/rpc/src/main/scala/Implicit.scala new file mode 100644 index 0000000..876bab5 --- /dev/null +++ b/rpc/src/main/scala/Implicit.scala @@ -0,0 +1,46 @@ +package kr.ac.postech.paranode.rpc + +import com.google.protobuf.ByteString +import kr.ac.postech.paranode.core.Block +import kr.ac.postech.paranode.core.Key +import kr.ac.postech.paranode.core.KeyRange +import kr.ac.postech.paranode.core.WorkerMetadata + +import scala.language.implicitConversions + +import common.{ + KeyRange => RpcKeyRange, + WorkerMetadata => RpcWorkerMetadata, + Node => RpcNode +} + +object Implicit { + implicit def toKeyRange(rpcKeyRange: RpcKeyRange): KeyRange = KeyRange( + Key.fromByteString(rpcKeyRange.from), + Key.fromByteString(rpcKeyRange.to) + ) + + implicit def toWorkerMetadata( + rpcWorkerMetadata: RpcWorkerMetadata + ): WorkerMetadata = WorkerMetadata( + rpcWorkerMetadata.node.get.host, + rpcWorkerMetadata.node.get.port, + rpcWorkerMetadata.keyRange.map(toKeyRange) + ) + + implicit def toWorkerMetadata( + rpcNode: RpcNode + ): WorkerMetadata = WorkerMetadata( + rpcNode.host, + rpcNode.port, + None + ) + + implicit def toWorkerMetadata( + rpcWorkerMetadata: Seq[RpcWorkerMetadata] + ): Seq[WorkerMetadata] = rpcWorkerMetadata.map(toWorkerMetadata) + + implicit def toBlock( + rpcBlock: ByteString + ): Block = Block.fromBytes(LazyList.from(rpcBlock.toByteArray)) +} diff --git a/rpc/src/main/scala/MasterServer.scala b/rpc/src/main/scala/MasterServer.scala index af3f9f9..901346a 100644 --- a/rpc/src/main/scala/MasterServer.scala +++ b/rpc/src/main/scala/MasterServer.scala @@ -11,6 +11,7 @@ import scala.concurrent.Future import scala.concurrent.Promise import master.{MasterGrpc, RegisterReply, RegisterRequest} +import Implicit._ class MasterServer(executionContext: ExecutionContext, val port: Int = 50051) extends Logging { self => @@ -63,8 +64,8 @@ class MasterServer(executionContext: ExecutionContext, val port: Int = 50051) Future { logger.debug(s"[MasterServer] Register ($request)") - val workerMetadata = - WorkerMetadata(request.worker.get.host, request.worker.get.port, None) + val workerMetadata: WorkerMetadata = request.worker.get + addWorkerInfo(workerMetadata) }(executionContext) diff --git a/rpc/src/main/scala/WorkerServer.scala b/rpc/src/main/scala/WorkerServer.scala index ce39dde..fb0dbba 100644 --- a/rpc/src/main/scala/WorkerServer.scala +++ b/rpc/src/main/scala/WorkerServer.scala @@ -15,7 +15,7 @@ import scala.concurrent.Promise import scala.reflect.io.Directory import scala.reflect.io.Path -import common.{WorkerMetadata => RpcWorkerMetadata} +import Implicit._ import worker._ class WorkerServer( @@ -66,19 +66,6 @@ class WorkerServer( } private class WorkerImpl extends WorkerGrpc.Worker { - private def toWorkerMetadata(workers: Seq[RpcWorkerMetadata]) = - workers.map(worker => - WorkerMetadata( - worker.node.get.host, - worker.node.get.port, - worker.keyRange.map(keyRange => - KeyRange( - Key.fromByteString(keyRange.from), - Key.fromByteString(keyRange.to) - ) - ) - ) - ) override def sample(request: SampleRequest): Future[SampleReply] = { val promise = Promise[SampleReply] @@ -130,7 +117,7 @@ class WorkerServer( Future { logger.debug(s"[WorkerServer] Partition ($request)") - val workers = toWorkerMetadata(request.workers) + val workers: Seq[WorkerMetadata] = request.workers inputFiles .map(path => { @@ -173,7 +160,7 @@ class WorkerServer( Future { logger.debug(s"[WorkerServer] Exchange ($request)") - val workers = toWorkerMetadata(request.workers) + val workers: Seq[WorkerMetadata] = request.workers inputFiles.foreach(path => { val block = Block.fromPath(path) @@ -206,9 +193,7 @@ class WorkerServer( Future { logger.debug(s"[WorkerServer] SaveBlock ($request)") - val block = Block.fromBytes( - LazyList.from(request.block.toByteArray) - ) + val block: Block = request.block val path = outputDirectory / UUID.randomUUID().toString From f0e1360e1aa3046d3ef39d260302ff00eb8bcc45 Mon Sep 17 00:00:00 2001 From: Minjae Gwon Date: Mon, 27 Nov 2023 01:04:24 +0900 Subject: [PATCH 31/39] feat: wrap mutable state --- rpc/src/main/scala/MasterService.scala | 3 +++ utils/src/main/scala/MutableState.scala | 7 +++++++ 2 files changed, 10 insertions(+) create mode 100644 rpc/src/main/scala/MasterService.scala create mode 100644 utils/src/main/scala/MutableState.scala diff --git a/rpc/src/main/scala/MasterService.scala b/rpc/src/main/scala/MasterService.scala new file mode 100644 index 0000000..770d1f6 --- /dev/null +++ b/rpc/src/main/scala/MasterService.scala @@ -0,0 +1,3 @@ +package kr.ac.postech.paranode.rpc class MasterService { + +} diff --git a/utils/src/main/scala/MutableState.scala b/utils/src/main/scala/MutableState.scala new file mode 100644 index 0000000..921a881 --- /dev/null +++ b/utils/src/main/scala/MutableState.scala @@ -0,0 +1,7 @@ +package kr.ac.postech.paranode.utils + +class MutableState[A](var underlying: A) { + def update(f: A => A) = new MutableState[A](f(underlying)) + + def get: A = underlying +} From fb9161122512d446e603ae263d504fa23883c67c Mon Sep 17 00:00:00 2001 From: Minjae Gwon Date: Mon, 27 Nov 2023 01:04:42 +0900 Subject: [PATCH 32/39] feat: wrap workers with mutable state --- master/src/main/scala/Master.scala | 6 ++-- rpc/src/main/scala/MasterServer.scala | 46 ++++++++++---------------- rpc/src/main/scala/MasterService.scala | 30 ++++++++++++++++- 3 files changed, 50 insertions(+), 32 deletions(-) diff --git a/master/src/main/scala/Master.scala b/master/src/main/scala/Master.scala index ffdb17f..d2084fc 100644 --- a/master/src/main/scala/Master.scala +++ b/master/src/main/scala/Master.scala @@ -30,13 +30,13 @@ object Master extends Logging { server.start() - println(masterHost + ":" + server.port) + println(masterHost + ":" + masterPort) - while (server.getWorkerDetails.size < masterArguments.numberOfWorkers) { + while (server.registeredWorkers.size < masterArguments.numberOfWorkers) { Thread.sleep(1000) } - val workerInfo: List[WorkerMetadata] = server.getWorkerDetails + val workerInfo: List[WorkerMetadata] = server.registeredWorkers println(workerInfo.map(_.host).mkString(", ")) diff --git a/rpc/src/main/scala/MasterServer.scala b/rpc/src/main/scala/MasterServer.scala index 901346a..c335c07 100644 --- a/rpc/src/main/scala/MasterServer.scala +++ b/rpc/src/main/scala/MasterServer.scala @@ -9,24 +9,28 @@ import scala.collection.mutable.ListBuffer import scala.concurrent.ExecutionContext import scala.concurrent.Future import scala.concurrent.Promise - import master.{MasterGrpc, RegisterReply, RegisterRequest} import Implicit._ -class MasterServer(executionContext: ExecutionContext, val port: Int = 50051) - extends Logging { self => - private[this] val server: Server = ServerBuilder - .forPort(port) - .addService(MasterGrpc.bindService(new MasterImpl, executionContext)) - .build() +import kr.ac.postech.paranode.utils.MutableState - private val workerDetails: ListBuffer[WorkerMetadata] = ListBuffer() +class MasterServer(executionContext: ExecutionContext, port: Int = 50051) + extends Logging { - def addWorkerInfo(workerMetadata: WorkerMetadata): Unit = synchronized { - workerDetails += workerMetadata - } + private[this] val workers: MutableState[List[WorkerMetadata]] = + new MutableState(List.empty) - def getWorkerDetails: List[WorkerMetadata] = workerDetails.toList + val server: Server = + ServerBuilder + .forPort(port) + .addService( + MasterGrpc + .bindService( + new MasterService(executionContext, workers), + executionContext + ) + ) + .build() def start(): Unit = { server.start() @@ -40,7 +44,7 @@ class MasterServer(executionContext: ExecutionContext, val port: Int = 50051) logger.error( "[MasterServer] shutting down gRPC server since JVM is shutting down" ) - self.stop() + stop() logger.error("[MasterServer] server shut down") } } @@ -57,19 +61,5 @@ class MasterServer(executionContext: ExecutionContext, val port: Int = 50051) } } - private class MasterImpl extends MasterGrpc.Master { - override def register(request: RegisterRequest): Future[RegisterReply] = { - val promise = Promise[RegisterReply] - - Future { - logger.debug(s"[MasterServer] Register ($request)") - - val workerMetadata: WorkerMetadata = request.worker.get - - addWorkerInfo(workerMetadata) - }(executionContext) - - promise.future - } - } + def registeredWorkers: List[WorkerMetadata] = workers.get } diff --git a/rpc/src/main/scala/MasterService.scala b/rpc/src/main/scala/MasterService.scala index 770d1f6..75245e2 100644 --- a/rpc/src/main/scala/MasterService.scala +++ b/rpc/src/main/scala/MasterService.scala @@ -1,3 +1,31 @@ -package kr.ac.postech.paranode.rpc class MasterService { +package kr.ac.postech.paranode.rpc +import master.{MasterGrpc, RegisterReply, RegisterRequest} + +import kr.ac.postech.paranode.core.WorkerMetadata +import org.apache.logging.log4j.scala.Logging + +import scala.concurrent.{ExecutionContext, Future, Promise} +import Implicit._ + +import kr.ac.postech.paranode.utils.MutableState + +class MasterService( + executionContext: ExecutionContext, + workers: MutableState[List[WorkerMetadata]] +) extends MasterGrpc.Master + with Logging { + override def register(request: RegisterRequest): Future[RegisterReply] = { + val promise = Promise[RegisterReply] + + Future { + logger.debug(s"[MasterServer] Register ($request)") + + val worker: WorkerMetadata = request.worker.get + + workers.update(_ :+ worker) + }(executionContext) + + promise.future + } } From 7f5c81126e53d8b112ed7834679324a3a63f8732 Mon Sep 17 00:00:00 2001 From: Minjae Gwon Date: Mon, 27 Nov 2023 01:05:16 +0900 Subject: [PATCH 33/39] chore: lint --- rpc/src/main/scala/MasterServer.scala | 8 ++------ rpc/src/main/scala/MasterService.scala | 11 ++++++----- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/rpc/src/main/scala/MasterServer.scala b/rpc/src/main/scala/MasterServer.scala index c335c07..4410015 100644 --- a/rpc/src/main/scala/MasterServer.scala +++ b/rpc/src/main/scala/MasterServer.scala @@ -3,16 +3,12 @@ package kr.ac.postech.paranode.rpc import io.grpc.Server import io.grpc.ServerBuilder import kr.ac.postech.paranode.core.WorkerMetadata +import kr.ac.postech.paranode.utils.MutableState import org.apache.logging.log4j.scala.Logging -import scala.collection.mutable.ListBuffer import scala.concurrent.ExecutionContext -import scala.concurrent.Future -import scala.concurrent.Promise -import master.{MasterGrpc, RegisterReply, RegisterRequest} -import Implicit._ -import kr.ac.postech.paranode.utils.MutableState +import master.MasterGrpc class MasterServer(executionContext: ExecutionContext, port: Int = 50051) extends Logging { diff --git a/rpc/src/main/scala/MasterService.scala b/rpc/src/main/scala/MasterService.scala index 75245e2..72db7c6 100644 --- a/rpc/src/main/scala/MasterService.scala +++ b/rpc/src/main/scala/MasterService.scala @@ -1,14 +1,15 @@ package kr.ac.postech.paranode.rpc -import master.{MasterGrpc, RegisterReply, RegisterRequest} - import kr.ac.postech.paranode.core.WorkerMetadata +import kr.ac.postech.paranode.utils.MutableState import org.apache.logging.log4j.scala.Logging -import scala.concurrent.{ExecutionContext, Future, Promise} -import Implicit._ +import scala.concurrent.ExecutionContext +import scala.concurrent.Future +import scala.concurrent.Promise -import kr.ac.postech.paranode.utils.MutableState +import master.{MasterGrpc, RegisterReply, RegisterRequest} +import Implicit._ class MasterService( executionContext: ExecutionContext, From 6844f924a0cae7c07dccd98940a34884a44254a2 Mon Sep 17 00:00:00 2001 From: Minjae Gwon Date: Mon, 27 Nov 2023 01:26:28 +0900 Subject: [PATCH 34/39] fix: race --- master/src/main/scala/Master.scala | 1 + utils/src/main/scala/MutableState.scala | 9 +++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/master/src/main/scala/Master.scala b/master/src/main/scala/Master.scala index d2084fc..4e9c526 100644 --- a/master/src/main/scala/Master.scala +++ b/master/src/main/scala/Master.scala @@ -33,6 +33,7 @@ object Master extends Logging { println(masterHost + ":" + masterPort) while (server.registeredWorkers.size < masterArguments.numberOfWorkers) { + logger.debug(s"${server.registeredWorkers}") Thread.sleep(1000) } diff --git a/utils/src/main/scala/MutableState.scala b/utils/src/main/scala/MutableState.scala index 921a881..e4034a3 100644 --- a/utils/src/main/scala/MutableState.scala +++ b/utils/src/main/scala/MutableState.scala @@ -1,7 +1,12 @@ package kr.ac.postech.paranode.utils -class MutableState[A](var underlying: A) { - def update(f: A => A) = new MutableState[A](f(underlying)) +class MutableState[A](var underlying: A) { self => + def update(f: A => A): MutableState[A] = { + synchronized { + underlying = f(underlying) + } + self + } def get: A = underlying } From 7c10c41b404594ca50f7fa33e240a7f1d16b3e7f Mon Sep 17 00:00:00 2001 From: Minjae Gwon Date: Mon, 27 Nov 2023 01:26:58 +0900 Subject: [PATCH 35/39] refactor: separate services --- rpc/src/main/scala/WorkerServer.scala | 201 +--------------------- rpc/src/main/scala/WorkerService.scala | 221 +++++++++++++++++++++++++ 2 files changed, 227 insertions(+), 195 deletions(-) create mode 100644 rpc/src/main/scala/WorkerService.scala diff --git a/rpc/src/main/scala/WorkerServer.scala b/rpc/src/main/scala/WorkerServer.scala index fb0dbba..cc094b7 100644 --- a/rpc/src/main/scala/WorkerServer.scala +++ b/rpc/src/main/scala/WorkerServer.scala @@ -1,21 +1,11 @@ package kr.ac.postech.paranode.rpc - -import com.google.protobuf.ByteString import io.grpc.Server import io.grpc.ServerBuilder -import kr.ac.postech.paranode.core._ import org.apache.logging.log4j.scala.Logging -import java.util.UUID -import scala.concurrent.Await import scala.concurrent.ExecutionContext -import scala.concurrent.ExecutionContext.Implicits.global -import scala.concurrent.Future -import scala.concurrent.Promise import scala.reflect.io.Directory -import scala.reflect.io.Path -import Implicit._ import worker._ class WorkerServer( @@ -26,13 +16,14 @@ class WorkerServer( ) extends Logging { self => private[this] val server: Server = ServerBuilder .forPort(port) - .addService(WorkerGrpc.bindService(new WorkerImpl, executionContext)) + .addService( + WorkerGrpc.bindService( + new WorkerService(executionContext, inputDirectories, outputDirectory), + executionContext + ) + ) .build() - private def inputFiles = inputDirectories.flatMap(_.files) - - private def outputFiles = outputDirectory.files.toList - def start(): Unit = { server.start() @@ -40,7 +31,6 @@ class WorkerServer( "[WorkerServer] \n" + s"port: $port\n" + s"inputDirectories: ${inputDirectories.mkString(", ")}\n" + - s"inputFiles: ${inputFiles.mkString(", ")}\n" + s"outputDirectory: $outputDirectory\n" ) @@ -65,183 +55,4 @@ class WorkerServer( } } - private class WorkerImpl extends WorkerGrpc.Worker { - - override def sample(request: SampleRequest): Future[SampleReply] = { - val promise = Promise[SampleReply] - - Future { - logger.debug(s"[WorkerServer] Sample ($request)") - - val sampledKeys = inputFiles - .map(f => Block.fromPath(f.path)) - .flatMap(_.sample(request.numberOfKeys)) - .map(key => ByteString.copyFrom(key.underlying)) - - promise.success(SampleReply(sampledKeys)) - }(executionContext) - - promise.future - } - - override def sort(request: SortRequest): Future[SortReply] = { - val promise = Promise[SortReply] - - Future { - logger.debug(s"[WorkerServer] Sort ($request)") - - inputFiles - .foreach(path => { - val block = Block.fromPath(path) - - val sortedBlock = block.sorted - - logger.debug(s"[WorkerServer] Writing sorted block to $path") - - sortedBlock.writeTo(path) - - logger.debug(s"[WorkerServer] Wrote sorted block to $path") - }) - - promise.success(new SortReply()) - } - - promise.future - } - - override def partition( - request: PartitionRequest - ): Future[PartitionReply] = { - val promise = Promise[PartitionReply] - - Future { - logger.debug(s"[WorkerServer] Partition ($request)") - - val workers: Seq[WorkerMetadata] = request.workers - - inputFiles - .map(path => { - val block = Block.fromPath(path) - workers - .map(_.keyRange.get) - .map(block.partition) - .map({ case (keyRange, partition) => - val partitionPath = Path( - s"$path.${keyRange.from.hex}-${keyRange.to.hex}" - ) - - logger.debug( - s"[WorkerServer] Writing partition to $partitionPath" - ) - - partition.writeTo(partitionPath) - - logger.debug( - s"[WorkerServer] Wrote partition to $partitionPath" - ) - - if (path.exists && path.isFile) { - val result = path.delete() - logger.debug(s"[WorkerServer] Deleted $path: $result") - } - }) - - }) - - promise.success(new PartitionReply()) - }(executionContext) - - promise.future - } - - override def exchange(request: ExchangeRequest): Future[ExchangeReply] = { - val promise = Promise[ExchangeReply] - - Future { - logger.debug(s"[WorkerServer] Exchange ($request)") - - val workers: Seq[WorkerMetadata] = request.workers - - inputFiles.foreach(path => { - val block = Block.fromPath(path) - val targetWorkers = workers - .filter(_.keyRange.get.includes(block.records.head.key)) - - logger.debug(s"[WorkerServer] Sending $block to $targetWorkers") - - Await.result( - Future.sequence( - targetWorkers - .map(worker => WorkerClient(worker.host, worker.port)) - .map(_.saveBlock(block)) - ), - scala.concurrent.duration.Duration.Inf - ) - }) - - promise.success(new ExchangeReply()) - } - - promise.future - } - - override def saveBlock( - request: SaveBlockRequest - ): Future[SaveBlockReply] = { - val promise = Promise[SaveBlockReply] - - Future { - logger.debug(s"[WorkerServer] SaveBlock ($request)") - - val block: Block = request.block - - val path = outputDirectory / UUID.randomUUID().toString - - logger.debug(s"[WorkerServer] Writing block to $path") - - block.writeTo(path) - - logger.debug(s"[WorkerServer] Wrote block to $path") - - promise.success(new SaveBlockReply()) - } - - promise.future - } - - override def merge(request: MergeRequest): Future[MergeReply] = { - val promise = Promise[MergeReply] - - Future { - logger.debug(s"[WorkerServer] Merge ($request)") - val targetFiles = outputFiles - - val blocks = targetFiles.map(path => Block.fromPath(path)) - - logger.debug("[WorkerServer] Merging blocks") - - val mergedBlock = blocks.merged - - val results = mergedBlock.writeTo(outputDirectory / "result") - - targetFiles.foreach(file => { - val result = file.delete() - - logger.debug( - s"[WorkerServer] Deleted $file: $result" - ) - }) - - logger.debug( - s"[WorkerServer] Merged blocks: $results" - ) - - promise.success(new MergeReply()) - }(executionContext) - - promise.future - } - - } - } diff --git a/rpc/src/main/scala/WorkerService.scala b/rpc/src/main/scala/WorkerService.scala new file mode 100644 index 0000000..70bee1d --- /dev/null +++ b/rpc/src/main/scala/WorkerService.scala @@ -0,0 +1,221 @@ +package kr.ac.postech.paranode.rpc + +import com.google.protobuf.ByteString +import kr.ac.postech.paranode.core.Block +import kr.ac.postech.paranode.core.WorkerMetadata +import org.apache.logging.log4j.scala.Logging + +import java.util.UUID +import scala.concurrent.Await +import scala.concurrent.ExecutionContext +import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.Future +import scala.concurrent.Promise +import scala.reflect.io.Directory +import scala.reflect.io.File +import scala.reflect.io.Path + +import worker.{ + ExchangeReply, + ExchangeRequest, + MergeReply, + MergeRequest, + PartitionReply, + PartitionRequest, + SampleReply, + SampleRequest, + SaveBlockReply, + SaveBlockRequest, + SortReply, + SortRequest, + WorkerGrpc +} +import Implicit._ + +class WorkerService( + executionContext: ExecutionContext, + inputDirectories: Array[Directory], + outputDirectory: Directory +) extends WorkerGrpc.Worker + with Logging { + + private def inputFiles: Array[File] = inputDirectories.flatMap(_.files) + + private def outputFiles: List[File] = outputDirectory.files.toList + + override def sample(request: SampleRequest): Future[SampleReply] = { + val promise = Promise[SampleReply] + + Future { + logger.debug(s"[WorkerServer] Sample ($request)") + + val sampledKeys = inputFiles + .map(f => Block.fromPath(f.path)) + .flatMap(_.sample(request.numberOfKeys)) + .map(key => ByteString.copyFrom(key.underlying)) + + promise.success(SampleReply(sampledKeys)) + }(executionContext) + + promise.future + } + + override def sort(request: SortRequest): Future[SortReply] = { + val promise = Promise[SortReply] + + Future { + logger.debug(s"[WorkerServer] Sort ($request)") + + inputFiles + .foreach(path => { + val block = Block.fromPath(path) + + val sortedBlock = block.sorted + + logger.debug(s"[WorkerServer] Writing sorted block to $path") + + sortedBlock.writeTo(path) + + logger.debug(s"[WorkerServer] Wrote sorted block to $path") + }) + + promise.success(new SortReply()) + }(executionContext) + + promise.future + } + + override def partition( + request: PartitionRequest + ): Future[PartitionReply] = { + val promise = Promise[PartitionReply] + + Future { + logger.debug(s"[WorkerServer] Partition ($request)") + + val workers: Seq[WorkerMetadata] = request.workers + + inputFiles + .map(path => { + val block = Block.fromPath(path) + workers + .map(_.keyRange.get) + .map(block.partition) + .map({ case (keyRange, partition) => + val partitionPath = Path( + s"$path.${keyRange.from.hex}-${keyRange.to.hex}" + ) + + logger.debug( + s"[WorkerServer] Writing partition to $partitionPath" + ) + + partition.writeTo(partitionPath) + + logger.debug( + s"[WorkerServer] Wrote partition to $partitionPath" + ) + + if (path.exists && path.isFile) { + val result = path.delete() + logger.debug(s"[WorkerServer] Deleted $path: $result") + } + }) + + }) + + promise.success(new PartitionReply()) + }(executionContext) + + promise.future + } + + override def exchange(request: ExchangeRequest): Future[ExchangeReply] = { + val promise = Promise[ExchangeReply] + + Future { + logger.debug(s"[WorkerServer] Exchange ($request)") + + val workers: Seq[WorkerMetadata] = request.workers + + inputFiles.foreach(path => { + val block = Block.fromPath(path) + val targetWorkers = workers + .filter(_.keyRange.get.includes(block.records.head.key)) + + logger.debug(s"[WorkerServer] Sending $block to $targetWorkers") + + Await.result( + Future.sequence( + targetWorkers + .map(worker => WorkerClient(worker.host, worker.port)) + .map(_.saveBlock(block)) + ), + scala.concurrent.duration.Duration.Inf + ) + }) + + promise.success(new ExchangeReply()) + }(executionContext) + + promise.future + } + + override def saveBlock( + request: SaveBlockRequest + ): Future[SaveBlockReply] = { + val promise = Promise[SaveBlockReply] + + Future { + logger.debug(s"[WorkerServer] SaveBlock ($request)") + + val block: Block = request.block + + val path = outputDirectory / UUID.randomUUID().toString + + logger.debug(s"[WorkerServer] Writing block to $path") + + block.writeTo(path) + + logger.debug(s"[WorkerServer] Wrote block to $path") + + promise.success(new SaveBlockReply()) + }(executionContext) + + promise.future + } + + override def merge(request: MergeRequest): Future[MergeReply] = { + val promise = Promise[MergeReply] + + Future { + logger.debug(s"[WorkerServer] Merge ($request)") + val targetFiles = outputFiles + + val blocks = targetFiles.map(path => Block.fromPath(path)) + + logger.debug("[WorkerServer] Merging blocks") + + val mergedBlock = blocks.merged + + val results = mergedBlock.writeTo(outputDirectory / "result") + + targetFiles.foreach(file => { + val result = file.delete() + + logger.debug( + s"[WorkerServer] Deleted $file: $result" + ) + }) + + logger.debug( + s"[WorkerServer] Merged blocks: $results" + ) + + promise.success(new MergeReply()) + }(executionContext) + + promise.future + } + +} From 8db71c060d097254a385de64cbc786e9f5851af2 Mon Sep 17 00:00:00 2001 From: Minjae Gwon Date: Mon, 27 Nov 2023 01:52:05 +0900 Subject: [PATCH 36/39] refactor: implicit conversion --- master/src/main/scala/Master.scala | 66 ++++++++-------------- rpc/src/main/scala/Implicit.scala | 22 ++++++++ rpc/src/main/scala/WorkerClient.scala | 81 +++++++++++++++------------ 3 files changed, 92 insertions(+), 77 deletions(-) diff --git a/master/src/main/scala/Master.scala b/master/src/main/scala/Master.scala index 4e9c526..e834454 100644 --- a/master/src/main/scala/Master.scala +++ b/master/src/main/scala/Master.scala @@ -8,11 +8,24 @@ import kr.ac.postech.paranode.rpc.WorkerClient import org.apache.logging.log4j.scala.Logging import java.net._ -import scala.concurrent.Await -import scala.concurrent.ExecutionContext.Implicits.global -import scala.concurrent.Future object Master extends Logging { + private def workersWithKeyRange( + keys: List[Key], + workers: List[WorkerMetadata] + ): List[WorkerMetadata] = + keys + .sliding( + keys.size / workers.size, + keys.size / workers.size + ) + .toList + .map(keys => KeyRange.tupled(keys.head, keys.last)) + .zip(workers) + .map { case (keyRange, worker) => + worker.copy(keyRange = Some(keyRange)) + } + def main(args: Array[String]): Unit = { val masterArguments = new MasterArguments(args) val masterHost = InetAddress.getLocalHost.getHostAddress @@ -45,11 +58,8 @@ object Master extends Logging { WorkerClient(worker.host, worker.port) } - val sampledKeys = Await - .result( - Future.sequence(clients.map(_.sample(64))), - scala.concurrent.duration.Duration.Inf - ) + val sampledKeys = clients + .sample(64) .flatMap(_.sampledKeys) .map(Key.fromByteString) @@ -59,59 +69,31 @@ object Master extends Logging { logger.debug(s"[Master] Sorted Sampled keys: $sortedSampledKeys") - val keyRanges = sortedSampledKeys - .sliding( - sortedSampledKeys.size / masterArguments.numberOfWorkers, - sortedSampledKeys.size / masterArguments.numberOfWorkers - ) - .toList - .map(keys => (keys.head, keys.last)) - - val keyRangesWithWorker = workerInfo.zip(keyRanges.map(KeyRange.tupled)) + val workers = workersWithKeyRange(sortedSampledKeys, workerInfo) - logger.debug(s"[Master] Key ranges with worker: $keyRangesWithWorker") + logger.debug(s"[Master] Key ranges with worker: $workers") logger.debug("[Master] Sort started") - Await.result( - Future.sequence( - clients.map(_.sort()) - ), - scala.concurrent.duration.Duration.Inf - ) + clients.sort() logger.debug("[Master] Sort finished") logger.debug("[Master] Partition started") - Await.result( - Future.sequence( - clients.map(_.partition(keyRangesWithWorker)) - ), - scala.concurrent.duration.Duration.Inf - ) + clients.partition(workers) logger.debug("[Master] Partition finished") logger.debug("[Master] Exchange started") - Await.result( - Future.sequence( - clients.map(_.exchange(keyRangesWithWorker)) - ), - scala.concurrent.duration.Duration.Inf - ) + clients.exchange(workers) logger.debug("[Master] Exchange finished") logger.debug("[Master] Merge started") - Await.result( - Future.sequence( - clients.map(_.merge()) - ), - scala.concurrent.duration.Duration.Inf - ) + clients.merge() logger.debug("[Master] Merge finished") diff --git a/rpc/src/main/scala/Implicit.scala b/rpc/src/main/scala/Implicit.scala index 876bab5..64810d5 100644 --- a/rpc/src/main/scala/Implicit.scala +++ b/rpc/src/main/scala/Implicit.scala @@ -43,4 +43,26 @@ object Implicit { implicit def toBlock( rpcBlock: ByteString ): Block = Block.fromBytes(LazyList.from(rpcBlock.toByteArray)) + + implicit def toRpcKeyRange( + keyRange: KeyRange + ): RpcKeyRange = RpcKeyRange( + ByteString.copyFrom(keyRange.from.underlying), + ByteString.copyFrom(keyRange.to.underlying) + ) + + implicit def toRpcWorkerMetadata( + workerMetadata: WorkerMetadata + ): RpcWorkerMetadata = RpcWorkerMetadata( + Some(RpcNode(workerMetadata.host, workerMetadata.port)), + workerMetadata.keyRange.map(toRpcKeyRange) + ) + + implicit def toRpcWorkerMetadata( + workerMetadata: List[WorkerMetadata] + ): List[RpcWorkerMetadata] = workerMetadata.map(toRpcWorkerMetadata) + + implicit def toByteString( + block: Block + ): ByteString = ByteString.copyFrom(block.toChars.map(_.toByte).toArray) } diff --git a/rpc/src/main/scala/WorkerClient.scala b/rpc/src/main/scala/WorkerClient.scala index 4b71752..2d4054c 100644 --- a/rpc/src/main/scala/WorkerClient.scala +++ b/rpc/src/main/scala/WorkerClient.scala @@ -1,25 +1,57 @@ package kr.ac.postech.paranode.rpc - -import com.google.protobuf.ByteString import io.grpc.ManagedChannel import io.grpc.ManagedChannelBuilder import kr.ac.postech.paranode.core.Block -import kr.ac.postech.paranode.core.KeyRange import kr.ac.postech.paranode.core.WorkerMetadata import java.util.concurrent.TimeUnit import java.util.logging.Logger +import scala.concurrent.Await +import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.Future import worker._ import worker.WorkerGrpc.WorkerStub -import common.{ - Node, - KeyRange => RpcKeyRange, - WorkerMetadata => RpcWorkerMetadata -} +import Implicit._ object WorkerClient { + + implicit class WorkerClients(val clients: List[WorkerClient]) { + def sample(numberOfKeys: Int): List[SampleReply] = + Await.result( + Future.sequence(clients.map(_.sample(numberOfKeys))), + scala.concurrent.duration.Duration.Inf + ) + + def sort(): List[SortReply] = + Await.result( + Future.sequence(clients.map(_.sort())), + scala.concurrent.duration.Duration.Inf + ) + + def partition( + keyRanges: List[WorkerMetadata] + ): List[PartitionReply] = + Await.result( + Future.sequence(clients.map(_.partition(keyRanges))), + scala.concurrent.duration.Duration.Inf + ) + + def exchange( + keyRanges: List[WorkerMetadata] + ): List[ExchangeReply] = + Await.result( + Future.sequence(clients.map(_.exchange(keyRanges))), + scala.concurrent.duration.Duration.Inf + ) + + def merge(): List[MergeReply] = + Await.result( + Future.sequence(clients.map(_.merge())), + scala.concurrent.duration.Duration.Inf + ) + } + def apply(host: String, port: Int): WorkerClient = { val channel = ManagedChannelBuilder .forAddress(host, port) @@ -28,6 +60,7 @@ object WorkerClient { val stub = WorkerGrpc.stub(channel) new WorkerClient(channel, stub) } + } class WorkerClient private ( @@ -55,37 +88,17 @@ class WorkerClient private ( } def partition( - workers: List[(WorkerMetadata, KeyRange)] + workers: List[WorkerMetadata] ): Future[PartitionReply] = { - val request = PartitionRequest(workers.map({ case (worker, keyRange) => - RpcWorkerMetadata( - Some(Node(worker.host, worker.port)), - Some( - RpcKeyRange( - ByteString.copyFrom(keyRange.from.underlying), - ByteString.copyFrom(keyRange.to.underlying) - ) - ) - ) - })) + val request = PartitionRequest(workers) stub.partition(request) } def exchange( - workers: List[(WorkerMetadata, KeyRange)] + workers: List[WorkerMetadata] ): Future[ExchangeReply] = { - val request = ExchangeRequest(workers.map({ case (worker, keyRange) => - RpcWorkerMetadata( - Some(Node(worker.host, worker.port)), - Some( - RpcKeyRange( - ByteString.copyFrom(keyRange.from.underlying), - ByteString.copyFrom(keyRange.to.underlying) - ) - ) - ) - })) + val request = ExchangeRequest(workers) stub.exchange(request) } @@ -93,9 +106,7 @@ class WorkerClient private ( def saveBlock( block: Block ): Future[SaveBlockReply] = { - val request = SaveBlockRequest( - ByteString.copyFrom(block.toChars.map(_.toByte).toArray) - ) + val request = SaveBlockRequest(block) stub.saveBlock(request) } From 593fee0f4e594888cdb9c8af1ea6a441e2fdffe3 Mon Sep 17 00:00:00 2001 From: Minjae Gwon Date: Mon, 27 Nov 2023 16:27:39 +0900 Subject: [PATCH 37/39] feat: use custom execution context --- master/src/main/scala/Master.scala | 5 +++ rpc/src/main/scala/WorkerClient.scala | 44 ++++++++++++++------- utils/src/main/scala/GenericBuildFrom.scala | 24 +++++++++++ 3 files changed, 59 insertions(+), 14 deletions(-) create mode 100644 utils/src/main/scala/GenericBuildFrom.scala diff --git a/master/src/main/scala/Master.scala b/master/src/main/scala/Master.scala index e834454..0f25bbe 100644 --- a/master/src/main/scala/Master.scala +++ b/master/src/main/scala/Master.scala @@ -8,6 +8,7 @@ import kr.ac.postech.paranode.rpc.WorkerClient import org.apache.logging.log4j.scala.Logging import java.net._ +import scala.concurrent.ExecutionContextExecutor object Master extends Logging { private def workersWithKeyRange( @@ -58,6 +59,10 @@ object Master extends Logging { WorkerClient(worker.host, worker.port) } + implicit val requestExecutionContext: ExecutionContextExecutor = scala.concurrent.ExecutionContext.fromExecutor( + java.util.concurrent.Executors.newFixedThreadPool(workerInfo.size) + ) + val sampledKeys = clients .sample(64) .flatMap(_.sampledKeys) diff --git a/rpc/src/main/scala/WorkerClient.scala b/rpc/src/main/scala/WorkerClient.scala index 2d4054c..7e08e93 100644 --- a/rpc/src/main/scala/WorkerClient.scala +++ b/rpc/src/main/scala/WorkerClient.scala @@ -6,48 +6,64 @@ import kr.ac.postech.paranode.core.WorkerMetadata import java.util.concurrent.TimeUnit import java.util.logging.Logger -import scala.concurrent.Await -import scala.concurrent.ExecutionContext.Implicits.global -import scala.concurrent.Future - +import scala.concurrent.{Await, ExecutionContext, Future} import worker._ import worker.WorkerGrpc.WorkerStub import Implicit._ +import kr.ac.postech.paranode.utils.GenericBuildFrom + object WorkerClient { implicit class WorkerClients(val clients: List[WorkerClient]) { - def sample(numberOfKeys: Int): List[SampleReply] = + def sample( + numberOfKeys: Int + )(implicit executionContext: ExecutionContext): List[SampleReply] = Await.result( - Future.sequence(clients.map(_.sample(numberOfKeys))), + Future.traverse(clients)(_.sample(numberOfKeys))( + GenericBuildFrom[WorkerClient, SampleReply], + executionContext + ), scala.concurrent.duration.Duration.Inf ) - def sort(): List[SortReply] = + def sort()(implicit executionContext: ExecutionContext): List[SortReply] = Await.result( - Future.sequence(clients.map(_.sort())), + Future.traverse(clients)(_.sort())( + GenericBuildFrom[WorkerClient, SortReply], + executionContext + ), scala.concurrent.duration.Duration.Inf ) def partition( keyRanges: List[WorkerMetadata] - ): List[PartitionReply] = + )(implicit executionContext: ExecutionContext): List[PartitionReply] = Await.result( - Future.sequence(clients.map(_.partition(keyRanges))), + Future.traverse(clients)(_.partition(keyRanges))( + GenericBuildFrom[WorkerClient, PartitionReply], + executionContext + ), scala.concurrent.duration.Duration.Inf ) def exchange( keyRanges: List[WorkerMetadata] - ): List[ExchangeReply] = + )(implicit executionContext: ExecutionContext): List[ExchangeReply] = Await.result( - Future.sequence(clients.map(_.exchange(keyRanges))), + Future.traverse(clients)(_.exchange(keyRanges))( + GenericBuildFrom[WorkerClient, ExchangeReply], + executionContext + ), scala.concurrent.duration.Duration.Inf ) - def merge(): List[MergeReply] = + def merge()(implicit executionContext: ExecutionContext): List[MergeReply] = Await.result( - Future.sequence(clients.map(_.merge())), + Future.traverse(clients)(_.merge())( + GenericBuildFrom[WorkerClient, MergeReply], + executionContext + ), scala.concurrent.duration.Duration.Inf ) } diff --git a/utils/src/main/scala/GenericBuildFrom.scala b/utils/src/main/scala/GenericBuildFrom.scala new file mode 100644 index 0000000..e774896 --- /dev/null +++ b/utils/src/main/scala/GenericBuildFrom.scala @@ -0,0 +1,24 @@ +package kr.ac.postech.paranode.utils + +import scala.collection.{BuildFrom, mutable} + +object GenericBuildFrom { + def apply[A, B]: BuildFrom[List[A], B, List[B]] = + new BuildFrom[List[A], B, List[B]] { + override def fromSpecific(from: List[A])( + it: IterableOnce[B] + ): List[B] = { + val b = newBuilder(from) + b ++= it + b.result() + } + + override def newBuilder( + from: List[A] + ): mutable.Builder[B, List[B]] = { + val b = List.newBuilder[B] + b.sizeHint(from) + b + } + } +} From e5dfbfdeadc0e3270f376105612f2001b7f87d3c Mon Sep 17 00:00:00 2001 From: Minjae Gwon Date: Mon, 27 Nov 2023 17:05:39 +0900 Subject: [PATCH 38/39] fix: adjust log level --- core/src/main/resources/log4j2.properties | 18 +--------- core/src/main/scala/Block.scala | 2 +- core/src/main/scala/Record.scala | 4 --- docker/master/Dockerfile | 2 +- docker/worker/Dockerfile | 2 +- log4j2.properties | 17 ++++++++++ master/src/main/resources/log4j2.properties | 18 +--------- master/src/main/scala/Master.scala | 37 +++++++++++---------- rpc/src/main/resources/log4j2.properties | 18 +--------- rpc/src/main/scala/MasterServer.scala | 2 +- rpc/src/main/scala/MasterService.scala | 2 +- rpc/src/main/scala/WorkerServer.scala | 2 +- rpc/src/main/scala/WorkerService.scala | 34 +++++++++---------- utils/src/main/resources/log4j2.properties | 18 +--------- worker/src/main/resources/log4j2.properties | 18 +--------- worker/src/main/scala/Worker.scala | 2 +- 16 files changed, 66 insertions(+), 130 deletions(-) mode change 100644 => 120000 core/src/main/resources/log4j2.properties create mode 100644 log4j2.properties mode change 100644 => 120000 master/src/main/resources/log4j2.properties mode change 100644 => 120000 rpc/src/main/resources/log4j2.properties mode change 100644 => 120000 utils/src/main/resources/log4j2.properties mode change 100644 => 120000 worker/src/main/resources/log4j2.properties diff --git a/core/src/main/resources/log4j2.properties b/core/src/main/resources/log4j2.properties deleted file mode 100644 index 59b1671..0000000 --- a/core/src/main/resources/log4j2.properties +++ /dev/null @@ -1,17 +0,0 @@ -# Set to debug or trace if log4j initialization is failing -status = warn - -# Name of the configuration -name = ConsoleLogConfigDemo - -# Console appender configuration -appender.console.type = Console -appender.console.name = consoleLogger -appender.console.layout.type = PatternLayout -appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n - -# Root logger level -rootLogger.level = debug - -# Root logger referring to console appender -rootLogger.appenderRef.stdout.ref = consoleLogger diff --git a/core/src/main/resources/log4j2.properties b/core/src/main/resources/log4j2.properties new file mode 120000 index 0000000..e4f686e --- /dev/null +++ b/core/src/main/resources/log4j2.properties @@ -0,0 +1 @@ +../../../../log4j2.properties \ No newline at end of file diff --git a/core/src/main/scala/Block.scala b/core/src/main/scala/Block.scala index 2735202..8d8b892 100644 --- a/core/src/main/scala/Block.scala +++ b/core/src/main/scala/Block.scala @@ -36,7 +36,7 @@ object Block extends Logging { keyLength: Int = 10, valueLength: Int = 90 ): Block = { - logger.debug(s"[Block] Reading block from $path") + logger.info(s"[Block] Reading block from $path") Block.fromSource(Source.fromURI(path.toURI), keyLength, valueLength) } diff --git a/core/src/main/scala/Record.scala b/core/src/main/scala/Record.scala index 1728f57..8c5e067 100644 --- a/core/src/main/scala/Record.scala +++ b/core/src/main/scala/Record.scala @@ -45,10 +45,6 @@ object Record extends Logging { val sortedListOfRecords = listOfRecords.sorted(Ordering.by((_: LazyList[Record]).head.key)) - logger.debug( - s"[Record] Sorted list of records: ${sortedListOfRecords.map(_.head.key.hex).mkString(", ")}" - ) - sortedListOfRecords.head.head #:: merged( (sortedListOfRecords.head.tail :: sortedListOfRecords.tail).filter( _.nonEmpty diff --git a/docker/master/Dockerfile b/docker/master/Dockerfile index d35f6e8..0cb6fa4 100644 --- a/docker/master/Dockerfile +++ b/docker/master/Dockerfile @@ -8,7 +8,7 @@ RUN mkdir -p /app WORKDIR /app -COPY build.sbt ./ +COPY build.sbt log4j2.properties ./ COPY core/build.sbt ./core/ COPY master/build.sbt ./master/ COPY rpc/build.sbt ./rpc/ diff --git a/docker/worker/Dockerfile b/docker/worker/Dockerfile index af28466..f44807b 100644 --- a/docker/worker/Dockerfile +++ b/docker/worker/Dockerfile @@ -14,7 +14,7 @@ COPY docker/worker/data /data WORKDIR /app -COPY build.sbt ./ +COPY build.sbt log4j2.properties ./ COPY core/build.sbt ./core/ COPY master/build.sbt ./master/ COPY rpc/build.sbt ./rpc/ diff --git a/log4j2.properties b/log4j2.properties new file mode 100644 index 0000000..6687ea3 --- /dev/null +++ b/log4j2.properties @@ -0,0 +1,17 @@ +# Set to debug or trace if log4j initialization is failing +status = warn + +# Name of the configuration +name = ConsoleLogConfigDemo + +# Console appender configuration +appender.console.type = Console +appender.console.name = consoleLogger +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n + +# Root logger level +rootLogger.level = info + +# Root logger referring to console appender +rootLogger.appenderRef.stdout.ref = consoleLogger diff --git a/master/src/main/resources/log4j2.properties b/master/src/main/resources/log4j2.properties deleted file mode 100644 index 59b1671..0000000 --- a/master/src/main/resources/log4j2.properties +++ /dev/null @@ -1,17 +0,0 @@ -# Set to debug or trace if log4j initialization is failing -status = warn - -# Name of the configuration -name = ConsoleLogConfigDemo - -# Console appender configuration -appender.console.type = Console -appender.console.name = consoleLogger -appender.console.layout.type = PatternLayout -appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n - -# Root logger level -rootLogger.level = debug - -# Root logger referring to console appender -rootLogger.appenderRef.stdout.ref = consoleLogger diff --git a/master/src/main/resources/log4j2.properties b/master/src/main/resources/log4j2.properties new file mode 120000 index 0000000..e4f686e --- /dev/null +++ b/master/src/main/resources/log4j2.properties @@ -0,0 +1 @@ +../../../../log4j2.properties \ No newline at end of file diff --git a/master/src/main/scala/Master.scala b/master/src/main/scala/Master.scala index 0f25bbe..4544aeb 100644 --- a/master/src/main/scala/Master.scala +++ b/master/src/main/scala/Master.scala @@ -32,7 +32,7 @@ object Master extends Logging { val masterHost = InetAddress.getLocalHost.getHostAddress val masterPort = sys.env.getOrElse("MASTER_PORT", "50051").toInt - logger.debug( + logger.info( "[Master] Arguments: \n" + s"masterHost: $masterHost\n" + s"masterPort: $masterPort\n" + @@ -47,7 +47,7 @@ object Master extends Logging { println(masterHost + ":" + masterPort) while (server.registeredWorkers.size < masterArguments.numberOfWorkers) { - logger.debug(s"${server.registeredWorkers}") + logger.info(s"${server.registeredWorkers}") Thread.sleep(1000) } @@ -59,48 +59,51 @@ object Master extends Logging { WorkerClient(worker.host, worker.port) } - implicit val requestExecutionContext: ExecutionContextExecutor = scala.concurrent.ExecutionContext.fromExecutor( - java.util.concurrent.Executors.newFixedThreadPool(workerInfo.size) - ) + implicit val requestExecutionContext: ExecutionContextExecutor = + scala.concurrent.ExecutionContext.fromExecutor( + java.util.concurrent.Executors.newFixedThreadPool(workerInfo.size) + ) + + logger.info(s"[Master] Clients: $clients") + + logger.info(s"[Master] Sample Requested") val sampledKeys = clients .sample(64) .flatMap(_.sampledKeys) .map(Key.fromByteString) - logger.debug(s"[Master] Sampled keys: $sampledKeys") + logger.info(s"[Master] Sampled") val sortedSampledKeys = sampledKeys.sorted - logger.debug(s"[Master] Sorted Sampled keys: $sortedSampledKeys") - val workers = workersWithKeyRange(sortedSampledKeys, workerInfo) - logger.debug(s"[Master] Key ranges with worker: $workers") + logger.info(s"[Master] Key ranges with worker: $workers") - logger.debug("[Master] Sort started") + logger.info("[Master] Sort started") clients.sort() - logger.debug("[Master] Sort finished") + logger.info("[Master] Sort finished") - logger.debug("[Master] Partition started") + logger.info("[Master] Partition started") clients.partition(workers) - logger.debug("[Master] Partition finished") + logger.info("[Master] Partition finished") - logger.debug("[Master] Exchange started") + logger.info("[Master] Exchange started") clients.exchange(workers) - logger.debug("[Master] Exchange finished") + logger.info("[Master] Exchange finished") - logger.debug("[Master] Merge started") + logger.info("[Master] Merge started") clients.merge() - logger.debug("[Master] Merge finished") + logger.info("[Master] Merge finished") server.blockUntilShutdown() } diff --git a/rpc/src/main/resources/log4j2.properties b/rpc/src/main/resources/log4j2.properties deleted file mode 100644 index 59b1671..0000000 --- a/rpc/src/main/resources/log4j2.properties +++ /dev/null @@ -1,17 +0,0 @@ -# Set to debug or trace if log4j initialization is failing -status = warn - -# Name of the configuration -name = ConsoleLogConfigDemo - -# Console appender configuration -appender.console.type = Console -appender.console.name = consoleLogger -appender.console.layout.type = PatternLayout -appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n - -# Root logger level -rootLogger.level = debug - -# Root logger referring to console appender -rootLogger.appenderRef.stdout.ref = consoleLogger diff --git a/rpc/src/main/resources/log4j2.properties b/rpc/src/main/resources/log4j2.properties new file mode 120000 index 0000000..e4f686e --- /dev/null +++ b/rpc/src/main/resources/log4j2.properties @@ -0,0 +1 @@ +../../../../log4j2.properties \ No newline at end of file diff --git a/rpc/src/main/scala/MasterServer.scala b/rpc/src/main/scala/MasterServer.scala index 4410015..9d99025 100644 --- a/rpc/src/main/scala/MasterServer.scala +++ b/rpc/src/main/scala/MasterServer.scala @@ -31,7 +31,7 @@ class MasterServer(executionContext: ExecutionContext, port: Int = 50051) def start(): Unit = { server.start() - logger.debug( + logger.info( "[MasterServer] \n" + s"port: $port\n" ) diff --git a/rpc/src/main/scala/MasterService.scala b/rpc/src/main/scala/MasterService.scala index 72db7c6..7616b13 100644 --- a/rpc/src/main/scala/MasterService.scala +++ b/rpc/src/main/scala/MasterService.scala @@ -20,7 +20,7 @@ class MasterService( val promise = Promise[RegisterReply] Future { - logger.debug(s"[MasterServer] Register ($request)") + logger.info(s"[MasterServer] Register ($request)") val worker: WorkerMetadata = request.worker.get diff --git a/rpc/src/main/scala/WorkerServer.scala b/rpc/src/main/scala/WorkerServer.scala index cc094b7..a574903 100644 --- a/rpc/src/main/scala/WorkerServer.scala +++ b/rpc/src/main/scala/WorkerServer.scala @@ -27,7 +27,7 @@ class WorkerServer( def start(): Unit = { server.start() - logger.debug( + logger.info( "[WorkerServer] \n" + s"port: $port\n" + s"inputDirectories: ${inputDirectories.mkString(", ")}\n" + diff --git a/rpc/src/main/scala/WorkerService.scala b/rpc/src/main/scala/WorkerService.scala index 70bee1d..028fd94 100644 --- a/rpc/src/main/scala/WorkerService.scala +++ b/rpc/src/main/scala/WorkerService.scala @@ -47,7 +47,7 @@ class WorkerService( val promise = Promise[SampleReply] Future { - logger.debug(s"[WorkerServer] Sample ($request)") + logger.info(s"[WorkerServer] Sample ($request)") val sampledKeys = inputFiles .map(f => Block.fromPath(f.path)) @@ -64,7 +64,7 @@ class WorkerService( val promise = Promise[SortReply] Future { - logger.debug(s"[WorkerServer] Sort ($request)") + logger.info(s"[WorkerServer] Sort ($request)") inputFiles .foreach(path => { @@ -72,11 +72,11 @@ class WorkerService( val sortedBlock = block.sorted - logger.debug(s"[WorkerServer] Writing sorted block to $path") + logger.info(s"[WorkerServer] Writing sorted block to $path") sortedBlock.writeTo(path) - logger.debug(s"[WorkerServer] Wrote sorted block to $path") + logger.info(s"[WorkerServer] Wrote sorted block to $path") }) promise.success(new SortReply()) @@ -91,7 +91,7 @@ class WorkerService( val promise = Promise[PartitionReply] Future { - logger.debug(s"[WorkerServer] Partition ($request)") + logger.info(s"[WorkerServer] Partition ($request)") val workers: Seq[WorkerMetadata] = request.workers @@ -106,19 +106,19 @@ class WorkerService( s"$path.${keyRange.from.hex}-${keyRange.to.hex}" ) - logger.debug( + logger.info( s"[WorkerServer] Writing partition to $partitionPath" ) partition.writeTo(partitionPath) - logger.debug( + logger.info( s"[WorkerServer] Wrote partition to $partitionPath" ) if (path.exists && path.isFile) { val result = path.delete() - logger.debug(s"[WorkerServer] Deleted $path: $result") + logger.info(s"[WorkerServer] Deleted $path: $result") } }) @@ -134,7 +134,7 @@ class WorkerService( val promise = Promise[ExchangeReply] Future { - logger.debug(s"[WorkerServer] Exchange ($request)") + logger.info(s"[WorkerServer] Exchange ($request)") val workers: Seq[WorkerMetadata] = request.workers @@ -143,7 +143,7 @@ class WorkerService( val targetWorkers = workers .filter(_.keyRange.get.includes(block.records.head.key)) - logger.debug(s"[WorkerServer] Sending $block to $targetWorkers") + logger.info(s"[WorkerServer] Sending $block to $targetWorkers") Await.result( Future.sequence( @@ -167,17 +167,17 @@ class WorkerService( val promise = Promise[SaveBlockReply] Future { - logger.debug(s"[WorkerServer] SaveBlock ($request)") + logger.info(s"[WorkerServer] SaveBlock ($request)") val block: Block = request.block val path = outputDirectory / UUID.randomUUID().toString - logger.debug(s"[WorkerServer] Writing block to $path") + logger.info(s"[WorkerServer] Writing block to $path") block.writeTo(path) - logger.debug(s"[WorkerServer] Wrote block to $path") + logger.info(s"[WorkerServer] Wrote block to $path") promise.success(new SaveBlockReply()) }(executionContext) @@ -189,12 +189,12 @@ class WorkerService( val promise = Promise[MergeReply] Future { - logger.debug(s"[WorkerServer] Merge ($request)") + logger.info(s"[WorkerServer] Merge ($request)") val targetFiles = outputFiles val blocks = targetFiles.map(path => Block.fromPath(path)) - logger.debug("[WorkerServer] Merging blocks") + logger.info("[WorkerServer] Merging blocks") val mergedBlock = blocks.merged @@ -203,12 +203,12 @@ class WorkerService( targetFiles.foreach(file => { val result = file.delete() - logger.debug( + logger.info( s"[WorkerServer] Deleted $file: $result" ) }) - logger.debug( + logger.info( s"[WorkerServer] Merged blocks: $results" ) diff --git a/utils/src/main/resources/log4j2.properties b/utils/src/main/resources/log4j2.properties deleted file mode 100644 index 59b1671..0000000 --- a/utils/src/main/resources/log4j2.properties +++ /dev/null @@ -1,17 +0,0 @@ -# Set to debug or trace if log4j initialization is failing -status = warn - -# Name of the configuration -name = ConsoleLogConfigDemo - -# Console appender configuration -appender.console.type = Console -appender.console.name = consoleLogger -appender.console.layout.type = PatternLayout -appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n - -# Root logger level -rootLogger.level = debug - -# Root logger referring to console appender -rootLogger.appenderRef.stdout.ref = consoleLogger diff --git a/utils/src/main/resources/log4j2.properties b/utils/src/main/resources/log4j2.properties new file mode 120000 index 0000000..e4f686e --- /dev/null +++ b/utils/src/main/resources/log4j2.properties @@ -0,0 +1 @@ +../../../../log4j2.properties \ No newline at end of file diff --git a/worker/src/main/resources/log4j2.properties b/worker/src/main/resources/log4j2.properties deleted file mode 100644 index 59b1671..0000000 --- a/worker/src/main/resources/log4j2.properties +++ /dev/null @@ -1,17 +0,0 @@ -# Set to debug or trace if log4j initialization is failing -status = warn - -# Name of the configuration -name = ConsoleLogConfigDemo - -# Console appender configuration -appender.console.type = Console -appender.console.name = consoleLogger -appender.console.layout.type = PatternLayout -appender.console.layout.pattern = %d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n - -# Root logger level -rootLogger.level = debug - -# Root logger referring to console appender -rootLogger.appenderRef.stdout.ref = consoleLogger diff --git a/worker/src/main/resources/log4j2.properties b/worker/src/main/resources/log4j2.properties new file mode 120000 index 0000000..e4f686e --- /dev/null +++ b/worker/src/main/resources/log4j2.properties @@ -0,0 +1 @@ +../../../../log4j2.properties \ No newline at end of file diff --git a/worker/src/main/scala/Worker.scala b/worker/src/main/scala/Worker.scala index 449b4c9..44215ed 100644 --- a/worker/src/main/scala/Worker.scala +++ b/worker/src/main/scala/Worker.scala @@ -18,7 +18,7 @@ object Worker extends Logging { val workerPort = Using(new ServerSocket(0))(_.getLocalPort).get val workerMetadata = WorkerMetadata(workerHost, workerPort, None) - logger.debug( + logger.info( "[Worker] Arguments: \n" + s"workerHost: $workerHost\n" + s"workerPort: $workerPort\n" + From e106129964e67bf6c05613ce68b583c76b143fdc Mon Sep 17 00:00:00 2001 From: Minjae Gwon Date: Mon, 27 Nov 2023 17:06:35 +0900 Subject: [PATCH 39/39] chore: lint --- master/src/main/scala/Master.scala | 4 ++-- rpc/src/main/scala/WorkerClient.scala | 8 +++++--- utils/src/main/scala/GenericBuildFrom.scala | 3 ++- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/master/src/main/scala/Master.scala b/master/src/main/scala/Master.scala index 4544aeb..2bbc7a2 100644 --- a/master/src/main/scala/Master.scala +++ b/master/src/main/scala/Master.scala @@ -66,14 +66,14 @@ object Master extends Logging { logger.info(s"[Master] Clients: $clients") - logger.info(s"[Master] Sample Requested") + logger.info("[Master] Sample Requested") val sampledKeys = clients .sample(64) .flatMap(_.sampledKeys) .map(Key.fromByteString) - logger.info(s"[Master] Sampled") + logger.info("[Master] Sampled") val sortedSampledKeys = sampledKeys.sorted diff --git a/rpc/src/main/scala/WorkerClient.scala b/rpc/src/main/scala/WorkerClient.scala index 7e08e93..ce56e8b 100644 --- a/rpc/src/main/scala/WorkerClient.scala +++ b/rpc/src/main/scala/WorkerClient.scala @@ -3,16 +3,18 @@ import io.grpc.ManagedChannel import io.grpc.ManagedChannelBuilder import kr.ac.postech.paranode.core.Block import kr.ac.postech.paranode.core.WorkerMetadata +import kr.ac.postech.paranode.utils.GenericBuildFrom import java.util.concurrent.TimeUnit import java.util.logging.Logger -import scala.concurrent.{Await, ExecutionContext, Future} +import scala.concurrent.Await +import scala.concurrent.ExecutionContext +import scala.concurrent.Future + import worker._ import worker.WorkerGrpc.WorkerStub import Implicit._ -import kr.ac.postech.paranode.utils.GenericBuildFrom - object WorkerClient { implicit class WorkerClients(val clients: List[WorkerClient]) { diff --git a/utils/src/main/scala/GenericBuildFrom.scala b/utils/src/main/scala/GenericBuildFrom.scala index e774896..67ac4d4 100644 --- a/utils/src/main/scala/GenericBuildFrom.scala +++ b/utils/src/main/scala/GenericBuildFrom.scala @@ -1,6 +1,7 @@ package kr.ac.postech.paranode.utils -import scala.collection.{BuildFrom, mutable} +import scala.collection.BuildFrom +import scala.collection.mutable object GenericBuildFrom { def apply[A, B]: BuildFrom[List[A], B, List[B]] =