Feature/netty poc

build.sbt

@@ -27,6 +27,10 @@ lazy val s2rest_play = project.enablePlugins(PlayScala)
   .dependsOn(s2core, s2counter_core)
   .settings(commonSettings: _*)
 
+lazy val s2rest_netty = project
+  .dependsOn(s2core)
+  .settings(commonSettings: _*)
+
 lazy val s2core = project.settings(commonSettings: _*)
 
 lazy val spark = project.settings(commonSettings: _*)
@@ -41,3 +45,5 @@ lazy val s2counter_loader = project.dependsOn(s2counter_core, spark)
   .settings(commonSettings: _*)
 
 lazy val s2ml = project.settings(commonSettings: _*)
+
+Revolver.settings

project/plugins.sbt

@@ -3,3 +3,8 @@ addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.3.10")
 
 // http://www.scalastyle.org/sbt.html
 addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "0.7.0")
+
+// sbt revolver
+addSbtPlugin("io.spray" % "sbt-revolver" % "0.7.2")
+
+addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.0.3")

s2rest_netty/build.sbt

@@ -0,0 +1,8 @@
+name := "s2rest_netty"
+
+enablePlugins(JavaAppPackaging)
+
+libraryDependencies ++= Seq(
+  "io.netty" % "netty-all" % "4.0.33.Final",
+  "com.typesafe.play" %% "play-ws" % Common.playVersion
+)

s2rest_netty/conf/logger.xml

@@ -0,0 +1,83 @@
+<configuration>
+
+    <conversionRule conversionWord="coloredLevel" converterClass="play.api.Logger$ColoredLevel"/>
+
+    <appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>logs/application.log</file>
+
+        <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
+            <fileNamePattern>logs/application.%i.log</fileNamePattern>
+            <minIndex>1</minIndex>
+            <maxIndex>9</maxIndex>
+        </rollingPolicy>
+
+        <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
+            <maxFileSize>500MB</maxFileSize>
+        </triggeringPolicy>
+        <encoder>
+            <pattern>%date [%level] [%logger] [%thread] - %message %xException%n</pattern>
+        </encoder>
+    </appender>
+
+
+    <appender name="ERROR" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>logs/error.log</file>
+        <append>true</append>
+        <encoder>
+            <pattern>%date [%level] [%logger] [%thread] - %message %xException%n</pattern>
+        </encoder>
+        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+            <fileNamePattern>logs/error.log.%d.%i</fileNamePattern>
+            <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
+                <maxFileSize>500MB</maxFileSize>
+            </timeBasedFileNamingAndTriggeringPolicy>
+            <maxHistory>3</maxHistory>
+        </rollingPolicy>
+    </appender>
+
+    <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+        <encoder>
+            <pattern>%coloredLevel %logger{15} - %message%n%xException%n</pattern>
+        </encoder>
+    </appender>
+
+    <appender name="ACTOR" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <file>logs/actor.log</file>
+        <append>true</append>
+        <encoder>
+            <pattern>%date [%level] [%logger] [%thread] - %message %xException%n</pattern>
+        </encoder>
+        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+            <fileNamePattern>logs/actor.log.%d.%i</fileNamePattern>
+            <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
+                <maxFileSize>200MB</maxFileSize>
+            </timeBasedFileNamingAndTriggeringPolicy>
+            <maxHistory>7</maxHistory>
+        </rollingPolicy>
+    </appender>
+
+    <logger name="play" level="INFO">
+        <appender-ref ref="STDOUT"/>
+        <appender-ref ref="FILE"/>
+    </logger>
+
+    <logger name="application" level="INFO">
+        <appender-ref ref="STDOUT"/>
+        <appender-ref ref="FILE"/>
+    </logger>
+
+    <logger name="error" level="INFO">
+        <appender-ref ref="STDOUT"/>
+        <appender-ref ref="ERROR"/>
+    </logger>
+
+    <logger name="actor" level="INFO">
+        <appender-ref ref="ACTOR"/>
+    </logger>
+
+    <logger name="akka" level="INFO">
+        <appender-ref ref="STDOUT"/>
+        <appender-ref ref="FILE"/>
+    </logger>
+
+</configuration>

s2rest_netty/conf/reference.conf

@@ -0,0 +1,129 @@
+# This is the main configuration file for the application.
+# ~~~~~
+
+# Secret key
+# ~~~~~
+# The secret key is used to secure cryptographics functions.
+#
+# This must be changed for production, but we recommend not changing it in this file.
+#
+# See http://www.playframework.com/documentation/latest/ApplicationSecret for more details.
+application.secret="/`==g^yr2DNnZGK_L^rguLZeR`60uLOVgY@OhyTv:maatl:Tl>9or/d1xME3b/Pi"
+
+# The application languages
+# ~~~~~
+application.langs="en"
+
+# Global object class
+# ~~~~~
+# Define the Global object class for this application.
+# Default to Global in the root package.
+# application.global=Global
+
+# Router
+# ~~~~~
+# Define the Router object to use for this application.
+# This router will be looked up first when the application is starting up,
+# so make sure this is the entry point.
+# Furthermore, it's assumed your route file is named properly.
+# So for an application router like `my.application.Router`,
+# you may need to define a router file `conf/my.application.routes`.
+# Default to Routes in the root package (and conf/routes)
+# application.router=my.application.Routes
+
+# Database configuration
+# ~~~~~
+# You can declare as many datasources as you want.
+# By convention, the default datasource is named `default`
+#
+# db.default.driver=org.h2.Driver
+# db.default.url="jdbc:h2:mem:play"
+# db.default.user=sa
+# db.default.password=""
+
+# Evolutions
+# ~~~~~
+# You can disable evolutions if needed
+# evolutionplugin=disabled
+
+# Logger
+# ~~~~~
+# You can also configure logback (http://logback.qos.ch/),
+# by providing an application-logger.xml file in the conf directory.
+
+# Root logger:
+logger.root=ERROR
+
+# Logger used by the framework:
+logger.play=INFO
+
+# Logger provided to your application:
+logger.application=DEBUG
+
+# APP PHASE
+phase=dev
+host=localhost
+
+# DB
+s2graph.models.table.name="models-dev"
+hbase.zookeeper.quorum=${host}
+db.default.url="jdbc:mysql://"${host}":3306/graph_dev"
+# Query server
+is.query.server=true
+is.write.server=true
+query.hard.limit=100000
+
+# Local Cache
+cache.ttl.seconds=60
+cache.max.size=100000
+
+# HBASE
+#hbase.client.operation.timeout=1000
+#async.hbase.client.flush.interval=100
+hbase.table.compression.algorithm="gz"
+
+# Asynchbase
+hbase.client.retries.number=100
+hbase.rpcs.buffered_flush_interval=100
+hbase.rpc.timeout=0
+#hbase.nsre.high_watermark=1000000
+#hbase.timer.tick=5
+#hbase.timer.ticks_per_wheel=5
+
+# Kafka
+kafka.metadata.broker.list=${host}
+kafka.producer.pool.size=0
+
+# HTTP
+parsers.text.maxLength=512K
+parsers.json.maxLength=512K
+trustxforwarded=false
+
+# Local Queue Actor
+local.queue.actor.max.queue.size=100000
+local.queue.actor.rate.limit=1000000
+
+# local retry number
+max.retry.number=100
+max.back.off=50
+delete.all.fetch.size=10000
+hbase.fail.prob=-1.0
+
+# max allowd edges for deleteAll is multiply of above two configuration.
+
+# set global obejct package, TODO: remove global
+application.global=com.kakao.s2graph.rest.Global
+
+akka {
+  loggers = ["akka.event.slf4j.Slf4jLogger"]
+  loglevel = "DEBUG"
+}
+
+
+# Future cache.
+future.cache.max.size=1000000
+future.cache.max.idle.ttl=10000
+
+# Counter
+redis.instances = [${host}]
+

s2rest_netty/conf/routes

@@ -0,0 +1,124 @@
+# Routes
+# This file defines all application routes (Higher priority routes first)
+# ~~~~
+
+
+# publish
+#POST   /publish/:topic                                                        controllers.PublishController.publish(topic)
+POST        /publish/:topic                                               controllers.PublishController.mutateBulk(topic)
+POST        /publishOnly/:topic                                           controllers.PublishController.publishOnly(topic)
+
+#### Health Check
+#GET    /health_check.html                                                     controllers.Assets.at(path="/public", file="health_check.html")
+GET         /health_check.html                                            controllers.ApplicationController.healthCheck()
+PUT         /health_check/:isHealthy                                      controllers.ApplicationController.updateHealthCheck(isHealthy: Boolean)
+
+## Edge
+POST        /graphs/edges/insert                                          controllers.EdgeController.inserts()
+POST        /graphs/edges/insertWithWait                                  controllers.EdgeController.insertsWithWait()
+POST        /graphs/edges/insertBulk                                      controllers.EdgeController.insertsBulk()
+POST        /graphs/edges/delete                                          controllers.EdgeController.deletes()
+POST        /graphs/edges/deleteWithWait                                  controllers.EdgeController.deletesWithWait()
+POST        /graphs/edges/deleteAll                                       controllers.EdgeController.deleteAll()
+POST        /graphs/edges/update                                          controllers.EdgeController.updates()
+POST        /graphs/edges/updateWithWait                                  controllers.EdgeController.updatesWithWait()
+POST        /graphs/edges/increment                                       controllers.EdgeController.increments()
+POST        /graphs/edges/incrementCount                                  controllers.EdgeController.incrementCounts()
+POST        /graphs/edges/bulk                                            controllers.EdgeController.mutateBulk()
+
+## Vertex
+POST        /graphs/vertices/insert                                       controllers.VertexController.inserts()
+POST        /graphs/vertices/insertWithWait                               controllers.VertexController.insertsWithWait()
+POST        /graphs/vertices/insert/:serviceName/:columnName              controllers.VertexController.insertsSimple(serviceName, columnName)
+POST        /graphs/vertices/delete                                       controllers.VertexController.deletes()
+POST        /graphs/vertices/deleteWithWait                               controllers.VertexController.deletesWithWait()
+POST        /graphs/vertices/delete/:serviceName/:columnName              controllers.VertexController.deletesSimple(serviceName, columnName)
+POST        /graphs/vertices/deleteAll                                    controllers.VertexController.deletesAll()
+POST        /graphs/vertices/deleteAll/:serviceName/:columnName           controllers.VertexController.deletesAllSimple(serviceName, columnName)
+
+
+### SELECT Edges
+POST        /graphs/getEdges                                              controllers.QueryController.getEdges()
+POST        /graphs/getEdges/grouped                                      controllers.QueryController.getEdgesWithGrouping()
+POST        /graphs/getEdgesExcluded                                      controllers.QueryController.getEdgesExcluded()
+POST        /graphs/getEdgesExcluded/grouped                              controllers.QueryController.getEdgesExcludedWithGrouping()
+POST        /graphs/checkEdges                                            controllers.QueryController.checkEdges()
+
+### this will be deprecated
+POST        /graphs/getEdgesGrouped                                       controllers.QueryController.getEdgesGrouped()
+POST        /graphs/getEdgesGroupedExcluded                               controllers.QueryController.getEdgesGroupedExcluded()
+POST        /graphs/getEdgesGroupedExcludedFormatted                      controllers.QueryController.getEdgesGroupedExcludedFormatted()
+GET         /graphs/getEdge/:srcId/:tgtId/:labelName/:direction           controllers.QueryController.getEdge(srcId, tgtId, labelName, direction)
+
+
+### SELECT Vertices
+#POST   /graphs/getVertex                                                      controllers.QueryController.getVertex()
+POST        /graphs/getVertices                                           controllers.QueryController.getVertices()
+
+
+#### ADMIN
+POST        /graphs/createService                                         controllers.AdminController.createService()
+GET         /graphs/getService/:serviceName                               controllers.AdminController.getService(serviceName)
+GET         /graphs/getLabels/:serviceName                                controllers.AdminController.getLabels(serviceName)
+POST        /graphs/createLabel                                           controllers.AdminController.createLabel()
+POST        /graphs/addIndex                                              controllers.AdminController.addIndex()
+GET         /graphs/getLabel/:labelName                                   controllers.AdminController.getLabel(labelName)
+PUT         /graphs/deleteLabel/:labelName                                controllers.AdminController.deleteLabel(labelName)
+
+POST        /graphs/addProp/:labelName                                    controllers.AdminController.addProp(labelName)
+POST        /graphs/createServiceColumn                                   controllers.AdminController.createServiceColumn()
+PUT         /graphs/deleteServiceColumn/:serviceName/:columnName          controllers.AdminController.deleteServiceColumn(serviceName, columnName)
+POST        /graphs/addServiceColumnProp/:serviceName/:columnName         controllers.AdminController.addServiceColumnProp(serviceName, columnName)
+POST        /graphs/addServiceColumnProps/:serviceName/:columnName        controllers.AdminController.addServiceColumnProps(serviceName, columnName)
+GET         /graphs/getServiceColumn/:serviceName/:columnName             controllers.AdminController.getServiceColumn(serviceName, columnName)
+POST        /graphs/createHTable                                          controllers.AdminController.createHTable()
+
+
+
+
+#### TEST
+#GET    /graphs/testGetEdges/:label/:limit/:friendCntStep                       controllers.QueryController.testGetEdges(label, limit: Int, friendCntStep: Int)
+#GET    /graphs/testGetEdges2/:label1/:limit1/:label2/:limit2                   controllers.QueryController.testGetEdges2(label1, limit1: Int, label2, limit2: Int)
+#GET    /graphs/testGetEdges3/:label1/:limit1/:label2/:limit2/:label3/:limit3   controllers.QueryController.testGetEdges3(label1, limit1: Int, label2, limit2: Int, label3, limit3: Int)
+POST        /ping                                                         controllers.TestController.ping()
+POST        /pingAsync                                                    controllers.TestController.pingAsync()
+GET         /graphs/testId                                                controllers.TestController.getRandomId()
+
+# Map static resources from the /public folder to the /assets URL path
+GET         /images/*file                                                 controllers.Assets.at(path="/public/images", file)
+GET         /javascripts/*file                                            controllers.Assets.at(path="/public/javascripts", file)
+GET         /stylesheets/*file                                            controllers.Assets.at(path="/public/stylesheets", file)
+GET         /font-awesome-4.1.0/*file                                     controllers.Assets.at(path="/public/font-awesome-4.1.0", file)
+GET         /swagger/*file                                                controllers.Assets.at(path="/public/swagger-ui", file)
+
+
+# AdminController API
+#GET    /admin/services                                                         controllers.AdminController.allServices
+GET         /admin/labels/:serviceName                                    controllers.AdminController.getLabels(serviceName)
+#POST   /admin/labels/delete/:zkAddr/:tableName/:labelIds/:minTs/:maxTs         controllers.AdminController.deleteEdges(zkAddr, tableName, labelIds, minTs: Long, maxTs: Long)
+#POST   /admin/labels/deleteAll/:zkAddr/:tableName/:minTs/:maxTs                controllers.AdminController.deleteAllEdges(zkAddr, tableName, minTs: Long, maxTs: Long)
+#POST   /admin/swapLabel/:oldLabelName/:newLabelName                            controllers.AdminController.swapLabel(oldLabelName, newLabelName)
+#GET    /admin/reloadLabel/:labelName                                           controllers.AdminController.reloadLabel(labelName)
+#POST        /admin/getEdges                                                              controllers.AdminController.getEdges()
+POST        /graphs/copyLabel/:oldLabelName/:newLabelName                 controllers.AdminController.copyLabel(oldLabelName, newLabelName)
+POST        /graphs/renameLabel/:oldLabelName/:newLabelName               controllers.AdminController.renameLabel(oldLabelName, newLabelName)
+POST        /graphs/updateHTable/:labelName/:newHTableName                controllers.AdminController.updateHTable(labelName, newHTableName)
+PUT         /graphs/loadCache                                             controllers.AdminController.loadCache()
+
+
+# Counter Admin API
+POST    /counter/v1/:service/:action                                    controllers.CounterController.createAction(service, action)
+GET     /counter/v1/:service/:action                                    controllers.CounterController.getAction(service, action)
+PUT     /counter/v1/:service/:action                                    controllers.CounterController.updateAction(service, action)
+PUT     /counter/v1/:service/:action/prepare                            controllers.CounterController.prepareAction(service, action)
+DELETE  /counter/v1/:service/:action                                    controllers.CounterController.deleteAction(service, action)
+
+# Counter API
+GET     /counter/v1/:service/:action/ranking                            controllers.CounterController.getRankingCountAsync(service, action)
+DELETE  /counter/v1/:service/:action/ranking                            controllers.CounterController.deleteRankingCount(service, action)
+GET     /counter/v1/:service/:action/:item                              controllers.CounterController.getExactCountAsync(service, action, item)
+PUT     /counter/v1/:service/:action/:item                              controllers.CounterController.incrementCount(service, action, item)
+POST    /counter/v1/mget                                                controllers.CounterController.getExactCountAsyncMulti()
+
+# Experiment API
+POST        /graphs/experiment/:accessToken/:experimentName/:uuid         controllers.ExperimentController.experiment(accessToken, experimentName, uuid)

s2rest_netty/conf/test.conf

@@ -0,0 +1,2 @@
+max.retry.number=10000
+hbase.fail.prob=0.1