[New Scheduler] Add a centralized watcher for etcd data (#5069)

* Add a centralized watcher for etcd data

* Fix class name format

* Fix compilation error

* Enable test for WatcherService

common/scala/src/main/scala/org/apache/openwhisk/core/service/WatcherService.scala

@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.openwhisk.core.service
+
+import akka.actor.{Actor, ActorRef, ActorSystem, Props}
+import com.ibm.etcd.api.Event.EventType
+import com.ibm.etcd.client.kv.WatchUpdate
+import org.apache.openwhisk.common.Logging
+import org.apache.openwhisk.core.etcd.EtcdClient
+import org.apache.openwhisk.core.etcd.EtcdType._
+import scala.collection.JavaConverters._
+import scala.collection.concurrent.TrieMap
+
+// messages received by this actor
+case class WatchEndpoint(key: String,
+ value: String,
+ isPrefix: Boolean,
+ name: String,
+ listenEvents: Set[EtcdEvent] = Set.empty)
+case class UnwatchEndpoint(watchKey: String, isPrefix: Boolean, watchName: String, needFeedback: Boolean = false)
+
+// the watchKey is the string user want to watch, it can be a prefix, the key is a record's key in Etcd
+// so if `isPrefix = true`, the `watchKey != key`, else the `watchKey == key`
+sealed abstract class WatchEndpointOperation(val watchKey: String,
+ val key: String,
+ val value: String,
+ val isPrefix: Boolean)
+case class WatchEndpointRemoved(override val watchKey: String,
+ override val key: String,
+ override val value: String,
+ override val isPrefix: Boolean)
+ extends WatchEndpointOperation(watchKey, key, value, isPrefix)
+case class WatchEndpointInserted(override val watchKey: String,
+ override val key: String,
+ override val value: String,
+ override val isPrefix: Boolean)
+ extends WatchEndpointOperation(watchKey, key, value, isPrefix)
+case class WatcherClosed(key: String, isPrefix: Boolean)
+
+sealed trait EtcdEvent
+case object PutEvent extends EtcdEvent
+case object DeleteEvent extends EtcdEvent
+
+// there may be several watchers for a same watcher key, so add a watcherName to distinguish them
+case class WatcherKey(watchKey: String, watchName: String)
+
+class WatcherService(etcdClient: EtcdClient)(implicit logging: Logging, actorSystem: ActorSystem) extends Actor {
+
+ implicit val ec = context.dispatcher
+
+ private[service] val putWatchers = TrieMap[WatcherKey, ActorRef]()
+ private[service] val deleteWatchers = TrieMap[WatcherKey, ActorRef]()
+ private[service] val prefixPutWatchers = TrieMap[WatcherKey, ActorRef]()
+ private[service] val prefixDeleteWatchers = TrieMap[WatcherKey, ActorRef]()
+
+ private val watcher = etcdClient.watchAllKeys { res: WatchUpdate =>
+ res.getEvents.asScala.foreach { event =>
+ event.getType match {
+ case EventType.DELETE =>
+ val key = ByteStringToString(event.getPrevKv.getKey)
+ val value = ByteStringToString(event.getPrevKv.getValue)
+ val watchEvent = WatchEndpointRemoved(key, key, value, false)
+ deleteWatchers
+ .foreach { watcher =>
+ if (watcher._1.watchKey == key) {
+ watcher._2 ! watchEvent
+ }
+ }
+ prefixDeleteWatchers
+ .foreach { watcher =>
+ if (key.startsWith(watcher._1.watchKey)) {
+ watcher._2 ! WatchEndpointRemoved(watcher._1.watchKey, key, value, true)
+ }
+ }
+ case EventType.PUT =>
+ val key = ByteStringToString(event.getKv.getKey)
+ val value = ByteStringToString(event.getKv.getValue)
+ val watchEvent = WatchEndpointInserted(key, key, value, false)
+ putWatchers
+ .foreach { watcher =>
+ if (watcher._1.watchKey == key) {
+ watcher._2 ! watchEvent
+ }
+ }
+ prefixPutWatchers
+ .foreach { watcher =>
+ if (key.startsWith(watcher._1.watchKey)) {
+ watcher._2 ! WatchEndpointInserted(watcher._1.watchKey, key, value, true)
+ }
+ }
+ case msg =>
+ logging.debug(this, s"watch event received: $msg.")
+ }
+ }
+
+ }
+
+ override def receive: Receive = {
+ case request: WatchEndpoint =>
+ logging.info(this, s"watch endpoint: $request")
+ val watcherKey = WatcherKey(request.key, request.name)
+ if (request.listenEvents.contains(PutEvent))
+ if (request.isPrefix)
+ prefixPutWatchers.update(watcherKey, sender())
+ else
+ putWatchers.update(watcherKey, sender())
+
+ if (request.listenEvents.contains(DeleteEvent))
+ if (request.isPrefix)
+ prefixDeleteWatchers.update(watcherKey, sender())
+ else
+ deleteWatchers.update(watcherKey, sender())
+
+ case request: UnwatchEndpoint =>
+ val watcherKey = WatcherKey(request.watchKey, request.watchName)
+ if (request.isPrefix) {
+ prefixPutWatchers.remove(watcherKey)
+ prefixDeleteWatchers.remove(watcherKey)
+ } else {
+ putWatchers.remove(watcherKey)
+ deleteWatchers.remove(watcherKey)
+ }
+
+ // always send WatcherClosed back to sender if it need a feedback
+ if (request.needFeedback)
+ sender ! WatcherClosed(request.watchKey, request.isPrefix)
+ }
+}
+
+object WatcherService {
+ def props(etcdClient: EtcdClient)(implicit logging: Logging, actorSystem: ActorSystem): Props = {
+ Props(new WatcherService(etcdClient))
+ }
+}

tests/build.gradle

@@ -90,6 +90,7 @@ ext.testSets = [
  "includes" : [
  "org/apache/openwhisk/common/etcd/**",
  "org/apache/openwhisk/core/scheduler/**",
+ "org/apache/openwhisk/core/service/**",
  ]
  ],
  "REQUIRE_MULTI_RUNTIME" : [