use playwright to wrap a browser to grab race monitor data events

build.gradle.kts

@@ -65,6 +65,8 @@ dependencies {
  testImplementation("org.testcontainers:junit-jupiter:1.19.0")
  testImplementation("org.testcontainers:gcloud:1.19.0")
 
+ implementation("com.microsoft.playwright:playwright:1.47.0")
+
 }
 
 dependencyManagement {

gradle.properties

@@ -2,4 +2,4 @@ protobufVersion=3.25.3
 protobufPluginVersion=0.9.4
 # setting this to match lemon-pi
 grpcVersion=1.57.0
-ktorVersion=1.6.7
+ktorVersion=2.3.12

src/main/kotlin/com/normtronix/meringue/AdminService.kt

@@ -80,7 +80,7 @@ class AdminService : AdminServiceGrpcKt.AdminServiceCoroutineImplBase(), Initial
  var raceDataSourceFactoryFn : (MeringueAdmin.RaceDataProvider, String) -> RaceDataSource =
  fun(provider: MeringueAdmin.RaceDataProvider, x:String) : RaceDataSource {
  return when (provider) {
- MeringueAdmin.RaceDataProvider.PROVIDER_RM -> DataSource1(x)
+ MeringueAdmin.RaceDataProvider.PROVIDER_RM -> DataSource1b(x)
  MeringueAdmin.RaceDataProvider.PROVIDER_RH -> DataSource2(x)
  else -> throw RuntimeException("unknown race provider")
  }

src/main/kotlin/com/normtronix/meringue/racedata/DataSource1.kt

@@ -4,9 +4,9 @@ import com.google.gson.JsonParser
 import com.normtronix.meringue.event.*
 import io.ktor.client.*
 import io.ktor.client.engine.cio.*
-import io.ktor.client.features.*
-import io.ktor.client.features.websocket.*
-import io.ktor.http.cio.websocket.*
+import io.ktor.client.plugins.*
+import io.ktor.client.plugins.websocket.*
+import io.ktor.websocket.*
 import kotlinx.coroutines.channels.ClosedReceiveChannelException
 import kotlinx.coroutines.delay
 import kotlinx.coroutines.launch
@@ -66,7 +66,7 @@ open class DataSource1(val raceId:String) : EventHandler, RaceDataSource {
  client.webSocket(streamUrl) {
  log.info("connected to url, timeout = ${this.timeoutMillis}, ping_interval = ${this.pingIntervalMillis}")
  val joinMsg = "\$JOIN,${fields["instance"]},${fields["token"]}}"
- val joinMessage = Frame.byType(true, FrameType.TEXT, joinMsg.encodeToByteArray())
+ val joinMessage = Frame.byType(true, FrameType.TEXT, joinMsg.encodeToByteArray(), false, false, false)
  outgoing.send(joinMessage)
  while (!stopped) {
  val joinedMessage = StringBuilder()
@@ -88,6 +88,8 @@ open class DataSource1(val raceId:String) : EventHandler, RaceDataSource {
  launch {
  handler.handleWebSocketMessage(line)
  }
+ } else if (line.trim().isNotEmpty()) {
+ log.warn("IGNORING >> $line")
  }
  }
  if (logRaceData) {

src/main/kotlin/com/normtronix/meringue/racedata/DataSource1b.kt

@@ -0,0 +1,95 @@
+package com.normtronix.meringue.racedata
+
+import com.normtronix.meringue.event.*
+import com.microsoft.playwright.Page
+import com.microsoft.playwright.Playwright
+import com.microsoft.playwright.PlaywrightException
+import kotlinx.coroutines.*
+import org.slf4j.Logger
+import org.slf4j.LoggerFactory
+import java.io.File
+import java.io.IOException
+
+open class DataSource1b(val raceId:String) : EventHandler, RaceDataSource {
+
+ val provider = "race-monitor"
+ var page: Page? = null
+ override var logRaceData = false
+
+ override fun connect() : String {
+ // return the url to the resource
+ return "https://api.${provider}.com/Timing/?raceid=${raceId}&source=${provider}.com"
+ }
+
+ override suspend fun stream(context: Any, baseHandler: BaseDataSourceHandler) {
+ val handler = baseHandler as DataSourceHandler
+ val streamUrl = context as String
+ log.info("connecting to $streamUrl")
+ File("logs").mkdir()
+
+ Events.register(
+ RaceDisconnectEvent::class.java, this,
+ filter = { it is RaceDisconnectEvent && it.trackCode == handler.trackCode })
+
+ try {
+ Playwright.create().use { playwright ->
+ coroutineScope {
+ log.info("launching headless browser")
+ val browser = playwright.chromium().launch()
+ log.info("browser running")
+ page = browser.newPage()
+
+ page?.onWebSocket {
+ it.onFrameReceived {
+ val lines = it.text().split("\n")
+ for (line in lines) {
+ if (line.startsWith("$")) {
+ launch(Dispatchers.IO) {
+ handler.handleWebSocketMessage(line)
+ }
+ } else if (line.trim().isNotEmpty()) {
+ log.warn("IGNORING >> $line")
+ }
+ }
+ if (logRaceData) {
+ try {
+ File("logs/race-$raceId.log").appendText(it.text())
+ } catch (e: IOException) {
+ log.warn("failed to log racedata : ${e.message}")
+ }
+ }
+ }
+ it.onClose {
+ log.warn("websocket closed")
+ page?.close()
+ }
+ it.onSocketError {
+ log.warn("websocket socket error")
+ page?.close()
+ }
+ }
+
+ page?.navigate(streamUrl)
+ page?.setDefaultTimeout(86400000.0)
+ page?.waitForWebSocket {
+ log.info("websocket working")
+ }
+ }
+ }
+ } catch (e: PlaywrightException) {
+ log.warn("caught playwrite exception", e)
+ }
+ }
+
+ companion object {
+ val log: Logger = LoggerFactory.getLogger(DataSource1b::class.java)
+ }
+
+ override suspend fun handleEvent(e: Event) {
+ if (e is RaceDisconnectEvent) {
+ page?.apply { if (!this.isClosed) { this.close() }}
+ }
+ }
+
+}
+

src/main/kotlin/com/normtronix/meringue/racedata/DataSource2Handler.kt

@@ -32,9 +32,13 @@ class DataSource2Handler(private val leaderboard: RaceOrder,
  private fun convertFlagStatus(flag: FlagRH?): String {
  return when (flag?.color?.lowercase()) {
  "green" -> "green"
+ "green flag" -> "green"
  "red" -> "red"
+ "red flag" -> "red"
  "yellow" -> "yellow"
+ "yellow flag" -> "yellow"
  "black" -> "black"
+ "black flag" -> "black"
  else -> "unknown"
  }
  }

src/main/kotlin/com/normtronix/meringue/racedata/DataSourceHandler.kt

@@ -75,6 +75,10 @@ class DataSourceHandler(private val leaderboard: RaceOrder,
  // note : these appear to arrive in the order J, G, H
  "\$G" -> {
  // $G,25,"10",34,"01:52:31.206"
+ // [1] 25 position in race
+ // [2] "10" car number
+ // [3] 34 laps completed
+ // [4] "01:52:31.206" time spent on track
  if (bits.size == 5) {
  val carNumber = bits[2].trim('"')
  // if we get updates saying they completed null laps then ignore it
@@ -93,13 +97,23 @@ class DataSourceHandler(private val leaderboard: RaceOrder,
  }
  }
  "\$H" -> {
+ // Handle fastest lap information
  // $H,66,"10",27,"00:02:35.467"
+ // [1] 66 is position in class (or overall?) for fastest lap
+ // [2] "10" is car number
+ // [3] 27 is fastest lap
+ // [4] "00:02:35.467" is their fastest lap time
  if (bits.size == 5) {
  val carNumber = bits[2].trim('"')
  leaderboard.updateFastestLap(carNumber, bits[3].trim('"').toInt(), convertToSeconds(bits[4]))
  }
  }
  "\$J" -> {
+ // Last lap time
+ // $J,"71","00:03:42.155","02:58:04.545"
+ // [1] "71" car number
+ // [2] "00:03:42.155" last lap
+ // [3] "02:58:04.545" total time on track
  if (bits.size == 4) {
  val carNumber = bits[1].trim('"')
  leaderboard.updateLastLap(carNumber, convertToSeconds(bits[2]))

src/test/kotlin/com/normtronix/meringue/racedata/DebugRaceDataStream.kt

@@ -0,0 +1,23 @@
+package com.normtronix.meringue.racedata
+
+import kotlinx.coroutines.runBlocking
+import org.junit.jupiter.api.Test
+import java.io.File
+
+class DebugRaceDataStream {
+
+ // manual test
+ // @Test
+ fun readLiveRaceData() {
+ runBlocking {
+ val raceId = "24-hours-of-lemons-pacific-raceway"
+ val race = RaceOrder()
+ val handler = DataSource2Handler(race, "test1", setOf("236"))
+ val ds2 = DataSource2(raceId)
+ ds2.logRaceData = true
+ val url = ds2.connect()
+ println(url)
+ ds2.stream(url, handler)
+ }
+ }
+}

src/test/kotlin/com/normtronix/meringue/racedata/PlaywrightTest.kt

@@ -0,0 +1,72 @@
+package com.normtronix.meringue.racedata
+
+import com.microsoft.playwright.Page
+import com.microsoft.playwright.Playwright
+import kotlinx.coroutines.*
+import org.junit.jupiter.api.Test
+
+
+class PlaywrightTest {
+
+ // @Test
+ fun openWindow() {
+ Playwright.create().use { playwright ->
+ val browser = playwright.chromium().launch()
+ val page: Page = browser.newPage()
+ // page.navigate("https://api.race-monitor.com/Timing/?raceid=149008&source=web")
+ page.onWebSocket {
+ it.onFrameReceived {
+ println(it.text())
+ }
+ it.onClose {
+ println("closed")
+ }
+ it.onSocketError {
+ println("socket error")
+ }
+ }
+ // coroutineScope {
+// launch {
+// println("sleeping for a minute")
+// delay(5000)
+// // hook this up to the disconnect request
+// println("closing page")
+// page.close()
+// println("page closed")
+// }
+// launch {
+ println("navigating")
+ page.navigate("https://api.race-monitor.com/Timing/?raceid=37872&source=race-monitor.com")
+ println("completed")
+ page.setDefaultTimeout(86400000.0)
+ page.waitForWebSocket(Page.WaitForWebSocketOptions().setTimeout(86400000.0)) {
+ println("page closed")
+ }
+ println("set timeout")
+
+// page.waitForWebSocket {
+// println("got a websocket")
+// }
+// }
+// }
+
+ println("you never get here")
+
+ //Thread.sleep(3000)
+ //page.screenshot(Page.ScreenshotOptions().setPath(Paths.get("example.png")))
+ }
+ }
+
+ // @Test
+ fun testDataSource1b() {
+ val ds = DataSource1b("37872")
+ val url = ds.connect()
+ val race = RaceOrder()
+ val handler = DataSourceHandler(race, "test1", 150, setOf("236"))
+ runBlocking {
+ ds.stream(url, handler)
+ println("finished")
+ }
+ }
+
+}