Hash submitters in the deduplication key [DPP-347]

ledger/participant-integration-api/src/main/scala/platform/indexer/parallel/UpdateToDBDTOV1.scala

@@ -4,14 +4,14 @@
 package com.daml.platform.indexer.parallel
 
 import java.util.UUID
-
 import com.daml.ledger.api.domain
 import com.daml.ledger.participant.state.v1.{Configuration, Offset, ParticipantId, Update}
 import com.daml.lf.engine.Blinding
 import com.daml.lf.ledger.EventId
 import com.daml.platform.store.Conversions
 import com.daml.platform.store.appendonlydao.events._
 import com.daml.platform.store.appendonlydao.JdbcLedgerDao
+import com.daml.platform.store.dao.DeduplicationKeyMaker
 
 // TODO append-only: target to separation per update-type to it's own function + unit tests
 object UpdateToDBDTOV1 {
@@ -36,7 +36,7 @@ object UpdateToDBDTOV1 {
             status_message = Some(u.reason.description),
           ),
           new DBDTOV1.CommandDeduplication(
-            JdbcLedgerDao.deduplicationKey(
+            DeduplicationKeyMaker.make(
               domain.CommandId(u.submitterInfo.commandId),
               u.submitterInfo.actAs,
             )

ledger/participant-integration-api/src/main/scala/platform/store/appendonlydao/JdbcLedgerDao.scala

@@ -5,7 +5,6 @@ package com.daml.platform.store.appendonlydao
 import java.sql.Connection
 import java.time.Instant
 import java.util.Date
-
 import akka.NotUsed
 import akka.stream.scaladsl.Source
 import anorm.SqlParser._
@@ -46,6 +45,7 @@ import com.daml.platform.store.appendonlydao.events.{
 }
 import com.daml.platform.store.dao.events.TransactionsWriter.PreparedInsert
 import com.daml.platform.store.dao.{
+  DeduplicationKeyMaker,
   LedgerDao,
   LedgerReadDao,
   MeteredLedgerDao,
@@ -531,7 +531,7 @@ private class JdbcLedgerDao(
       deduplicateUntil: Instant,
   )(implicit loggingContext: LoggingContext): Future[CommandDeduplicationResult] =
     dbDispatcher.executeSql(metrics.daml.index.db.deduplicateCommandDbMetrics) { implicit conn =>
-      val key = deduplicationKey(commandId, submitters)
+      val key = DeduplicationKeyMaker.make(commandId, submitters)
       // Insert a new deduplication entry, or update an expired entry
       val updated = SQL(queries.SQL_INSERT_COMMAND)
         .on(
@@ -579,7 +579,7 @@ private class JdbcLedgerDao(
       commandId: domain.CommandId,
       submitters: List[Party],
   )(implicit conn: Connection): Unit = {
-    val key = deduplicationKey(commandId, submitters)
+    val key = DeduplicationKeyMaker.make(commandId, submitters)
     SQL_DELETE_COMMAND
       .on("deduplicationKey" -> key)
       .execute()
@@ -881,18 +881,6 @@ private[platform] object JdbcLedgerDao {
     ): Unit = ()
   }
 
-  def deduplicationKey(
-      commandId: domain.CommandId,
-      submitters: List[Ref.Party],
-  ): String = {
-    val submitterPart =
-      if (submitters.length == 1)
-        submitters.head
-      else
-        submitters.sorted(Ordering.String).distinct.mkString("%")
-    commandId.unwrap + "%" + submitterPart
-  }
-
   val acceptType = "accept"
   val rejectType = "reject"
 }

ledger/participant-integration-api/src/main/scala/platform/store/dao/DeduplicationKeyMaker.scala

@@ -0,0 +1,22 @@
+// Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package com.daml.platform.store.dao
+
+import com.daml.ledger.api.domain
+import com.daml.lf.data.Ref
+
+import java.security.MessageDigest
+import scalaz.syntax.tag._
+
+object DeduplicationKeyMaker {
+  def make(commandId: domain.CommandId, submitters: List[Ref.Party]): String =
+    commandId.unwrap + "%" + hashSubmitters(submitters.sorted(Ordering.String).distinct)
+
+  private def hashSubmitters(submitters: List[Ref.Party]): String = {
+    MessageDigest
+      .getInstance("SHA-256")
+      .digest(submitters.mkString.getBytes)
+      .mkString
+  }
+}

ledger/participant-integration-api/src/main/scala/platform/store/dao/JdbcLedgerDao.scala

@@ -809,26 +809,14 @@ private class JdbcLedgerDao(
       "deduplicate_until"
     )
 
-  private def deduplicationKey(
-      commandId: domain.CommandId,
-      submitters: List[Ref.Party],
-  ): String = {
-    val submitterPart =
-      if (submitters.length == 1)
-        submitters.head
-      else
-        submitters.sorted(Ordering.String).distinct.mkString("%")
-    commandId.unwrap + "%" + submitterPart
-  }
-
   override def deduplicateCommand(
       commandId: domain.CommandId,
       submitters: List[Ref.Party],
       submittedAt: Instant,
       deduplicateUntil: Instant,
   )(implicit loggingContext: LoggingContext): Future[CommandDeduplicationResult] =
     dbDispatcher.executeSql(metrics.daml.index.db.deduplicateCommandDbMetrics) { implicit conn =>
-      val key = deduplicationKey(commandId, submitters)
+      val key = DeduplicationKeyMaker.make(commandId, submitters)
       // Insert a new deduplication entry, or update an expired entry
       val updated = SQL(queries.SQL_INSERT_COMMAND)
         .on(
@@ -876,7 +864,7 @@ private class JdbcLedgerDao(
       commandId: domain.CommandId,
       submitters: List[Party],
   )(implicit conn: Connection): Unit = {
-    val key = deduplicationKey(commandId, submitters)
+    val key = DeduplicationKeyMaker.make(commandId, submitters)
     SQL_DELETE_COMMAND
       .on("deduplicationKey" -> key)
       .execute()

ledger/participant-integration-api/src/test/suite/scala/platform/store/dao/DeduplicationKeyMakerSpec.scala

@@ -0,0 +1,79 @@
+// Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package com.daml.platform.store.dao
+
+import org.scalatest.matchers.should.Matchers
+import org.scalatest.wordspec.AnyWordSpec
+import com.daml.ledger.api.domain.CommandId
+import com.daml.lf.data.Ref
+import org.scalatest.prop.TableDrivenPropertyChecks
+
+import java.util.UUID
+import scala.util.Random
+import scalaz.syntax.tag._
+
+class DeduplicationKeyMakerSpec extends AnyWordSpec with Matchers with TableDrivenPropertyChecks {
+
+  val commandId: CommandId = CommandId(Ref.LedgerString.assertFromString(UUID.randomUUID.toString))
+
+  "DeduplicationKeyMaker" should {
+    "make a deduplication key starting with a command ID in plain-text" in {
+      DeduplicationKeyMaker.make(commandId, List(aParty())) should startWith(commandId.unwrap)
+    }
+
+    "make different keys for different sets of submitters" in {
+      val aCommonParty = aParty()
+      val cases = Table(
+        ("Submitters for key1", "Submitters for key2"),
+        (List(aParty()), List(aParty())),
+        (List(aCommonParty, aParty()), List(aCommonParty, aParty())),
+        (List(aParty(), aParty()), List(aParty(), aParty())),
+      )
+
+      forAll(cases) { case (key1Submitters, key2Submitters) =>
+        val key1 = DeduplicationKeyMaker.make(commandId, key1Submitters)
+        val key2 = DeduplicationKeyMaker.make(commandId, key2Submitters)
+
+        key1 shouldNot equal(key2)
+      }
+    }
+
+    "make a deduplication key with a limited length for a large number of submitters" in {
+      val submitters = (1 to 50).map(_ => aParty()).toList
+
+      /** The motivation for the MaxKeyLength is to avoid problems with putting deduplication key in a database
+        * index (e.g. for Postgres the limit for the index row size is 2712).
+        * The value 200 is set arbitrarily to provide some space for other data.
+        */
+      val MaxKeyLength = 200
+      DeduplicationKeyMaker.make(commandId, submitters).length should be < MaxKeyLength
+    }
+
+    "make the same deduplication key for submitters of different order" in {
+      val submitter1 = aParty()
+      val submitter2 = aParty()
+      val submitter3 = aParty()
+
+      val key1 = DeduplicationKeyMaker.make(commandId, List(submitter1, submitter2, submitter3))
+      val key2 = DeduplicationKeyMaker.make(commandId, List(submitter1, submitter3, submitter2))
+
+      key1 shouldBe key2
+    }
+
+    "make the same deduplication key for duplicated submitters" in {
+      val submitter1 = aParty()
+      val submitter2 = aParty()
+
+      val key1 = DeduplicationKeyMaker.make(commandId, List(submitter1, submitter2))
+      val key2 = DeduplicationKeyMaker.make(
+        commandId,
+        List(submitter1, submitter1, submitter2, submitter2, submitter2),
+      )
+
+      key1 shouldBe key2
+    }
+
+    def aParty(): Ref.Party = Ref.Party.assertFromString(Random.alphanumeric.take(100).mkString)
+  }
+}