diff --git a/ledger/participant-integration-api/src/main/scala/platform/indexer/parallel/UpdateToDBDTOV1.scala b/ledger/participant-integration-api/src/main/scala/platform/indexer/parallel/UpdateToDBDTOV1.scala index a3f8ab17f292..7a3f232e66ad 100644 --- a/ledger/participant-integration-api/src/main/scala/platform/indexer/parallel/UpdateToDBDTOV1.scala +++ b/ledger/participant-integration-api/src/main/scala/platform/indexer/parallel/UpdateToDBDTOV1.scala @@ -4,7 +4,6 @@ package com.daml.platform.indexer.parallel import java.util.UUID - import com.daml.ledger.api.domain import com.daml.ledger.participant.state.v1.{Configuration, Offset, ParticipantId, Update} import com.daml.lf.engine.Blinding @@ -12,6 +11,7 @@ import com.daml.lf.ledger.EventId import com.daml.platform.store.Conversions import com.daml.platform.store.appendonlydao.events._ import com.daml.platform.store.appendonlydao.JdbcLedgerDao +import com.daml.platform.store.dao.DeduplicationKeyMaker // TODO append-only: target to separation per update-type to it's own function + unit tests object UpdateToDBDTOV1 { @@ -36,7 +36,7 @@ object UpdateToDBDTOV1 { status_message = Some(u.reason.description), ), new DBDTOV1.CommandDeduplication( - JdbcLedgerDao.deduplicationKey( + DeduplicationKeyMaker.make( domain.CommandId(u.submitterInfo.commandId), u.submitterInfo.actAs, ) diff --git a/ledger/participant-integration-api/src/main/scala/platform/store/appendonlydao/JdbcLedgerDao.scala b/ledger/participant-integration-api/src/main/scala/platform/store/appendonlydao/JdbcLedgerDao.scala index 8f72bf082744..f5f355616ef5 100644 --- a/ledger/participant-integration-api/src/main/scala/platform/store/appendonlydao/JdbcLedgerDao.scala +++ b/ledger/participant-integration-api/src/main/scala/platform/store/appendonlydao/JdbcLedgerDao.scala @@ -5,7 +5,6 @@ package com.daml.platform.store.appendonlydao import java.sql.Connection import java.time.Instant import java.util.Date - import akka.NotUsed import akka.stream.scaladsl.Source import anorm.SqlParser._ @@ -46,6 +45,7 @@ import com.daml.platform.store.appendonlydao.events.{ } import com.daml.platform.store.dao.events.TransactionsWriter.PreparedInsert import com.daml.platform.store.dao.{ + DeduplicationKeyMaker, LedgerDao, LedgerReadDao, MeteredLedgerDao, @@ -531,7 +531,7 @@ private class JdbcLedgerDao( deduplicateUntil: Instant, )(implicit loggingContext: LoggingContext): Future[CommandDeduplicationResult] = dbDispatcher.executeSql(metrics.daml.index.db.deduplicateCommandDbMetrics) { implicit conn => - val key = deduplicationKey(commandId, submitters) + val key = DeduplicationKeyMaker.make(commandId, submitters) // Insert a new deduplication entry, or update an expired entry val updated = SQL(queries.SQL_INSERT_COMMAND) .on( @@ -579,7 +579,7 @@ private class JdbcLedgerDao( commandId: domain.CommandId, submitters: List[Party], )(implicit conn: Connection): Unit = { - val key = deduplicationKey(commandId, submitters) + val key = DeduplicationKeyMaker.make(commandId, submitters) SQL_DELETE_COMMAND .on("deduplicationKey" -> key) .execute() @@ -881,18 +881,6 @@ private[platform] object JdbcLedgerDao { ): Unit = () } - def deduplicationKey( - commandId: domain.CommandId, - submitters: List[Ref.Party], - ): String = { - val submitterPart = - if (submitters.length == 1) - submitters.head - else - submitters.sorted(Ordering.String).distinct.mkString("%") - commandId.unwrap + "%" + submitterPart - } - val acceptType = "accept" val rejectType = "reject" } diff --git a/ledger/participant-integration-api/src/main/scala/platform/store/dao/DeduplicationKeyMaker.scala b/ledger/participant-integration-api/src/main/scala/platform/store/dao/DeduplicationKeyMaker.scala new file mode 100644 index 000000000000..516e5aaefff8 --- /dev/null +++ b/ledger/participant-integration-api/src/main/scala/platform/store/dao/DeduplicationKeyMaker.scala @@ -0,0 +1,22 @@ +// Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package com.daml.platform.store.dao + +import com.daml.ledger.api.domain +import com.daml.lf.data.Ref + +import java.security.MessageDigest +import scalaz.syntax.tag._ + +object DeduplicationKeyMaker { + def make(commandId: domain.CommandId, submitters: List[Ref.Party]): String = + commandId.unwrap + "%" + hashSubmitters(submitters.sorted(Ordering.String).distinct) + + private def hashSubmitters(submitters: List[Ref.Party]): String = { + MessageDigest + .getInstance("SHA-256") + .digest(submitters.mkString.getBytes) + .mkString + } +} diff --git a/ledger/participant-integration-api/src/main/scala/platform/store/dao/JdbcLedgerDao.scala b/ledger/participant-integration-api/src/main/scala/platform/store/dao/JdbcLedgerDao.scala index da1578f4ccc8..d608c953bd07 100644 --- a/ledger/participant-integration-api/src/main/scala/platform/store/dao/JdbcLedgerDao.scala +++ b/ledger/participant-integration-api/src/main/scala/platform/store/dao/JdbcLedgerDao.scala @@ -809,18 +809,6 @@ private class JdbcLedgerDao( "deduplicate_until" ) - private def deduplicationKey( - commandId: domain.CommandId, - submitters: List[Ref.Party], - ): String = { - val submitterPart = - if (submitters.length == 1) - submitters.head - else - submitters.sorted(Ordering.String).distinct.mkString("%") - commandId.unwrap + "%" + submitterPart - } - override def deduplicateCommand( commandId: domain.CommandId, submitters: List[Ref.Party], @@ -828,7 +816,7 @@ private class JdbcLedgerDao( deduplicateUntil: Instant, )(implicit loggingContext: LoggingContext): Future[CommandDeduplicationResult] = dbDispatcher.executeSql(metrics.daml.index.db.deduplicateCommandDbMetrics) { implicit conn => - val key = deduplicationKey(commandId, submitters) + val key = DeduplicationKeyMaker.make(commandId, submitters) // Insert a new deduplication entry, or update an expired entry val updated = SQL(queries.SQL_INSERT_COMMAND) .on( @@ -876,7 +864,7 @@ private class JdbcLedgerDao( commandId: domain.CommandId, submitters: List[Party], )(implicit conn: Connection): Unit = { - val key = deduplicationKey(commandId, submitters) + val key = DeduplicationKeyMaker.make(commandId, submitters) SQL_DELETE_COMMAND .on("deduplicationKey" -> key) .execute() diff --git a/ledger/participant-integration-api/src/test/suite/scala/platform/store/dao/DeduplicationKeyMakerSpec.scala b/ledger/participant-integration-api/src/test/suite/scala/platform/store/dao/DeduplicationKeyMakerSpec.scala new file mode 100644 index 000000000000..2958bd923cd4 --- /dev/null +++ b/ledger/participant-integration-api/src/test/suite/scala/platform/store/dao/DeduplicationKeyMakerSpec.scala @@ -0,0 +1,79 @@ +// Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package com.daml.platform.store.dao + +import org.scalatest.matchers.should.Matchers +import org.scalatest.wordspec.AnyWordSpec +import com.daml.ledger.api.domain.CommandId +import com.daml.lf.data.Ref +import org.scalatest.prop.TableDrivenPropertyChecks + +import java.util.UUID +import scala.util.Random +import scalaz.syntax.tag._ + +class DeduplicationKeyMakerSpec extends AnyWordSpec with Matchers with TableDrivenPropertyChecks { + + val commandId: CommandId = CommandId(Ref.LedgerString.assertFromString(UUID.randomUUID.toString)) + + "DeduplicationKeyMaker" should { + "make a deduplication key starting with a command ID in plain-text" in { + DeduplicationKeyMaker.make(commandId, List(aParty())) should startWith(commandId.unwrap) + } + + "make different keys for different sets of submitters" in { + val aCommonParty = aParty() + val cases = Table( + ("Submitters for key1", "Submitters for key2"), + (List(aParty()), List(aParty())), + (List(aCommonParty, aParty()), List(aCommonParty, aParty())), + (List(aParty(), aParty()), List(aParty(), aParty())), + ) + + forAll(cases) { case (key1Submitters, key2Submitters) => + val key1 = DeduplicationKeyMaker.make(commandId, key1Submitters) + val key2 = DeduplicationKeyMaker.make(commandId, key2Submitters) + + key1 shouldNot equal(key2) + } + } + + "make a deduplication key with a limited length for a large number of submitters" in { + val submitters = (1 to 50).map(_ => aParty()).toList + + /** The motivation for the MaxKeyLength is to avoid problems with putting deduplication key in a database + * index (e.g. for Postgres the limit for the index row size is 2712). + * The value 200 is set arbitrarily to provide some space for other data. + */ + val MaxKeyLength = 200 + DeduplicationKeyMaker.make(commandId, submitters).length should be < MaxKeyLength + } + + "make the same deduplication key for submitters of different order" in { + val submitter1 = aParty() + val submitter2 = aParty() + val submitter3 = aParty() + + val key1 = DeduplicationKeyMaker.make(commandId, List(submitter1, submitter2, submitter3)) + val key2 = DeduplicationKeyMaker.make(commandId, List(submitter1, submitter3, submitter2)) + + key1 shouldBe key2 + } + + "make the same deduplication key for duplicated submitters" in { + val submitter1 = aParty() + val submitter2 = aParty() + + val key1 = DeduplicationKeyMaker.make(commandId, List(submitter1, submitter2)) + val key2 = DeduplicationKeyMaker.make( + commandId, + List(submitter1, submitter1, submitter2, submitter2, submitter2), + ) + + key1 shouldBe key2 + } + + def aParty(): Ref.Party = Ref.Party.assertFromString(Random.alphanumeric.take(100).mkString) + } +}