From 507d242ed0f81a3fded0b7467b8b2db8b7e81984 Mon Sep 17 00:00:00 2001 From: Kamil Bozek Date: Wed, 14 Apr 2021 17:39:42 +0200 Subject: [PATCH 01/14] A ledger-api-test-tool test case for too large deduplication key failures --- .../suites/CommandDeduplicationIT.scala | 34 ++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/suites/CommandDeduplicationIT.scala b/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/suites/CommandDeduplicationIT.scala index 46801ca33cd3..9ae941324eb3 100644 --- a/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/suites/CommandDeduplicationIT.scala +++ b/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/suites/CommandDeduplicationIT.scala @@ -15,7 +15,7 @@ import io.grpc.Status import scala.concurrent.Future import scala.concurrent.duration.{DurationInt, FiniteDuration} -import scala.util.{Failure, Success} +import scala.util.{Failure, Random, Success} final class CommandDeduplicationIT(timeoutScaleFactor: Double, ledgerTimeInterval: FiniteDuration) extends LedgerTestSuite { @@ -98,6 +98,38 @@ final class CommandDeduplicationIT(timeoutScaleFactor: Double, ledgerTimeInterva } }) + test( + "CDLargeSubmittersNumber", + "Deduplicate commands with a large number of submitters", + allocate(NoParties), + )(implicit ec => { case Participants(Participant(ledger)) => + /** Postgres has a limit on the index row size of 2712. + * We test that a large submitters number doesn't cause failure because of that limit. + */ + for { + // Need to manually allocate parties to avoid db string compression + parties <- Future.traverse(1 to 50) { number => + ledger.allocateParty( + partyIdHint = + Some(s"deduplicationRandomParty_${number}_" + Random.alphanumeric.take(100).mkString), + displayName = Some(s"Clone $number"), + ) + } + request = ledger + .submitRequest( + actAs = parties.toList, + readAs = parties.toList, + commands = DummyWithAnnotation(parties.head, "First submission").create.command, + ) + .update( + _.commands.deduplicationTime := deduplicationTime.asProtobuf + ) + _ <- ledger.submit(request) + } yield { + () + } + }) + test( "CDStopOnSubmissionFailure", "Stop deduplicating commands on submission failure", From b30b09824b6b55bc5c1a1c196043af4819ba8129 Mon Sep 17 00:00:00 2001 From: Kamil Bozek Date: Wed, 14 Apr 2021 18:49:07 +0200 Subject: [PATCH 02/14] Hashing submitters in the deduplication key CHANGELOG_BEGIN - [Integration Kit] - deduplication key will contain hashed submitters instead of concatenated submitters CHANGELOG_END --- .../store/dao/DeduplicationKeyHashing.scala | 17 +++++++++++++++++ .../platform/store/dao/JdbcLedgerDao.scala | 10 ++++------ 2 files changed, 21 insertions(+), 6 deletions(-) create mode 100644 ledger/participant-integration-api/src/main/scala/platform/store/dao/DeduplicationKeyHashing.scala diff --git a/ledger/participant-integration-api/src/main/scala/platform/store/dao/DeduplicationKeyHashing.scala b/ledger/participant-integration-api/src/main/scala/platform/store/dao/DeduplicationKeyHashing.scala new file mode 100644 index 000000000000..83e7704cd0d0 --- /dev/null +++ b/ledger/participant-integration-api/src/main/scala/platform/store/dao/DeduplicationKeyHashing.scala @@ -0,0 +1,17 @@ +// Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package com.daml.platform.store.dao + +import java.security.MessageDigest + +import com.daml.lf.data.Ref.Party + +object DeduplicationKeyHashing { + def hashSubmitters(submitters: List[Party]): String = { + MessageDigest + .getInstance("SHA-256") + .digest(submitters.mkString.getBytes) + .mkString + } +} diff --git a/ledger/participant-integration-api/src/main/scala/platform/store/dao/JdbcLedgerDao.scala b/ledger/participant-integration-api/src/main/scala/platform/store/dao/JdbcLedgerDao.scala index da1578f4ccc8..0ad052622b25 100644 --- a/ledger/participant-integration-api/src/main/scala/platform/store/dao/JdbcLedgerDao.scala +++ b/ledger/participant-integration-api/src/main/scala/platform/store/dao/JdbcLedgerDao.scala @@ -813,12 +813,10 @@ private class JdbcLedgerDao( commandId: domain.CommandId, submitters: List[Ref.Party], ): String = { - val submitterPart = - if (submitters.length == 1) - submitters.head - else - submitters.sorted(Ordering.String).distinct.mkString("%") - commandId.unwrap + "%" + submitterPart + val hashedSubmitters = + DeduplicationKeyHashing.hashSubmitters(submitters.sorted(Ordering.String).distinct) + + commandId.unwrap + "%" + hashedSubmitters } override def deduplicateCommand( From 2d2cc4b94a3d2e644716d842b97eae505a2702b7 Mon Sep 17 00:00:00 2001 From: Kamil Bozek Date: Thu, 15 Apr 2021 16:49:44 +0200 Subject: [PATCH 03/14] Separate ValueLimitsIT test suite for the ledger-api-test-tool --- .../suites/CommandDeduplicationIT.scala | 34 +---------- .../api/testtool/suites/ValueLimitsIT.scala | 56 +++++++++++++++++++ .../ledger/api/testtool/tests/Tests.scala | 1 + 3 files changed, 58 insertions(+), 33 deletions(-) create mode 100644 ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/suites/ValueLimitsIT.scala diff --git a/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/suites/CommandDeduplicationIT.scala b/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/suites/CommandDeduplicationIT.scala index 9ae941324eb3..46801ca33cd3 100644 --- a/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/suites/CommandDeduplicationIT.scala +++ b/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/suites/CommandDeduplicationIT.scala @@ -15,7 +15,7 @@ import io.grpc.Status import scala.concurrent.Future import scala.concurrent.duration.{DurationInt, FiniteDuration} -import scala.util.{Failure, Random, Success} +import scala.util.{Failure, Success} final class CommandDeduplicationIT(timeoutScaleFactor: Double, ledgerTimeInterval: FiniteDuration) extends LedgerTestSuite { @@ -98,38 +98,6 @@ final class CommandDeduplicationIT(timeoutScaleFactor: Double, ledgerTimeInterva } }) - test( - "CDLargeSubmittersNumber", - "Deduplicate commands with a large number of submitters", - allocate(NoParties), - )(implicit ec => { case Participants(Participant(ledger)) => - /** Postgres has a limit on the index row size of 2712. - * We test that a large submitters number doesn't cause failure because of that limit. - */ - for { - // Need to manually allocate parties to avoid db string compression - parties <- Future.traverse(1 to 50) { number => - ledger.allocateParty( - partyIdHint = - Some(s"deduplicationRandomParty_${number}_" + Random.alphanumeric.take(100).mkString), - displayName = Some(s"Clone $number"), - ) - } - request = ledger - .submitRequest( - actAs = parties.toList, - readAs = parties.toList, - commands = DummyWithAnnotation(parties.head, "First submission").create.command, - ) - .update( - _.commands.deduplicationTime := deduplicationTime.asProtobuf - ) - _ <- ledger.submit(request) - } yield { - () - } - }) - test( "CDStopOnSubmissionFailure", "Stop deduplicating commands on submission failure", diff --git a/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/suites/ValueLimitsIT.scala b/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/suites/ValueLimitsIT.scala new file mode 100644 index 000000000000..21d5770c13cf --- /dev/null +++ b/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/suites/ValueLimitsIT.scala @@ -0,0 +1,56 @@ +// Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package com.daml.ledger.api.testtool.suites + +import com.daml.ledger.api.testtool.infrastructure.Allocation._ +import com.daml.ledger.api.testtool.infrastructure.LedgerTestSuite +import com.daml.ledger.api.testtool.infrastructure.ProtobufConverters._ +import com.daml.ledger.test.model.Test.DummyWithAnnotation + +import scala.concurrent.Future +import scala.concurrent.duration.{DurationInt, FiniteDuration} +import scala.util.Random + +final class ValueLimitsIT(timeoutScaleFactor: Double) extends LedgerTestSuite { + + /** Postgres has a limit on the index row size of 2712. + * We test that a large submitters number doesn't cause deduplication failure because of that limit. + * THIS TEST CASE DOES NOT TEST ANY OTHER FAILURES THAT MAY BE CAUSED BY A LARGE SUBMITTERS NUMBER + */ + test( + "VLDeduplicateWithLargeSubmittersNumber", + "Deduplicate commands with a large number of submitters", + allocate(NoParties), + )(implicit ec => { case Participants(Participant(ledger)) => + for { + // Need to manually allocate parties to avoid db string compression + parties <- Future.traverse(1 to 50) { number => + ledger.allocateParty( + partyIdHint = + Some(s"deduplicationRandomParty_${number}_" + Random.alphanumeric.take(100).mkString), + displayName = Some(s"Clone $number"), + ) + } + request = ledger + .submitRequest( + actAs = parties.toList, + readAs = parties.toList, + commands = DummyWithAnnotation(parties.head, "First submission").create.command, + ) + .update( + _.commands.deduplicationTime := deduplicationTime.asProtobuf + ) + _ <- ledger.submit(request) + } yield { + () + } + }) + + private val deduplicationTime = 3.seconds * timeoutScaleFactor match { + case duration: FiniteDuration => duration + case _ => + throw new IllegalArgumentException(s"Invalid timeout scale factor: $timeoutScaleFactor") + } + +} diff --git a/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/tests/Tests.scala b/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/tests/Tests.scala index 8bb2d3b35be6..b8e30a8aee09 100644 --- a/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/tests/Tests.scala +++ b/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/tests/Tests.scala @@ -35,6 +35,7 @@ object Tests { new RaceConditionIT, new SemanticTests, new TransactionServiceIT, + new ValueLimitsIT(timeoutScaleFactor), new WitnessesIT, new WronglyTypedContractIdIT, ) From 8ea6c8e0dad9254447da57694b07324703f048b2 Mon Sep 17 00:00:00 2001 From: Kamil Bozek Date: Thu, 15 Apr 2021 17:09:00 +0200 Subject: [PATCH 04/14] Excluded ValueLimitsIT for sandbox and ledger-on-sql conformance tests --- ledger/ledger-on-sql/BUILD.bazel | 1 + ledger/sandbox/BUILD.bazel | 2 ++ 2 files changed, 3 insertions(+) diff --git a/ledger/ledger-on-sql/BUILD.bazel b/ledger/ledger-on-sql/BUILD.bazel index 5b1c985e6834..178a0d60aefb 100644 --- a/ledger/ledger-on-sql/BUILD.bazel +++ b/ledger/ledger-on-sql/BUILD.bazel @@ -259,6 +259,7 @@ da_scala_test_suite( tags = db.get("conformance_test_tags", []), test_tool_args = db.get("conformance_test_tool_args", []) + [ "--verbose", + "--exclude=ValueLimitsIT", ], ) for db in supported_databases diff --git a/ledger/sandbox/BUILD.bazel b/ledger/sandbox/BUILD.bazel index 754ab59bf6ad..1fbefda30373 100644 --- a/ledger/sandbox/BUILD.bazel +++ b/ledger/sandbox/BUILD.bazel @@ -253,6 +253,7 @@ server_conformance_test( test_tool_args = [ "--open-world", "--exclude=ClosedWorldIT", + "--exclude=ValueLimitsIT", ], ) @@ -265,6 +266,7 @@ server_conformance_test( test_tool_args = [ "--open-world", "--exclude=ClosedWorldIT", + "--exclude=ValueLimitsIT", ], ) From 3c33fcf49e1b5c973f21877349db09915efeaee6 Mon Sep 17 00:00:00 2001 From: Kamil Bozek Date: Fri, 16 Apr 2021 18:01:38 +0200 Subject: [PATCH 05/14] Checking contract creation result in the test case --- .../ledger/api/testtool/suites/ValueLimitsIT.scala | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/suites/ValueLimitsIT.scala b/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/suites/ValueLimitsIT.scala index 21d5770c13cf..f60e7fe3200b 100644 --- a/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/suites/ValueLimitsIT.scala +++ b/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/suites/ValueLimitsIT.scala @@ -4,6 +4,7 @@ package com.daml.ledger.api.testtool.suites import com.daml.ledger.api.testtool.infrastructure.Allocation._ +import com.daml.ledger.api.testtool.infrastructure.Assertions._ import com.daml.ledger.api.testtool.infrastructure.LedgerTestSuite import com.daml.ledger.api.testtool.infrastructure.ProtobufConverters._ import com.daml.ledger.test.model.Test.DummyWithAnnotation @@ -14,13 +15,9 @@ import scala.util.Random final class ValueLimitsIT(timeoutScaleFactor: Double) extends LedgerTestSuite { - /** Postgres has a limit on the index row size of 2712. - * We test that a large submitters number doesn't cause deduplication failure because of that limit. - * THIS TEST CASE DOES NOT TEST ANY OTHER FAILURES THAT MAY BE CAUSED BY A LARGE SUBMITTERS NUMBER - */ test( - "VLDeduplicateWithLargeSubmittersNumber", - "Deduplicate commands with a large number of submitters", + "VLLargeSubmittersNumberCreateContract", + "Create a contract with a large submitters number", allocate(NoParties), )(implicit ec => { case Participants(Participant(ledger)) => for { @@ -42,7 +39,9 @@ final class ValueLimitsIT(timeoutScaleFactor: Double) extends LedgerTestSuite { _.commands.deduplicationTime := deduplicationTime.asProtobuf ) _ <- ledger.submit(request) + contracts <- ledger.activeContracts(parties.head) } yield { + assertSingleton("Single create contract expected", contracts) () } }) From 9ef8dcb90b5252c38140c50fce1c95d0702d4ee0 Mon Sep 17 00:00:00 2001 From: Kamil Bozek Date: Fri, 16 Apr 2021 18:06:09 +0200 Subject: [PATCH 06/14] Update index on participant_command_completions table Do not include submitters in the index to avoid issues with index row size limit --- .../db/migration/postgres/V44__update_completions_index.sql | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 ledger/participant-integration-api/src/main/resources/db/migration/postgres/V44__update_completions_index.sql diff --git a/ledger/participant-integration-api/src/main/resources/db/migration/postgres/V44__update_completions_index.sql b/ledger/participant-integration-api/src/main/resources/db/migration/postgres/V44__update_completions_index.sql new file mode 100644 index 000000000000..b766c7505507 --- /dev/null +++ b/ledger/participant-integration-api/src/main/resources/db/migration/postgres/V44__update_completions_index.sql @@ -0,0 +1,2 @@ +DROP INDEX IF EXISTS participant_command_completio_completion_offset_application_idx; +CREATE INDEX ON participant_command_completions(completion_offset, application_id); From 1ecaf9c09fff284c168f952fe70bb32f2fbad3d7 Mon Sep 17 00:00:00 2001 From: Kamil Bozek Date: Fri, 16 Apr 2021 20:00:53 +0200 Subject: [PATCH 07/14] Added a delay for more consistent results --- .../com/daml/ledger/api/testtool/suites/ValueLimitsIT.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/suites/ValueLimitsIT.scala b/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/suites/ValueLimitsIT.scala index f60e7fe3200b..bdf81a0c3e39 100644 --- a/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/suites/ValueLimitsIT.scala +++ b/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/suites/ValueLimitsIT.scala @@ -8,6 +8,7 @@ import com.daml.ledger.api.testtool.infrastructure.Assertions._ import com.daml.ledger.api.testtool.infrastructure.LedgerTestSuite import com.daml.ledger.api.testtool.infrastructure.ProtobufConverters._ import com.daml.ledger.test.model.Test.DummyWithAnnotation +import com.daml.timer.Delayed import scala.concurrent.Future import scala.concurrent.duration.{DurationInt, FiniteDuration} @@ -39,6 +40,7 @@ final class ValueLimitsIT(timeoutScaleFactor: Double) extends LedgerTestSuite { _.commands.deduplicationTime := deduplicationTime.asProtobuf ) _ <- ledger.submit(request) + _ <- Delayed.by(1.second)(()) contracts <- ledger.activeContracts(parties.head) } yield { assertSingleton("Single create contract expected", contracts) From 0b579a68d3db6b094c7a7c1ca007d70190f1ae50 Mon Sep 17 00:00:00 2001 From: Kamil Bozek Date: Mon, 19 Apr 2021 14:00:52 +0200 Subject: [PATCH 08/14] Fixed submitting a command --- .../com/daml/ledger/api/testtool/suites/ValueLimitsIT.scala | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/suites/ValueLimitsIT.scala b/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/suites/ValueLimitsIT.scala index bdf81a0c3e39..5eaf92dc32da 100644 --- a/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/suites/ValueLimitsIT.scala +++ b/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/suites/ValueLimitsIT.scala @@ -8,7 +8,6 @@ import com.daml.ledger.api.testtool.infrastructure.Assertions._ import com.daml.ledger.api.testtool.infrastructure.LedgerTestSuite import com.daml.ledger.api.testtool.infrastructure.ProtobufConverters._ import com.daml.ledger.test.model.Test.DummyWithAnnotation -import com.daml.timer.Delayed import scala.concurrent.Future import scala.concurrent.duration.{DurationInt, FiniteDuration} @@ -31,7 +30,7 @@ final class ValueLimitsIT(timeoutScaleFactor: Double) extends LedgerTestSuite { ) } request = ledger - .submitRequest( + .submitAndWaitRequest( actAs = parties.toList, readAs = parties.toList, commands = DummyWithAnnotation(parties.head, "First submission").create.command, @@ -39,8 +38,7 @@ final class ValueLimitsIT(timeoutScaleFactor: Double) extends LedgerTestSuite { .update( _.commands.deduplicationTime := deduplicationTime.asProtobuf ) - _ <- ledger.submit(request) - _ <- Delayed.by(1.second)(()) + _ <- ledger.submitAndWait(request) contracts <- ledger.activeContracts(parties.head) } yield { assertSingleton("Single create contract expected", contracts) From cd491035279882a8c518ec36e1f4d901770ae201 Mon Sep 17 00:00:00 2001 From: Kamil Bozek Date: Mon, 19 Apr 2021 14:14:04 +0200 Subject: [PATCH 09/14] Simplified the test case --- .../ledger/api/testtool/suites/ValueLimitsIT.scala | 13 +------------ .../com/daml/ledger/api/testtool/tests/Tests.scala | 2 +- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/suites/ValueLimitsIT.scala b/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/suites/ValueLimitsIT.scala index 5eaf92dc32da..186338fed0c3 100644 --- a/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/suites/ValueLimitsIT.scala +++ b/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/suites/ValueLimitsIT.scala @@ -6,14 +6,12 @@ package com.daml.ledger.api.testtool.suites import com.daml.ledger.api.testtool.infrastructure.Allocation._ import com.daml.ledger.api.testtool.infrastructure.Assertions._ import com.daml.ledger.api.testtool.infrastructure.LedgerTestSuite -import com.daml.ledger.api.testtool.infrastructure.ProtobufConverters._ import com.daml.ledger.test.model.Test.DummyWithAnnotation import scala.concurrent.Future -import scala.concurrent.duration.{DurationInt, FiniteDuration} import scala.util.Random -final class ValueLimitsIT(timeoutScaleFactor: Double) extends LedgerTestSuite { +final class ValueLimitsIT extends LedgerTestSuite { test( "VLLargeSubmittersNumberCreateContract", @@ -35,9 +33,6 @@ final class ValueLimitsIT(timeoutScaleFactor: Double) extends LedgerTestSuite { readAs = parties.toList, commands = DummyWithAnnotation(parties.head, "First submission").create.command, ) - .update( - _.commands.deduplicationTime := deduplicationTime.asProtobuf - ) _ <- ledger.submitAndWait(request) contracts <- ledger.activeContracts(parties.head) } yield { @@ -46,10 +41,4 @@ final class ValueLimitsIT(timeoutScaleFactor: Double) extends LedgerTestSuite { } }) - private val deduplicationTime = 3.seconds * timeoutScaleFactor match { - case duration: FiniteDuration => duration - case _ => - throw new IllegalArgumentException(s"Invalid timeout scale factor: $timeoutScaleFactor") - } - } diff --git a/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/tests/Tests.scala b/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/tests/Tests.scala index b8e30a8aee09..5868b6e26922 100644 --- a/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/tests/Tests.scala +++ b/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/tests/Tests.scala @@ -35,7 +35,7 @@ object Tests { new RaceConditionIT, new SemanticTests, new TransactionServiceIT, - new ValueLimitsIT(timeoutScaleFactor), + new ValueLimitsIT, new WitnessesIT, new WronglyTypedContractIdIT, ) From 3008af9983adf9f2faad70359c0d36edb73f5478 Mon Sep 17 00:00:00 2001 From: Kamil Bozek Date: Mon, 19 Apr 2021 18:10:05 +0200 Subject: [PATCH 10/14] Reverted changing indexes --- .../db/migration/postgres/V44__update_completions_index.sql | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 ledger/participant-integration-api/src/main/resources/db/migration/postgres/V44__update_completions_index.sql diff --git a/ledger/participant-integration-api/src/main/resources/db/migration/postgres/V44__update_completions_index.sql b/ledger/participant-integration-api/src/main/resources/db/migration/postgres/V44__update_completions_index.sql deleted file mode 100644 index b766c7505507..000000000000 --- a/ledger/participant-integration-api/src/main/resources/db/migration/postgres/V44__update_completions_index.sql +++ /dev/null @@ -1,2 +0,0 @@ -DROP INDEX IF EXISTS participant_command_completio_completion_offset_application_idx; -CREATE INDEX ON participant_command_completions(completion_offset, application_id); From 5b0fae817800dd5e70fe94ea68dc24ac021c070c Mon Sep 17 00:00:00 2001 From: Kamil Bozek Date: Mon, 19 Apr 2021 19:50:57 +0200 Subject: [PATCH 11/14] Dedicated object for making deduplication keys --- .../store/dao/DeduplicationKeyHashing.scala | 17 ------ .../store/dao/DeduplicationKeyMaker.scala | 22 ++++++++ .../platform/store/dao/JdbcLedgerDao.scala | 14 +---- .../store/dao/DeduplicationKeyMakerSpec.scala | 56 +++++++++++++++++++ 4 files changed, 80 insertions(+), 29 deletions(-) delete mode 100644 ledger/participant-integration-api/src/main/scala/platform/store/dao/DeduplicationKeyHashing.scala create mode 100644 ledger/participant-integration-api/src/main/scala/platform/store/dao/DeduplicationKeyMaker.scala create mode 100644 ledger/participant-integration-api/src/test/suite/scala/platform/store/dao/DeduplicationKeyMakerSpec.scala diff --git a/ledger/participant-integration-api/src/main/scala/platform/store/dao/DeduplicationKeyHashing.scala b/ledger/participant-integration-api/src/main/scala/platform/store/dao/DeduplicationKeyHashing.scala deleted file mode 100644 index 83e7704cd0d0..000000000000 --- a/ledger/participant-integration-api/src/main/scala/platform/store/dao/DeduplicationKeyHashing.scala +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package com.daml.platform.store.dao - -import java.security.MessageDigest - -import com.daml.lf.data.Ref.Party - -object DeduplicationKeyHashing { - def hashSubmitters(submitters: List[Party]): String = { - MessageDigest - .getInstance("SHA-256") - .digest(submitters.mkString.getBytes) - .mkString - } -} diff --git a/ledger/participant-integration-api/src/main/scala/platform/store/dao/DeduplicationKeyMaker.scala b/ledger/participant-integration-api/src/main/scala/platform/store/dao/DeduplicationKeyMaker.scala new file mode 100644 index 000000000000..516e5aaefff8 --- /dev/null +++ b/ledger/participant-integration-api/src/main/scala/platform/store/dao/DeduplicationKeyMaker.scala @@ -0,0 +1,22 @@ +// Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package com.daml.platform.store.dao + +import com.daml.ledger.api.domain +import com.daml.lf.data.Ref + +import java.security.MessageDigest +import scalaz.syntax.tag._ + +object DeduplicationKeyMaker { + def make(commandId: domain.CommandId, submitters: List[Ref.Party]): String = + commandId.unwrap + "%" + hashSubmitters(submitters.sorted(Ordering.String).distinct) + + private def hashSubmitters(submitters: List[Ref.Party]): String = { + MessageDigest + .getInstance("SHA-256") + .digest(submitters.mkString.getBytes) + .mkString + } +} diff --git a/ledger/participant-integration-api/src/main/scala/platform/store/dao/JdbcLedgerDao.scala b/ledger/participant-integration-api/src/main/scala/platform/store/dao/JdbcLedgerDao.scala index 0ad052622b25..d608c953bd07 100644 --- a/ledger/participant-integration-api/src/main/scala/platform/store/dao/JdbcLedgerDao.scala +++ b/ledger/participant-integration-api/src/main/scala/platform/store/dao/JdbcLedgerDao.scala @@ -809,16 +809,6 @@ private class JdbcLedgerDao( "deduplicate_until" ) - private def deduplicationKey( - commandId: domain.CommandId, - submitters: List[Ref.Party], - ): String = { - val hashedSubmitters = - DeduplicationKeyHashing.hashSubmitters(submitters.sorted(Ordering.String).distinct) - - commandId.unwrap + "%" + hashedSubmitters - } - override def deduplicateCommand( commandId: domain.CommandId, submitters: List[Ref.Party], @@ -826,7 +816,7 @@ private class JdbcLedgerDao( deduplicateUntil: Instant, )(implicit loggingContext: LoggingContext): Future[CommandDeduplicationResult] = dbDispatcher.executeSql(metrics.daml.index.db.deduplicateCommandDbMetrics) { implicit conn => - val key = deduplicationKey(commandId, submitters) + val key = DeduplicationKeyMaker.make(commandId, submitters) // Insert a new deduplication entry, or update an expired entry val updated = SQL(queries.SQL_INSERT_COMMAND) .on( @@ -874,7 +864,7 @@ private class JdbcLedgerDao( commandId: domain.CommandId, submitters: List[Party], )(implicit conn: Connection): Unit = { - val key = deduplicationKey(commandId, submitters) + val key = DeduplicationKeyMaker.make(commandId, submitters) SQL_DELETE_COMMAND .on("deduplicationKey" -> key) .execute() diff --git a/ledger/participant-integration-api/src/test/suite/scala/platform/store/dao/DeduplicationKeyMakerSpec.scala b/ledger/participant-integration-api/src/test/suite/scala/platform/store/dao/DeduplicationKeyMakerSpec.scala new file mode 100644 index 000000000000..5d7db33501d8 --- /dev/null +++ b/ledger/participant-integration-api/src/test/suite/scala/platform/store/dao/DeduplicationKeyMakerSpec.scala @@ -0,0 +1,56 @@ +// Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package com.daml.platform.store.dao + +import org.scalatest.matchers.should.Matchers +import org.scalatest.wordspec.AnyWordSpec +import com.daml.ledger.api.domain.CommandId +import com.daml.lf.data.Ref + +import java.util.UUID +import scala.util.Random +import scalaz.syntax.tag._ + +class DeduplicationKeyMakerSpec extends AnyWordSpec with Matchers { + + val commandId: CommandId = CommandId(Ref.LedgerString.assertFromString(UUID.randomUUID.toString)) + + "DeduplicationKeyMaker" should { + "make a deduplication key starting with a command ID in plain-text" in { + DeduplicationKeyMaker.make(commandId, List(aParty())) should startWith(commandId.unwrap) + } + + "make a deduplication key with a limited length for a large number of submitters" in { + val submitters = (1 to 50).map(_ => aParty()).toList + val MaxKeyLength = 200 + DeduplicationKeyMaker.make(commandId, submitters).length should be < MaxKeyLength + } + + "make the same deduplication key for submitters of different order" in { + val submitter1 = aParty() + val submitter2 = aParty() + val submitter3 = aParty() + + val key1 = DeduplicationKeyMaker.make(commandId, List(submitter1, submitter2, submitter3)) + val key2 = DeduplicationKeyMaker.make(commandId, List(submitter1, submitter3, submitter2)) + + key1 shouldBe key2 + } + + "make the same deduplication key for duplicated submitters" in { + val submitter1 = aParty() + val submitter2 = aParty() + + val key1 = DeduplicationKeyMaker.make(commandId, List(submitter1, submitter2)) + val key2 = DeduplicationKeyMaker.make( + commandId, + List(submitter1, submitter1, submitter2, submitter2, submitter2), + ) + + key1 shouldBe key2 + } + + def aParty(): Ref.Party = Ref.Party.assertFromString(Random.alphanumeric.take(100).mkString) + } +} From 488338387315db352c9156b119ceeb12a4acb7bd Mon Sep 17 00:00:00 2001 From: Kamil Bozek Date: Mon, 19 Apr 2021 19:56:02 +0200 Subject: [PATCH 12/14] Ported hashing deduplication keys to the appendonlydao --- .../indexer/parallel/UpdateToDBDTOV1.scala | 4 ++-- .../store/appendonlydao/JdbcLedgerDao.scala | 18 +++--------------- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/ledger/participant-integration-api/src/main/scala/platform/indexer/parallel/UpdateToDBDTOV1.scala b/ledger/participant-integration-api/src/main/scala/platform/indexer/parallel/UpdateToDBDTOV1.scala index a3f8ab17f292..7a3f232e66ad 100644 --- a/ledger/participant-integration-api/src/main/scala/platform/indexer/parallel/UpdateToDBDTOV1.scala +++ b/ledger/participant-integration-api/src/main/scala/platform/indexer/parallel/UpdateToDBDTOV1.scala @@ -4,7 +4,6 @@ package com.daml.platform.indexer.parallel import java.util.UUID - import com.daml.ledger.api.domain import com.daml.ledger.participant.state.v1.{Configuration, Offset, ParticipantId, Update} import com.daml.lf.engine.Blinding @@ -12,6 +11,7 @@ import com.daml.lf.ledger.EventId import com.daml.platform.store.Conversions import com.daml.platform.store.appendonlydao.events._ import com.daml.platform.store.appendonlydao.JdbcLedgerDao +import com.daml.platform.store.dao.DeduplicationKeyMaker // TODO append-only: target to separation per update-type to it's own function + unit tests object UpdateToDBDTOV1 { @@ -36,7 +36,7 @@ object UpdateToDBDTOV1 { status_message = Some(u.reason.description), ), new DBDTOV1.CommandDeduplication( - JdbcLedgerDao.deduplicationKey( + DeduplicationKeyMaker.make( domain.CommandId(u.submitterInfo.commandId), u.submitterInfo.actAs, ) diff --git a/ledger/participant-integration-api/src/main/scala/platform/store/appendonlydao/JdbcLedgerDao.scala b/ledger/participant-integration-api/src/main/scala/platform/store/appendonlydao/JdbcLedgerDao.scala index 8f72bf082744..f5f355616ef5 100644 --- a/ledger/participant-integration-api/src/main/scala/platform/store/appendonlydao/JdbcLedgerDao.scala +++ b/ledger/participant-integration-api/src/main/scala/platform/store/appendonlydao/JdbcLedgerDao.scala @@ -5,7 +5,6 @@ package com.daml.platform.store.appendonlydao import java.sql.Connection import java.time.Instant import java.util.Date - import akka.NotUsed import akka.stream.scaladsl.Source import anorm.SqlParser._ @@ -46,6 +45,7 @@ import com.daml.platform.store.appendonlydao.events.{ } import com.daml.platform.store.dao.events.TransactionsWriter.PreparedInsert import com.daml.platform.store.dao.{ + DeduplicationKeyMaker, LedgerDao, LedgerReadDao, MeteredLedgerDao, @@ -531,7 +531,7 @@ private class JdbcLedgerDao( deduplicateUntil: Instant, )(implicit loggingContext: LoggingContext): Future[CommandDeduplicationResult] = dbDispatcher.executeSql(metrics.daml.index.db.deduplicateCommandDbMetrics) { implicit conn => - val key = deduplicationKey(commandId, submitters) + val key = DeduplicationKeyMaker.make(commandId, submitters) // Insert a new deduplication entry, or update an expired entry val updated = SQL(queries.SQL_INSERT_COMMAND) .on( @@ -579,7 +579,7 @@ private class JdbcLedgerDao( commandId: domain.CommandId, submitters: List[Party], )(implicit conn: Connection): Unit = { - val key = deduplicationKey(commandId, submitters) + val key = DeduplicationKeyMaker.make(commandId, submitters) SQL_DELETE_COMMAND .on("deduplicationKey" -> key) .execute() @@ -881,18 +881,6 @@ private[platform] object JdbcLedgerDao { ): Unit = () } - def deduplicationKey( - commandId: domain.CommandId, - submitters: List[Ref.Party], - ): String = { - val submitterPart = - if (submitters.length == 1) - submitters.head - else - submitters.sorted(Ordering.String).distinct.mkString("%") - commandId.unwrap + "%" + submitterPart - } - val acceptType = "accept" val rejectType = "reject" } From 368b6d6040588f0acc10345967829cac6d1bf60d Mon Sep 17 00:00:00 2001 From: Kamil Bozek Date: Mon, 19 Apr 2021 20:09:08 +0200 Subject: [PATCH 13/14] Removed ValueLimitIT suite. The suite will be restored when database index fixes for large submitters sets are implemented --- .../api/testtool/suites/ValueLimitsIT.scala | 44 ------------------- .../ledger/api/testtool/tests/Tests.scala | 1 - ledger/ledger-on-sql/BUILD.bazel | 1 - ledger/sandbox/BUILD.bazel | 2 - 4 files changed, 48 deletions(-) delete mode 100644 ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/suites/ValueLimitsIT.scala diff --git a/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/suites/ValueLimitsIT.scala b/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/suites/ValueLimitsIT.scala deleted file mode 100644 index 186338fed0c3..000000000000 --- a/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/suites/ValueLimitsIT.scala +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright (c) 2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -package com.daml.ledger.api.testtool.suites - -import com.daml.ledger.api.testtool.infrastructure.Allocation._ -import com.daml.ledger.api.testtool.infrastructure.Assertions._ -import com.daml.ledger.api.testtool.infrastructure.LedgerTestSuite -import com.daml.ledger.test.model.Test.DummyWithAnnotation - -import scala.concurrent.Future -import scala.util.Random - -final class ValueLimitsIT extends LedgerTestSuite { - - test( - "VLLargeSubmittersNumberCreateContract", - "Create a contract with a large submitters number", - allocate(NoParties), - )(implicit ec => { case Participants(Participant(ledger)) => - for { - // Need to manually allocate parties to avoid db string compression - parties <- Future.traverse(1 to 50) { number => - ledger.allocateParty( - partyIdHint = - Some(s"deduplicationRandomParty_${number}_" + Random.alphanumeric.take(100).mkString), - displayName = Some(s"Clone $number"), - ) - } - request = ledger - .submitAndWaitRequest( - actAs = parties.toList, - readAs = parties.toList, - commands = DummyWithAnnotation(parties.head, "First submission").create.command, - ) - _ <- ledger.submitAndWait(request) - contracts <- ledger.activeContracts(parties.head) - } yield { - assertSingleton("Single create contract expected", contracts) - () - } - }) - -} diff --git a/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/tests/Tests.scala b/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/tests/Tests.scala index 5868b6e26922..8bb2d3b35be6 100644 --- a/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/tests/Tests.scala +++ b/ledger/ledger-api-test-tool/src/main/scala/com/daml/ledger/api/testtool/tests/Tests.scala @@ -35,7 +35,6 @@ object Tests { new RaceConditionIT, new SemanticTests, new TransactionServiceIT, - new ValueLimitsIT, new WitnessesIT, new WronglyTypedContractIdIT, ) diff --git a/ledger/ledger-on-sql/BUILD.bazel b/ledger/ledger-on-sql/BUILD.bazel index 178a0d60aefb..5b1c985e6834 100644 --- a/ledger/ledger-on-sql/BUILD.bazel +++ b/ledger/ledger-on-sql/BUILD.bazel @@ -259,7 +259,6 @@ da_scala_test_suite( tags = db.get("conformance_test_tags", []), test_tool_args = db.get("conformance_test_tool_args", []) + [ "--verbose", - "--exclude=ValueLimitsIT", ], ) for db in supported_databases diff --git a/ledger/sandbox/BUILD.bazel b/ledger/sandbox/BUILD.bazel index 1fbefda30373..754ab59bf6ad 100644 --- a/ledger/sandbox/BUILD.bazel +++ b/ledger/sandbox/BUILD.bazel @@ -253,7 +253,6 @@ server_conformance_test( test_tool_args = [ "--open-world", "--exclude=ClosedWorldIT", - "--exclude=ValueLimitsIT", ], ) @@ -266,7 +265,6 @@ server_conformance_test( test_tool_args = [ "--open-world", "--exclude=ClosedWorldIT", - "--exclude=ValueLimitsIT", ], ) From 3bf79126ba564ac9848e85425a106cd7bc224026 Mon Sep 17 00:00:00 2001 From: Kamil Bozek Date: Tue, 20 Apr 2021 10:44:59 +0200 Subject: [PATCH 14/14] Additional unit test for hashing deduplication keys --- .../store/dao/DeduplicationKeyMakerSpec.scala | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/ledger/participant-integration-api/src/test/suite/scala/platform/store/dao/DeduplicationKeyMakerSpec.scala b/ledger/participant-integration-api/src/test/suite/scala/platform/store/dao/DeduplicationKeyMakerSpec.scala index 5d7db33501d8..2958bd923cd4 100644 --- a/ledger/participant-integration-api/src/test/suite/scala/platform/store/dao/DeduplicationKeyMakerSpec.scala +++ b/ledger/participant-integration-api/src/test/suite/scala/platform/store/dao/DeduplicationKeyMakerSpec.scala @@ -7,12 +7,13 @@ import org.scalatest.matchers.should.Matchers import org.scalatest.wordspec.AnyWordSpec import com.daml.ledger.api.domain.CommandId import com.daml.lf.data.Ref +import org.scalatest.prop.TableDrivenPropertyChecks import java.util.UUID import scala.util.Random import scalaz.syntax.tag._ -class DeduplicationKeyMakerSpec extends AnyWordSpec with Matchers { +class DeduplicationKeyMakerSpec extends AnyWordSpec with Matchers with TableDrivenPropertyChecks { val commandId: CommandId = CommandId(Ref.LedgerString.assertFromString(UUID.randomUUID.toString)) @@ -21,8 +22,30 @@ class DeduplicationKeyMakerSpec extends AnyWordSpec with Matchers { DeduplicationKeyMaker.make(commandId, List(aParty())) should startWith(commandId.unwrap) } + "make different keys for different sets of submitters" in { + val aCommonParty = aParty() + val cases = Table( + ("Submitters for key1", "Submitters for key2"), + (List(aParty()), List(aParty())), + (List(aCommonParty, aParty()), List(aCommonParty, aParty())), + (List(aParty(), aParty()), List(aParty(), aParty())), + ) + + forAll(cases) { case (key1Submitters, key2Submitters) => + val key1 = DeduplicationKeyMaker.make(commandId, key1Submitters) + val key2 = DeduplicationKeyMaker.make(commandId, key2Submitters) + + key1 shouldNot equal(key2) + } + } + "make a deduplication key with a limited length for a large number of submitters" in { val submitters = (1 to 50).map(_ => aParty()).toList + + /** The motivation for the MaxKeyLength is to avoid problems with putting deduplication key in a database + * index (e.g. for Postgres the limit for the index row size is 2712). + * The value 200 is set arbitrarily to provide some space for other data. + */ val MaxKeyLength = 200 DeduplicationKeyMaker.make(commandId, submitters).length should be < MaxKeyLength }