diff --git a/CMakeLists.txt b/CMakeLists.txt index 1ae475960..12c831d48 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -205,7 +205,6 @@ endif() find_package(Doxygen) if(DOXYGEN_FOUND) add_custom_target(docs - ALL COMMENT "Build public and internal documentation" ) diff --git a/src/applications/bmqstoragetool/README.md b/src/applications/bmqstoragetool/README.md index 6c7a6b5b6..6be2f8747 100644 --- a/src/applications/bmqstoragetool/README.md +++ b/src/applications/bmqstoragetool/README.md @@ -18,10 +18,16 @@ Usage: bmqstoragetool [--journal-path ] [--data-file ] [--csl-file ] [--guid ]* + [--seqnum ]* + [--offset ]* [--queue-name ]* [--queue-key ]* [--timestamp-gt ] [--timestamp-lt ] + [--seqnum-gt ] + [--seqnum-lt ] + [--offset-gt ] + [--offset-lt ] [--outstanding] [--confirmed] [--partially-confirmed] @@ -42,6 +48,10 @@ Where: path to a .bmq_csl file --guid message guid + --seqnum + message composite sequence number + --offset + message offset --queue-name message queue name --queue-key @@ -50,6 +60,14 @@ Where: lower timestamp bound --timestamp-lt higher timestamp bound + --seqnum-gt + lower composite sequence number bound, defined in form , e.g. 123-456 + --seqnum-lt + higher composite sequence number bound, defined in form , e.g. 123-456 + --offset-gt + lower offset bound + --offset-lt + higher offset bound --outstanding show only outstanding (not deleted) messages --confirmed @@ -122,6 +140,22 @@ bmqstoragetool --journal-file= --guid= --guid= ``` NOTE: no other filters are allowed with this one +Filter messages with corresponding composite sequence numbers (defined in form ) +--------------------------------------------------------------------------------------------------------------- +Example: +```bash +bmqstoragetool --journal-file= --seqnum= --seqnum= +``` +NOTE: no other filters are allowed with this one + +Filter messages with corresponding record offsets +------------------------------------------------- +Example: +```bash +bmqstoragetool --journal-file= --offset= --offset= +``` +NOTE: no other filters are allowed with this one + Filter messages within time range --------------------------------- Example: @@ -131,6 +165,24 @@ bmqstoragetool --journal-file= --timestamp-gt= bmqstoragetool --journal-file= --timestamp-lt= --timestamp-gt= ``` +Filter messages within composite sequence numbers (primaryLeaseId, sequenceNumber) range +---------------------------------------------------------------------------------------- +Example: +```bash +bmqstoragetool --journal-file= --seqnum-lt= +bmqstoragetool --journal-file= --seqnum-gt= +bmqstoragetool --journal-file= --seqnum-lt= --seqnum-gt= +``` + +Filter messages within record offsets range +------------------------------------------- +Example: +```bash +bmqstoragetool --journal-file= --offset-lt= +bmqstoragetool --journal-file= --offset-gt= +bmqstoragetool --journal-file= --offset-lt= --offset-gt= +``` + Filter messages by queue key ---------------------------- Example: diff --git a/src/applications/bmqstoragetool/bmqstoragetool.m.cpp b/src/applications/bmqstoragetool/bmqstoragetool.m.cpp index 2fe44db4f..6b816cd6c 100644 --- a/src/applications/bmqstoragetool/bmqstoragetool.m.cpp +++ b/src/applications/bmqstoragetool/bmqstoragetool.m.cpp @@ -30,8 +30,10 @@ using namespace BloombergLP; using namespace m_bmqstoragetool; -static bool -parseArgs(CommandLineArguments& arguments, int argc, const char* argv[]) +static bool parseArgs(CommandLineArguments& arguments, + int argc, + const char* argv[], + bslma::Allocator* allocator) { bool showHelp = false; @@ -62,6 +64,16 @@ parseArgs(CommandLineArguments& arguments, int argc, const char* argv[]) "message guid", balcl::TypeInfo(&arguments.d_guid), balcl::OccurrenceInfo::e_OPTIONAL}, + {"seqnum", + "seqnum", + "message composite sequence number", + balcl::TypeInfo(&arguments.d_seqNum), + balcl::OccurrenceInfo::e_OPTIONAL}, + {"offset", + "offset", + "message offset", + balcl::TypeInfo(&arguments.d_offset), + balcl::OccurrenceInfo::e_OPTIONAL}, {"queue-name", "queue name", "message queue name", @@ -82,6 +94,28 @@ parseArgs(CommandLineArguments& arguments, int argc, const char* argv[]) "higher timestamp bound", balcl::TypeInfo(&arguments.d_timestampLt), balcl::OccurrenceInfo::e_OPTIONAL}, + {"seqnum-gt", + "message composite sequence number greater than", + "lower record sequence number bound, defined in form " + "", + balcl::TypeInfo(&arguments.d_seqNumGt), + balcl::OccurrenceInfo::e_OPTIONAL}, + {"seqnum-lt", + "message composite sequence number less than", + "higher sequence number bound, defined in form " + "", + balcl::TypeInfo(&arguments.d_seqNumLt), + balcl::OccurrenceInfo::e_OPTIONAL}, + {"offset-gt", + "message offset greater than", + "lower record offset bound", + balcl::TypeInfo(&arguments.d_offsetGt), + balcl::OccurrenceInfo::e_OPTIONAL}, + {"offset-lt", + "message offset less than", + "higher record offset bound", + balcl::TypeInfo(&arguments.d_offsetLt), + balcl::OccurrenceInfo::e_OPTIONAL}, {"outstanding", "only outstanding", "show only outstanding (not deleted) messages", @@ -130,7 +164,7 @@ parseArgs(CommandLineArguments& arguments, int argc, const char* argv[]) } bsl::string error; - if (!arguments.validate(&error)) { + if (!arguments.validate(&error, allocator)) { bsl::cerr << "Arguments validation failed:\n" << error; return false; // RETURN } @@ -157,7 +191,7 @@ int main(int argc, const char* argv[]) // Arguments parsing CommandLineArguments arguments(allocator); - if (!parseArgs(arguments, argc, argv)) { + if (!parseArgs(arguments, argc, argv, allocator)) { return rc_ARGUMENTS_PARSING_FAILED; // RETURN } diff --git a/src/applications/bmqstoragetool/m_bmqstoragetool_compositesequencenumber.cpp b/src/applications/bmqstoragetool/m_bmqstoragetool_compositesequencenumber.cpp new file mode 100644 index 000000000..261fe8fb5 --- /dev/null +++ b/src/applications/bmqstoragetool/m_bmqstoragetool_compositesequencenumber.cpp @@ -0,0 +1,129 @@ +// Copyright 2014-2023 Bloomberg Finance L.P. +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// bmqstoragetool +#include + +// BDE +#include +#include + +namespace BloombergLP { +namespace m_bmqstoragetool { + +// ============================= +// class CompositeSequenceNumber +// ============================= + +CompositeSequenceNumber::CompositeSequenceNumber() +: d_leaseId(0) +, d_seqNumber(0) +, d_isSet(false) +{ + // NOTHING +} + +CompositeSequenceNumber::CompositeSequenceNumber( + const unsigned int leaseId, + const bsls::Types::Uint64 sequenceNumber) +: d_leaseId(leaseId) +, d_seqNumber(sequenceNumber) +{ + BSLS_ASSERT(d_leaseId > 0 && d_seqNumber > 0); + d_isSet = d_leaseId > 0 && d_seqNumber > 0; +} + +CompositeSequenceNumber& +CompositeSequenceNumber::fromString(bsl::ostream& errorDescription, + const bsl::string& seqNumString) +{ + d_isSet = false; + + if (seqNumString.empty()) { + errorDescription << "Invalid input: empty string."; + return *this; // RETURN + } + + // Find the position of the separator + const size_t separatorPos = seqNumString.find('-'); + if (separatorPos == bsl::string::npos) { + errorDescription << "Invalid format: no '-' separator found."; + return *this; // RETURN + } + + // Extract parts + const bsl::string firstPart = seqNumString.substr(0, separatorPos); + const bsl::string secondPart = seqNumString.substr(separatorPos + 1); + + // Convert parts to numbers + try { + size_t posFirst, posSecond; + + unsigned long uLong = bsl::stoul(firstPart, &posFirst); + d_seqNumber = bsl::stoul(secondPart, &posSecond); + + if (posFirst != firstPart.size() || posSecond != secondPart.size()) { + throw bsl::invalid_argument(""); // THROW + } + + d_leaseId = static_cast(uLong); + if (uLong != d_leaseId) { + throw bsl::out_of_range(""); // THROW + } + + if (d_leaseId == 0 || d_seqNumber == 0) { + errorDescription << "Invalid input: zero values encountered."; + return *this; // RETURN + } + + d_isSet = true; + } + catch (const bsl::invalid_argument& e) { + errorDescription << "Invalid input: non-numeric values encountered."; + } + catch (const bsl::out_of_range& e) { + errorDescription << "Invalid input: number out of range."; + } + + return *this; +} + +bsl::ostream& CompositeSequenceNumber::print(bsl::ostream& stream, + int level, + int spacesPerLevel) const +{ + if (stream.bad()) { + return stream; // RETURN + } + + bdlb::Print::indent(stream, level, spacesPerLevel); + + if (isSet()) { + stream << "leaseId: " << leaseId() + << ", sequenceNumber: " << sequenceNumber(); + } + else { + stream << "** UNSET **"; + } + + if (spacesPerLevel >= 0) { + stream << '\n'; + } + + return stream; +} + +} // close package namespace +} // close enterprise namespace diff --git a/src/applications/bmqstoragetool/m_bmqstoragetool_compositesequencenumber.h b/src/applications/bmqstoragetool/m_bmqstoragetool_compositesequencenumber.h new file mode 100644 index 000000000..a8c5da817 --- /dev/null +++ b/src/applications/bmqstoragetool/m_bmqstoragetool_compositesequencenumber.h @@ -0,0 +1,218 @@ +// Copyright 2014-2023 Bloomberg Finance L.P. +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef INCLUDED_M_BMQSTORAGETOOL_COMPOSITESEQUENCENUMBER +#define INCLUDED_M_BMQSTORAGETOOL_COMPOSITESEQUENCENUMBER + +//@PURPOSE: Provide value-semantic type to represent composite sequence number +//(consists of primary lease Id and sequence number), +// which is used for message filtering. +// +//@CLASSES: +// m_bmqstoragetool::CompositeSequenceNumber: Value-semantic type to represent +// composite sequence number. +// +//@DESCRIPTION: 'CompositeSequenceNumber' provides value-semantic type to +// represent composite sequence number. +// There could be sequence numbers collision inside journal file for different +// lease Ids, so need to handle composite sequence number taking into account +// primary lease Id too. + +// BDE +#include +#include +#include + +namespace BloombergLP { +namespace m_bmqstoragetool { + +// ============================= +// class CompositeSequenceNumber +// ============================= + +class CompositeSequenceNumber { + private: + // DATA + unsigned int d_leaseId; + // Primary Lease Id + bsls::Types::Uint64 d_seqNumber; + // Sequence Number + bool d_isSet; + // Set to `true` if the value of this object is set + + public: + // CREATORS + + /// Create an un-initialized CompositeSequenceNumber. Note that + /// `isSet()` would return false. + CompositeSequenceNumber(); + + /// Create CompositeSequenceNumber from the specified `leaseId` and + /// `sequenceNumber` + CompositeSequenceNumber(const unsigned int leaseId, + const bsls::Types::Uint64 sequenceNumber); + + // MANIPULATORS + + /// Initialize this CompositeSequenceNumber from the specified + /// `seqNumString` representation in format `-`. + /// Return a reference offering modifiable access to this object. If + /// convertion is successfull, `isSet()` would return `true`. Otherwise, + /// `isSet()` would return `false` and specified `errorDescription` is + /// filled with error description. + CompositeSequenceNumber& fromString(bsl::ostream& errorDescription, + const bsl::string& seqNumString); + + // ACCESSORS + + /// Return `true` if the value of this object is not set. + bool isSet() const; + + /// Return Primary Lease Id value. + unsigned int leaseId() const; + + /// Return Sequence Number value. + bsls::Types::Uint64 sequenceNumber() const; + + /// Write the value of this object to the specified output `stream` in a + /// human-readable format, and return a reference to `stream`. + /// Optionally specify an initial indentation `level`. If `level` is + /// specified, optionally specify `spacesPerLevel`, whose absolute value + /// indicates the number of spaces per indentation level for this + /// object. If `level` is negative, suppress indentation of the first + /// line. If `spacesPerLevel` is negative, format the entire output on + /// one line, suppressing all but the initial indentation (as governed + /// by `level`). If `stream` is not valid on entry, this operation has + /// no effect. Note that this human-readable format is not fully + /// specified, and can change without notice. + bsl::ostream& + print(bsl::ostream& stream, int level = 0, int spacesPerLevel = 4) const; +}; + +// FREE OPERATORS + +// ----------------------------- +// class CompositeSequenceNumber +// ----------------------------- + +/// Write the value of the specified `rhs` object to the specified output +/// `stream` in a human-readable format, and return a reference to `stream`. +/// Note that this human-readable format is not fully specified, and can +/// change without notice. +bsl::ostream& operator<<(bsl::ostream& stream, + const CompositeSequenceNumber& rhs); + +/// Return true if the specified `lhs` instance is equal to the +/// specified `rhs` instance, false otherwise. +bool operator==(const CompositeSequenceNumber& lhs, + const CompositeSequenceNumber& rhs); + +/// Return true if the specified `lhs` instance is less than the +/// specified `rhs` instance, false otherwise. +bool operator<(const CompositeSequenceNumber& lhs, + const CompositeSequenceNumber& rhs); + +/// Return true if the specified `lhs` instance is less or equal to the +/// specified `rhs` instance, false otherwise. +bool operator<=(const CompositeSequenceNumber& lhs, + const CompositeSequenceNumber& rhs); + +// ============================================================================ +// INLINE DEFINITIONS +// ============================================================================ + +// ============================= +// class CompositeSequenceNumber +// ============================= + +// ACCESSORS + +inline bool CompositeSequenceNumber::isSet() const +{ + return d_isSet; +} + +inline unsigned int CompositeSequenceNumber::leaseId() const +{ + return d_leaseId; +} + +inline bsls::Types::Uint64 CompositeSequenceNumber::sequenceNumber() const +{ + return d_seqNumber; +} + +} // close package namespace + +// ============================= +// class CompositeSequenceNumber +// ============================= + +// FREE OPERATORS + +inline bsl::ostream& m_bmqstoragetool::operator<<( + bsl::ostream& stream, + const m_bmqstoragetool::CompositeSequenceNumber& rhs) +{ + // PRECONDITIONS + BSLS_ASSERT(rhs.isSet()); + + return rhs.print(stream, 0, -1); +} + +inline bool m_bmqstoragetool::operator==( + const m_bmqstoragetool::CompositeSequenceNumber& lhs, + const m_bmqstoragetool::CompositeSequenceNumber& rhs) +{ + // PRECONDITIONS + BSLS_ASSERT(lhs.isSet() && rhs.isSet()); + + return (lhs.leaseId() == rhs.leaseId() && + lhs.sequenceNumber() == rhs.sequenceNumber()); +} + +inline bool m_bmqstoragetool::operator<( + const m_bmqstoragetool::CompositeSequenceNumber& lhs, + const m_bmqstoragetool::CompositeSequenceNumber& rhs) +{ + // PRECONDITIONS + BSLS_ASSERT(lhs.isSet() && rhs.isSet()); + + // Check leaseId first + if (lhs.leaseId() < rhs.leaseId()) { + return true; // RETURN + } + else if (lhs.leaseId() == rhs.leaseId()) { + if (lhs.sequenceNumber() < rhs.sequenceNumber()) { + return true; // RETURN + } + } + + return false; +} + +inline bool m_bmqstoragetool::operator<=( + const m_bmqstoragetool::CompositeSequenceNumber& lhs, + const m_bmqstoragetool::CompositeSequenceNumber& rhs) +{ + // PRECONDITIONS + BSLS_ASSERT(lhs.isSet() && rhs.isSet()); + + return (lhs < rhs || lhs == rhs); +} + +} // close enterprise namespace + +#endif diff --git a/src/applications/bmqstoragetool/m_bmqstoragetool_compositesequencenumber.t.cpp b/src/applications/bmqstoragetool/m_bmqstoragetool_compositesequencenumber.t.cpp new file mode 100644 index 000000000..779386935 --- /dev/null +++ b/src/applications/bmqstoragetool/m_bmqstoragetool_compositesequencenumber.t.cpp @@ -0,0 +1,327 @@ +// Copyright 2023 Bloomberg Finance L.P. +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// bmqstoragetool +#include + +// BMQ +#include + +// TEST DRIVER +#include + +// CONVENIENCE +using namespace BloombergLP; +using namespace m_bmqstoragetool; +using namespace bsl; + +// ============================================================================ +// TESTS +// ---------------------------------------------------------------------------- + +static void test1_breathingTest() +// ------------------------------------------------------------------------ +// BREATHING TEST +// +// Concerns: +// Exercise the component initialization. +// +// Testing: +// Component initialization +// ------------------------------------------------------------------------ +{ + bmqtst::TestHelper::printTestName("BREATHING TEST"); + + { + CompositeSequenceNumber compositeSeqNum; + BMQTST_ASSERT(!compositeSeqNum.isSet()); + } + + { + CompositeSequenceNumber compositeSeqNum(1, 2); + BMQTST_ASSERT(compositeSeqNum.isSet()); + BMQTST_ASSERT_EQ(compositeSeqNum.leaseId(), 1ul); + BMQTST_ASSERT_EQ(compositeSeqNum.sequenceNumber(), 2ul); + } +} + +static void test2_fromStringTest() +// ------------------------------------------------------------------------ +// FROM STRING TEST +// +// Concerns: +// Exercise the functionality to initialize component from string +// representation. +// +// Testing: +// fromString method +// ------------------------------------------------------------------------ +{ + bmqtst::TestHelper::printTestName("FROM STRING TEST"); + + bmqu::MemOutStream errorDescription(bmqtst::TestHelperUtil::allocator()); + + // Valid string + { + CompositeSequenceNumber compositeSeqNum; + + bsl::string inputString("123-456", + bmqtst::TestHelperUtil::allocator()); + + compositeSeqNum.fromString(errorDescription, inputString); + BMQTST_ASSERT(compositeSeqNum.isSet()); + BMQTST_ASSERT_EQ(compositeSeqNum.leaseId(), 123u); + BMQTST_ASSERT_EQ(compositeSeqNum.sequenceNumber(), 456u); + BMQTST_ASSERT(errorDescription.str().empty()); + } + + // Valid string with leading zeros + { + CompositeSequenceNumber compositeSeqNum; + + bsl::string inputString("00123-000456", + bmqtst::TestHelperUtil::allocator()); + + compositeSeqNum.fromString(errorDescription, inputString); + BMQTST_ASSERT(compositeSeqNum.isSet()); + BMQTST_ASSERT_EQ(compositeSeqNum.leaseId(), 123u); + BMQTST_ASSERT_EQ(compositeSeqNum.sequenceNumber(), 456u); + BMQTST_ASSERT(errorDescription.str().empty()); + } + + // Empty string + { + CompositeSequenceNumber compositeSeqNum; + + bsl::string inputString("", bmqtst::TestHelperUtil::allocator()); + + compositeSeqNum.fromString(errorDescription, inputString); + BMQTST_ASSERT_EQ(compositeSeqNum.isSet(), false); + BMQTST_ASSERT(!errorDescription.str().empty()); + BMQTST_ASSERT_EQ(errorDescription.str(), + "Invalid input: empty string."); + } + + // Invalid string with missed separator + { + CompositeSequenceNumber compositeSeqNum; + + bsl::string inputString("123456", bmqtst::TestHelperUtil::allocator()); + errorDescription.reset(); + + compositeSeqNum.fromString(errorDescription, inputString); + BMQTST_ASSERT_EQ(compositeSeqNum.isSet(), false); + BMQTST_ASSERT(!errorDescription.str().empty()); + BMQTST_ASSERT_EQ(errorDescription.str(), + "Invalid format: no '-' separator found."); + } + + // Invalid string with wrong separator + { + CompositeSequenceNumber compositeSeqNum; + + bsl::string inputString("123_456", + bmqtst::TestHelperUtil::allocator()); + errorDescription.reset(); + + compositeSeqNum.fromString(errorDescription, inputString); + BMQTST_ASSERT_EQ(compositeSeqNum.isSet(), false); + BMQTST_ASSERT(!errorDescription.str().empty()); + BMQTST_ASSERT_EQ(errorDescription.str(), + "Invalid format: no '-' separator found."); + } + + // Invalid string with non-numeric value in first part + { + CompositeSequenceNumber compositeSeqNum; + + bsl::string inputString("1a23-456", + bmqtst::TestHelperUtil::allocator()); + errorDescription.reset(); + + compositeSeqNum.fromString(errorDescription, inputString); + BMQTST_ASSERT_EQ(compositeSeqNum.isSet(), false); + BMQTST_ASSERT(!errorDescription.str().empty()); + BMQTST_ASSERT_EQ(errorDescription.str(), + "Invalid input: non-numeric values encountered."); + } + + // Invalid string with non-numeric value in second part + { + CompositeSequenceNumber compositeSeqNum; + + bsl::string inputString("123-45a6", + bmqtst::TestHelperUtil::allocator()); + errorDescription.reset(); + + compositeSeqNum.fromString(errorDescription, inputString); + BMQTST_ASSERT_EQ(compositeSeqNum.isSet(), false); + BMQTST_ASSERT(!errorDescription.str().empty()); + BMQTST_ASSERT_EQ(errorDescription.str(), + "Invalid input: non-numeric values encountered."); + } + + // Invalid string with zero value in first part + { + CompositeSequenceNumber compositeSeqNum; + + bsl::string inputString("0-456", bmqtst::TestHelperUtil::allocator()); + errorDescription.reset(); + + compositeSeqNum.fromString(errorDescription, inputString); + BMQTST_ASSERT_EQ(compositeSeqNum.isSet(), false); + BMQTST_ASSERT(!errorDescription.str().empty()); + BMQTST_ASSERT_EQ(errorDescription.str(), + "Invalid input: zero values encountered."); + } + + // Invalid string with zero value in second part + { + CompositeSequenceNumber compositeSeqNum; + + bsl::string inputString("123-0", bmqtst::TestHelperUtil::allocator()); + errorDescription.reset(); + + compositeSeqNum.fromString(errorDescription, inputString); + BMQTST_ASSERT_EQ(compositeSeqNum.isSet(), false); + BMQTST_ASSERT(!errorDescription.str().empty()); + BMQTST_ASSERT_EQ(errorDescription.str(), + "Invalid input: zero values encountered."); + } + + // Invalid string with out of range value in first part + { + CompositeSequenceNumber compositeSeqNum; + + // Simulate unsigned int overflow + bsl::string inputString("11111111111-123", + bmqtst::TestHelperUtil::allocator()); + errorDescription.reset(); + + compositeSeqNum.fromString(errorDescription, inputString); + BMQTST_ASSERT_EQ(compositeSeqNum.isSet(), false); + BMQTST_ASSERT(!errorDescription.str().empty()); + BMQTST_ASSERT_EQ(errorDescription.str(), + "Invalid input: number out of range."); + } + + // Invalid string with out of range value in second part + { + CompositeSequenceNumber compositeSeqNum; + + // Simulate bsls::Types::Uint64 overflow + bsl::string inputString("123-111111111111111111111", + bmqtst::TestHelperUtil::allocator()); + errorDescription.reset(); + + compositeSeqNum.fromString(errorDescription, inputString); + BMQTST_ASSERT_EQ(compositeSeqNum.isSet(), false); + BMQTST_ASSERT(!errorDescription.str().empty()); + BMQTST_ASSERT_EQ(errorDescription.str(), + "Invalid input: number out of range."); + } +} + +static void test3_comparisonTest() +// ------------------------------------------------------------------------ +// COMPARISON TEST +// +// Concerns: +// Exercise the functionality to compare objects. +// +// Testing: +// operator<()==, operator<() and operator<=() methods +// ------------------------------------------------------------------------ +{ + bmqtst::TestHelper::printTestName("COMPARISON TEST"); + + // leaseId is less, seqNumber is greater + { + CompositeSequenceNumber lhs(1, 2); + CompositeSequenceNumber rhs(2, 1); + BMQTST_ASSERT(lhs < rhs); + } + + // leaseId is less, seqNumber is less + { + CompositeSequenceNumber lhs(1, 1); + CompositeSequenceNumber rhs(2, 2); + BMQTST_ASSERT(lhs < rhs); + } + + // leaseId is greater, seqNumber is greater + { + CompositeSequenceNumber lhs(3, 2); + CompositeSequenceNumber rhs(2, 1); + BMQTST_ASSERT_EQ((lhs < rhs), false); + } + + // leaseId is greater, seqNumber is less + { + CompositeSequenceNumber lhs(3, 1); + CompositeSequenceNumber rhs(2, 2); + BMQTST_ASSERT_EQ((lhs < rhs), false); + } + + // leaseId is equal, seqNumber is less + { + CompositeSequenceNumber lhs(1, 1); + CompositeSequenceNumber rhs(1, 2); + BMQTST_ASSERT(lhs < rhs); + } + + // leaseId is equal, seqNumber is greater + { + CompositeSequenceNumber lhs(1, 2); + CompositeSequenceNumber rhs(1, 1); + BMQTST_ASSERT_EQ((lhs < rhs), false); + } + + // Compare for equality: leaseId is equal, seqNumber is equal + { + CompositeSequenceNumber lhs(1, 2); + CompositeSequenceNumber rhs(1, 2); + BMQTST_ASSERT_EQ((lhs == rhs), true); + } + + // Compare for equality using '<=': leaseId is equal, seqNumber is equal + { + CompositeSequenceNumber lhs(1, 2); + CompositeSequenceNumber rhs(1, 2); + BMQTST_ASSERT_EQ((lhs <= rhs), true); + } +} + +// ============================================================================ +// MAIN PROGRAM +// ---------------------------------------------------------------------------- + +int main(int argc, char* argv[]) +{ + TEST_PROLOG(bmqtst::TestHelper::e_DEFAULT); + + switch (_testCase) { + case 0: + case 1: test1_breathingTest(); break; + case 2: test2_fromStringTest(); break; + case 3: test3_comparisonTest(); break; + default: { + cerr << "WARNING: CASE '" << _testCase << "' NOT FOUND." << endl; + bmqtst::TestHelperUtil::testStatus() = -1; + } break; + } + + TEST_EPILOG(bmqtst::TestHelper::e_CHECK_DEF_GBL_ALLOC); +} diff --git a/src/applications/bmqstoragetool/m_bmqstoragetool_filters.cpp b/src/applications/bmqstoragetool/m_bmqstoragetool_filters.cpp index 578cc86fb..925541948 100644 --- a/src/applications/bmqstoragetool/m_bmqstoragetool_filters.cpp +++ b/src/applications/bmqstoragetool/m_bmqstoragetool_filters.cpp @@ -13,6 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +// bmqstoragetool #include namespace BloombergLP { @@ -25,12 +26,10 @@ namespace m_bmqstoragetool { Filters::Filters(const bsl::vector& queueKeys, const bsl::vector& queueUris, const QueueMap& queueMap, - const bsls::Types::Int64 timestampGt, - const bsls::Types::Int64 timestampLt, + const Parameters::Range& range, bslma::Allocator* allocator) : d_queueKeys(allocator) -, d_timestampGt(static_cast(timestampGt)) -, d_timestampLt(static_cast(timestampLt)) +, d_range(range) { // Fill internal structures if (!queueKeys.empty()) { @@ -52,8 +51,10 @@ Filters::Filters(const bsl::vector& queueKeys, } } -bool Filters::apply(const mqbs::MessageRecord& record) +bool Filters::apply(const mqbs::MessageRecord& record, + bsls::Types::Uint64 offset) const { + // Apply `queue key` filter if (!d_queueKeys.empty()) { // Match by queueKey bsl::unordered_set::const_iterator it = bsl::find( @@ -65,10 +66,35 @@ bool Filters::apply(const mqbs::MessageRecord& record) return false; // RETURN } } - const bsls::Types::Uint64& ts = record.header().timestamp(); - if ((d_timestampGt > 0 && ts <= d_timestampGt) || - (d_timestampLt > 0 && ts >= d_timestampLt)) { - // Match by timestamp + + // Apply `range` filter + bsls::Types::Uint64 value, valueGt, valueLt; + switch (d_range.d_type) { + case Parameters::Range::e_TIMESTAMP: + value = record.header().timestamp(); + valueGt = d_range.d_timestampGt; + valueLt = d_range.d_timestampLt; + break; + case Parameters::Range::e_OFFSET: + value = offset; + valueGt = d_range.d_offsetGt; + valueLt = d_range.d_offsetLt; + break; + case Parameters::Range::e_SEQUENCE_NUM: { + CompositeSequenceNumber seqNum(record.header().primaryLeaseId(), + record.header().sequenceNumber()); + return !( + (d_range.d_seqNumGt.isSet() && seqNum <= d_range.d_seqNumGt) || + (d_range.d_seqNumLt.isSet() && + d_range.d_seqNumLt <= seqNum)); // RETURN + } break; + default: + // No range filter defined + return true; // RETURN + } + if ((valueGt > 0 && value <= valueGt) || + (valueLt > 0 && value >= valueLt)) { + // Not inside range return false; // RETURN } return true; diff --git a/src/applications/bmqstoragetool/m_bmqstoragetool_filters.h b/src/applications/bmqstoragetool/m_bmqstoragetool_filters.h index c637ff993..608f3fbd1 100644 --- a/src/applications/bmqstoragetool/m_bmqstoragetool_filters.h +++ b/src/applications/bmqstoragetool/m_bmqstoragetool_filters.h @@ -43,8 +43,7 @@ class Filters { private: // DATA bsl::unordered_set d_queueKeys; - const bsls::Types::Uint64 d_timestampGt; - const bsls::Types::Uint64 d_timestampLt; + const Parameters::Range d_range; public: // CREATORS @@ -53,15 +52,15 @@ class Filters { explicit Filters(const bsl::vector& queueKeys, const bsl::vector& queueUris, const QueueMap& queueMap, - const bsls::Types::Int64 timestampGt, - const bsls::Types::Int64 timestampLt, + const Parameters::Range& range, bslma::Allocator* allocator); - // MANIPULATORS + // ACCESSORS /// Apply filters at specified 'record' and return true if all filters /// are matched, false otherwise. - bool apply(const mqbs::MessageRecord& record); + bool apply(const mqbs::MessageRecord& record, + bsls::Types::Uint64 offset) const; }; } // close package namespace diff --git a/src/applications/bmqstoragetool/m_bmqstoragetool_journalfile.cpp b/src/applications/bmqstoragetool/m_bmqstoragetool_journalfile.cpp index aef61e6d1..a420382f2 100644 --- a/src/applications/bmqstoragetool/m_bmqstoragetool_journalfile.cpp +++ b/src/applications/bmqstoragetool/m_bmqstoragetool_journalfile.cpp @@ -602,6 +602,129 @@ void JournalFile::addJournalRecordsWithConfirmedMessagesWithDifferentOrder( } } +void JournalFile::addMultipleTypesRecordsWithMultipleLeaseId( + RecordsListType* records, + size_t numRecordsWithSameLeaseId) +{ + // PRECONDITIONS + BSLS_ASSERT(records); + + unsigned int leaseId = 0; + bsls::Types::Uint64 seqNumber = 1; + + for (unsigned int i = 1; i <= d_numRecords; ++i) { + unsigned int remainder = i % 4; + if (i % numRecordsWithSameLeaseId == 1) { + leaseId++; + seqNumber = 1; + } + if (0 == remainder) { + bmqt::MessageGUID guid; + mqbu::MessageGUIDUtil::generateGUID(&guid); + OffsetPtr rec(d_block, d_currPos); + new (rec.get()) MessageRecord(); + rec->header() + .setPrimaryLeaseId(leaseId) + .setSequenceNumber(seqNumber++) + .setTimestamp(i * d_timestampIncrement); + rec->setRefCount(i % FileStoreProtocol::k_MAX_MSG_REF_COUNT_HARD) + .setQueueKey( + mqbu::StorageKey(mqbu::StorageKey::BinaryRepresentation(), + "abcde")) + .setFileKey( + mqbu::StorageKey(mqbu::StorageKey::BinaryRepresentation(), + "12345")) + .setMessageOffsetDwords(i) + .setMessageGUID(guid) + .setCrc32c(i) + .setCompressionAlgorithmType( + bmqt::CompressionAlgorithmType::e_NONE) + .setMagic(RecordHeader::k_MAGIC); + + RecordBufferType buf; + bsl::memcpy(buf.buffer(), + rec.get(), + FileStoreProtocol::k_JOURNAL_RECORD_SIZE); + records->push_back(bsl::make_pair(RecordType::e_MESSAGE, buf)); + } + else if (1 == remainder) { + // ConfRec + bmqt::MessageGUID guid; + mqbu::MessageGUIDUtil::generateGUID(&guid); + OffsetPtr rec(d_block, d_currPos); + new (rec.get()) ConfirmRecord(); + rec->header() + .setPrimaryLeaseId(leaseId) + .setSequenceNumber(seqNumber++) + .setTimestamp(i * d_timestampIncrement); + rec->setReason(ConfirmReason::e_REJECTED) + .setQueueKey( + mqbu::StorageKey(mqbu::StorageKey::BinaryRepresentation(), + "abcde")) + .setAppKey( + mqbu::StorageKey(mqbu::StorageKey::BinaryRepresentation(), + "appid")) + .setMessageGUID(guid) + .setMagic(RecordHeader::k_MAGIC); + + RecordBufferType buf; + bsl::memcpy(buf.buffer(), + rec.get(), + FileStoreProtocol::k_JOURNAL_RECORD_SIZE); + records->push_back(bsl::make_pair(RecordType::e_CONFIRM, buf)); + } + else if (2 == remainder) { + // DelRec + bmqt::MessageGUID guid; + mqbu::MessageGUIDUtil::generateGUID(&guid); + OffsetPtr rec(d_block, d_currPos); + new (rec.get()) DeletionRecord(); + rec->header() + .setPrimaryLeaseId(leaseId) + .setSequenceNumber(seqNumber++) + .setTimestamp(i * d_timestampIncrement); + rec->setDeletionRecordFlag(DeletionRecordFlag::e_IMPLICIT_CONFIRM) + .setQueueKey( + mqbu::StorageKey(mqbu::StorageKey::BinaryRepresentation(), + "abcde")) + .setMessageGUID(guid) + .setMagic(RecordHeader::k_MAGIC); + + RecordBufferType buf; + bsl::memcpy(buf.buffer(), + rec.get(), + FileStoreProtocol::k_JOURNAL_RECORD_SIZE); + records->push_back(bsl::make_pair(RecordType::e_DELETION, buf)); + } + else { + // QueueOpRec + OffsetPtr rec(d_block, d_currPos); + new (rec.get()) QueueOpRecord(); + rec->header() + .setPrimaryLeaseId(leaseId) + .setSequenceNumber(seqNumber++) + .setTimestamp(i * d_timestampIncrement); + rec->setFlags(3) + .setQueueKey( + mqbu::StorageKey(mqbu::StorageKey::BinaryRepresentation(), + "abcde")) + .setAppKey( + mqbu::StorageKey(mqbu::StorageKey::BinaryRepresentation(), + "appid")) + .setType(QueueOpType::e_PURGE) + .setMagic(RecordHeader::k_MAGIC); + + RecordBufferType buf; + bsl::memcpy(buf.buffer(), + rec.get(), + FileStoreProtocol::k_JOURNAL_RECORD_SIZE); + records->push_back(bsl::make_pair(RecordType::e_QUEUE_OP, buf)); + } + + d_currPos += FileStoreProtocol::k_JOURNAL_RECORD_SIZE; + } +} + } // close package namespace } // close enterprise namespace diff --git a/src/applications/bmqstoragetool/m_bmqstoragetool_journalfile.h b/src/applications/bmqstoragetool/m_bmqstoragetool_journalfile.h index 1a6b42efe..080cb9406 100644 --- a/src/applications/bmqstoragetool/m_bmqstoragetool_journalfile.h +++ b/src/applications/bmqstoragetool/m_bmqstoragetool_journalfile.h @@ -157,6 +157,13 @@ class JournalFile { GuidVectorType* expectedGUIDs, size_t numMessages, bsl::vector& messageOffsets); + + /// Generate sequence of multiple types of records. Increase Primary Lease + /// Id after the specified `numRecordsWithSameLeaseId` records. Store list + /// of created records in the specified `records`. + void addMultipleTypesRecordsWithMultipleLeaseId( + RecordsListType* records, + size_t numRecordsWithSameLeaseId); }; // ============================================================================ diff --git a/src/applications/bmqstoragetool/m_bmqstoragetool_journalfileprocessor.cpp b/src/applications/bmqstoragetool/m_bmqstoragetool_journalfileprocessor.cpp index efdd5c4af..edd55f390 100644 --- a/src/applications/bmqstoragetool/m_bmqstoragetool_journalfileprocessor.cpp +++ b/src/applications/bmqstoragetool/m_bmqstoragetool_journalfileprocessor.cpp @@ -14,6 +14,8 @@ // limitations under the License. // bmqstoragetool +#include "m_bmqstoragetool_compositesequencenumber.h" +#include "m_bmqstoragetool_parameters.h" #include // BDE @@ -37,13 +39,14 @@ namespace BloombergLP { namespace m_bmqstoragetool { /// Move the journal iterator pointed by the specified 'jit' to the first -/// message whose timestamp is more then the specified 'timestamp'. Return '1' -/// on success, '0' if there are no such records or negative value if an error -/// was encountered. Note that if this method returns < 0, the specified 'jit' -/// is invalidated. Behavior is undefined unless last call to `nextRecord` or +/// record whose value is more then the range lower bound. The specified +/// `lessThanLowerBoundFn` functor is used for comparison. Return '1' on +/// success, '0' if there are no such records or negative value if an error was +/// encountered. Note that if this method returns < 0, the specified 'jit' is +/// invalidated. Behavior is undefined unless last call to `nextRecord` or /// 'advance' returned '1' and the iterator points to a valid record. int moveToLowerBound(mqbs::JournalFileIterator* jit, - const bsls::Types::Uint64& timestamp) + LessThanLowerBoundFn& lessThanLowerBoundFn) { // PRECONDITIONS BSLS_ASSERT(jit); @@ -57,7 +60,7 @@ int moveToLowerBound(mqbs::JournalFileIterator* jit, bsls::Types::Uint64 left = 0; bsls::Types::Uint64 right = recordsNumber; while (right > left + 1) { - const bool goBackwards = jit->recordHeader().timestamp() > timestamp; + const bool goBackwards = lessThanLowerBoundFn(jit, true); if (goBackwards != jit->isReverseMode()) { jit->flipDirection(); } @@ -81,7 +84,8 @@ int moveToLowerBound(mqbs::JournalFileIterator* jit, if (jit->isReverseMode()) { jit->flipDirection(); } - if (jit->recordHeader().timestamp() <= timestamp) { + // Move to next record if value <= lower bound) { + if (lessThanLowerBoundFn(jit) || !lessThanLowerBoundFn(jit, true)) { if (jit->recordIndex() < recordsNumber) { rc = jit->nextRecord(); } @@ -95,6 +99,42 @@ int moveToLowerBound(mqbs::JournalFileIterator* jit, return rc; // RETURN } +// ========================== +// class LessThanLowerBoundFn +// ========================== + +LessThanLowerBoundFn::LessThanLowerBoundFn(const Parameters::Range& range) +: d_range(range) +{ + BSLS_ASSERT(d_range.d_type != Parameters::Range::e_NONE); +} + +bool LessThanLowerBoundFn::operator()(const mqbs::JournalFileIterator* jit, + bool inverseOrder) const +{ + // PRECONDITIONS + BSLS_ASSERT(jit); + + bool res; + if (d_range.d_type == Parameters::Range::e_TIMESTAMP) { + res = inverseOrder + ? d_range.d_timestampGt < jit->recordHeader().timestamp() + : jit->recordHeader().timestamp() < d_range.d_timestampGt; + } + else if (d_range.d_type == Parameters::Range::e_OFFSET) { + res = inverseOrder ? d_range.d_offsetGt < jit->recordOffset() + : jit->recordOffset() < d_range.d_offsetGt; + } + else { + CompositeSequenceNumber seqNum(jit->recordHeader().primaryLeaseId(), + jit->recordHeader().sequenceNumber()); + res = inverseOrder ? d_range.d_seqNumGt < seqNum + : seqNum < d_range.d_seqNumGt; + } + + return res; +} + // ========================== // class JournalFileProcessor // ========================== @@ -121,12 +161,13 @@ void JournalFileProcessor::process() Filters filters(d_parameters->d_queueKey, d_parameters->d_queueName, d_parameters->d_queueMap, - d_parameters->d_timestampGt, - d_parameters->d_timestampLt, + d_parameters->d_range, d_allocator_p); - bool stopSearch = false; - bool needTimestampSearch = d_parameters->d_timestampGt > 0; + bool stopSearch = false; + bool needMoveToLowerBound = d_parameters->d_range.d_timestampGt > 0 || + d_parameters->d_range.d_offsetGt > 0 || + d_parameters->d_range.d_seqNumGt.isSet(); // Iterate through all Journal file records mqbs::JournalFileIterator* iter = d_fileManager->journalFileIterator(); @@ -140,24 +181,28 @@ void JournalFileProcessor::process() d_ostream << "Iteration aborted (exit status " << rc << ")."; return; // RETURN } - if (needTimestampSearch) { - rc = moveToLowerBound(iter, d_parameters->d_timestampGt); + + if (needMoveToLowerBound) { + LessThanLowerBoundFn lessThanLowerBoundFn(d_parameters->d_range); + rc = moveToLowerBound(iter, lessThanLowerBoundFn); if (rc == 0) { stopSearch = true; continue; // CONTINUE } else if (rc < 0) { - d_ostream << "Binary search by timesamp aborted (exit status " - << rc << ")."; + d_ostream << "Binary search aborted (exit status " << rc + << ")."; return; // RETURN } - needTimestampSearch = false; + needMoveToLowerBound = false; } + // MessageRecord if (iter->recordType() == mqbs::RecordType::e_MESSAGE) { const mqbs::MessageRecord& record = iter->asMessageRecord(); + // Apply filters - if (filters.apply(record)) { + if (filters.apply(record, iter->recordOffset())) { stopSearch = d_searchResult_p->processMessageRecord( record, iter->recordIndex(), diff --git a/src/applications/bmqstoragetool/m_bmqstoragetool_journalfileprocessor.h b/src/applications/bmqstoragetool/m_bmqstoragetool_journalfileprocessor.h index 08876adec..1abfc2d92 100644 --- a/src/applications/bmqstoragetool/m_bmqstoragetool_journalfileprocessor.h +++ b/src/applications/bmqstoragetool/m_bmqstoragetool_journalfileprocessor.h @@ -40,8 +40,33 @@ namespace BloombergLP { namespace m_bmqstoragetool { +// ========================== +// class LessThanLowerBoundFn +// ========================== + +// Functor to perform comparison with range lower bound. +class LessThanLowerBoundFn { + private: + // PRIVATE DATA + const Parameters::Range d_range; + + public: + // CREATORS + + explicit LessThanLowerBoundFn(const Parameters::Range& range); + + // ACCESSORS + + bool operator()(const mqbs::JournalFileIterator* jit, + bool inverseOrder = false) const; + // Return true if value specified by `jit` is less than range lower bound + // when the specified `inverseOrder` is false, false otherwise. + // Return true if range lower bound is less than value specified by `jit` + // when the specified `inverseOrder` is true, false otherwise. +}; + int moveToLowerBound(mqbs::JournalFileIterator* jit, - const bsls::Types::Uint64& timestamp); + LessThanLowerBoundFn& lessThanLowerBoundFn); // ========================== // class JournalFileProcessor diff --git a/src/applications/bmqstoragetool/m_bmqstoragetool_journalfileprocessor.t.cpp b/src/applications/bmqstoragetool/m_bmqstoragetool_journalfileprocessor.t.cpp index 7bfe25df4..7cf062f9e 100644 --- a/src/applications/bmqstoragetool/m_bmqstoragetool_journalfileprocessor.t.cpp +++ b/src/applications/bmqstoragetool/m_bmqstoragetool_journalfileprocessor.t.cpp @@ -14,6 +14,8 @@ // limitations under the License. // bmqstoragetool +#include "m_bmqstoragetool_compositesequencenumber.h" +#include "m_bmqstoragetool_parameters.h" #include #include #include @@ -863,8 +865,10 @@ static void test11_searchMessagesByTimestamp() // Configure parameters to search messages by timestamps Parameters params(bmqtst::TestHelperUtil::allocator()); - params.d_timestampGt = ts1; - params.d_timestampLt = ts2; + params.d_range.d_timestampGt = ts1; + params.d_range.d_timestampLt = ts2; + params.d_range.d_type = Parameters::Range::e_TIMESTAMP; + // Prepare file manager bslma::ManagedPtr fileManager( new (*bmqtst::TestHelperUtil::allocator()) @@ -1233,9 +1237,21 @@ static void test15_timestampSearchTest() false); // Move the iterator to the beginning of the file BMQTST_ASSERT_EQ(journalFileIt.nextRecord(), 1); - BMQTST_ASSERT_EQ(m_bmqstoragetool::moveToLowerBound(&journalFileIt, - ts), - 1); + + Parameters::Range range; + range.d_type = Parameters::Range::e_TIMESTAMP; + range.d_timestampGt = ts; + LessThanLowerBoundFn lessThanLowerBoundFn(range); + + BMQTST_ASSERT_EQ( + m_bmqstoragetool::moveToLowerBound(&journalFileIt, + lessThanLowerBoundFn), + 1); + BMQTST_ASSERT_EQ(journalFileIt.nextRecord(), 1); + BMQTST_ASSERT_EQ( + m_bmqstoragetool::moveToLowerBound(&journalFileIt, + lessThanLowerBoundFn), + 1); ResultChecker::check(journalFileIt, ts); } @@ -1255,17 +1271,28 @@ static void test15_timestampSearchTest() // Find record with lower timestamp than the record pointed by the // specified iterator, which is initially forward BMQTST_ASSERT_GT(journalFileIt.recordHeader().timestamp(), ts1); - BMQTST_ASSERT_EQ(m_bmqstoragetool::moveToLowerBound(&journalFileIt, - ts1), - 1); + + Parameters::Range range; + range.d_type = Parameters::Range::e_TIMESTAMP; + range.d_timestampGt = ts1; + LessThanLowerBoundFn lessThanLowerBoundFn(range); + + BMQTST_ASSERT_EQ( + m_bmqstoragetool::moveToLowerBound(&journalFileIt, + lessThanLowerBoundFn), + 1); ResultChecker::check(journalFileIt, ts1); // Find record with higher timestamp than the record pointed by the // specified iterator, which is initially forward BMQTST_ASSERT_LT(journalFileIt.recordHeader().timestamp(), ts2); - BMQTST_ASSERT_EQ(m_bmqstoragetool::moveToLowerBound(&journalFileIt, - ts2), - 1); + range.d_timestampGt = ts2; + + LessThanLowerBoundFn lessThanLowerBoundFn2(range); + BMQTST_ASSERT_EQ( + m_bmqstoragetool::moveToLowerBound(&journalFileIt, + lessThanLowerBoundFn2), + 1); ResultChecker::check(journalFileIt, ts2); // Find record with lower timestamp than the record pointed by the @@ -1273,9 +1300,10 @@ static void test15_timestampSearchTest() BMQTST_ASSERT_GT(journalFileIt.recordHeader().timestamp(), ts1); journalFileIt.flipDirection(); BMQTST_ASSERT(journalFileIt.isReverseMode()); - BMQTST_ASSERT_EQ(m_bmqstoragetool::moveToLowerBound(&journalFileIt, - ts1), - 1); + BMQTST_ASSERT_EQ( + m_bmqstoragetool::moveToLowerBound(&journalFileIt, + lessThanLowerBoundFn), + 1); ResultChecker::check(journalFileIt, ts1); // Find record with higher timestamp than the record pointed by the @@ -1283,9 +1311,10 @@ static void test15_timestampSearchTest() BMQTST_ASSERT_LT(journalFileIt.recordHeader().timestamp(), ts2); journalFileIt.flipDirection(); BMQTST_ASSERT(journalFileIt.isReverseMode()); - BMQTST_ASSERT_EQ(m_bmqstoragetool::moveToLowerBound(&journalFileIt, - ts2), - 1); + BMQTST_ASSERT_EQ( + m_bmqstoragetool::moveToLowerBound(&journalFileIt, + lessThanLowerBoundFn2), + 1); ResultChecker::check(journalFileIt, ts2); } @@ -1299,9 +1328,16 @@ static void test15_timestampSearchTest() false); // Move the iterator to the beginning of the file BMQTST_ASSERT_EQ(journalFileIt.nextRecord(), 1); - BMQTST_ASSERT_EQ(m_bmqstoragetool::moveToLowerBound(&journalFileIt, - ts), - 0); + + Parameters::Range range; + range.d_type = Parameters::Range::e_TIMESTAMP; + range.d_timestampGt = ts; + LessThanLowerBoundFn lessThanLowerBoundFn(range); + + BMQTST_ASSERT_EQ( + m_bmqstoragetool::moveToLowerBound(&journalFileIt, + lessThanLowerBoundFn), + 0); BMQTST_ASSERT_EQ(journalFileIt.recordIndex(), k_NUM_RECORDS - 1); BMQTST_ASSERT_LT(journalFileIt.recordHeader().timestamp(), ts); BMQTST_ASSERT(!journalFileIt.isReverseMode()); @@ -1316,15 +1352,283 @@ static void test15_timestampSearchTest() false); // Move the iterator to the beginning of the file BMQTST_ASSERT_EQ(journalFileIt.nextRecord(), 1); - BMQTST_ASSERT_EQ(m_bmqstoragetool::moveToLowerBound(&journalFileIt, - ts), - 1); + + Parameters::Range range; + range.d_type = Parameters::Range::e_TIMESTAMP; + range.d_timestampGt = ts; + LessThanLowerBoundFn lessThanLowerBoundFn(range); + + BMQTST_ASSERT_EQ( + m_bmqstoragetool::moveToLowerBound(&journalFileIt, + lessThanLowerBoundFn), + 1); BMQTST_ASSERT_EQ(journalFileIt.recordIndex(), 0U); BMQTST_ASSERT_GT(journalFileIt.recordHeader().timestamp(), ts); BMQTST_ASSERT(!journalFileIt.isReverseMode()); } } +static void test16_sequenceNumberLowerBoundTest() +// ------------------------------------------------------------------------ +// MOVE TO SEQUENCE NUMBER LOWER BOUND TEST +// +// Concerns: +// Find the first message in journal file with sequence number more than the +// specified 'valueGt' and move the specified JournalFileIterator to it. +// +// Testing: +// m_bmqstoragetool::moveToLowerBound() +// ------------------------------------------------------------------------ +{ + bmqtst::TestHelper::printTestName( + "MOVE TO SEQUENCE NUMBER LOWER BOUND TEST"); + + struct Test { + int d_line; + size_t d_numRecords; + size_t d_numRecordsWithSameLeaseId; + unsigned int d_leaseIdGt; + bsls::Types::Uint64 d_seqNumberGt; + } k_DATA[] = { + {L_, 32, 4, 3, 2}, + {L_, 3, 2, 1, 2}, + {L_, 300, 10, 3, 2}, + {L_, 300, 11, 3, 2}, + {L_, 300, 11, 3, 1}, // edge case (first seqNum inside leaseId) + {L_, 300, 11, 3, 11}, // edge case (last seqNum inside leaseId) + {L_, 300, 11, 1, 1}, // edge case (left seqNum edge inside first + // leaseId) + {L_, 330, 11, 30, 10}, // edge case (prev before last seqNum inside + // last leaseId) + }; + + const size_t k_NUM_DATA = sizeof(k_DATA) / sizeof(*k_DATA); + + for (size_t idx = 0; idx < k_NUM_DATA; ++idx) { + const Test& test = k_DATA[idx]; + + // Simulate journal file + JournalFile::RecordsListType records( + bmqtst::TestHelperUtil::allocator()); + JournalFile journalFile(test.d_numRecords, + bmqtst::TestHelperUtil::allocator()); + journalFile.addMultipleTypesRecordsWithMultipleLeaseId( + &records, + test.d_numRecordsWithSameLeaseId); + + mqbs::JournalFileIterator journalFileIt( + &journalFile.mappedFileDescriptor(), + journalFile.fileHeader(), + false); + + CompositeSequenceNumber seqNumGt(test.d_leaseIdGt, test.d_seqNumberGt); + unsigned int expectedLeaseId = + test.d_leaseIdGt + + (test.d_seqNumberGt == test.d_numRecordsWithSameLeaseId ? 1 : 0); + bsls::Types::Uint64 expectedSeqNumber = + test.d_seqNumberGt == test.d_numRecordsWithSameLeaseId + ? 1 + : (test.d_seqNumberGt + 1); + + // Move the iterator to the beginning of the file + BMQTST_ASSERT_EQ(journalFileIt.nextRecord(), 1); + + Parameters::Range range; + range.d_type = Parameters::Range::e_SEQUENCE_NUM; + range.d_seqNumGt = seqNumGt; + LessThanLowerBoundFn lessThanLowerBoundFn(range); + + BMQTST_ASSERT_EQ_D( + test.d_line, + m_bmqstoragetool::moveToLowerBound(&journalFileIt, + lessThanLowerBoundFn), + 1); + BMQTST_ASSERT_EQ_D(test.d_line, + journalFileIt.recordHeader().primaryLeaseId(), + expectedLeaseId); + BMQTST_ASSERT_EQ_D(test.d_line, + journalFileIt.recordHeader().sequenceNumber(), + expectedSeqNumber); + } + + // Edge case: not in the range (greater then the last record) + { + const size_t k_NUM_RECORDS = 30; + JournalFile::RecordsListType records( + bmqtst::TestHelperUtil::allocator()); + JournalFile journalFile(k_NUM_RECORDS, + bmqtst::TestHelperUtil::allocator()); + journalFile.addMultipleTypesRecordsWithMultipleLeaseId(&records, + k_NUM_RECORDS); + + mqbs::JournalFileIterator journalFileIt( + &journalFile.mappedFileDescriptor(), + journalFile.fileHeader(), + false); + + // Move the iterator to the beginning of the file + BMQTST_ASSERT_EQ(journalFileIt.nextRecord(), 1); + + CompositeSequenceNumber seqNumGt(1, k_NUM_RECORDS); + Parameters::Range range; + range.d_type = Parameters::Range::e_SEQUENCE_NUM; + range.d_seqNumGt = seqNumGt; + LessThanLowerBoundFn lessThanLowerBoundFn(range); + + BMQTST_ASSERT_EQ( + m_bmqstoragetool::moveToLowerBound(&journalFileIt, + lessThanLowerBoundFn), + 0); + BMQTST_ASSERT_EQ(journalFileIt.recordHeader().primaryLeaseId(), 1u); + BMQTST_ASSERT_EQ(journalFileIt.recordHeader().sequenceNumber(), + k_NUM_RECORDS); + } +} + +static void test17_searchMessagesBySequenceNumbersRange() +// ------------------------------------------------------------------------ +// SEARCH MESSAGES BY SEQUENCE NUMBERS RANGE TEST +// +// Concerns: +// Search messages by sequence numbers range in journal file and output +// GUIDs. +// +// Testing: +// JournalFileProcessor::process() +// ------------------------------------------------------------------------ +{ + bmqtst::TestHelper::printTestName( + "SEARCH MESSAGES BY SEQUENCE NUMBERS RANGE TEST"); + + // Simulate journal file + const size_t k_NUM_RECORDS = 100; + JournalFile::RecordsListType records(bmqtst::TestHelperUtil::allocator()); + JournalFile journalFile(k_NUM_RECORDS, + bmqtst::TestHelperUtil::allocator()); + journalFile.addMultipleTypesRecordsWithMultipleLeaseId(&records, 10); + const CompositeSequenceNumber seqNumGt(3, 3); + const CompositeSequenceNumber seqNumLt(4, 6); + + // Configure parameters to search messages by sequence number range + Parameters params(bmqtst::TestHelperUtil::allocator()); + params.d_range.d_seqNumGt = seqNumGt; + params.d_range.d_seqNumLt = seqNumLt; + params.d_range.d_type = Parameters::Range::e_SEQUENCE_NUM; + // Prepare file manager + bslma::ManagedPtr fileManager( + new (*bmqtst::TestHelperUtil::allocator()) + FileManagerMock(journalFile), + bmqtst::TestHelperUtil::allocator()); + + // Get GUIDs of messages inside sequence numbers range and prepare expected + // output + bmqu::MemOutStream expectedStream(bmqtst::TestHelperUtil::allocator()); + + bsl::list::const_iterator recordIter = + records.begin(); + bsl::size_t msgCnt = 0; + for (; recordIter != records.end(); ++recordIter) { + RecordType::Enum rtype = recordIter->first; + if (rtype == RecordType::e_MESSAGE) { + const MessageRecord& msg = *reinterpret_cast( + recordIter->second.buffer()); + const CompositeSequenceNumber seqNum( + msg.header().primaryLeaseId(), + msg.header().sequenceNumber()); + if (seqNumGt < seqNum && seqNum < seqNumLt) { + outputGuidString(expectedStream, msg.messageGUID()); + msgCnt++; + } + } + } + expectedStream << msgCnt << " message GUID(s) found." << bsl::endl; + + // Run search + bmqu::MemOutStream resultStream(bmqtst::TestHelperUtil::allocator()); + bslma::ManagedPtr searchProcessor = + CommandProcessorFactory::createCommandProcessor( + ¶ms, + fileManager, + resultStream, + bmqtst::TestHelperUtil::allocator()); + searchProcessor->process(); + + BMQTST_ASSERT_EQ(resultStream.str(), expectedStream.str()); +} + +static void test18_searchMessagesByOffsetsRange() +// ------------------------------------------------------------------------ +// SEARCH MESSAGES BY OFFSETS RANGE TEST +// +// Concerns: +// Search messages by offsets range in journal file and output GUIDs. +// +// Testing: +// JournalFileProcessor::process() +// ------------------------------------------------------------------------ +{ + bmqtst::TestHelper::printTestName("SEARCH MESSAGES BY OFFSETS RANGE TEST"); + + // Simulate journal file + const size_t k_NUM_RECORDS = 50; + JournalFile::RecordsListType records(bmqtst::TestHelperUtil::allocator()); + JournalFile journalFile(k_NUM_RECORDS, + bmqtst::TestHelperUtil::allocator()); + journalFile.addAllTypesRecords(&records); + const size_t k_HEADER_SIZE = sizeof(mqbs::FileHeader) + + sizeof(mqbs::JournalFileHeader); + const bsls::Types::Uint64 offsetGt = + mqbs::FileStoreProtocol::k_JOURNAL_RECORD_SIZE * 15 + k_HEADER_SIZE; + const bsls::Types::Uint64 offsetLt = + mqbs::FileStoreProtocol::k_JOURNAL_RECORD_SIZE * 35 + k_HEADER_SIZE; + + // Configure parameters to search messages by timestamps + Parameters params(bmqtst::TestHelperUtil::allocator()); + params.d_range.d_offsetGt = offsetGt; + params.d_range.d_offsetLt = offsetLt; + params.d_range.d_type = Parameters::Range::e_OFFSET; + // Prepare file manager + bslma::ManagedPtr fileManager( + new (*bmqtst::TestHelperUtil::allocator()) + FileManagerMock(journalFile), + bmqtst::TestHelperUtil::allocator()); + + // Get GUIDs of messages within offsets range and prepare expected + // output + bmqu::MemOutStream expectedStream(bmqtst::TestHelperUtil::allocator()); + + bsl::list::const_iterator recordIter = + records.begin(); + bsl::size_t msgCnt = 0; + for (; recordIter != records.end(); ++recordIter) { + RecordType::Enum rtype = recordIter->first; + if (rtype == RecordType::e_MESSAGE) { + const MessageRecord& msg = *reinterpret_cast( + recordIter->second.buffer()); + const bsls::Types::Uint64& offset = + msg.header().sequenceNumber() * + mqbs::FileStoreProtocol::k_JOURNAL_RECORD_SIZE; + if (offset > offsetGt && offset < offsetLt) { + outputGuidString(expectedStream, msg.messageGUID()); + msgCnt++; + } + } + } + expectedStream << msgCnt << " message GUID(s) found." << bsl::endl; + + // Run search + bmqu::MemOutStream resultStream(bmqtst::TestHelperUtil::allocator()); + bslma::ManagedPtr searchProcessor = + CommandProcessorFactory::createCommandProcessor( + ¶ms, + fileManager, + resultStream, + bmqtst::TestHelperUtil::allocator()); + searchProcessor->process(); + + BMQTST_ASSERT_EQ(resultStream.str(), expectedStream.str()); +} + // ============================================================================ // MAIN PROGRAM // ---------------------------------------------------------------------------- @@ -1350,6 +1654,9 @@ int main(int argc, char* argv[]) case 13: test13_searchMessagesWithPayloadDumpTest(); break; case 14: test14_summaryTest(); break; case 15: test15_timestampSearchTest(); break; + case 16: test16_sequenceNumberLowerBoundTest(); break; + case 17: test17_searchMessagesBySequenceNumbersRange(); break; + case 18: test18_searchMessagesByOffsetsRange(); break; default: { cerr << "WARNING: CASE '" << _testCase << "' NOT FOUND." << endl; bmqtst::TestHelperUtil::testStatus() = -1; diff --git a/src/applications/bmqstoragetool/m_bmqstoragetool_parameters.cpp b/src/applications/bmqstoragetool/m_bmqstoragetool_parameters.cpp index 79c7e3028..7d0f950c4 100644 --- a/src/applications/bmqstoragetool/m_bmqstoragetool_parameters.cpp +++ b/src/applications/bmqstoragetool/m_bmqstoragetool_parameters.cpp @@ -77,11 +77,17 @@ bool isValidQueueKeyHexRepresentation(const char* queueKeyBuf) CommandLineArguments::CommandLineArguments(bslma::Allocator* allocator) : d_timestampGt(0) , d_timestampLt(0) +, d_seqNumGt(allocator) +, d_seqNumLt(allocator) +, d_offsetGt(0) +, d_offsetLt(0) , d_journalPath(allocator) , d_journalFile(allocator) , d_dataFile(allocator) , d_cslFile(allocator) , d_guid(allocator) +, d_seqNum(allocator) +, d_offset(allocator) , d_queueKey(allocator) , d_queueName(allocator) , d_dumpLimit(0) @@ -94,9 +100,10 @@ CommandLineArguments::CommandLineArguments(bslma::Allocator* allocator) { } -bool CommandLineArguments::validate(bsl::string* error) +bool CommandLineArguments::validate(bsl::string* error, + bslma::Allocator* allocator) { - bmqu::MemOutStream ss; + bmqu::MemOutStream ss(allocator); if (d_journalPath.empty() && d_journalFile.empty()) { ss << "Neither journal path nor journal file are specified\n"; @@ -167,13 +174,96 @@ bool CommandLineArguments::validate(bsl::string* error) (d_timestampLt > 0 && d_timestampGt >= d_timestampLt)) { ss << "Invalid timestamp range specified\n"; } + if (!d_seqNumLt.empty() || !d_seqNumGt.empty()) { + bmqu::MemOutStream errorDescr(allocator); + CompositeSequenceNumber seqNumLt, seqNumGt; + if (!d_seqNumLt.empty()) { + seqNumLt.fromString(errorDescr, d_seqNumLt); + if (!seqNumLt.isSet()) { + ss << "--seqnum-lt: " << errorDescr.str() << "\n"; + errorDescr.reset(); + } + } + if (!d_seqNumGt.empty()) { + seqNumGt.fromString(errorDescr, d_seqNumGt); + if (!seqNumGt.isSet()) { + ss << "--seqnum-gt: " << errorDescr.str() << "\n"; + } + } + + if (seqNumLt.isSet() && seqNumGt.isSet()) { + if (seqNumLt <= seqNumGt) { + ss << "Invalid sequence number range specified\n"; + } + } + } + if (d_offsetLt < 0 || d_offsetGt < 0 || + (d_offsetLt > 0 && d_offsetGt >= d_offsetLt)) { + ss << "Invalid offset range specified\n"; + } + // Check that only one range type is selected + bsl::size_t rangesCnt = 0; + if (d_timestampLt || d_timestampGt) { + rangesCnt++; + } + if (!d_seqNumLt.empty() || !d_seqNumGt.empty()) { + rangesCnt++; + } + if (d_offsetLt || d_offsetGt) { + rangesCnt++; + } + if (rangesCnt > 1) { + ss << "Only one range type can be selected: timestamp, seqnum or " + "offset\n"; + } + if (!d_guid.empty() && - (!d_queueKey.empty() || !d_queueName.empty() || d_outstanding || - d_confirmed || d_partiallyConfirmed || d_timestampGt > 0 || - d_timestampLt > 0 || d_summary)) { + (!d_queueKey.empty() || !d_queueName.empty() || !d_seqNum.empty() || + !d_offset.empty() || d_outstanding || d_confirmed || + d_partiallyConfirmed || rangesCnt > 0 || d_summary)) { ss << "Giud filter can't be combined with any other filters, as it is " "specific enough to find a particular message\n"; } + if (!d_seqNum.empty() && + (!d_queueKey.empty() || !d_queueName.empty() || !d_guid.empty() || + !d_offset.empty() || d_outstanding || d_confirmed || + d_partiallyConfirmed || rangesCnt > 0 || d_summary)) { + ss << "Secnum filter can't be combined with any other filters, as it " + "is " + "specific enough to find a particular message\n"; + } + if (!d_offset.empty() && + (!d_queueKey.empty() || !d_queueName.empty() || !d_guid.empty() || + !d_seqNum.empty() || d_outstanding || d_confirmed || + d_partiallyConfirmed || rangesCnt > 0 || d_summary)) { + ss << "Offset filter can't be combined with any other filters, as it " + "is " + "specific enough to find a particular message\n"; + } + if (!d_seqNum.empty()) { + CompositeSequenceNumber seqNum; + bmqu::MemOutStream errorDescr(allocator); + for (bsl::vector::const_iterator cit = d_seqNum.begin(); + cit != d_seqNum.end(); + ++cit) { + seqNum.fromString(errorDescr, *cit); + if (!seqNum.isSet()) { + ss << "--seqnum: " << errorDescr.str() << "\n"; + errorDescr.reset(); + } + } + } + if (!d_offset.empty()) { + for (bsl::vector::const_iterator cit = + d_offset.begin(); + cit != d_offset.end(); + ++cit) { + if (*cit < 0) { + ss << "--offset: " << *cit << " cannot be negative\n"; + } + } + } + if (d_summary && (d_outstanding || d_confirmed || d_partiallyConfirmed || d_details)) { ss << "'--summary' can't be combined with '--outstanding', " @@ -209,11 +299,24 @@ bool CommandLineArguments::validate(bsl::string* error) return error->empty(); } -Parameters::Parameters(bslma::Allocator* allocator) -: d_queueMap(allocator) +Parameters::Range::Range() +: d_type(Range::e_NONE) , d_timestampGt(0) , d_timestampLt(0) +, d_offsetGt(0) +, d_offsetLt(0) +, d_seqNumGt() +, d_seqNumLt() +{ + // NOTHING +} + +Parameters::Parameters(bslma::Allocator* allocator) +: d_queueMap(allocator) +, d_range() , d_guid(allocator) +, d_seqNum(allocator) +, d_offset(allocator) , d_queueKey(allocator) , d_queueName(allocator) , d_dumpLimit(0) @@ -224,14 +327,16 @@ Parameters::Parameters(bslma::Allocator* allocator) , d_confirmed(false) , d_partiallyConfirmed(false) { + // NOTHING } Parameters::Parameters(const CommandLineArguments& arguments, bslma::Allocator* allocator) : d_queueMap(allocator) -, d_timestampGt(arguments.d_timestampGt) -, d_timestampLt(arguments.d_timestampLt) +, d_range() , d_guid(arguments.d_guid, allocator) +, d_seqNum(allocator) +, d_offset(arguments.d_offset, allocator) , d_queueKey(arguments.d_queueKey, allocator) , d_queueName(arguments.d_queueName, allocator) , d_dumpLimit(arguments.d_dumpLimit) @@ -242,6 +347,44 @@ Parameters::Parameters(const CommandLineArguments& arguments, , d_confirmed(arguments.d_confirmed) , d_partiallyConfirmed(arguments.d_partiallyConfirmed) { + // Set search range type and values if present + if (arguments.d_timestampLt || arguments.d_timestampGt) { + d_range.d_type = Range::e_TIMESTAMP; + d_range.d_timestampLt = static_cast( + arguments.d_timestampLt); + d_range.d_timestampGt = static_cast( + arguments.d_timestampGt); + } + else if (arguments.d_offsetLt || arguments.d_offsetGt) { + d_range.d_type = Range::e_OFFSET; + d_range.d_offsetLt = static_cast( + arguments.d_offsetLt); + d_range.d_offsetGt = static_cast( + arguments.d_offsetGt); + } + else if (!arguments.d_seqNumLt.empty() || !arguments.d_seqNumGt.empty()) { + d_range.d_type = Range::e_SEQUENCE_NUM; + bmqu::MemOutStream errorDescr(allocator); + if (!arguments.d_seqNumLt.empty()) { + d_range.d_seqNumLt.fromString(errorDescr, arguments.d_seqNumLt); + } + if (!arguments.d_seqNumGt.empty()) { + d_range.d_seqNumGt.fromString(errorDescr, arguments.d_seqNumGt); + } + } + + // Set specific sequence numbers if present + if (!arguments.d_seqNum.empty()) { + CompositeSequenceNumber seqNum; + bmqu::MemOutStream errorDescr(allocator); + for (bsl::vector::const_iterator cit = + arguments.d_seqNum.begin(); + cit != arguments.d_seqNum.end(); + ++cit) { + seqNum.fromString(errorDescr, *cit); + d_seqNum.push_back(seqNum); + } + } } void Parameters::validateQueueNames(bslma::Allocator* allocator) const diff --git a/src/applications/bmqstoragetool/m_bmqstoragetool_parameters.h b/src/applications/bmqstoragetool/m_bmqstoragetool_parameters.h index 25ba703c0..d9a465413 100644 --- a/src/applications/bmqstoragetool/m_bmqstoragetool_parameters.h +++ b/src/applications/bmqstoragetool/m_bmqstoragetool_parameters.h @@ -26,6 +26,7 @@ // command-line parameters for the 'bmqtool' program. // bmqstoragetool +#include #include // MQB @@ -53,75 +54,122 @@ namespace m_bmqstoragetool { struct CommandLineArguments { // PUBLIC DATA + /// Filter messages by minimum timestamp bsls::Types::Int64 d_timestampGt; - // Filter messages by minimum timestamp + /// Filter messages by maximum timestamp bsls::Types::Int64 d_timestampLt; - // Filter messages by maximum timestamp + /// Filter messages by minimum record composite sequence number + bsl::string d_seqNumGt; + /// Filter messages by maximum record composite sequence number + bsl::string d_seqNumLt; + /// Filter messages by minimum record offset + bsls::Types::Int64 d_offsetGt; + /// Filter messages by maximum record offset + bsls::Types::Int64 d_offsetLt; + /// Path to find all files from bsl::string d_journalPath; - // Path to find all files from + /// Path to read journal files from bsl::string d_journalFile; - // Path to read journal files from + /// Path to read data files from bsl::string d_dataFile; - // Path to read data files from + /// Path to read CSL files from bsl::string d_cslFile; - // Path to read CSL files from + /// Filter messages by message guids bsl::vector d_guid; - // Filter messages by message guids + /// Filter messages by record composite sequence numbers + bsl::vector d_seqNum; + /// Filter messages by record offsets + bsl::vector d_offset; + /// Filter messages by queue keys bsl::vector d_queueKey; - // Filter messages by queue keys + /// Filter messages by queue names bsl::vector d_queueName; - // Filter messages by queue names + /// Limit number of bytes to int d_dumpLimit; - // Limit number of bytes to + /// Print message details bool d_details; - // Print message details + /// Print message payload bool d_dumpPayload; - // Print message payload + /// Print summary of messages bool d_summary; - // Print summary of messages + /// Show only outstanding messages (not deleted) bool d_outstanding; - // Show only outstanding messages (not deleted) + /// Show only messages, confirmed by all the appId's bool d_confirmed; - // Show only messages, confirmed by all the appId's + /// Show only messages, confirmed by some of the appId's bool d_partiallyConfirmed; - // Show only messages, confirmed by some of the appId's // CREATORS explicit CommandLineArguments(bslma::Allocator* allocator = 0); // MANIPULATORS /// Validate the consistency of all settings. - bool validate(bsl::string* error); + bool validate(bsl::string* error, bslma::Allocator* allocator = 0); }; struct Parameters { + // PUBLIC TYPES + + /// VST representing search range parameters + struct Range { + // PUBLIC TYPES + enum Type { + e_NONE = 0, + e_TIMESTAMP = 1, + e_SEQUENCE_NUM = 2, + e_OFFSET = 3 + }; + + // PUBLIC DATA + /// Range type + Type d_type; + /// Filter messages greater than timestamp value + bsls::Types::Uint64 d_timestampGt; + /// Filter messages less than timestamp value + bsls::Types::Uint64 d_timestampLt; + /// Filter messages greater than offset value + bsls::Types::Uint64 d_offsetGt; + /// Filter messages less than offset value + bsls::Types::Uint64 d_offsetLt; + /// Filter messages greater than sequence number + CompositeSequenceNumber d_seqNumGt; + /// Filter messages less than sequence number + CompositeSequenceNumber d_seqNumLt; + + // CREATORS + /// Default constructor + explicit Range(); + }; + // PUBLIC DATA + /// Queue map containing uri to key and key to info mappings QueueMap d_queueMap; - // Queue map containing uri to key and key to info mappings - bsls::Types::Int64 d_timestampGt; - // Filter messages by minimum timestamp - bsls::Types::Int64 d_timestampLt; - // Filter messages by maximum timestamp + /// Range parameters for filtering + Range d_range; + /// Filter messages by message guids bsl::vector d_guid; - // Filter messages by message guids + /// Filter messages by message sequence number + bsl::vector d_seqNum; + /// Filter messages by message offsets + bsl::vector d_offset; + /// Filter messages by queue keys bsl::vector d_queueKey; - // Filter messages by queue keys + /// Filter messages by queue names bsl::vector d_queueName; - // Filter messages by queue names + /// Limit number of bytes to dump unsigned int d_dumpLimit; - // Limit number of bytes to dump + /// Print message details bool d_details; - // Print message details + /// Print message payload bool d_dumpPayload; - // Print message payload + /// Print summary of messages bool d_summary; - // Print summary of messages + /// Show only outstanding messages (not deleted) bool d_outstanding; - // Show only outstanding messages (not deleted) + /// Show only messages, confirmed by all the appId's bool d_confirmed; - // Show only messages, confirmed by all the appId's + /// Show only messages, confirmed by some of the appId's bool d_partiallyConfirmed; - // Show only messages, confirmed by some of the appId's // CREATORS /// Default constructor diff --git a/src/applications/bmqstoragetool/m_bmqstoragetool_searchresult.cpp b/src/applications/bmqstoragetool/m_bmqstoragetool_searchresult.cpp index 039d9fcb7..db71719fb 100644 --- a/src/applications/bmqstoragetool/m_bmqstoragetool_searchresult.cpp +++ b/src/applications/bmqstoragetool/m_bmqstoragetool_searchresult.cpp @@ -14,6 +14,7 @@ // limitations under the License. // bmqstoragetool +#include "m_bmqstoragetool_compositesequencenumber.h" #include // MQB @@ -241,18 +242,24 @@ void SearchResultDecorator::outputResult(const GuidsList& guidFilter) d_searchResult->outputResult(guidFilter); } +bool SearchResultDecorator::hasCache() const +{ + return d_searchResult->hasCache(); +} + // ==================================== // class SearchResultTimestampDecorator // ==================================== -bool SearchResultTimestampDecorator::stop(bsls::Types::Uint64 timestamp) const +bool SearchResultTimestampDecorator::stop( + const bsls::Types::Uint64 timestamp) const { return timestamp >= d_timestampLt && !SearchResultDecorator::hasCache(); } SearchResultTimestampDecorator::SearchResultTimestampDecorator( const bsl::shared_ptr& component, - bsls::Types::Uint64 timestampLt, + const bsls::Types::Uint64 timestampLt, bslma::Allocator* allocator) : SearchResultDecorator(component, allocator) , d_timestampLt(timestampLt) @@ -292,6 +299,115 @@ bool SearchResultTimestampDecorator::processDeletionRecord( stop(record.header().timestamp()); } +// ================================= +// class SearchResultOffsetDecorator +// ================================= + +bool SearchResultOffsetDecorator::stop(const bsls::Types::Uint64 offset) const +{ + return offset >= d_offsetLt && !SearchResultDecorator::hasCache(); +} + +SearchResultOffsetDecorator::SearchResultOffsetDecorator( + const bsl::shared_ptr& component, + const bsls::Types::Uint64 offsetLt, + bslma::Allocator* allocator) +: SearchResultDecorator(component, allocator) +, d_offsetLt(offsetLt) +{ + // NOTHING +} + +bool SearchResultOffsetDecorator::processMessageRecord( + const mqbs::MessageRecord& record, + bsls::Types::Uint64 recordIndex, + bsls::Types::Uint64 recordOffset) +{ + return SearchResultDecorator::processMessageRecord(record, + recordIndex, + recordOffset) || + stop(recordOffset); +} + +bool SearchResultOffsetDecorator::processConfirmRecord( + const mqbs::ConfirmRecord& record, + bsls::Types::Uint64 recordIndex, + bsls::Types::Uint64 recordOffset) +{ + return SearchResultDecorator::processConfirmRecord(record, + recordIndex, + recordOffset) || + stop(recordOffset); +} + +bool SearchResultOffsetDecorator::processDeletionRecord( + const mqbs::DeletionRecord& record, + bsls::Types::Uint64 recordIndex, + bsls::Types::Uint64 recordOffset) +{ + return SearchResultDecorator::processDeletionRecord(record, + recordIndex, + recordOffset) || + stop(recordOffset); +} + +// ========================================= +// class SearchResultSequenceNumberDecorator +// ========================================= + +bool SearchResultSequenceNumberDecorator::stop( + const CompositeSequenceNumber& sequenceNumber) const +{ + return sequenceNumberLt <= sequenceNumber && + !SearchResultDecorator::hasCache(); +} + +SearchResultSequenceNumberDecorator::SearchResultSequenceNumberDecorator( + const bsl::shared_ptr& component, + const CompositeSequenceNumber& sequenceNumberLt, + bslma::Allocator* allocator) +: SearchResultDecorator(component, allocator) +, sequenceNumberLt(sequenceNumberLt) +{ + // NOTHING +} + +bool SearchResultSequenceNumberDecorator::processMessageRecord( + const mqbs::MessageRecord& record, + bsls::Types::Uint64 recordIndex, + bsls::Types::Uint64 recordOffset) +{ + return SearchResultDecorator::processMessageRecord(record, + recordIndex, + recordOffset) || + stop(CompositeSequenceNumber(record.header().primaryLeaseId(), + record.header().sequenceNumber())); +} + +bool SearchResultSequenceNumberDecorator::processConfirmRecord( + const mqbs::ConfirmRecord& record, + bsls::Types::Uint64 recordIndex, + bsls::Types::Uint64 recordOffset) +{ + return SearchResultDecorator::processConfirmRecord(record, + recordIndex, + recordOffset) || + stop(CompositeSequenceNumber(record.header().primaryLeaseId(), + record.header().sequenceNumber())); +} + +bool SearchResultSequenceNumberDecorator::processDeletionRecord( + const mqbs::DeletionRecord& record, + bsls::Types::Uint64 recordIndex, + bsls::Types::Uint64 recordOffset) +{ + return SearchResultDecorator::processDeletionRecord(record, + recordIndex, + recordOffset) || + stop(CompositeSequenceNumber(record.header().primaryLeaseId(), + record.header().sequenceNumber())); +} + // ======================= // class SearchShortResult // ======================= @@ -784,6 +900,138 @@ void SearchGuidDecorator::outputResult() } } +// =========================== +// class SearchOffsetDecorator +// =========================== +SearchOffsetDecorator::SearchOffsetDecorator( + const bsl::shared_ptr& component, + const bsl::vector& offsets, + bsl::ostream& ostream, + bool withDetails, + bslma::Allocator* allocator) +: SearchResultDecorator(component, allocator) +, d_offsets(offsets, allocator) +, d_ostream(ostream) +, d_withDetails(withDetails) +{ + // NOTHING +} + +bool SearchOffsetDecorator::processMessageRecord( + const mqbs::MessageRecord& record, + bsls::Types::Uint64 recordIndex, + bsls::Types::Uint64 recordOffset) +{ + bsl::vector::const_iterator it = + bsl::find(d_offsets.cbegin(), d_offsets.cend(), recordOffset); + if (it != d_offsets.cend()) { + SearchResultDecorator::processMessageRecord(record, + recordIndex, + recordOffset); + // Remove processed offset. + d_offsets.erase(it); + } + + // return true (stop search) if no detail is needed and d_offsets is empty. + return (!d_withDetails && d_offsets.empty()); +} + +bool SearchOffsetDecorator::processDeletionRecord( + const mqbs::DeletionRecord& record, + bsls::Types::Uint64 recordIndex, + bsls::Types::Uint64 recordOffset) +{ + SearchResultDecorator::processDeletionRecord(record, + recordIndex, + recordOffset); + // return true (stop search) when details needed and search is done + // (d_offsets is empty). + return (d_withDetails && d_offsets.empty()); +} + +void SearchOffsetDecorator::outputResult() +{ + SearchResultDecorator::outputResult(); + // Print non found offsets + if (!d_offsets.empty()) { + d_ostream << '\n' + << "The following " << d_offsets.size() + << " offset(s) not found:" << '\n'; + bsl::vector::const_iterator it = + d_offsets.cbegin(); + for (; it != d_offsets.cend(); ++it) { + d_ostream << *it << '\n'; + } + } +} + +// =================================== +// class SearchSequenceNumberDecorator +// =================================== +SearchSequenceNumberDecorator::SearchSequenceNumberDecorator( + const bsl::shared_ptr& component, + const bsl::vector& seqNums, + bsl::ostream& ostream, + bool withDetails, + bslma::Allocator* allocator) +: SearchResultDecorator(component, allocator) +, d_seqNums(seqNums, allocator) +, d_ostream(ostream) +, d_withDetails(withDetails) +{ + // NOTHING +} + +bool SearchSequenceNumberDecorator::processMessageRecord( + const mqbs::MessageRecord& record, + bsls::Types::Uint64 recordIndex, + bsls::Types::Uint64 recordOffset) +{ + CompositeSequenceNumber seqNum(record.header().primaryLeaseId(), + record.header().sequenceNumber()); + bsl::vector::const_iterator it = + bsl::find(d_seqNums.cbegin(), d_seqNums.cend(), seqNum); + if (it != d_seqNums.cend()) { + SearchResultDecorator::processMessageRecord(record, + recordIndex, + recordOffset); + // Remove processed sequence number. + d_seqNums.erase(it); + } + + // return true (stop search) if no detail is needed and d_seqNums is empty. + return (!d_withDetails && d_seqNums.empty()); +} + +bool SearchSequenceNumberDecorator::processDeletionRecord( + const mqbs::DeletionRecord& record, + bsls::Types::Uint64 recordIndex, + bsls::Types::Uint64 recordOffset) +{ + SearchResultDecorator::processDeletionRecord(record, + recordIndex, + recordOffset); + // return true (stop search) when details needed and search is done + // (d_seqNums is empty). + return (d_withDetails && d_seqNums.empty()); +} + +void SearchSequenceNumberDecorator::outputResult() +{ + SearchResultDecorator::outputResult(); + // Print non found offsets + if (!d_seqNums.empty()) { + d_ostream << '\n' + << "The following " << d_seqNums.size() + << " sequence number(s) not found:" << '\n'; + bsl::vector::const_iterator it = + d_seqNums.cbegin(); + for (; it != d_seqNums.cend(); ++it) { + d_ostream << *it << '\n'; + } + } +} + // ====================== // class SummaryProcessor // ====================== diff --git a/src/applications/bmqstoragetool/m_bmqstoragetool_searchresult.h b/src/applications/bmqstoragetool/m_bmqstoragetool_searchresult.h index 16da30901..eaf111474 100644 --- a/src/applications/bmqstoragetool/m_bmqstoragetool_searchresult.h +++ b/src/applications/bmqstoragetool/m_bmqstoragetool_searchresult.h @@ -28,6 +28,10 @@ // search processor. // m_bmqstoragetool::SearchResultTimestampDecorator: // provides decorator to handle timestamps. +// m_bmqstoragetool::SearchResultOffsetDecorator: +// provides decorator to handle offsets. +// m_bmqstoragetool::SearchResultSequenceNumberDecorator: +// provides decorator to handle composite sequence numbers. // m_bmqstoragetool::SearchAllDecorator: provides decorator to handle all // messages. // m_bmqstoragetool::SearchOutstandingDecorator: provides decorator @@ -36,6 +40,10 @@ // handle partially confirmed messages. // m_bmqstoragetool::SearchGuidDecorator: // provides decorator to handle search of given GUIDs. +// m_bmqstoragetool::SearchOffsetDecorator: +// provides decorator to handle search of given offsets. +// m_bmqstoragetool::SearchSequenceNumberDecorator: +// provides decorator to handle search of given composite sequence numbers. // m_bmqstoragetool::SummaryProcessor: provides logic to collect summary of // journal file. // @@ -43,6 +51,7 @@ // a logic of search and output results. // bmqstoragetool +#include "m_bmqstoragetool_compositesequencenumber.h" #include #include #include @@ -77,11 +86,6 @@ class SearchResult { GuidsMap; // Hash map of message guids to iterators of GuidsList. - virtual bool hasCache() const { return false; } - // Return `false` if all required data is processed, e.g. all given GUIDs - // are output and search could be stopped. Return `true` to indicate that - // there is incomplete data. - public: // CREATORS @@ -109,6 +113,11 @@ class SearchResult { virtual void outputResult() = 0; /// Output result of a search filtered by the specified GUIDs filter. virtual void outputResult(const GuidsList& guidFilter) = 0; + + /// Return `false` if all required data is processed, e.g. all given GUIDs + /// are output and search could be stopped. Return `true` to indicate that + /// there is incomplete data. + virtual bool hasCache() const { return false; } }; // ======================= @@ -159,13 +168,6 @@ class SearchShortResult : public SearchResult { void outputGuidData(const GuidData& guidData); // Output result in short format (only GUIDs). - // PRIVATE ACCESSORS - - bool hasCache() const BSLS_KEYWORD_OVERRIDE; - // Return 'false' if all required data is processed, e.g. all given GUIDs - // are output and search could be stopped. Return 'true' to indicate that - // there is incomplete data. - public: // CREATORS @@ -206,6 +208,13 @@ class SearchShortResult : public SearchResult { void outputResult() BSLS_KEYWORD_OVERRIDE; /// Output result of a search filtered by the specified GUIDs filter. void outputResult(const GuidsList& guidFilter) BSLS_KEYWORD_OVERRIDE; + + // ACCESSORS + + /// Return 'false' if all required data is processed, e.g. all given GUIDs + /// are output and search could be stopped. Return 'true' to indicate that + /// there is incomplete data. + bool hasCache() const BSLS_KEYWORD_OVERRIDE; }; // ======================== @@ -265,13 +274,6 @@ class SearchDetailResult : public SearchResult { void outputMessageDetails(const MessageDetails& messageDetails); // Output message details with the specified 'messageDetails'. - // PRIVATE ACCESSORS - - bool hasCache() const BSLS_KEYWORD_OVERRIDE; - // Return 'false' if all required data is processed, e.g. all given GUIDs - // are output and search could be stopped. Return 'true' to indicate that - // there is incomplete data. - public: // CREATORS @@ -309,6 +311,13 @@ class SearchDetailResult : public SearchResult { void outputResult() BSLS_KEYWORD_OVERRIDE; /// Output result of a search filtered by the specified GUIDs filter. void outputResult(const GuidsList& guidFilter) BSLS_KEYWORD_OVERRIDE; + + // ACCESSORS + + /// Return 'false' if all required data is processed, e.g. all given GUIDs + /// are output and search could be stopped. Return 'true' to indicate that + /// there is incomplete data. + bool hasCache() const BSLS_KEYWORD_OVERRIDE; }; // =========================== @@ -355,6 +364,13 @@ class SearchResultDecorator : public SearchResult { void outputResult() BSLS_KEYWORD_OVERRIDE; /// Output result of a search filtered by the specified GUIDs filter. void outputResult(const GuidsList& guidFilter) BSLS_KEYWORD_OVERRIDE; + + // ACCESSORS + + /// Return 'false' if all required data is processed, e.g. all given GUIDs + /// are output and search could be stopped. Return 'true' to indicate that + /// there is incomplete data. + bool hasCache() const BSLS_KEYWORD_OVERRIDE; }; // ==================================== @@ -369,9 +385,9 @@ class SearchResultTimestampDecorator : public SearchResultDecorator { // ACCESSORS - bool stop(bsls::Types::Uint64 timestamp) const; - // Return 'true' if the specified 'timestamp' is greated than - // 'd_timestampLt' and internal cache is empty. + /// Return 'true' if the specified 'timestamp' is greater than + /// 'd_timestampLt' and internal cache is empty. + bool stop(const bsls::Types::Uint64 timestamp) const; public: // CREATORS @@ -405,6 +421,101 @@ class SearchResultTimestampDecorator : public SearchResultDecorator { BSLS_KEYWORD_OVERRIDE; }; +// ================================= +// class SearchResultOffsetDecorator +// ================================= + +/// This class provides decorator to handle offsets. +class SearchResultOffsetDecorator : public SearchResultDecorator { + private: + /// Higher bound offset. + const bsls::Types::Uint64 d_offsetLt; + + // ACCESSORS + + /// Return 'true' if the specified 'offset' is greater than + /// 'd_offsetLt' and internal cache is empty. + bool stop(const bsls::Types::Uint64 offset) const; + + public: + // CREATORS + + /// Constructor using the specified `component`, `offsetLt` and + /// `allocator`. + SearchResultOffsetDecorator(const bsl::shared_ptr& component, + const bsls::Types::Uint64 offsetLt, + bslma::Allocator* allocator); + + // MANIPULATORS + + /// Process `message` record with the specified `record`, `recordIndex` and + /// `recordOffset`. + bool processMessageRecord(const mqbs::MessageRecord& record, + bsls::Types::Uint64 recordIndex, + bsls::Types::Uint64 recordOffset) + BSLS_KEYWORD_OVERRIDE; + /// Process `confirm` record with the specified `record`, `recordIndex` and + /// `recordOffset`. + bool processConfirmRecord(const mqbs::ConfirmRecord& record, + bsls::Types::Uint64 recordIndex, + bsls::Types::Uint64 recordOffset) + BSLS_KEYWORD_OVERRIDE; + /// Process `deletion` record with the specified `record`, `recordIndex` + /// and `recordOffset`. + bool processDeletionRecord(const mqbs::DeletionRecord& record, + bsls::Types::Uint64 recordIndex, + bsls::Types::Uint64 recordOffset) + BSLS_KEYWORD_OVERRIDE; +}; + +// ========================================= +// class SearchResultSequenceNumberDecorator +// ========================================= + +/// This class provides decorator to handle composite sequence numbers. +class SearchResultSequenceNumberDecorator : public SearchResultDecorator { + private: + /// Higher bound sequence number. + const CompositeSequenceNumber sequenceNumberLt; + + // ACCESSORS + + /// Return 'true' if the specified 'sequenceNumber' is greater than + /// 'sequenceNumberLt' and internal cache is empty. + bool stop(const CompositeSequenceNumber& sequenceNumber) const; + + public: + // CREATORS + + /// Constructor using the specified `component`, `seqNumberLt` and + /// `allocator`. + SearchResultSequenceNumberDecorator( + const bsl::shared_ptr& component, + const CompositeSequenceNumber& seqNumberLt, + bslma::Allocator* allocator); + + // MANIPULATORS + + /// Process `message` record with the specified `record`, `recordIndex` and + /// `recordOffset`. + bool processMessageRecord(const mqbs::MessageRecord& record, + bsls::Types::Uint64 recordIndex, + bsls::Types::Uint64 recordOffset) + BSLS_KEYWORD_OVERRIDE; + /// Process `confirm` record with the specified `record`, `recordIndex` and + /// `recordOffset`. + bool processConfirmRecord(const mqbs::ConfirmRecord& record, + bsls::Types::Uint64 recordIndex, + bsls::Types::Uint64 recordOffset) + BSLS_KEYWORD_OVERRIDE; + /// Process `deletion` record with the specified `record`, `recordIndex` + /// and `recordOffset`. + bool processDeletionRecord(const mqbs::DeletionRecord& record, + bsls::Types::Uint64 recordIndex, + bsls::Types::Uint64 recordOffset) + BSLS_KEYWORD_OVERRIDE; +}; + // ======================== // class SearchAllDecorator // ======================== @@ -577,6 +688,96 @@ class SearchGuidDecorator : public SearchResultDecorator { void outputResult() BSLS_KEYWORD_OVERRIDE; }; +// =========================== +// class SearchOffsetDecorator +// =========================== + +/// This class provides decorator to handle search of given offsets. +class SearchOffsetDecorator : public SearchResultDecorator { + private: + // PRIVATE DATA + /// List of offsets to search for. + bsl::vector d_offsets; + /// Reference to output stream. + bsl::ostream& d_ostream; + // If 'true', output detailed result, output short one otherwise. + bool d_withDetails; + + public: + // CREATORS + + /// Constructor using the specified `component`, `offsets`, `ostream`, + /// `withDetails` and `allocator`. + SearchOffsetDecorator(const bsl::shared_ptr& component, + const bsl::vector& offsets, + bsl::ostream& ostream, + bool withDetails, + bslma::Allocator* allocator); + + // MANIPULATORS + + /// Process `message` record with the specified `record`, `recordIndex` and + /// `recordOffset`. + bool processMessageRecord(const mqbs::MessageRecord& record, + bsls::Types::Uint64 recordIndex, + bsls::Types::Uint64 recordOffset) + BSLS_KEYWORD_OVERRIDE; + /// Process `deletion` record with the specified `record`, `recordIndex` + /// and `recordOffset`. + bool processDeletionRecord(const mqbs::DeletionRecord& record, + bsls::Types::Uint64 recordIndex, + bsls::Types::Uint64 recordOffset) + BSLS_KEYWORD_OVERRIDE; + /// Output result of a search. + void outputResult() BSLS_KEYWORD_OVERRIDE; +}; + +// =================================== +// class SearchSequenceNumberDecorator +// =================================== + +/// This class provides decorator to handle search of given composite sequence +/// numbers. +class SearchSequenceNumberDecorator : public SearchResultDecorator { + private: + // PRIVATE DATA + bsl::vector d_seqNums; + // List of composite sequence numbers to search for. + bsl::ostream& d_ostream; + // Reference to output stream. + bool d_withDetails; + // If 'true', output detailed result, output short one otherwise. + + public: + // CREATORS + + /// Constructor using the specified `component`, `seqNums`, `ostream`, + /// `withDetails` and `allocator`. + SearchSequenceNumberDecorator( + const bsl::shared_ptr& component, + const bsl::vector& seqNums, + bsl::ostream& ostream, + bool withDetails, + bslma::Allocator* allocator); + + // MANIPULATORS + + /// Process `message` record with the specified `record`, `recordIndex` and + /// `recordOffset`. + bool processMessageRecord(const mqbs::MessageRecord& record, + bsls::Types::Uint64 recordIndex, + bsls::Types::Uint64 recordOffset) + BSLS_KEYWORD_OVERRIDE; + /// Process `deletion` record with the specified `record`, `recordIndex` + /// and `recordOffset`. + bool processDeletionRecord(const mqbs::DeletionRecord& record, + bsls::Types::Uint64 recordIndex, + bsls::Types::Uint64 recordOffset) + BSLS_KEYWORD_OVERRIDE; + /// Output result of a search. + void outputResult() BSLS_KEYWORD_OVERRIDE; +}; + // ====================== // class SummaryProcessor // ====================== diff --git a/src/applications/bmqstoragetool/m_bmqstoragetool_searchresultfactory.cpp b/src/applications/bmqstoragetool/m_bmqstoragetool_searchresultfactory.cpp index 04fe3127e..3caec9eb1 100644 --- a/src/applications/bmqstoragetool/m_bmqstoragetool_searchresultfactory.cpp +++ b/src/applications/bmqstoragetool/m_bmqstoragetool_searchresultfactory.cpp @@ -14,6 +14,7 @@ // limitations under the License. // bmqstoragetool +#include "m_bmqstoragetool_parameters.h" #include namespace BloombergLP { @@ -91,6 +92,25 @@ bsl::shared_ptr SearchResultFactory::createSearchResult( alloc), alloc); } + else if (!params->d_seqNum.empty()) { + // Search offsets + searchResult.reset(new (*alloc) + SearchSequenceNumberDecorator(searchResult, + params->d_seqNum, + ostream, + details, + alloc), + alloc); + } + else if (!params->d_offset.empty()) { + // Search composite sequence numbers + searchResult.reset(new (*alloc) SearchOffsetDecorator(searchResult, + params->d_offset, + ostream, + details, + alloc), + alloc); + } else if (params->d_summary) { // Summary searchResult.reset( @@ -122,13 +142,35 @@ bsl::shared_ptr SearchResultFactory::createSearchResult( alloc); } - // Add TimestampDecorator if 'timestampLt' is given. - if (params->d_timestampLt > 0) { - searchResult.reset( - new (*alloc) SearchResultTimestampDecorator(searchResult, - params->d_timestampLt, - alloc), - alloc); + // Add TimestampDecorator if 'timestampLt' is given and value type is + // `e_TIMESTAMP`. + if (params->d_range.d_type == Parameters::Range::e_TIMESTAMP && + params->d_range.d_timestampLt > 0) { + searchResult.reset(new (*alloc) SearchResultTimestampDecorator( + searchResult, + params->d_range.d_timestampLt, + alloc), + alloc); + } + else if (params->d_range.d_type == Parameters::Range::e_OFFSET && + params->d_range.d_offsetLt > 0) { + // Add OffsetDecorator if 'offsetLt' is given and value type is + // `e_OFFSET`. + searchResult.reset(new (*alloc) SearchResultOffsetDecorator( + searchResult, + params->d_range.d_offsetLt, + alloc), + alloc); + } + else if (params->d_range.d_type == Parameters::Range::e_SEQUENCE_NUM && + params->d_range.d_seqNumLt.isSet()) { + // Add SequenceNumberDecorator if 'seqNumLt' is given and value type is + // `e_SEQUENCE_NUM`. + searchResult.reset(new (*alloc) SearchResultSequenceNumberDecorator( + searchResult, + params->d_range.d_seqNumLt, + alloc), + alloc); } BSLS_ASSERT(searchResult); diff --git a/src/applications/bmqstoragetool/package/bmqstoragetool.mem b/src/applications/bmqstoragetool/package/bmqstoragetool.mem index 497ae4f23..b6ed97056 100644 --- a/src/applications/bmqstoragetool/package/bmqstoragetool.mem +++ b/src/applications/bmqstoragetool/package/bmqstoragetool.mem @@ -1,5 +1,6 @@ m_bmqstoragetool_commandprocessor m_bmqstoragetool_commandprocessorfactory +m_bmqstoragetool_compositesequencenumber m_bmqstoragetool_filemanager m_bmqstoragetool_filemanagermock m_bmqstoragetool_filters diff --git a/src/groups/mqb/mqba/mqba_clientsession.cpp b/src/groups/mqb/mqba/mqba_clientsession.cpp index 636eec316..cc8988a65 100644 --- a/src/groups/mqb/mqba/mqba_clientsession.cpp +++ b/src/groups/mqb/mqba/mqba_clientsession.cpp @@ -584,9 +584,11 @@ void ClientSession::sendAck(bmqt::AckResult::Enum status, // If queue is found, report locally generated NACK if (isSelfGenerated) { - queueState->d_handle_p->queue()->stats()->onEvent( - mqbstat::QueueStatsDomain::EventType::e_NACK, - 1); + queueState->d_handle_p->queue() + ->stats() + ->onEvent( + + 1); } } @@ -1658,9 +1660,9 @@ void ClientSession::onAckEvent(const mqbi::DispatcherAckEvent& event) // Calculate time delta between PUT and ACK const bsls::Types::Int64 timeDelta = bmqsys::Time::highResolutionTimer() - cit->second.d_timeStamp; - queue->stats()->onEvent( - mqbstat::QueueStatsDomain::EventType::e_ACK_TIME, - timeDelta); + queue->stats() + ->onEvent( + timeDelta); if (!d_isClientGeneratingGUIDs) { // Legacy client. @@ -2693,8 +2695,8 @@ ClientSession::ClientSession( this, bdlf::PlaceHolders::_1)); // type - mqbstat::BrokerStats::instance().onEvent( - mqbstat::BrokerStats::EventType::e_CLIENT_CREATED); + mqbstat::BrokerStats::instance() + .onEvent(); BALL_LOG_INFO << description() << ": created " << "[dispatcherProcessor: " << processor @@ -2714,8 +2716,8 @@ ClientSession::~ClientSession() BALL_LOG_INFO << description() << ": destructor"; - mqbstat::BrokerStats::instance().onEvent( - mqbstat::BrokerStats::EventType::e_CLIENT_DESTROYED); + mqbstat::BrokerStats::instance() + .onEvent(); // Unregister from the dispatcher dispatcher()->unregisterClient(this); diff --git a/src/groups/mqb/mqbblp/mqbblp_cluster.cpp b/src/groups/mqb/mqbblp/mqbblp_cluster.cpp index 059227215..a8b82702d 100644 --- a/src/groups/mqb/mqbblp/mqbblp_cluster.cpp +++ b/src/groups/mqb/mqbblp/mqbblp_cluster.cpp @@ -418,9 +418,9 @@ void Cluster::sendAck(bmqt::AckResult::Enum status, // If queue exists, report self generated NACK if (isSelfGenerated) { - it->second.d_handle_p->queue()->stats()->onEvent( - mqbstat::QueueStatsDomain::EventType::e_NACK, - 1); + it->second.d_handle_p->queue() + ->stats() + ->onEvent(1); } } else if (!isSelfGenerated) { @@ -467,9 +467,9 @@ void Cluster::sendAck(bmqt::AckResult::Enum status, cit->second.d_subQueueInfosMap.findBySubIdSafe( bmqp::QueueId::k_DEFAULT_SUBQUEUE_ID); if (subQueueCiter != cit->second.d_subQueueInfosMap.end()) { - subQueueCiter->value().d_clientStats->onEvent( - mqbstat::ClusterNodeStats::EventType::e_ACK, - 1); + subQueueCiter->value() + .d_clientStats + ->onEvent(1); } // In the case of Strong Consistency, a Receipt can arrive and trigger // an ACK after Producer closes subStream. @@ -548,7 +548,7 @@ void Cluster::generateNack(bmqt::AckResult::Enum status, options); // Report locally generated NACK - queue->stats()->onEvent(mqbstat::QueueStatsDomain::EventType::e_NACK, 1); + queue->stats()->onEvent(1); bmqu::MemOutStream os; os << description() << ": Failed to relay PUT message " @@ -1206,9 +1206,11 @@ void Cluster::onPutEvent(const mqbi::DispatcherPutEvent& event) queueState.d_subQueueInfosMap.findBySubId( bmqp::QueueId::k_DEFAULT_SUBQUEUE_ID); - subQueueCiter->value().d_clientStats->onEvent( - mqbstat::ClusterNodeStats::EventType::e_PUT, - appDataSp->length()); + subQueueCiter->value() + .d_clientStats + ->onEvent( + + appDataSp->length()); // TBD: groupId: Similar to 'appDataSp' above, load 'optionsSp' here, // using something like PutMessageIterator::loadOptions(). @@ -1644,9 +1646,9 @@ Cluster::validateMessage(mqbi::QueueHandle** queueHandle, if (eventType == bmqp::EventType::e_CONFIRM) { // Update client stats - subQueueIt->value().d_clientStats->onEvent( - mqbstat::ClusterNodeStats::EventType::e_CONFIRM, - 1); + subQueueIt->value() + .d_clientStats + ->onEvent(1); } return ValidationResult::k_SUCCESS; @@ -1912,9 +1914,10 @@ void Cluster::onPushEvent(const mqbi::DispatcherPushEvent& event) queueState.d_subQueueInfosMap.findBySubscriptionId( event.subQueueInfos()[i].id()); - subQueueCiter->value().d_clientStats->onEvent( - mqbstat::ClusterNodeStats::EventType::e_PUSH, - event.blob() ? event.blob()->length() : 0); + subQueueCiter->value() + .d_clientStats + ->onEvent( + event.blob() ? event.blob()->length() : 0); } bmqt::GenericResult::Enum rc = bmqt::GenericResult::e_SUCCESS; diff --git a/src/groups/mqb/mqbblp/mqbblp_domain.cpp b/src/groups/mqb/mqbblp/mqbblp_domain.cpp index 3fd90fb12..d2d5583ef 100644 --- a/src/groups/mqb/mqbblp/mqbblp_domain.cpp +++ b/src/groups/mqb/mqbblp/mqbblp_domain.cpp @@ -430,10 +430,10 @@ int Domain::configure(bsl::ostream& errorDescription, d_capacityMeter.setLimits(limits.messages(), limits.bytes()) .setWatermarkThresholds(limits.messagesWatermarkRatio(), limits.bytesWatermarkRatio()); - d_domainsStats.onEvent(mqbstat::DomainStats::EventType::e_CFG_MSGS, - limits.messages()); - d_domainsStats.onEvent(mqbstat::DomainStats::EventType::e_CFG_BYTES, - limits.bytes()); + d_domainsStats.onEvent( + limits.messages()); + d_domainsStats.onEvent( + limits.bytes()); if (isReconfigure) { BSLS_ASSERT_OPT(oldConfig.has_value()); diff --git a/src/groups/mqb/mqbblp/mqbblp_localqueue.cpp b/src/groups/mqb/mqbblp/mqbblp_localqueue.cpp index 208ad4205..d10ecaafe 100644 --- a/src/groups/mqb/mqbblp/mqbblp_localqueue.cpp +++ b/src/groups/mqb/mqbblp/mqbblp_localqueue.cpp @@ -173,17 +173,17 @@ int LocalQueue::configure(bsl::ostream& errorDescription, bool isReconfigure) d_state_p->uri(), d_state_p->partitionId()); - d_state_p->stats()->onEvent( - mqbstat::QueueStatsDomain::EventType::e_CHANGE_ROLE, - mqbstat::QueueStatsDomain::Role::e_PRIMARY); + d_state_p->stats() + ->onEvent( + mqbstat::QueueStatsDomain::Role::e_PRIMARY); - d_state_p->stats()->onEvent( - mqbstat::QueueStatsDomain::EventType::e_CFG_MSGS, - domainCfg.storage().queueLimits().messages()); + d_state_p->stats() + ->onEvent( + domainCfg.storage().queueLimits().messages()); - d_state_p->stats()->onEvent( - mqbstat::QueueStatsDomain::EventType::e_CFG_BYTES, - domainCfg.storage().queueLimits().bytes()); + d_state_p->stats() + ->onEvent( + domainCfg.storage().queueLimits().bytes()); if (isReconfigure) { if (domainCfg.mode().isFanoutValue()) { @@ -482,9 +482,9 @@ void LocalQueue::postMessage(const bmqp::PutHeader& putHeader, // Calculate time delta between PUT and ACK const bsls::Types::Int64 timeDelta = bmqsys::Time::highResolutionTimer() - timePoint; - d_state_p->stats()->onEvent( - mqbstat::QueueStatsDomain::EventType::e_ACK_TIME, - timeDelta); + d_state_p->stats() + ->onEvent( + timeDelta); if (res != mqbi::StorageResult::e_SUCCESS || doAck) { bmqp::AckMessage ackMessage; ackMessage @@ -509,9 +509,9 @@ void LocalQueue::postMessage(const bmqp::PutHeader& putHeader, // flushed (which occurs in 'flush' routine). In no case should // 'afterNewMessage' be called here. - d_state_p->stats()->onEvent( - mqbstat::QueueStatsDomain::EventType::e_PUT, - appData->length()); + d_state_p->stats() + ->onEvent( + appData->length()); } else { BSLS_PERFORMANCEHINT_UNLIKELY_HINT; @@ -525,9 +525,8 @@ void LocalQueue::postMessage(const bmqp::PutHeader& putHeader, } } else { - d_state_p->stats()->onEvent( - mqbstat::QueueStatsDomain::EventType::e_NACK, - 1); + d_state_p->stats() + ->onEvent(1); } } } @@ -548,9 +547,8 @@ void LocalQueue::onReceipt(const bmqt::MessageGUID& msgGUID, const bsls::Types::Int64 timeDelta = bmqsys::Time::highResolutionTimer() - arrivalTimepoint; - d_state_p->stats()->onEvent( - mqbstat::QueueStatsDomain::EventType::e_ACK_TIME, - timeDelta); + d_state_p->stats() + ->onEvent(timeDelta); if (d_state_p->handleCatalog().hasHandle(qH)) { // Send acknowledgement @@ -572,8 +570,8 @@ void LocalQueue::onRemoval(const bmqt::MessageGUID& msgGUID, // TODO: do we need to update NACK stats considering that downstream can // NACK the same GUID as well? - d_state_p->stats()->onEvent(mqbstat::QueueStatsDomain::EventType::e_NACK, - 1); + d_state_p->stats()->onEvent( + 1); if (d_state_p->handleCatalog().hasHandle(qH)) { // Send negative acknowledgement diff --git a/src/groups/mqb/mqbblp/mqbblp_pushstream.h b/src/groups/mqb/mqbblp/mqbblp_pushstream.h index 4a39a99f0..eb1c6854f 100644 --- a/src/groups/mqb/mqbblp/mqbblp_pushstream.h +++ b/src/groups/mqb/mqbblp/mqbblp_pushstream.h @@ -120,10 +120,20 @@ struct PushStream { struct App { Elements d_elements; bsl::shared_ptr d_app; + /// Replica deduplicates PUSH for the same App in the same batch. + bmqt::MessageGUID d_lastGUID; App(const bsl::shared_ptr& app); void add(Element* element); void remove(Element* element); + + /// Return 'true' if the specified `guid` is the same as in the last + /// `setLastPush` call. + bool isLastPush(const bmqt::MessageGUID& guid); + + /// Cache the specified `guid` for subsequent checks by `isLastPush`. + void setLastPush(const bmqt::MessageGUID& guid); + const Element* last() const; }; @@ -144,9 +154,10 @@ struct PushStream { const Apps::iterator d_iteratorApp; public: - Element(const bmqp::SubQueueInfo& subscription, - const iterator& iterator, - const Apps::iterator& iteratorApp); + Element(const bmqp::RdaInfo& rda, + unsigned int subscriptionId, + const iterator& iterator, + const Apps::iterator& iteratorApp); /// Return a modifiable reference to the App state associated with this /// Element. @@ -215,11 +226,13 @@ struct PushStream { /// Remove all Elements, Apps, and GUIDs. unsigned int removeAll(); - /// Create new Element associated with the specified `info`, - // `upstreamSubQueueId`, and `iterator`. - Element* create(const bmqp::SubQueueInfo& info, - const iterator& iterator, - const Apps::iterator& iteratorApp); + /// Create new Element associated with the specified `rda`, + /// 'subscriptionId`, `iterator` pointing to the corresponding GUID, and + /// `iteratorApp` pointing to the corresponding App. + Element* create(const bmqp::RdaInfo& rda, + unsigned int subscriptionId, + const iterator& iterator, + const Apps::iterator& iteratorApp); }; // ======================== @@ -430,14 +443,15 @@ inline PushStream::ElementBase::ElementBase() // NOTHING } -inline PushStream::Element::Element(const bmqp::SubQueueInfo& subscription, - const iterator& iterator, - const Apps::iterator& iteratorApp) -: d_app(subscription.rdaInfo()) +inline PushStream::Element::Element(const bmqp::RdaInfo& rda, + unsigned int subscriptionId, + const iterator& iterator, + const Apps::iterator& iteratorApp) +: d_app(rda) , d_iteratorGuid(iterator) , d_iteratorApp(iteratorApp) { - d_app.d_subscriptionId = subscription.id(); + d_app.d_subscriptionId = subscriptionId; } inline void PushStream::Element::eraseGuid(PushStream::Stream& stream) @@ -610,6 +624,16 @@ inline void PushStream::App::remove(Element* element) d_elements.remove(element, e_APP); } +inline bool PushStream::App::isLastPush(const bmqt::MessageGUID& lastGUID) +{ + return d_lastGUID == lastGUID; +} + +inline void PushStream::App::setLastPush(const bmqt::MessageGUID& lastGUID) +{ + d_lastGUID = lastGUID; +} + inline const PushStream::Element* PushStream::App::last() const { return d_elements.back(); @@ -620,14 +644,15 @@ inline const PushStream::Element* PushStream::App::last() const // ----------------- inline PushStream::Element* -PushStream::create(const bmqp::SubQueueInfo& subscription, - const iterator& it, - const Apps::iterator& iteratorApp) +PushStream::create(const bmqp::RdaInfo& rda, + unsigned int subscriptionId, + const iterator& it, + const Apps::iterator& iteratorApp) { BSLS_ASSERT_SAFE(it != d_stream.end()); Element* element = new (d_pushElementsPool_sp->allocate()) - Element(subscription, it, iteratorApp); + Element(rda, subscriptionId, it, iteratorApp); return element; } diff --git a/src/groups/mqb/mqbblp/mqbblp_pushstream.t.cpp b/src/groups/mqb/mqbblp/mqbblp_pushstream.t.cpp index 30f93846e..472db5c61 100644 --- a/src/groups/mqb/mqbblp/mqbblp_pushstream.t.cpp +++ b/src/groups/mqb/mqbblp/mqbblp_pushstream.t.cpp @@ -55,9 +55,8 @@ static void test1_basic() mqbblp::PushStream::Apps::iterator itApp = ps.d_apps.emplace(subQueueId, app).first; - mqbblp::PushStream::Element* element = ps.create(subscription, - itGuid, - itApp); + mqbblp::PushStream::Element* element = + ps.create(subscription.rdaInfo(), subscription.id(), itGuid, itApp); ps.add(element); ps.remove(element, true); @@ -85,7 +84,8 @@ static void test2_iterations() mqbblp::PushStream::Apps::iterator itApp1 = ps.d_apps.emplace(subQueueId1, unused).first; - mqbblp::PushStream::Element* element1 = ps.create(subscription1, + mqbblp::PushStream::Element* element1 = ps.create(subscription1.rdaInfo(), + subscription1.id(), itGuid1, itApp1); @@ -97,19 +97,22 @@ static void test2_iterations() mqbblp::PushStream::Apps::iterator itApp2 = ps.d_apps.emplace(subQueueId2, unused).first; - mqbblp::PushStream::Element* element2 = ps.create(subscription2, + mqbblp::PushStream::Element* element2 = ps.create(subscription2.rdaInfo(), + subscription2.id(), itGuid2, itApp2); ps.add(element2); - mqbblp::PushStream::Element* element3 = ps.create(subscription2, + mqbblp::PushStream::Element* element3 = ps.create(subscription2.rdaInfo(), + subscription2.id(), itGuid1, itApp2); ps.add(element3); - mqbblp::PushStream::Element* element4 = ps.create(subscription1, + mqbblp::PushStream::Element* element4 = ps.create(subscription1.rdaInfo(), + subscription1.id(), itGuid2, itApp1); diff --git a/src/groups/mqb/mqbblp/mqbblp_queueenginetester.cpp b/src/groups/mqb/mqbblp/mqbblp_queueenginetester.cpp index 85763337d..0266e1f52 100644 --- a/src/groups/mqb/mqbblp/mqbblp_queueenginetester.cpp +++ b/src/groups/mqb/mqbblp/mqbblp_queueenginetester.cpp @@ -916,25 +916,26 @@ void QueueEngineTester::post(const bslstl::StringRef& messages, bsl::vector msgs(d_allocator_p); parseMessages(&msgs, messages); - bmqp::Protocol::SubQueueInfosArray subscriptions(d_allocator_p); + for (unsigned int i = 0; i < msgs.size(); ++i) { + // Each message must have its own 'subscriptions'. + bmqp::Protocol::SubQueueInfosArray subscriptions(d_allocator_p); - if (d_subIds.empty()) { - subscriptions.push_back( - bmqp::SubQueueInfo(bmqp::Protocol::k_DEFAULT_SUBSCRIPTION_ID)); - } - else { - // Assume, RelayQueueEngine will use upstreamSubQueueIds as the - // subscriptionIds. - // This needs to be in accord with the 'configureHandle' logic. - - for (SubIdsMap::const_iterator cit = d_subIds.cbegin(); - cit != d_subIds.cend(); - ++cit) { - subscriptions.push_back(bmqp::SubQueueInfo(cit->second)); + if (d_subIds.empty()) { + subscriptions.push_back( + bmqp::SubQueueInfo(bmqp::Protocol::k_DEFAULT_SUBSCRIPTION_ID)); + } + else { + // Assume, RelayQueueEngine will use upstreamSubQueueIds as the + // subscriptionIds. + // This needs to be in accord with the 'configureHandle' logic. + + for (SubIdsMap::const_iterator cit = d_subIds.cbegin(); + cit != d_subIds.cend(); + ++cit) { + subscriptions.push_back(bmqp::SubQueueInfo(cit->second)); + } } - } - for (unsigned int i = 0; i < msgs.size(); ++i) { // Put in storage bmqt::MessageGUID msgGUID; mqbi::StorageMessageAttributes msgAttributes; diff --git a/src/groups/mqb/mqbblp/mqbblp_queueengineutil.cpp b/src/groups/mqb/mqbblp/mqbblp_queueengineutil.cpp index c1bd5fc1e..69c6412b4 100644 --- a/src/groups/mqb/mqbblp/mqbblp_queueengineutil.cpp +++ b/src/groups/mqb/mqbblp/mqbblp_queueengineutil.cpp @@ -1452,9 +1452,9 @@ void QueueEngineUtil_AppState::reportStats( message->attributes()); // First report 'queue time' metric for the entire queue - d_queue_p->stats()->onEvent( - mqbstat::QueueStatsDomain::EventType::e_QUEUE_TIME, - timeDelta); + d_queue_p->stats() + ->onEvent( + timeDelta); // Then report 'queue time' metric for appId d_queue_p->stats()->onEvent( diff --git a/src/groups/mqb/mqbblp/mqbblp_queuehandle.cpp b/src/groups/mqb/mqbblp/mqbblp_queuehandle.cpp index 42ffca0c7..f3a240ad9 100644 --- a/src/groups/mqb/mqbblp/mqbblp_queuehandle.cpp +++ b/src/groups/mqb/mqbblp/mqbblp_queuehandle.cpp @@ -299,9 +299,9 @@ QueueHandle::updateMonitor(const bsl::shared_ptr& subStream, // The message exist in the storage, we likely are in situation // (3), so it's a legit confirm and therefore, update the domain // stats. - d_domainStats_p->onEvent( - mqbstat::QueueStatsDomain::EventType::e_CONFIRM, - msgSize); + d_domainStats_p + ->onEvent( + msgSize); } BALL_LOG_INFO_BLOCK @@ -404,15 +404,15 @@ mqbu::ResourceUsageMonitorStateTransition::Enum QueueHandle::updateMonitor( if (type == bmqp::EventType::e_CONFIRM) { // Update domain stats - d_domainStats_p->onEvent( - mqbstat::QueueStatsDomain::EventType::e_CONFIRM, - msgSize); + d_domainStats_p + ->onEvent( + msgSize); // Report CONFIRM time only at first hop // Note that we update metric per entire queue and also per `appId` if (d_clientContext_sp->isFirstHop()) { - d_domainStats_p->onEvent( - mqbstat::QueueStatsDomain::EventType::e_CONFIRM_TIME, + d_domainStats_p->onEvent< + mqbstat::QueueStatsDomain::EventType::e_CONFIRM_TIME>( timeDelta); d_domainStats_p->onEvent( mqbstat::QueueStatsDomain::EventType::e_CONFIRM_TIME, @@ -498,8 +498,8 @@ void QueueHandle::deliverMessageImpl( BSLS_ASSERT_SAFE(subQueueInfos.size() >= 1 && subQueueInfos.size() <= d_subscriptions.size()); - d_domainStats_p->onEvent(mqbstat::QueueStatsDomain::EventType::e_PUSH, - msgSize); + d_domainStats_p->onEvent( + msgSize); // Create an event to dispatch delivery of the message to the client mqbi::DispatcherClient* client = d_clientContext_sp->client(); @@ -1208,7 +1208,7 @@ void QueueHandle::onAckMessage(const bmqp::AckMessage& ackMessage) ackMsg.setQueueId(id()); client->dispatcher()->dispatchEvent(event, client); - d_domainStats_p->onEvent(mqbstat::QueueStatsDomain::EventType::e_ACK, 1); + d_domainStats_p->onEvent(1); } bool QueueHandle::canDeliver(unsigned int downstreamSubscriptionId) const diff --git a/src/groups/mqb/mqbblp/mqbblp_queuestate.cpp b/src/groups/mqb/mqbblp/mqbblp_queuestate.cpp index 6070ba067..86ec95caa 100644 --- a/src/groups/mqb/mqbblp/mqbblp_queuestate.cpp +++ b/src/groups/mqb/mqbblp/mqbblp_queuestate.cpp @@ -88,14 +88,14 @@ QueueState::QueueState(mqbi::Queue* queue, // NOTE: The 'description' will be set by the owner of this object. - mqbstat::BrokerStats::instance().onEvent( - mqbstat::BrokerStats::EventType::e_QUEUE_CREATED); + mqbstat::BrokerStats::instance() + .onEvent(); } QueueState::~QueueState() { - mqbstat::BrokerStats::instance().onEvent( - mqbstat::BrokerStats::EventType::e_QUEUE_DESTROYED); + mqbstat::BrokerStats::instance() + .onEvent(); } void QueueState::add(const bmqp_ctrlmsg::QueueHandleParameters& params) diff --git a/src/groups/mqb/mqbblp/mqbblp_relayqueueengine.cpp b/src/groups/mqb/mqbblp/mqbblp_relayqueueengine.cpp index 5171c0b95..52b481d46 100644 --- a/src/groups/mqb/mqbblp/mqbblp_relayqueueengine.cpp +++ b/src/groups/mqb/mqbblp/mqbblp_relayqueueengine.cpp @@ -590,6 +590,7 @@ void RelayQueueEngine::deliverMessages() App_State* app = element->app().d_app.get(); BSLS_ASSERT_SAFE(app); + if (!app->isAuthorized()) { // This App got the PUSH (recorded in the PushStream) BMQ_LOGTHROTTLE_ERROR() @@ -602,13 +603,24 @@ void RelayQueueEngine::deliverMessages() d_storageIter_mp->removeCurrentElement(); } - else if (d_appsDeliveryContext.processApp(*app, i)) { - // The current element has made it either to delivery or - // putAside and it can be removed + else if (element->app().isLastPush(d_storageIter_mp->guid())) { + // This `app` has already seen this message. d_storageIter_mp->removeCurrentElement(); } - // Else, the current element has made it to resumePoint and it - // cannot be removed. + else { + element->app().setLastPush(d_storageIter_mp->guid()); + + if (d_appsDeliveryContext.processApp(*app, i)) { + // The current element has made it either to delivery or + // putAside and it can be removed + d_storageIter_mp->removeCurrentElement(); + } + else { + // The current element has made it to resumePoint and it + // cannot be removed. + element->app().setLastPush(bmqt::MessageGUID()); + } + } } d_appsDeliveryContext.deliverMessage(); } @@ -1725,22 +1737,28 @@ bool RelayQueueEngine::subscriptionId2upstreamSubQueueId( } unsigned int -RelayQueueEngine::push(mqbi::StorageMessageAttributes* attributes, - const bmqt::MessageGUID& msgGUID, - const bsl::shared_ptr& appData, - const bmqp::Protocol::SubQueueInfosArray& subscriptions, - bool isOutOfOrder) +RelayQueueEngine::push(mqbi::StorageMessageAttributes* attributes, + const bmqt::MessageGUID& msgGUID, + const bsl::shared_ptr& appData, + bmqp::Protocol::SubQueueInfosArray& subscriptions, + bool isOutOfOrder) { if (isOutOfOrder) { BSLS_ASSERT_SAFE(subscriptions.size() == 1); // No guarantee of uniqueness. Cannot use PushStream. - unsigned int upstreamSubQueueId; + unsigned int subQueueId; + + unsigned int subscriptionId = subscriptions.begin()->id(); + unsigned int ordinalPlusOne = 0; // Invalid value + + // Reusing 'subscriptions' to 'setPushState()' below. + subscriptions.begin()->setId(ordinalPlusOne); if (subscriptionId2upstreamSubQueueId(msgGUID, - &upstreamSubQueueId, - subscriptions.begin()->id())) { - App_State* app = findApp(upstreamSubQueueId); + &subQueueId, + subscriptionId)) { + App_State* app = findApp(subQueueId); if (app == 0) { BMQ_LOGTHROTTLE_ERROR() @@ -1748,7 +1766,7 @@ RelayQueueEngine::push(mqbi::StorageMessageAttributes* attributes, << "Remote queue: " << d_queueState_p->uri() << " (id: " << d_queueState_p->id() << ") discarding a PUSH message for guid " << msgGUID - << ", with unknown App Id " << upstreamSubQueueId; + << ", with unknown App Id " << subQueueId; return 0; // RETURN } @@ -1761,10 +1779,14 @@ RelayQueueEngine::push(mqbi::StorageMessageAttributes* attributes, attributes->setRefCount(1); - storePush(attributes, msgGUID, appData, true); + // Reusing 'subscriptions' to 'setPushState()' below. + ordinalPlusOne = 1 + app->ordinal(); + subscriptions.begin()->setId(ordinalPlusOne); + + storePush(attributes, msgGUID, appData, subscriptions, true); // Attempt to deliver - processAppRedelivery(upstreamSubQueueId, app); + processAppRedelivery(subQueueId, app); return 1; // RETURN } @@ -1777,17 +1799,21 @@ RelayQueueEngine::push(mqbi::StorageMessageAttributes* attributes, PushStream::iterator itGuid = d_pushStream.findOrAppendMessage(msgGUID); unsigned int count = 0; - for (bmqp::Protocol::SubQueueInfosArray::const_iterator cit = + for (bmqp::Protocol::SubQueueInfosArray::iterator it = subscriptions.begin(); - cit != subscriptions.end(); - ++cit) { - const bmqp::SubQueueInfo& subscription = *cit; - + it != subscriptions.end(); + ++it) { unsigned int subQueueId; + unsigned int subscriptionId = it->id(); + unsigned int ordinalPlusOne = 0; // Invalid value + + // Reusing 'subscriptions' to 'setPushState()' below. + it->setId(ordinalPlusOne); + if (!subscriptionId2upstreamSubQueueId(msgGUID, &subQueueId, - subscription.id())) { + subscriptionId)) { continue; // CONTINUE } @@ -1802,7 +1828,7 @@ RelayQueueEngine::push(mqbi::StorageMessageAttributes* attributes, << "Remote queue: " << d_queueState_p->uri() << " (id: " << d_queueState_p->id() << ") discarding a PUSH message for guid " << msgGUID - << ", with unknown App Id " << subscription.id(); + << ", with unknown App Id " << subscriptionId; continue; // CONTINUE } @@ -1827,9 +1853,12 @@ RelayQueueEngine::push(mqbi::StorageMessageAttributes* attributes, } } - PushStream::Element* element = d_pushStream.create(subscription, - itGuid, - itApp); + PushStream::Element* element = + d_pushStream.create(it->rdaInfo(), subscriptionId, itGuid, itApp); + + // Reusing 'subscriptions' to 'setPushState()' below. + ordinalPlusOne = 1 + itApp->second.d_app->ordinal(); + it->setId(ordinalPlusOne); d_pushStream.add(element); ++count; @@ -1838,7 +1867,7 @@ RelayQueueEngine::push(mqbi::StorageMessageAttributes* attributes, if (count) { // Pass correct ref count attributes->setRefCount(count); - storePush(attributes, msgGUID, appData, false); + storePush(attributes, msgGUID, appData, subscriptions, false); } return count; } @@ -1862,10 +1891,7 @@ bool RelayQueueEngine::checkForDuplicate(const App_State* app, mqbi::AppMessage& appState = d_realStorageIter_mp->appMessageState( app->ordinal()); - if (!appState.isPushing()) { - appState.setPushState(); - } - else { + if (appState.isPushing()) { BMQ_LOGTHROTTLE_INFO() << "Remote queue: " << d_queueState_p->uri() << " (id: " << d_queueState_p->id() << ", App '" @@ -1878,10 +1904,13 @@ bool RelayQueueEngine::checkForDuplicate(const App_State* app, return true; } -void RelayQueueEngine::storePush(mqbi::StorageMessageAttributes* attributes, - const bmqt::MessageGUID& msgGUID, - const bsl::shared_ptr& appData, - bool isOutOfOrder) +void RelayQueueEngine::storePush( + mqbi::StorageMessageAttributes* attributes, + const bmqt::MessageGUID& msgGUID, + const bsl::shared_ptr& appData, + const bmqp::Protocol::SubQueueInfosArray& subscriptions, + + bool isOutOfOrder) { if (d_queueState_p->domain()->cluster()->isRemote()) { // Save the message along with the subIds in the storage. Note that @@ -1889,11 +1918,14 @@ void RelayQueueEngine::storePush(mqbi::StorageMessageAttributes* attributes, // in 'options' is subQueueInfos, and we won't store the specified // 'options' in the storage. + mqbi::DataStreamMessage* dataStreamMessage = 0; + mqbi::StorageResult::Enum result = storage()->put( attributes, msgGUID, appData, - bsl::shared_ptr()); // No options + bsl::shared_ptr(), + &dataStreamMessage); // No options if (BSLS_PERFORMANCEHINT_PREDICT_UNLIKELY( result != mqbi::StorageResult::e_SUCCESS)) { @@ -1908,6 +1940,17 @@ void RelayQueueEngine::storePush(mqbi::StorageMessageAttributes* attributes, // A redelivery PUSH for one App in the presence of another App // can result in 'e_GUID_NOT_UNIQUE'. } + else { + // Reusing previously cached ordinals. + for (bmqp::Protocol::SubQueueInfosArray::const_iterator cit = + subscriptions.begin(); + cit != subscriptions.end(); + ++cit) { + if (cit->id() > 0) { + dataStreamMessage->app(cit->id() - 1).setPushState(); + } + } + } } } diff --git a/src/groups/mqb/mqbblp/mqbblp_relayqueueengine.h b/src/groups/mqb/mqbblp/mqbblp_relayqueueengine.h index 710e13182..2435a9ad5 100644 --- a/src/groups/mqb/mqbblp/mqbblp_relayqueueengine.h +++ b/src/groups/mqb/mqbblp/mqbblp_relayqueueengine.h @@ -351,10 +351,11 @@ class RelayQueueEngine BSLS_KEYWORD_FINAL : public mqbi::QueueEngine { bool checkForDuplicate(const App_State* app, const bmqt::MessageGUID& msgGUID); - void storePush(mqbi::StorageMessageAttributes* attributes, - const bmqt::MessageGUID& msgGUID, - const bsl::shared_ptr& appData, - bool isOutOfOrder); + void storePush(mqbi::StorageMessageAttributes* attributes, + const bmqt::MessageGUID& msgGUID, + const bsl::shared_ptr& appData, + const bmqp::Protocol::SubQueueInfosArray& subscriptions, + bool isOutOfOrder); void beforeOneAppRemoved(unsigned int upstreamSubQueueId); @@ -541,11 +542,11 @@ class RelayQueueEngine BSLS_KEYWORD_FINAL : public mqbi::QueueEngine { // (`mqbi::AppMessage`, `upstreamSubQueueId`) pairs for each recognized App /// in the specified `subscriptions`. /// Return number of inserted PushStream Elements. - unsigned int push(mqbi::StorageMessageAttributes* attributes, - const bmqt::MessageGUID& msgGUID, - const bsl::shared_ptr& appData, - const bmqp::Protocol::SubQueueInfosArray& subscriptions, - bool isOutOfOrder); + unsigned int push(mqbi::StorageMessageAttributes* attributes, + const bmqt::MessageGUID& msgGUID, + const bsl::shared_ptr& appData, + bmqp::Protocol::SubQueueInfosArray& subscriptions, + bool isOutOfOrder); // ACCESSORS /// Return the reference count that should be applied to a message diff --git a/src/groups/mqb/mqbblp/mqbblp_remotequeue.cpp b/src/groups/mqb/mqbblp/mqbblp_remotequeue.cpp index 9c8057c5e..06795b7be 100644 --- a/src/groups/mqb/mqbblp/mqbblp_remotequeue.cpp +++ b/src/groups/mqb/mqbblp/mqbblp_remotequeue.cpp @@ -158,9 +158,9 @@ int RemoteQueue::configureAsProxy(bsl::ostream& errorDescription, return 10 * rc + rc_QUEUE_ENGINE_CFG_FAILURE; // RETURN } - d_state_p->stats()->onEvent( - mqbstat::QueueStatsDomain::EventType::e_CHANGE_ROLE, - mqbstat::QueueStatsDomain::Role::e_PROXY); + d_state_p->stats() + ->onEvent( + mqbstat::QueueStatsDomain::Role::e_PROXY); BALL_LOG_INFO << "Created a ProxyRemoteQueue " @@ -276,9 +276,9 @@ int RemoteQueue::configureAsClusterMember(bsl::ostream& errorDescription, d_state_p->storageManager()->setQueueRaw(queue, d_state_p->uri(), d_state_p->partitionId()); - d_state_p->stats()->onEvent( - mqbstat::QueueStatsDomain::EventType::e_CHANGE_ROLE, - mqbstat::QueueStatsDomain::Role::e_REPLICA); + d_state_p->stats() + ->onEvent( + mqbstat::QueueStatsDomain::Role::e_REPLICA); BALL_LOG_INFO << d_state_p->domain()->cluster()->name() << ": Created a ClusterMemberRemoteQueue " @@ -922,9 +922,8 @@ void RemoteQueue::postMessage(const bmqp::PutHeader& putHeaderIn, bmqt::AckResult::e_REFUSED)); ackMessage.setMessageGUID(putHeader.messageGUID()); - d_state_p->stats()->onEvent( - mqbstat::QueueStatsDomain::EventType::e_NACK, - 1); + d_state_p->stats() + ->onEvent(1); // CorrelationId & QueueId are left unset as those fields // will be filled downstream. @@ -996,8 +995,9 @@ void RemoteQueue::postMessage(const bmqp::PutHeader& putHeaderIn, // the time the message is actually sent upstream, i.e. in // cluster/clusterProxy) for the most exact accuracy, but doing it here is // good enough. - d_state_p->stats()->onEvent(mqbstat::QueueStatsDomain::EventType::e_PUT, - appData->length()); + + d_state_p->stats()->onEvent( + appData->length()); } void RemoteQueue::confirmMessage(const bmqt::MessageGUID& msgGUID, @@ -1496,8 +1496,8 @@ RemoteQueue::Puts::iterator& RemoteQueue::nack(Puts::iterator& it, { ackMessage.setMessageGUID(it->first); - d_state_p->stats()->onEvent(mqbstat::QueueStatsDomain::EventType::e_NACK, - 1); + d_state_p->stats()->onEvent( + 1); // CorrelationId & QueueId are left unset as those fields // will be filled downstream. diff --git a/src/groups/mqb/mqbblp/mqbblp_rootqueueengine.cpp b/src/groups/mqb/mqbblp/mqbblp_rootqueueengine.cpp index 68435607b..06114746d 100644 --- a/src/groups/mqb/mqbblp/mqbblp_rootqueueengine.cpp +++ b/src/groups/mqb/mqbblp/mqbblp_rootqueueengine.cpp @@ -1301,9 +1301,10 @@ void RootQueueEngine::afterNewMessage( } if (!d_appsDeliveryContext.isEmpty()) { // Report 'queue time' metric for the entire queue - d_queueState_p->queue()->stats()->onEvent( - mqbstat::QueueStatsDomain::EventType::e_QUEUE_TIME, - d_appsDeliveryContext.timeDelta()); + d_queueState_p->queue() + ->stats() + ->onEvent( + d_appsDeliveryContext.timeDelta()); } d_appsDeliveryContext.deliverMessage(); } @@ -1518,9 +1519,9 @@ int RootQueueEngine::onRejectMessage(mqbi::QueueHandle* handle, d_queueState_p, d_allocator_p)); } - d_queueState_p->stats()->onEvent( - mqbstat::QueueStatsDomain::EventType::e_REJECT, - 1); + d_queueState_p->stats() + ->onEvent( + 1); // Lastly, if message reached a ref count of zero in the // storage (i.e., all appIds have confirmed the message), diff --git a/src/groups/mqb/mqbc/mqbc_clusterstate.cpp b/src/groups/mqb/mqbc/mqbc_clusterstate.cpp index 3d47083e0..88d5a7b5e 100644 --- a/src/groups/mqb/mqbc/mqbc_clusterstate.cpp +++ b/src/groups/mqb/mqbc/mqbc_clusterstate.cpp @@ -47,6 +47,7 @@ bsl::ostream& ClusterStateQueueInfo::print(bsl::ostream& stream, printer.printAttribute("queueKey", key()); printer.printAttribute("partitionId", partitionId()); printer.printAttribute("appIdInfos", appInfos()); + printer.printAttribute("stateOfAssignment", state()); printer.end(); return stream; @@ -94,7 +95,7 @@ void ClusterStateObserver::onQueueUpdated( } void ClusterStateObserver::onPartitionOrphanThreshold( - BSLS_ANNOTATION_UNUSED size_t partitiondId) + BSLS_ANNOTATION_UNUSED size_t partitionId) { // NOTHING } @@ -552,9 +553,10 @@ void ClusterState::DomainState::adjustQueueCount(int by) d_numAssignedQueues += by; if (d_domain_p != 0) { - d_domain_p->domainStats()->onEvent( - mqbstat::DomainStats::EventType::e_QUEUE_COUNT, - d_numAssignedQueues); + d_domain_p->domainStats() + ->onEvent( + + d_numAssignedQueues); } } diff --git a/src/groups/mqb/mqbc/mqbc_clusterstate.h b/src/groups/mqb/mqbc/mqbc_clusterstate.h index 52bdeb054..a6aaf967e 100644 --- a/src/groups/mqb/mqbc/mqbc_clusterstate.h +++ b/src/groups/mqb/mqbc/mqbc_clusterstate.h @@ -155,9 +155,6 @@ class ClusterStatePartitionInfo { /// `bmqp_ctrlmsg::QueueInfo`. Doing vice versa will not be possible /// because we don't want to edit generated file. Perhaps we can place the /// converter routine in `ClusterUtil`. -/// -/// TBD: When should AppIds and AppKeys come from? Should the leader/primary -/// generate them, or should we hardcode them in the domain config? class ClusterStateQueueInfo { public: // TYPES @@ -171,10 +168,10 @@ class ClusterStateQueueInfo { // Assigning following unassigning is also supported. // On Replica, the only possible state is k_ASSIGNED. - k_NONE, - k_ASSIGNING, - k_ASSIGNED, - k_UNASSIGNING + k_NONE = 0, + k_ASSIGNING = -1, + k_ASSIGNED = -2, + k_UNASSIGNING = -3 }; private: @@ -272,6 +269,18 @@ class ClusterStateQueueInfo { bsl::ostream& operator<<(bsl::ostream& stream, const ClusterStateQueueInfo& rhs); +/// Return `true` if the specified `rhs` object contains the value of the +/// same type as contained in the specified `lhs` object and the value +/// itself is the same in both objects, return false otherwise. +bool operator==(const ClusterStateQueueInfo& lhs, + const ClusterStateQueueInfo& rhs); + +/// Return `false` if the specified `rhs` object contains the value of the +/// same type as contained in the specified `lhs` object and the value +/// itself is the same in both objects, return `true` otherwise. +bool operator!=(const ClusterStateQueueInfo& lhs, + const ClusterStateQueueInfo& rhs); + // ========================== // class ClusterStateObserver // ========================== @@ -343,7 +352,7 @@ class ClusterStateObserver { /// /// THREAD: This method is invoked in the associated cluster's /// dispatcher thread. - virtual void onPartitionOrphanThreshold(size_t partitiondId); + virtual void onPartitionOrphanThreshold(size_t partitionId); /// Callback invoked when the specified `node` has been unavailable /// above a certain threshold amount of time. @@ -1163,6 +1172,20 @@ inline bsl::ostream& mqbc::operator<<(bsl::ostream& stream, return rhs.print(stream, 0, -1); } +inline bool mqbc::operator==(const ClusterStateQueueInfo& lhs, + const ClusterStateQueueInfo& rhs) +{ + return lhs.uri() == rhs.uri() && lhs.key() == rhs.key() && + lhs.partitionId() == rhs.partitionId() && + lhs.appInfos() == rhs.appInfos() && lhs.state() == rhs.state(); +} + +inline bool mqbc::operator!=(const ClusterStateQueueInfo& lhs, + const ClusterStateQueueInfo& rhs) +{ + return !(lhs == rhs); +} + } // close enterprise namespace #endif diff --git a/src/groups/mqb/mqbc/mqbc_clusterutil.cpp b/src/groups/mqb/mqbc/mqbc_clusterutil.cpp index 95f0c6408..c3988cd51 100644 --- a/src/groups/mqb/mqbc/mqbc_clusterutil.cpp +++ b/src/groups/mqb/mqbc/mqbc_clusterutil.cpp @@ -313,28 +313,6 @@ void getNextPrimarys(NumNewPartitionsMap* numNewPartitions, } } -void printQueues(bsl::ostream& out, const ClusterState& state) -{ - out << '\n' - << "-------------------------" << '\n' - << "QUEUES IN CLUSTER STATE :" << '\n' - << "-------------------------"; - for (ClusterState::DomainStatesCIter domCit = - state.domainStates().cbegin(); - domCit != state.domainStates().cend(); - ++domCit) { - for (ClusterState::UriToQueueInfoMapCIter citer = - domCit->second->queuesInfo().cbegin(); - citer != domCit->second->queuesInfo().cend(); - ++citer) { - const bsl::shared_ptr& info = citer->second; - bdlb::Print::newlineAndIndent(out, 1); - out << "[key: " << info->key() << ", uri: " << info->uri() - << ", partitionId: " << info->partitionId() << "]"; - } - } -} - /// If the specified `status` is SUCCESS, load the specified `domain` into /// the specified `domainState`. void createDomainCb(const bmqp_ctrlmsg::Status& status, @@ -1683,30 +1661,27 @@ void ClusterUtil::apply(mqbc::ClusterState* clusterState, } } -int ClusterUtil::validateState(bsl::ostream& errorDescription, - const mqbc::ClusterState& state, - const mqbc::ClusterState& reference) +int ClusterUtil::validateState(bsl::ostream& errorDescription, + const ClusterState& state, + const ClusterState& reference) { // PRECONDITIONS BSLS_ASSERT_SAFE(state.partitions().size() == reference.partitions().size()); - bool seenIncorrectPartitionInfo = false; - bool seenIncorrectQueueInfo = false; - bool seenMissingQueue = false; - bool seenExtraQueue = false; - bmqu::MemOutStream out; const int level = 0; - // Check incorrect partition information + // Validate partition information + bsl::vector incorrectPartitions; for (size_t pid = 0; pid < state.partitions().size(); ++pid) { - const mqbc::ClusterStatePartitionInfo& stateInfo = - state.partitions()[pid]; - const mqbc::ClusterStatePartitionInfo& referenceInfo = + const ClusterStatePartitionInfo& stateInfo = state.partitions()[pid]; + BSLS_ASSERT_SAFE(stateInfo.partitionId() == pid); + + const ClusterStatePartitionInfo& referenceInfo = reference.partitions()[pid]; - if (stateInfo.partitionId() != referenceInfo.partitionId() || - stateInfo.primaryLeaseId() != referenceInfo.primaryLeaseId()) { + BSLS_ASSERT_SAFE(referenceInfo.partitionId() == pid); + if (stateInfo.primaryLeaseId() != referenceInfo.primaryLeaseId()) { // Partition information mismatch. Note that we don't compare // primaryNodeIds here because 'state' is initialized with cluster // state ledger's contents and will likely have a valid @@ -1716,28 +1691,49 @@ int ClusterUtil::validateState(bsl::ostream& errorDescription, // primaryNodeId, specially if a primary has not yet been assigned // in the startup sequence. If if a primary has been assigned, its // nodeId may be different one. - if (!seenIncorrectPartitionInfo) { - bdlb::Print::newlineAndIndent(out, level); - out << "---------------------------"; - bdlb::Print::newlineAndIndent(out, level); - out << "Incorrect Partition Infos :"; - bdlb::Print::newlineAndIndent(out, level); - out << "---------------------------"; - seenIncorrectPartitionInfo = true; - } + incorrectPartitions.push_back(stateInfo); + } + } + + if (!incorrectPartitions.empty()) { + bdlb::Print::newlineAndIndent(out, level); + out << "---------------------------"; + bdlb::Print::newlineAndIndent(out, level); + out << "Incorrect Partition Infos :"; + bdlb::Print::newlineAndIndent(out, level); + out << "---------------------------"; + for (bsl::vector::const_iterator citer = + incorrectPartitions.cbegin(); + citer != incorrectPartitions.cend(); + ++citer) { bdlb::Print::newlineAndIndent(out, level + 1); - out << "[partitionId: " << stateInfo.partitionId() - << ", primaryLeaseId: " << stateInfo.primaryLeaseId() - << ", primaryNodeId: " << stateInfo.primaryNodeId() << "]" - << " (Correct: " - << "[partitionId: " << referenceInfo.partitionId() - << ", primaryLeaseId: " << referenceInfo.primaryLeaseId() - << ", primaryNodeId: " << referenceInfo.primaryNodeId() << "]" - << ")"; + out << "Partition [" << citer->partitionId() + << "]: " << " primaryLeaseId: " << citer->primaryLeaseId() + << ", primaryNodeId: " << citer->primaryNodeId(); + } + + bdlb::Print::newlineAndIndent(out, level); + out << "--------------------------------"; + bdlb::Print::newlineAndIndent(out, level); + out << "Partition Infos In Cluster State:"; + bdlb::Print::newlineAndIndent(out, level); + out << "--------------------------------"; + for (size_t pid = 0; pid < state.partitions().size(); ++pid) { + const ClusterStatePartitionInfo& referenceInfo = + reference.partitions()[pid]; + BSLS_ASSERT_SAFE(referenceInfo.partitionId() == pid); + bdlb::Print::newlineAndIndent(out, level + 1); + out << "Partition [" << pid << "]: " << " primaryLeaseId: " + << referenceInfo.primaryLeaseId() + << ", primaryNodeId: " << referenceInfo.primaryNodeId(); } } - // Check incorrect queue information + // Check incorrect or extra queues + bsl::vector, + bsl::shared_ptr > > + incorrectQueues; + bsl::vector > extraQueues; for (ClusterState::DomainStatesCIter domCit = state.domainStates().begin(); domCit != state.domainStates().end(); ++domCit) { @@ -1746,7 +1742,13 @@ int ClusterUtil::validateState(bsl::ostream& errorDescription, ClusterState::DomainStatesCIter refDomCit = reference.domainStates().find(domainName); if (refDomCit == reference.domainStates().cend()) { - // Domain not found in both states + // Entire domain is extra + for (ClusterState::UriToQueueInfoMapCIter citer = + domCit->second->queuesInfo().cbegin(); + citer != domCit->second->queuesInfo().cend(); + ++citer) { + extraQueues.push_back(citer->second); + } continue; // CONTINUE } @@ -1759,37 +1761,55 @@ int ClusterUtil::validateState(bsl::ostream& errorDescription, ClusterState::UriToQueueInfoMapCIter refCiter = refDomCit->second->queuesInfo().find(uri); if (refCiter == refDomCit->second->queuesInfo().cend()) { - // Queue not found in both states - continue; // CONTINUE + // Extra queue + extraQueues.push_back(citer->second); } - - const bsl::shared_ptr& referenceInfo = - refCiter->second; - const bsl::shared_ptr& info = citer->second; - if (info->uri() == referenceInfo->uri() && - info->key() == referenceInfo->key() && - info->partitionId() == referenceInfo->partitionId()) { - continue; // CONTINUE + else { + const bsl::shared_ptr& info = + citer->second; + const bsl::shared_ptr& referenceInfo = + refCiter->second; + if (*info != *referenceInfo) { + // Incorrect queue information + incorrectQueues.push_back( + bsl::make_pair(info, referenceInfo)); + } } + } + } - if (!seenIncorrectQueueInfo) { - bdlb::Print::newlineAndIndent(out, level); - out << "-----------------------------"; - bdlb::Print::newlineAndIndent(out, level); - out << "Incorrect Queue Information :"; - bdlb::Print::newlineAndIndent(out, level); - out << "-----------------------------"; - seenIncorrectQueueInfo = true; - } + if (!incorrectQueues.empty()) { + bdlb::Print::newlineAndIndent(out, level); + out << "-----------------"; + bdlb::Print::newlineAndIndent(out, level); + out << "Incorrect Queues:"; + bdlb::Print::newlineAndIndent(out, level); + out << "-----------------"; + for (bsl::vector, + bsl::shared_ptr > >:: + const_iterator citer = incorrectQueues.cbegin(); + citer != incorrectQueues.cend(); + ++citer) { + bdlb::Print::newlineAndIndent(out, level + 1); + out << citer->first; + bdlb::Print::newlineAndIndent(out, level + 1); + out << "(correct queue info) " << citer->second; + } + } + if (!extraQueues.empty()) { + bdlb::Print::newlineAndIndent(out, level); + out << "--------------"; + bdlb::Print::newlineAndIndent(out, level); + out << "Extra queues :"; + bdlb::Print::newlineAndIndent(out, level); + out << "--------------"; + for (bsl::vector >:: + const_iterator citer = extraQueues.cbegin(); + citer != extraQueues.cend(); + ++citer) { bdlb::Print::newlineAndIndent(out, level + 1); - out << "[key: " << info->key() << ", uri: " << info->uri() - << ", partitionId: " << info->partitionId() << "]" - << " (Correct: " - << "[key: " << referenceInfo->key() - << ", uri: " << referenceInfo->uri() - << ", partitionId: " << referenceInfo->partitionId() << "]" - << ")"; + out << *citer; } } @@ -1804,7 +1824,7 @@ int ClusterUtil::validateState(bsl::ostream& errorDescription, ClusterState::DomainStatesCIter domCit = state.domainStates().find( domainName); if (domCit == state.domainStates().cend()) { - // Entire domain of queues is not found + // Entire domain is missing for (ClusterState::UriToQueueInfoMapCIter refCiter = refDomCit->second->queuesInfo().cbegin(); refCiter != refDomCit->second->queuesInfo().cend(); @@ -1823,16 +1843,13 @@ int ClusterUtil::validateState(bsl::ostream& errorDescription, ClusterState::UriToQueueInfoMapCIter citer = domCit->second->queuesInfo().find(uri); - if (citer != domCit->second->queuesInfo().cend()) { - // Queue is found in both states - continue; // CONTINUE + if (citer == domCit->second->queuesInfo().cend()) { + // Missing queue + missingQueues.push_back(refCiter->second); } - - missingQueues.push_back(refCiter->second); } } - // Queue is missing from the cluster state if (!missingQueues.empty()) { bdlb::Print::newlineAndIndent(out, level); out << "----------------"; @@ -1840,78 +1857,40 @@ int ClusterUtil::validateState(bsl::ostream& errorDescription, out << "Missing queues :"; bdlb::Print::newlineAndIndent(out, level); out << "----------------"; - seenMissingQueue = true; - } - - for (bsl::vector >::const_iterator - citer = missingQueues.cbegin(); - citer != missingQueues.cend(); - ++citer) { - bdlb::Print::newlineAndIndent(out, level + 1); - out << "[key: " << (*citer)->key() << ", uri: " << (*citer)->uri() - << ", partitionId: " << (*citer)->partitionId() << "]"; + for (bsl::vector >:: + const_iterator citer = missingQueues.cbegin(); + citer != missingQueues.cend(); + ++citer) { + bdlb::Print::newlineAndIndent(out, level + 1); + out << *citer; + } } - // Check extra queues - bsl::vector > extraQueues; - for (ClusterState::DomainStatesCIter domCit = state.domainStates().begin(); - domCit != state.domainStates().end(); - ++domCit) { - const bsl::string& domainName = domCit->first; - - ClusterState::DomainStatesCIter refDomCit = - reference.domainStates().find(domainName); - if (refDomCit == reference.domainStates().cend()) { - // Entire domain of queues is not found + const bool queueInfoMismatch = !incorrectQueues.empty() || + !extraQueues.empty() || + !missingQueues.empty(); + if (queueInfoMismatch) { + bdlb::Print::newlineAndIndent(out, level); + out << "-------------------------"; + bdlb::Print::newlineAndIndent(out, level); + out << "QUEUES IN CLUSTER STATE :"; + bdlb::Print::newlineAndIndent(out, level); + out << "-------------------------"; + for (ClusterState::DomainStatesCIter domCit = + reference.domainStates().cbegin(); + domCit != reference.domainStates().cend(); + ++domCit) { for (ClusterState::UriToQueueInfoMapCIter citer = domCit->second->queuesInfo().cbegin(); citer != domCit->second->queuesInfo().cend(); ++citer) { - extraQueues.push_back(citer->second); + bdlb::Print::newlineAndIndent(out, level + 1); + out << citer->second; } - - continue; // CONTINUE } - - for (ClusterState::UriToQueueInfoMapCIter citer = - domCit->second->queuesInfo().cbegin(); - citer != domCit->second->queuesInfo().cend(); - ++citer) { - const bmqt::Uri& uri = citer->first; - - ClusterState::UriToQueueInfoMapCIter refCiter = - refDomCit->second->queuesInfo().find(uri); - if (refCiter != refDomCit->second->queuesInfo().cend()) { - // Queue is found in both states - continue; // CONTINUE - } - - extraQueues.push_back(citer->second); - } - } - - // Extra queue in the cluster state - if (!extraQueues.empty()) { - bdlb::Print::newlineAndIndent(out, level); - out << "--------------"; - bdlb::Print::newlineAndIndent(out, level); - out << "Extra queues :"; - bdlb::Print::newlineAndIndent(out, level); - out << "--------------"; - seenExtraQueue = true; } - for (bsl::vector >::const_iterator - citer = extraQueues.cbegin(); - citer != extraQueues.cend(); - ++citer) { - bdlb::Print::newlineAndIndent(out, level + 1); - out << "[key: " << (*citer)->key() << ", uri: " << (*citer)->uri() - << ", partitionId: " << (*citer)->partitionId() << "]"; - } - - if (seenIncorrectPartitionInfo || seenIncorrectQueueInfo || - seenMissingQueue || seenExtraQueue) { + if (!incorrectPartitions.empty() || queueInfoMismatch) { // Inconsistency detected errorDescription << out.str(); return -1; // RETURN @@ -1958,14 +1937,11 @@ void ClusterUtil::validateClusterStateLedger(mqbi::Cluster* cluster, bmqu::MemOutStream errorDescription; rc = validateState(errorDescription, tempState, clusterState); if (rc != 0) { - BALL_LOG_WARN_BLOCK - { - BALL_LOG_OUTPUT_STREAM << clusterData.identity().description() - << ": Cluster state ledger's contents are" - << " different from the cluster state: " - << errorDescription.str(); - printQueues(BALL_LOG_OUTPUT_STREAM, clusterState); - } + BMQTSK_ALARMLOG_ALARM("CLUSTER") + << clusterData.identity().description() + << ": Cluster state ledger's contents are" + << " different from the cluster state: " << errorDescription.str() + << BMQTSK_ALARMLOG_END; } } diff --git a/src/groups/mqb/mqbi/mqbi_storage.h b/src/groups/mqb/mqbi/mqbi_storage.h index 95267c8af..076de2a7c 100644 --- a/src/groups/mqb/mqbi/mqbi_storage.h +++ b/src/groups/mqb/mqbi/mqbi_storage.h @@ -292,6 +292,26 @@ struct AppMessage { bool isPushing() const; }; +struct DataStreamMessage { + // VST to track the state associated with a GUID (for all Apps). + + int d_size; + // The message size + + bsl::vector d_apps; + // App states for the message + + DataStreamMessage(int size, bslma::Allocator* allocator); + + /// Return reference to the modifiable state of the App corresponding + /// to the specified 'ordinal. + mqbi::AppMessage& app(unsigned int appOrdinal); + + /// Return reference to the non-modifiable state of the App + /// corresponding to the specified 'ordinal. + const mqbi::AppMessage& app(unsigned int appOrdinal) const; +}; + // ===================== // class StorageIterator // ===================== @@ -437,13 +457,16 @@ class Storage { /// Save the message contained in the specified `appData`, `options` and /// the associated `attributes` and `msgGUID` into this storage and the /// associated virtual storage. The `attributes` is an in/out parameter - /// and storage layer can populate certain fields of that struct. + /// and storage layer can populate certain fields of that struct. If the + /// optionally specified `out` is not zero, load the created + /// `DataStreamMessage` into the 'out'. /// Return 0 on success or an non-zero error code on failure. virtual StorageResult::Enum put(StorageMessageAttributes* attributes, const bmqt::MessageGUID& msgGUID, const bsl::shared_ptr& appData, - const bsl::shared_ptr& options) = 0; + const bsl::shared_ptr& options, + mqbi::DataStreamMessage** out = 0) = 0; /// Update the App state corresponding to the specified `msgGUID` and the /// specified `appKey` in the DataStream. Decrement the reference count of @@ -698,6 +721,33 @@ inline bool AppMessage::isPushing() const return d_state == e_PUSH; } +// ----------------------- +// class DataStreamMessage +// ----------------------- + +inline DataStreamMessage::DataStreamMessage(int size, + bslma::Allocator* allocator) +: d_size(size) +, d_apps(allocator) +{ + // NOTHING +} + +inline mqbi::AppMessage& DataStreamMessage::app(unsigned int appOrdinal) +{ + BSLS_ASSERT_SAFE(appOrdinal < d_apps.size()); + + return d_apps[appOrdinal]; +} + +inline const mqbi::AppMessage& +DataStreamMessage::app(unsigned int appOrdinal) const +{ + BSLS_ASSERT_SAFE(appOrdinal < d_apps.size()); + + return d_apps[appOrdinal]; +} + // ------------------------------ // class StorageMessageAttributes // ------------------------------ diff --git a/src/groups/mqb/mqbs/mqbs_filebackedstorage.cpp b/src/groups/mqb/mqbs/mqbs_filebackedstorage.cpp index 1dff4b762..5ea7fd38c 100644 --- a/src/groups/mqb/mqbs/mqbs_filebackedstorage.cpp +++ b/src/groups/mqb/mqbs/mqbs_filebackedstorage.cpp @@ -89,11 +89,11 @@ void FileBackedStorage::purgeCommon(const mqbu::StorageKey& appKey) // Update stats d_capacityMeter.clear(); - d_queueStats_sp->onEvent(mqbstat::QueueStatsDomain::EventType::e_PURGE, - 0); - d_queueStats_sp->onEvent( - mqbstat::QueueStatsDomain::EventType::e_UPDATE_HISTORY, - d_handles.historySize()); + d_queueStats_sp + ->onEvent(0); + d_queueStats_sp + ->onEvent( + d_handles.historySize()); } } @@ -289,7 +289,8 @@ mqbi::StorageResult::Enum FileBackedStorage::put(mqbi::StorageMessageAttributes* attributes, const bmqt::MessageGUID& msgGUID, const bsl::shared_ptr& appData, - const bsl::shared_ptr& options) + const bsl::shared_ptr& options, + mqbi::DataStreamMessage** out) { const int msgSize = appData->length(); @@ -340,28 +341,32 @@ FileBackedStorage::put(mqbi::StorageMessageAttributes* attributes, // if we keep `irc` (like we keep 'DataStoreRecordHandle'). if (d_autoConfirms.empty()) { - d_virtualStorageCatalog.put(msgGUID, msgSize); + d_virtualStorageCatalog.put(msgGUID, msgSize, out); } else { - VirtualStorage::DataStreamMessage* dataStreamMessage = 0; - d_virtualStorageCatalog.put(msgGUID, msgSize, &dataStreamMessage); + mqbi::DataStreamMessage* dataStreamMessage = 0; + if (out == 0) { + out = &dataStreamMessage; + } + d_virtualStorageCatalog.put(msgGUID, msgSize, out); // Move auto confirms to the data record for (AutoConfirms::const_iterator it = d_autoConfirms.begin(); it != d_autoConfirms.end(); ++it) { irc.first->second.d_array.push_back(it->d_confirmRecordHandle); - d_virtualStorageCatalog.autoConfirm(dataStreamMessage, - it->d_appKey); + d_virtualStorageCatalog.autoConfirm(*out, it->d_appKey); } d_autoConfirms.clear(); } d_currentlyAutoConfirming = bmqt::MessageGUID(); BSLS_ASSERT_SAFE(queue()); - queue()->stats()->onEvent( - mqbstat::QueueStatsDomain::EventType::e_ADD_MESSAGE, - msgSize); + queue() + ->stats() + ->onEvent( + + msgSize); d_isEmpty.storeRelaxed(0); @@ -479,9 +484,9 @@ FileBackedStorage::releaseRef(const bmqt::MessageGUID& guid) if (queue()) { queue()->queueEngine()->beforeMessageRemoved(guid); } - d_queueStats_sp->onEvent( - mqbstat::QueueStatsDomain::EventType::e_DEL_MESSAGE, - msgLen); + d_queueStats_sp + ->onEvent( + msgLen); // There is not really a need to remove the guid from all virtual // storages, because we can be here only if guid doesn't exist in @@ -496,9 +501,9 @@ FileBackedStorage::releaseRef(const bmqt::MessageGUID& guid) d_capacityMeter.remove(1, msgLen); d_handles.erase(it); - d_queueStats_sp->onEvent( - mqbstat::QueueStatsDomain::EventType::e_UPDATE_HISTORY, - d_handles.historySize()); + d_queueStats_sp + ->onEvent( + d_handles.historySize()); return mqbi::StorageResult::e_ZERO_REFERENCES; } @@ -544,12 +549,16 @@ FileBackedStorage::remove(const bmqt::MessageGUID& msgGUID, int* msgSize) d_capacityMeter.remove(1, msgLen); BSLS_ASSERT_SAFE(queue()); - queue()->stats()->onEvent( - mqbstat::QueueStatsDomain::EventType::e_DEL_MESSAGE, - msgLen); - queue()->stats()->onEvent( - mqbstat::QueueStatsDomain::EventType::e_UPDATE_HISTORY, - d_handles.historySize()); + queue() + ->stats() + ->onEvent( + + msgLen); + queue() + ->stats() + ->onEvent( + + d_handles.historySize()); if (msgSize) { *msgSize = msgLen; @@ -604,9 +613,9 @@ FileBackedStorage::removeAll(const mqbu::StorageKey& appKey) d_isEmpty.storeRelaxed(1); } - d_queueStats_sp->onEvent( - mqbstat::QueueStatsDomain::EventType::e_UPDATE_HISTORY, - d_handles.historySize()); + d_queueStats_sp + ->onEvent( + d_handles.historySize()); return mqbi::StorageResult::e_SUCCESS; } @@ -694,9 +703,9 @@ int FileBackedStorage::gcExpiredMessages( if (queue()) { queue()->queueEngine()->beforeMessageRemoved(cit->first); } - d_queueStats_sp->onEvent( - mqbstat::QueueStatsDomain::EventType::e_DEL_MESSAGE, - msgLen); + d_queueStats_sp + ->onEvent( + msgLen); // Remove message from all virtual storages. d_virtualStorageCatalog.gc(cit->first); @@ -715,18 +724,18 @@ int FileBackedStorage::gcExpiredMessages( if (numMsgsDeleted > 0) { if (numMsgsDeleted > numMsgsUnreceipted) { - d_queueStats_sp->onEvent( - mqbstat::QueueStatsDomain::EventType::e_GC_MESSAGE, - numMsgsDeleted - numMsgsUnreceipted); + d_queueStats_sp + ->onEvent( + numMsgsDeleted - numMsgsUnreceipted); } if (numMsgsUnreceipted) { - d_queueStats_sp->onEvent( - mqbstat::QueueStatsDomain::EventType::e_NO_SC_MESSAGE, + d_queueStats_sp->onEvent< + mqbstat::QueueStatsDomain::EventType::e_NO_SC_MESSAGE>( numMsgsUnreceipted); } - d_queueStats_sp->onEvent( - mqbstat::QueueStatsDomain::EventType::e_UPDATE_HISTORY, - d_handles.historySize()); + d_queueStats_sp + ->onEvent( + d_handles.historySize()); } if (d_handles.empty()) { @@ -741,9 +750,9 @@ bool FileBackedStorage::gcHistory() bool hasMoreToGc = d_handles.gc(bmqsys::Time::highResolutionTimer(), k_GC_MESSAGES_BATCH_SIZE); - d_queueStats_sp->onEvent( - mqbstat::QueueStatsDomain::EventType::e_UPDATE_HISTORY, - d_handles.historySize()); + d_queueStats_sp + ->onEvent( + d_handles.historySize()); return hasMoreToGc; } @@ -770,7 +779,7 @@ void FileBackedStorage::processMessageRecord( else { if (!d_currentlyAutoConfirming.isUnset()) { if (d_currentlyAutoConfirming == guid) { - VirtualStorage::DataStreamMessage* dataStreamMessage = 0; + mqbi::DataStreamMessage* dataStreamMessage = 0; d_virtualStorageCatalog.put(guid, msgLen, &dataStreamMessage); @@ -789,7 +798,6 @@ void FileBackedStorage::processMessageRecord( else { clearSelection(); } - d_currentlyAutoConfirming = bmqt::MessageGUID(); } d_autoConfirms.clear(); } @@ -798,9 +806,9 @@ void FileBackedStorage::processMessageRecord( // Update the messages & bytes monitors, and the stats. d_capacityMeter.forceCommit(1, msgLen); // Return value ignored. - d_queueStats_sp->onEvent( - mqbstat::QueueStatsDomain::EventType::e_ADD_MESSAGE, - msgLen); + d_queueStats_sp + ->onEvent( + msgLen); d_isEmpty.storeRelaxed(0); } @@ -914,9 +922,8 @@ void FileBackedStorage::processDeletionRecord(const bmqt::MessageGUID& guid) if (queue()) { queue()->queueEngine()->beforeMessageRemoved(guid); } - d_queueStats_sp->onEvent( - mqbstat::QueueStatsDomain::EventType::e_DEL_MESSAGE, - msgLen); + d_queueStats_sp + ->onEvent(msgLen); // Delete 'guid' from all virtual storages, if any. Note that 'guid' // should have already been removed from each virtual storage when confirm @@ -942,9 +949,9 @@ void FileBackedStorage::processDeletionRecord(const bmqt::MessageGUID& guid) d_isEmpty.storeRelaxed(1); } - d_queueStats_sp->onEvent( - mqbstat::QueueStatsDomain::EventType::e_UPDATE_HISTORY, - d_handles.historySize()); + d_queueStats_sp + ->onEvent( + d_handles.historySize()); } void FileBackedStorage::addQueueOpRecordHandle( @@ -1014,9 +1021,9 @@ FileBackedStorage::autoConfirm(const mqbu::StorageKey& appKey, void FileBackedStorage::setPrimary() { - d_queueStats_sp->onEvent( - mqbstat::QueueStatsDomain::EventType::e_CHANGE_ROLE, - mqbstat::QueueStatsDomain::Role::e_PRIMARY); + d_queueStats_sp + ->onEvent( + mqbstat::QueueStatsDomain::Role::e_PRIMARY); } void FileBackedStorage::clearSelection() diff --git a/src/groups/mqb/mqbs/mqbs_filebackedstorage.h b/src/groups/mqb/mqbs/mqbs_filebackedstorage.h index 382bf7170..cb67cc36d 100644 --- a/src/groups/mqb/mqbs/mqbs_filebackedstorage.h +++ b/src/groups/mqb/mqbs/mqbs_filebackedstorage.h @@ -381,13 +381,16 @@ class FileBackedStorage BSLS_KEYWORD_FINAL : public ReplicatedStorage { /// Save the message contained in the specified 'appData', 'options' and /// the associated 'attributes' and 'msgGUID' into this storage and the /// associated virtual storage. The 'attributes' is an in/out parameter - /// and storage layer can populate certain fields of that struct. + /// and storage layer can populate certain fields of that struct. If the + /// optionally specified `out` is not zero, load the created + /// `DataStreamMessage` into the 'out'. /// Return 0 on success or an non-zero error code on failure. mqbi::StorageResult::Enum put(mqbi::StorageMessageAttributes* attributes, const bmqt::MessageGUID& msgGUID, const bsl::shared_ptr& appData, - const bsl::shared_ptr& options) BSLS_KEYWORD_OVERRIDE; + const bsl::shared_ptr& options, + mqbi::DataStreamMessage** out = 0) BSLS_KEYWORD_OVERRIDE; /// Get an iterator for data stored in the virtual storage identified by /// the specified 'appKey'. diff --git a/src/groups/mqb/mqbs/mqbs_inmemorystorage.cpp b/src/groups/mqb/mqbs/mqbs_inmemorystorage.cpp index 37bd3450e..82a06a3e3 100644 --- a/src/groups/mqb/mqbs/mqbs_inmemorystorage.cpp +++ b/src/groups/mqb/mqbs/mqbs_inmemorystorage.cpp @@ -166,7 +166,8 @@ mqbi::StorageResult::Enum InMemoryStorage::put(mqbi::StorageMessageAttributes* attributes, const bmqt::MessageGUID& msgGUID, const bsl::shared_ptr& appData, - const bsl::shared_ptr& options) + const bsl::shared_ptr& options, + mqbi::DataStreamMessage** out) { const int msgSize = appData->length(); @@ -201,18 +202,20 @@ InMemoryStorage::put(mqbi::StorageMessageAttributes* attributes, attributes->arrivalTimepoint()); if (d_autoConfirms.empty()) { - d_virtualStorageCatalog.put(msgGUID, msgSize); + d_virtualStorageCatalog.put(msgGUID, msgSize, out); } else { - VirtualStorage::DataStreamMessage* dataStreamMessage = 0; - d_virtualStorageCatalog.put(msgGUID, msgSize, &dataStreamMessage); + mqbi::DataStreamMessage* dataStreamMessage = 0; + if (out == 0) { + out = &dataStreamMessage; + } + d_virtualStorageCatalog.put(msgGUID, msgSize, out); // Move auto confirms to the data record for (AutoConfirms::const_iterator it = d_autoConfirms.begin(); it != d_autoConfirms.end(); ++it) { - d_virtualStorageCatalog.autoConfirm(dataStreamMessage, - it->d_appKey); + d_virtualStorageCatalog.autoConfirm(*out, it->d_appKey); } d_autoConfirms.clear(); } @@ -220,9 +223,11 @@ InMemoryStorage::put(mqbi::StorageMessageAttributes* attributes, d_currentlyAutoConfirming = bmqt::MessageGUID(); if (queue()) { - queue()->stats()->onEvent( - mqbstat::QueueStatsDomain::EventType::e_ADD_MESSAGE, - msgSize); + queue() + ->stats() + ->onEvent( + + msgSize); } d_isEmpty.storeRelaxed(0); @@ -243,12 +248,20 @@ InMemoryStorage::put(mqbi::StorageMessageAttributes* attributes, mqbi::StorageMessageAttributes& existing = it->second.attributes(); existing.setRefCount(existing.refCount() + attributes->refCount()); // Bump up + + if (out) { + // Proxy can detect duplicates by inspecting 'DataStreamMessage'. + VirtualStorage::DataStreamIterator data = + d_virtualStorageCatalog.get(msgGUID); + BSLS_ASSERT_SAFE(data != d_virtualStorageCatalog.end()); + *out = &data->second; + } } else { d_items.insert(bsl::make_pair(msgGUID, Item(appData, options, *attributes)), attributes->arrivalTimepoint()); - d_virtualStorageCatalog.put(msgGUID, msgSize); + d_virtualStorageCatalog.put(msgGUID, msgSize, out); } // We don't verify uniqueness of the insertion because in the case of a @@ -309,9 +322,10 @@ InMemoryStorage::releaseRef(const bmqt::MessageGUID& guid) d_capacityMeter.remove(1, msgLen); if (queue()) { queue()->queueEngine()->beforeMessageRemoved(guid); - queue()->stats()->onEvent( - mqbstat::QueueStatsDomain::EventType::e_DEL_MESSAGE, - msgLen); + queue() + ->stats() + ->onEvent( + msgLen); } // There is not really a need to remove the guid from all virtual @@ -324,9 +338,11 @@ InMemoryStorage::releaseRef(const bmqt::MessageGUID& guid) d_items.erase(it); if (queue()) { - queue()->stats()->onEvent( - mqbstat::QueueStatsDomain::EventType::e_UPDATE_HISTORY, - d_items.historySize()); + queue() + ->stats() + ->onEvent< + mqbstat::QueueStatsDomain::EventType::e_UPDATE_HISTORY>( + d_items.historySize()); } return mqbi::StorageResult::e_ZERO_REFERENCES; // RETURN @@ -353,12 +369,14 @@ InMemoryStorage::remove(const bmqt::MessageGUID& msgGUID, int* msgSize) d_capacityMeter.remove(1, msgLen); if (queue()) { - queue()->stats()->onEvent( - mqbstat::QueueStatsDomain::EventType::e_DEL_MESSAGE, - msgLen); - queue()->stats()->onEvent( - mqbstat::QueueStatsDomain::EventType::e_UPDATE_HISTORY, - d_items.historySize()); + queue() + ->stats() + ->onEvent( + msgLen); + queue() + ->stats() + ->onEvent( + d_items.historySize()); } if (msgSize) { @@ -383,9 +401,9 @@ InMemoryStorage::removeAll(const mqbu::StorageKey& appKey) d_capacityMeter.clear(); if (queue()) { - queue()->stats()->onEvent( - mqbstat::QueueStatsDomain::EventType::e_PURGE, - 0); + queue() + ->stats() + ->onEvent(0); } d_isEmpty.storeRelaxed(1); @@ -413,9 +431,10 @@ InMemoryStorage::removeAll(const mqbu::StorageKey& appKey) } if (queue()) { - queue()->stats()->onEvent( - mqbstat::QueueStatsDomain::EventType::e_UPDATE_HISTORY, - d_items.historySize()); + queue() + ->stats() + ->onEvent( + d_items.historySize()); } return mqbi::StorageResult::e_SUCCESS; @@ -454,9 +473,10 @@ int InMemoryStorage::gcExpiredMessages( d_capacityMeter.remove(1, msgLen); if (queue()) { queue()->queueEngine()->beforeMessageRemoved(cit->first); - queue()->stats()->onEvent( - mqbstat::QueueStatsDomain::EventType::e_DEL_MESSAGE, - msgLen); + queue() + ->stats() + ->onEvent( + msgLen); } // Remove message from all virtual storages and the physical (this) @@ -467,12 +487,14 @@ int InMemoryStorage::gcExpiredMessages( } if (queue() && (numMsgsDeleted > 0)) { - queue()->stats()->onEvent( - mqbstat::QueueStatsDomain::EventType::e_GC_MESSAGE, - numMsgsDeleted); - queue()->stats()->onEvent( - mqbstat::QueueStatsDomain::EventType::e_UPDATE_HISTORY, - d_items.historySize()); + queue() + ->stats() + ->onEvent( + numMsgsDeleted); + queue() + ->stats() + ->onEvent( + d_items.historySize()); } if (d_items.empty()) { @@ -488,9 +510,10 @@ bool InMemoryStorage::gcHistory() k_GC_MESSAGES_BATCH_SIZE); if (queue()) { - queue()->stats()->onEvent( - mqbstat::QueueStatsDomain::EventType::e_UPDATE_HISTORY, - d_items.historySize()); + queue() + ->stats() + ->onEvent( + d_items.historySize()); } return hasMoreToGc; diff --git a/src/groups/mqb/mqbs/mqbs_inmemorystorage.h b/src/groups/mqb/mqbs/mqbs_inmemorystorage.h index 5ccbac8be..9aa83c5ed 100644 --- a/src/groups/mqb/mqbs/mqbs_inmemorystorage.h +++ b/src/groups/mqb/mqbs/mqbs_inmemorystorage.h @@ -302,13 +302,16 @@ class InMemoryStorage BSLS_KEYWORD_FINAL : public ReplicatedStorage { /// Save the message contained in the specified 'appData', 'options' and /// the associated 'attributes' and 'msgGUID' into this storage and the /// associated virtual storage. The 'attributes' is an in/out parameter - /// and storage layer can populate certain fields of that struct. + /// and storage layer can populate certain fields of that struct. If the + /// optionally specified `out` is not zero, load the created + /// `DataStreamMessage` into the 'out'. /// Return 0 on success or an non-zero error code on failure. mqbi::StorageResult::Enum put(mqbi::StorageMessageAttributes* attributes, const bmqt::MessageGUID& msgGUID, const bsl::shared_ptr& appData, - const bsl::shared_ptr& options) BSLS_KEYWORD_OVERRIDE; + const bsl::shared_ptr& options, + mqbi::DataStreamMessage** out = 0) BSLS_KEYWORD_OVERRIDE; /// Update the App state corresponding to the specified 'msgGUID' and the /// specified 'appKey' in the DataStream. Decrement the reference count of diff --git a/src/groups/mqb/mqbs/mqbs_virtualstorage.cpp b/src/groups/mqb/mqbs/mqbs_virtualstorage.cpp index dc9b9594c..defd3899d 100644 --- a/src/groups/mqb/mqbs/mqbs_virtualstorage.cpp +++ b/src/groups/mqb/mqbs/mqbs_virtualstorage.cpp @@ -58,7 +58,7 @@ VirtualStorage::~VirtualStorage() // MANIPULATORS mqbi::StorageResult::Enum -VirtualStorage::confirm(DataStreamMessage* dataStreamMessage) +VirtualStorage::confirm(mqbi::DataStreamMessage* dataStreamMessage) { mqbi::AppMessage& appMessage = dataStreamMessage->app(ordinal()); @@ -77,7 +77,7 @@ VirtualStorage::confirm(DataStreamMessage* dataStreamMessage) } mqbi::StorageResult::Enum -VirtualStorage::remove(DataStreamMessage* dataStreamMessage) +VirtualStorage::remove(mqbi::DataStreamMessage* dataStreamMessage) { mqbi::AppMessage& appMessage = dataStreamMessage->app(ordinal()); @@ -95,7 +95,7 @@ VirtualStorage::remove(DataStreamMessage* dataStreamMessage) } } -void VirtualStorage::onGC(const DataStreamMessage& dataStreamMessage) +void VirtualStorage::onGC(const mqbi::DataStreamMessage& dataStreamMessage) { if (!dataStreamMessage.d_apps.empty()) { const mqbi::AppMessage& appMessage = dataStreamMessage.app(ordinal()); @@ -191,7 +191,6 @@ void StorageIterator::reset(const bmqt::MessageGUID& where) { clear(); - // Reset iterator to beginning d_iterator = d_owner_p->begin(where); } @@ -210,8 +209,7 @@ StorageIterator::appMessageView(unsigned int appOrdinal) const // PRECONDITIONS BSLS_ASSERT_SAFE(!atEnd()); - const VirtualStorage::DataStreamMessage& dataStreamMessage = - d_iterator->second; + const mqbi::DataStreamMessage& dataStreamMessage = d_iterator->second; if (dataStreamMessage.d_apps.size() > appOrdinal) { return d_iterator->second.app(appOrdinal); @@ -224,7 +222,7 @@ mqbi::AppMessage& StorageIterator::appMessageState(unsigned int appOrdinal) // PRECONDITIONS BSLS_ASSERT_SAFE(!atEnd()); - VirtualStorage::DataStreamMessage* dataStreamMessage = &d_iterator->second; + mqbi::DataStreamMessage* dataStreamMessage = &d_iterator->second; d_owner_p->setup(dataStreamMessage); diff --git a/src/groups/mqb/mqbs/mqbs_virtualstorage.h b/src/groups/mqb/mqbs/mqbs_virtualstorage.h index c8eb10cf6..37d1dccfa 100644 --- a/src/groups/mqb/mqbs/mqbs_virtualstorage.h +++ b/src/groups/mqb/mqbs/mqbs_virtualstorage.h @@ -71,29 +71,11 @@ class VirtualStorage { // This Mechanism represents one App in a Storage (FileBased or InMemory) public: - struct DataStreamMessage { - int d_size; - // The message size - - bsl::vector d_apps; - // App states for the message - - DataStreamMessage(int size, bslma::Allocator* allocator); - - /// Return reference to the modifiable state of the App corresponding - /// to the specified 'ordinal. - mqbi::AppMessage& app(unsigned int appOrdinal); - - /// Return reference to the non-modifiable state of the App - /// corresponding to the specified 'ordinal. - const mqbi::AppMessage& app(unsigned int appOrdinal) const; - }; - /// msgGUID -> MessageContext /// Must be a container in which iteration order is same as insertion /// order. typedef bmqc::OrderedHashMap > DataStream; @@ -178,15 +160,17 @@ class VirtualStorage { /// Change the state of this App in the specified 'dataStreamMessage' to /// indicate CONFIRM. - mqbi::StorageResult::Enum confirm(DataStreamMessage* dataStreamMessage); + mqbi::StorageResult::Enum + confirm(mqbi::DataStreamMessage* dataStreamMessage); /// Change the state of this App in the specified 'dataStreamMessage' to /// indicate removal (by a purge or unregistration). - mqbi::StorageResult::Enum remove(DataStreamMessage* dataStreamMessage); + mqbi::StorageResult::Enum + remove(mqbi::DataStreamMessage* dataStreamMessage); /// Observe removal of this App from the specified 'dataStreamMessage' by /// GC and update bytes and messages counts if needed. - void onGC(const DataStreamMessage& dataStreamMessage); + void onGC(const mqbi::DataStreamMessage& dataStreamMessage); /// Reset bytes and messages counts as in the case of purging all Apps. void resetStats(); @@ -360,35 +344,6 @@ class VirtualStorageIterator : public StorageIterator { // INLINE DEFINITIONS // ============================================================================ -// --------------------------------------- -// class VirtualStorage::DataStreamMessage -// --------------------------------------- - -inline VirtualStorage::DataStreamMessage::DataStreamMessage( - int size, - bslma::Allocator* allocator) -: d_size(size) -, d_apps(allocator) -{ - // NOTHING -} - -inline mqbi::AppMessage& -VirtualStorage::DataStreamMessage::app(unsigned int appOrdinal) -{ - BSLS_ASSERT_SAFE(appOrdinal < d_apps.size()); - - return d_apps[appOrdinal]; -} - -inline const mqbi::AppMessage& -VirtualStorage::DataStreamMessage::app(unsigned int appOrdinal) const -{ - BSLS_ASSERT_SAFE(appOrdinal < d_apps.size()); - - return d_apps[appOrdinal]; -} - // -------------------- // class VirtualStorage // -------------------- diff --git a/src/groups/mqb/mqbs/mqbs_virtualstoragecatalog.cpp b/src/groups/mqb/mqbs/mqbs_virtualstoragecatalog.cpp index fabab605c..07c77d43c 100644 --- a/src/groups/mqb/mqbs/mqbs_virtualstoragecatalog.cpp +++ b/src/groups/mqb/mqbs/mqbs_virtualstoragecatalog.cpp @@ -145,7 +145,7 @@ VirtualStorageCatalog::get(const bmqt::MessageGUID& msgGUID) return it; } -void VirtualStorageCatalog::setup(VirtualStorage::DataStreamMessage* data) +void VirtualStorageCatalog::setup(mqbi::DataStreamMessage* data) { // The only case for subsequent resize is proxy receiving subsequent PUSH // messages for the same GUID and different apps @@ -155,14 +155,14 @@ void VirtualStorageCatalog::setup(VirtualStorage::DataStreamMessage* data) } mqbi::StorageResult::Enum -VirtualStorageCatalog::put(const bmqt::MessageGUID& msgGUID, - int msgSize, - VirtualStorage::DataStreamMessage** out) +VirtualStorageCatalog::put(const bmqt::MessageGUID& msgGUID, + int msgSize, + mqbi::DataStreamMessage** out) { bsl::pair insertResult = - d_dataStream.insert(bsl::make_pair( - msgGUID, - VirtualStorage::DataStreamMessage(msgSize, d_allocator_p))); + d_dataStream.insert( + bsl::make_pair(msgGUID, + mqbi::DataStreamMessage(msgSize, d_allocator_p))); if (!insertResult.second) { // Duplicate GUID @@ -270,7 +270,6 @@ VirtualStorageCatalog::confirm(const bmqt::MessageGUID& msgGUID, VirtualStoragesIter it = d_virtualStorages.findByKey2(appKey); BSLS_ASSERT_SAFE(it != d_virtualStorages.end()); - setup(&data->second); const mqbi::StorageResult::Enum rc = it->value()->confirm(&data->second); if (queue() && mqbi::StorageResult::e_SUCCESS == rc) { queue()->stats()->onEvent( @@ -328,7 +327,7 @@ VirtualStorageCatalog::removeAll(const mqbu::StorageKey& appKey) itData != d_dataStream.end();) { mqbi::StorageResult::Enum result = mqbi::StorageResult::e_SUCCESS; - VirtualStorage::DataStreamMessage* data = &itData->second; + mqbi::DataStreamMessage* data = &itData->second; setup(data); if (itVs->value()->remove(data) == @@ -503,8 +502,8 @@ VirtualStorageCatalog::virtualStorage(const mqbu::StorageKey& appKey) } void VirtualStorageCatalog::autoConfirm( - VirtualStorage::DataStreamMessage* dataStreamMessage, - const mqbu::StorageKey& appKey) + mqbi::DataStreamMessage* dataStreamMessage, + const mqbu::StorageKey& appKey) { // PRECONDITIONS BSLS_ASSERT_SAFE(dataStreamMessage); diff --git a/src/groups/mqb/mqbs/mqbs_virtualstoragecatalog.h b/src/groups/mqb/mqbs/mqbs_virtualstoragecatalog.h index 1b37dfc6a..1b7824328 100644 --- a/src/groups/mqb/mqbs/mqbs_virtualstoragecatalog.h +++ b/src/groups/mqb/mqbs/mqbs_virtualstoragecatalog.h @@ -171,14 +171,14 @@ class VirtualStorageCatalog { DataStreamIterator get(const bmqt::MessageGUID& msgGUID); /// Allocate space for all Apps states in the specified 'data' if needed. - void setup(VirtualStorage::DataStreamMessage* data); + void setup(mqbi::DataStreamMessage* data); /// Save the message having the specified 'msgGUID' and 'msgSize' to the /// DataStream. If the specified 'out' is not '0', allocate space for all /// Apps states and load the created object into the 'out'. - mqbi::StorageResult::Enum put(const bmqt::MessageGUID& msgGUID, - int msgSize, - VirtualStorage::DataStreamMessage** out = 0); + mqbi::StorageResult::Enum put(const bmqt::MessageGUID& msgGUID, + int msgSize, + mqbi::DataStreamMessage** out = 0); /// Get an iterator for items stored in the DataStream identified by the /// specified 'appKey'. @@ -245,8 +245,8 @@ class VirtualStorageCatalog { /// (Auto)Confirm the specified 'msgGUID' for the specified 'appKey'. /// Behavior is undefined unless there is an App with the 'appKey'. - void autoConfirm(VirtualStorage::DataStreamMessage* dataStreamMessage, - const mqbu::StorageKey& appKey); + void autoConfirm(mqbi::DataStreamMessage* dataStreamMessage, + const mqbu::StorageKey& appKey); /// Set the default RDA according to the specified 'maxDeliveryAttempts'. void setDefaultRda(int maxDeliveryAttempts); diff --git a/src/groups/mqb/mqbstat/mqbstat_brokerstats.cpp b/src/groups/mqb/mqbstat/mqbstat_brokerstats.cpp index ad1fbb04c..ffc58d672 100644 --- a/src/groups/mqb/mqbstat/mqbstat_brokerstats.cpp +++ b/src/groups/mqb/mqbstat/mqbstat_brokerstats.cpp @@ -40,16 +40,6 @@ namespace { /// Name of the stat context to create (holding all broker's statistics) static const char k_BROKER_STAT_NAME[] = "broker"; -//------------------------ -// struct BrokerStatsIndex -//------------------------ - -/// Namespace for the constants of stat values that applies to the queues -/// from the clients -struct BrokerStatsIndex { - enum Enum { e_STAT_QUEUE_COUNT, e_STAT_CLIENT_COUNT }; -}; - } // close unnamed namespace // ----------------- @@ -110,30 +100,6 @@ void BrokerStats::initialize(bmqst::StatContext* brokerStatContext) d_statContext_p = brokerStatContext; } -void BrokerStats::onEvent(EventType::Enum type) -{ - BSLS_ASSERT_SAFE(d_statContext_p && "initialize was not called"); - - switch (type) { - case EventType::e_CLIENT_CREATED: { - d_statContext_p->adjustValue(BrokerStatsIndex::e_STAT_CLIENT_COUNT, 1); - } break; - case EventType::e_CLIENT_DESTROYED: { - d_statContext_p->adjustValue(BrokerStatsIndex::e_STAT_CLIENT_COUNT, - -1); - } break; - case EventType::e_QUEUE_CREATED: { - d_statContext_p->adjustValue(BrokerStatsIndex::e_STAT_QUEUE_COUNT, 1); - } break; - case EventType::e_QUEUE_DESTROYED: { - d_statContext_p->adjustValue(BrokerStatsIndex::e_STAT_QUEUE_COUNT, -1); - } break; - default: { - BSLS_ASSERT_SAFE(false && "Unknown event type"); - } break; - }; -} - // --------------------- // class BrokerStatsUtil // --------------------- diff --git a/src/groups/mqb/mqbstat/mqbstat_brokerstats.h b/src/groups/mqb/mqbstat/mqbstat_brokerstats.h index c803f66bf..005e7c018 100644 --- a/src/groups/mqb/mqbstat/mqbstat_brokerstats.h +++ b/src/groups/mqb/mqbstat/mqbstat_brokerstats.h @@ -30,6 +30,7 @@ // MQB // BMQ +#include #include // BDE @@ -42,11 +43,6 @@ namespace BloombergLP { -// FORWARD DECLARATION -namespace bmqst { -class StatContext; -} - namespace mqbstat { // ================= @@ -84,6 +80,14 @@ class BrokerStats { // DATA bmqst::StatContext* d_statContext_p; // StatContext + // PRIVATE TYPES + + /// Namespace for the constants of stat values that applies to the queues + /// from the clients + struct BrokerStatsIndex { + enum Enum { e_STAT_QUEUE_COUNT, e_STAT_CLIENT_COUNT }; + }; + private: // NOT IMPLEMENTED BrokerStats(const BrokerStats&) BSLS_CPP11_DELETED; @@ -124,7 +128,8 @@ class BrokerStats { /// Update statistics for the event of the specified `type` and with the /// specified `value` (depending on the `type`, `value` can represent /// the number of bytes, a counter, ... - void onEvent(EventType::Enum type); + template + void onEvent(); /// Return a pointer to the statcontext. bmqst::StatContext* statContext(); @@ -159,6 +164,42 @@ inline bmqst::StatContext* BrokerStats::statContext() return d_statContext_p; } +template <> +inline void +BrokerStats::onEvent() +{ + BSLS_ASSERT_SAFE(d_statContext_p && "initialize was not called"); + + d_statContext_p->adjustValue(BrokerStatsIndex::e_STAT_CLIENT_COUNT, 1); +} + +template <> +inline void +BrokerStats::onEvent() +{ + BSLS_ASSERT_SAFE(d_statContext_p && "initialize was not called"); + + d_statContext_p->adjustValue(BrokerStatsIndex::e_STAT_CLIENT_COUNT, -1); +} + +template <> +inline void +BrokerStats::onEvent() +{ + BSLS_ASSERT_SAFE(d_statContext_p && "initialize was not called"); + + d_statContext_p->adjustValue(BrokerStatsIndex::e_STAT_QUEUE_COUNT, 1); +} + +template <> +inline void +BrokerStats::onEvent() +{ + BSLS_ASSERT_SAFE(d_statContext_p && "initialize was not called"); + + d_statContext_p->adjustValue(BrokerStatsIndex::e_STAT_QUEUE_COUNT, -1); +} + } // close package namespace } // close enterprise namespace diff --git a/src/groups/mqb/mqbstat/mqbstat_clusterstats.cpp b/src/groups/mqb/mqbstat/mqbstat_clusterstats.cpp index 3550d813a..42d3c5c97 100644 --- a/src/groups/mqb/mqbstat/mqbstat_clusterstats.cpp +++ b/src/groups/mqb/mqbstat/mqbstat_clusterstats.cpp @@ -85,35 +85,6 @@ struct ClusterStatsIndex { }; }; -// ---------------------------- -// struct ClusterNodeStatsIndex -// ---------------------------- - -/// Namespace for the constants of stat values that applies to the -/// cluster node -struct ClusterNodeStatsIndex { - enum Enum { - /// Value: Number of ack messages delivered to the client - e_STAT_ACK - - , - e_STAT_CONFIRM - // Value: Number of confirm messages delivered to the client - - , - e_STAT_PUSH - // Value: Accumulated bytes of all messages ever pushed to - // the client - // Increments: Number of messages ever pushed to the client - - , - e_STAT_PUT - // Value: Accumulated bytes of all messages ever received from - // the client - // Increments: Number of messages ever received from the client - }; -}; - //------------------------- // struct ClusterStatsIndex //------------------------- @@ -517,34 +488,6 @@ void ClusterNodeStats::initialize(const bmqt::Uri& uri, bmqst::StatContextConfiguration(uri.asString(), &localAllocator)); } -void ClusterNodeStats::onEvent(EventType::Enum type, bsls::Types::Int64 value) -{ - BSLS_ASSERT_SAFE(d_statContext_mp && "initialize was not called"); - - switch (type) { - case EventType::e_ACK: { - // For ACK, we don't have any bytes value, but we also wouldn't care .. - d_statContext_mp->adjustValue(ClusterNodeStatsIndex::e_STAT_ACK, 1); - } break; - case EventType::e_CONFIRM: { - // For CONFIRM, we don't care about the bytes value .. - d_statContext_mp->adjustValue(ClusterNodeStatsIndex::e_STAT_CONFIRM, - 1); - } break; - case EventType::e_PUSH: { - d_statContext_mp->adjustValue(ClusterNodeStatsIndex::e_STAT_PUSH, - value); - } break; - case EventType::e_PUT: { - d_statContext_mp->adjustValue(ClusterNodeStatsIndex::e_STAT_PUT, - value); - } break; - default: { - BSLS_ASSERT_SAFE(false && "Unknown event type"); - } break; - }; -} - // ---------------------- // class ClusterStatsUtil // ---------------------- diff --git a/src/groups/mqb/mqbstat/mqbstat_clusterstats.h b/src/groups/mqb/mqbstat/mqbstat_clusterstats.h index 3016166a2..3d7d5f78c 100644 --- a/src/groups/mqb/mqbstat/mqbstat_clusterstats.h +++ b/src/groups/mqb/mqbstat/mqbstat_clusterstats.h @@ -30,6 +30,7 @@ // utility namespace exposing methods to initialize the stat contexts. // BMQ +#include #include // MQB @@ -47,11 +48,6 @@ namespace BloombergLP { -// FORWARD DECLARATION -namespace bmqst { -class StatContext; -} - namespace mqbstat { // ================== @@ -303,6 +299,33 @@ class ClusterNodeStats { bslma::ManagedPtr d_statContext_mp; // StatContext + // PRIVATE TYPES + + /// Namespace for the constants of stat values that applies to the + /// cluster node + struct ClusterNodeStatsIndex { + enum Enum { + /// Value: Number of ack messages delivered to the client + e_STAT_ACK + + , + e_STAT_CONFIRM + // Value: Number of confirm messages delivered to the client + + , + e_STAT_PUSH + // Value: Accumulated bytes of all messages ever pushed to + // the client + // Increments: Number of messages ever pushed to the client + + , + e_STAT_PUT + // Value: Accumulated bytes of all messages ever received from + // the client + // Increments: Number of messages ever received from the client + }; + }; + private: // NOT IMPLEMENTED ClusterNodeStats(const ClusterNodeStats&) BSLS_CPP11_DELETED; @@ -341,7 +364,8 @@ class ClusterNodeStats { /// Update statistics for the event of the specified `type` and with the /// specified `value` (depending on the `type`, `value` can represent /// the number of bytes, a counter, ... - void onEvent(EventType::Enum type, bsls::Types::Int64 value); + template + void onEvent(bsls::Types::Int64 value); /// Return a pointer to the statcontext. bmqst::StatContext* statContext(); @@ -393,6 +417,38 @@ inline bmqst::StatContext* ClusterNodeStats::statContext() return d_statContext_mp.get(); } +template <> +inline void +ClusterNodeStats::onEvent( + BSLS_ANNOTATION_UNUSED bsls::Types::Int64 value) +{ + d_statContext_mp->adjustValue(ClusterNodeStatsIndex::e_STAT_ACK, 1); +} + +template <> +inline void +ClusterNodeStats::onEvent( + BSLS_ANNOTATION_UNUSED bsls::Types::Int64 value) +{ + d_statContext_mp->adjustValue(ClusterNodeStatsIndex::e_STAT_CONFIRM, 1); +} + +template <> +inline void +ClusterNodeStats::onEvent( + bsls::Types::Int64 value) +{ + d_statContext_mp->adjustValue(ClusterNodeStatsIndex::e_STAT_PUSH, value); +} + +template <> +inline void +ClusterNodeStats::onEvent( + bsls::Types::Int64 value) +{ + d_statContext_mp->adjustValue(ClusterNodeStatsIndex::e_STAT_PUT, value); +} + } // close package namespace } // close enterprise namespace diff --git a/src/groups/mqb/mqbstat/mqbstat_domainstats.cpp b/src/groups/mqb/mqbstat/mqbstat_domainstats.cpp index afbd25e88..5b0b5a31d 100644 --- a/src/groups/mqb/mqbstat/mqbstat_domainstats.cpp +++ b/src/groups/mqb/mqbstat/mqbstat_domainstats.cpp @@ -42,16 +42,6 @@ namespace { /// Name of the stat context to create (holding all domain's statistics) static const char k_DOMAIN_STAT_NAME[] = "domains"; -//------------------------ -// struct DomainStatsIndex -//------------------------ - -/// Namespace for the constants of stat values that applies to the queues -/// from the clients -struct DomainStatsIndex { - enum Enum { e_STAT_CFG_MSGS, e_STAT_CFG_BYTES, e_STAT_QUEUE_COUNT }; -}; - } // close unnamed namespace // ------------------ @@ -143,27 +133,6 @@ void DomainStats::initialize(mqbi::Domain* domain, datum->adopt(builder.commit()); } -void DomainStats::onEvent(EventType::Enum type, bsls::Types::Int64 value) -{ - BSLS_ASSERT_SAFE(d_statContext_mp && "initialize was not called"); - - switch (type) { - case EventType::e_CFG_MSGS: { - d_statContext_mp->setValue(DomainStatsIndex::e_STAT_CFG_MSGS, value); - } break; - case EventType::e_CFG_BYTES: { - d_statContext_mp->setValue(DomainStatsIndex::e_STAT_CFG_BYTES, value); - } break; - case EventType::e_QUEUE_COUNT: { - d_statContext_mp->setValue(DomainStatsIndex::e_STAT_QUEUE_COUNT, - value); - } break; - default: { - BSLS_ASSERT_SAFE(false && "Unknown event type"); - } break; - }; -} - // --------------------- // class DomainStatsUtil // --------------------- diff --git a/src/groups/mqb/mqbstat/mqbstat_domainstats.h b/src/groups/mqb/mqbstat/mqbstat_domainstats.h index 91c7e96fa..8f19a4d81 100644 --- a/src/groups/mqb/mqbstat/mqbstat_domainstats.h +++ b/src/groups/mqb/mqbstat/mqbstat_domainstats.h @@ -28,6 +28,7 @@ // exposing methods to initialize the stat contexts. // MQB +#include // BDE #include @@ -43,10 +44,6 @@ namespace BloombergLP { namespace mqbi { class Domain; } -namespace bmqst { -class StatContext; -} - namespace mqbstat { // ================= @@ -77,6 +74,14 @@ class DomainStats { bslma::ManagedPtr d_statContext_mp; // StatContext + // PRIVATE TYPES + + /// Namespace for the constants of stat values that applies to the queues + /// from the clients + struct DomainStatsIndex { + enum Enum { e_STAT_CFG_MSGS, e_STAT_CFG_BYTES, e_STAT_QUEUE_COUNT }; + }; + private: // NOT IMPLEMENTED DomainStats(const DomainStats&) BSLS_CPP11_DELETED; @@ -115,7 +120,8 @@ class DomainStats { /// Update statistics for the event of the specified `type` and with the /// specified `value` (depending on the `type`, `value` can represent /// the number of bytes, a counter, ... - void onEvent(EventType::Enum type, bsls::Types::Int64 value); + template + void onEvent(bsls::Types::Int64 value); /// Return a pointer to the statcontext. bmqst::StatContext* statContext(); @@ -150,6 +156,30 @@ inline bmqst::StatContext* DomainStats::statContext() return d_statContext_mp.get(); } +template <> +inline void DomainStats::onEvent( + bsls::Types::Int64 value) +{ + BSLS_ASSERT_SAFE(d_statContext_mp && "initialize was not called"); + d_statContext_mp->setValue(DomainStatsIndex::e_STAT_CFG_MSGS, value); +} + +template <> +inline void DomainStats::onEvent( + bsls::Types::Int64 value) +{ + BSLS_ASSERT_SAFE(d_statContext_mp && "initialize was not called"); + d_statContext_mp->setValue(DomainStatsIndex::e_STAT_CFG_BYTES, value); +} + +template <> +inline void DomainStats::onEvent( + bsls::Types::Int64 value) +{ + BSLS_ASSERT_SAFE(d_statContext_mp && "initialize was not called"); + d_statContext_mp->setValue(DomainStatsIndex::e_STAT_QUEUE_COUNT, value); +} + } // close package namespace } // close enterprise namespace diff --git a/src/groups/mqb/mqbstat/mqbstat_queuestats.cpp b/src/groups/mqb/mqbstat/mqbstat_queuestats.cpp index 7943f4b57..ab0e36b8e 100644 --- a/src/groups/mqb/mqbstat/mqbstat_queuestats.cpp +++ b/src/groups/mqb/mqbstat/mqbstat_queuestats.cpp @@ -63,104 +63,6 @@ const int k_MAX_INSTANT_MESSAGES = 10; const bsls::Types::Int64 k_NS_PER_MESSAGE = bdlt::TimeUnitRatio::k_NANOSECONDS_PER_MINUTE / k_MAX_INSTANT_MESSAGES; -// ----------------------- -// struct DomainQueueStats -// ----------------------- - -/// Namespace for the constants of stat values that applies to the queues on -/// the domain -struct DomainQueueStats { - enum Enum { - /// Value: Current number of clients who opened the queue with - /// the `WRITE` flag - e_STAT_NB_PRODUCER - - , - /// Value: Current number of clients who opened the queue with - /// the 'READ' flag - e_STAT_NB_CONSUMER - - , - /// Value: Current number of messages in the queue - e_STAT_MESSAGES - - , - /// Value: Accumulated bytes of all messages currently in the - /// queue - e_STAT_BYTES - - , - /// Value: Number of ack messages delivered by this queue - e_STAT_ACK - - , - /// Value: The time between PUT and ACK (in nanoseconds). - e_STAT_ACK_TIME - - , - /// Value: Number of NACK messages generated for this queue - e_STAT_NACK - - , - /// Value: Number of CONFIRM messages received by this queue - e_STAT_CONFIRM - - , - /// Value: The time between PUSH and CONFIRM (in nanoseconds). - e_STAT_CONFIRM_TIME - - , - /// Value: Number of messages rejected by this queue (RDA - /// reaching zero) - e_STAT_REJECT - - , - /// Value: The time spent by the message in the queue (in - /// nanoseconds). - e_STAT_QUEUE_TIME - - , - /// Value: Accumulated bytes of all messages ever pushed from - /// the queue - /// Increment: Number of messages ever pushed from the queue - e_STAT_PUSH - - , - /// Value: Accumulated bytes of all messages ever put in the - /// queue - /// Increment: Number of messages ever put in the queue - e_STAT_PUT - - , - /// Value: Accumulated number of messages ever GC'ed in the - /// queue - e_STAT_GC_MSGS - - , - /// Value: Role (Unknown, Primary, Replica, Proxy) - e_STAT_ROLE - - , - /// Value: The configured queue messages capacity - e_CFG_MSGS - - , - /// Value: The configured queue bytes capacity - e_CFG_BYTES - - , - /// Value: Accumulated number of messages in the strong - /// consistency queue expired before receiving quorum - /// Receipts - e_STAT_NO_SC_MSGS - - , - // Value: Current number of GUIDs stored in queue's history - // (does not include messages in the queue) - e_STAT_HISTORY - }; -}; - // ------------------ // struct ClientStats // ------------------ @@ -512,101 +414,6 @@ QueueStatsDomain& QueueStatsDomain::setWriterCount(int writerCount) return *this; } -void QueueStatsDomain::onEvent(EventType::Enum type, bsls::Types::Int64 value) -{ - BSLS_ASSERT_SAFE(d_statContext_mp && "initialize was not called"); - - switch (type) { - case EventType::e_ACK: { - // For ACK, we don't have any bytes value, but we also wouldn't care .. - d_statContext_mp->adjustValue(DomainQueueStats::e_STAT_ACK, 1); - } break; - case EventType::e_ACK_TIME: { - d_statContext_mp->reportValue(DomainQueueStats::e_STAT_ACK_TIME, - value); - } break; - case EventType::e_NACK: { - // For NACK, we don't care about the bytes value .. - d_statContext_mp->adjustValue(DomainQueueStats::e_STAT_NACK, 1); - } break; - case EventType::e_CONFIRM: { - // For CONFIRM, we don't care about the bytes value .. - d_statContext_mp->adjustValue(DomainQueueStats::e_STAT_CONFIRM, 1); - } break; - case EventType::e_CONFIRM_TIME: { - d_statContext_mp->reportValue(DomainQueueStats::e_STAT_CONFIRM_TIME, - value); - } break; - case EventType::e_REJECT: { - d_statContext_mp->adjustValue(DomainQueueStats::e_STAT_REJECT, 1); - } break; - case EventType::e_QUEUE_TIME: { - d_statContext_mp->reportValue(DomainQueueStats::e_STAT_QUEUE_TIME, - value); - } break; - case EventType::e_PUSH: { - d_statContext_mp->adjustValue(DomainQueueStats::e_STAT_PUSH, value); - } break; - case EventType::e_PUT: { - d_statContext_mp->adjustValue(DomainQueueStats::e_STAT_PUT, value); - } break; - case EventType::e_ADD_MESSAGE: { - d_statContext_mp->adjustValue(DomainQueueStats::e_STAT_BYTES, value); - d_statContext_mp->adjustValue(DomainQueueStats::e_STAT_MESSAGES, 1); - if (!d_subContextsHolder.empty()) { - bsl::list::iterator it = - d_subContextsHolder.begin(); - while (it != d_subContextsHolder.end()) { - it->get()->adjustValue(DomainQueueStats::e_STAT_BYTES, value); - it->get()->adjustValue(DomainQueueStats::e_STAT_MESSAGES, 1); - ++it; - } - } - } break; - case EventType::e_DEL_MESSAGE: { - d_statContext_mp->adjustValue(DomainQueueStats::e_STAT_BYTES, -value); - d_statContext_mp->adjustValue(DomainQueueStats::e_STAT_MESSAGES, -1); - } break; - case EventType::e_GC_MESSAGE: { - d_statContext_mp->adjustValue(DomainQueueStats::e_STAT_GC_MSGS, value); - } break; - case EventType::e_PURGE: { - // NOTE: Setting the value like that will cause weird results if using - // the stat to get rates - d_statContext_mp->setValue(DomainQueueStats::e_STAT_BYTES, 0); - d_statContext_mp->setValue(DomainQueueStats::e_STAT_MESSAGES, 0); - if (!d_subContextsHolder.empty()) { - bsl::list::iterator it = - d_subContextsHolder.begin(); - while (it != d_subContextsHolder.end()) { - it->get()->setValue(DomainQueueStats::e_STAT_BYTES, 0); - it->get()->setValue(DomainQueueStats::e_STAT_MESSAGES, 0); - ++it; - } - } - } break; - case EventType::e_CHANGE_ROLE: { - d_statContext_mp->setValue(DomainQueueStats::e_STAT_ROLE, value); - } break; - case EventType::e_CFG_MSGS: { - d_statContext_mp->setValue(DomainQueueStats::e_CFG_MSGS, value); - } break; - case EventType::e_CFG_BYTES: { - d_statContext_mp->setValue(DomainQueueStats::e_CFG_BYTES, value); - } break; - case EventType::e_NO_SC_MESSAGE: { - d_statContext_mp->adjustValue(DomainQueueStats::e_STAT_NO_SC_MSGS, - value); - } break; - case EventType::e_UPDATE_HISTORY: { - d_statContext_mp->setValue(DomainQueueStats::e_STAT_HISTORY, value); - } break; - default: { - BSLS_ASSERT_SAFE(false && "Unknown event type"); - } break; - }; -} - void QueueStatsDomain::onEvent(EventType::Enum type, bsls::Types::Int64 value, const bsl::string& appId) diff --git a/src/groups/mqb/mqbstat/mqbstat_queuestats.h b/src/groups/mqb/mqbstat/mqbstat_queuestats.h index 7d4a389b8..d6fcd8e5b 100644 --- a/src/groups/mqb/mqbstat/mqbstat_queuestats.h +++ b/src/groups/mqb/mqbstat/mqbstat_queuestats.h @@ -258,7 +258,8 @@ class QueueStatsDomain { /// Update statistics for the event of the specified `type` and with the /// specified `value`. Depending on the `type`, `value` can represent /// the number of bytes, a counter, ... - void onEvent(EventType::Enum type, bsls::Types::Int64 value); + template + void onEvent(bsls::Types::Int64 value); /// Update statistics for the event of the specified `type` and with the /// specified `value` for the specified `appId`. Depending on the `type`, @@ -414,6 +415,104 @@ struct QueueStatsUtil { bmqst::StatContext* statContext); }; +// ----------------------- +// struct DomainQueueStats +// ----------------------- + +/// Namespace for the constants of stat values that applies to the queues +/// on the domain +struct DomainQueueStats { + enum Enum { + /// Value: Current number of clients who opened the queue with + /// the `WRITE` flag + e_STAT_NB_PRODUCER + + , + /// Value: Current number of clients who opened the queue with + /// the 'READ' flag + e_STAT_NB_CONSUMER + + , + /// Value: Current number of messages in the queue + e_STAT_MESSAGES + + , + /// Value: Accumulated bytes of all messages currently in the + /// queue + e_STAT_BYTES + + , + /// Value: Number of ack messages delivered by this queue + e_STAT_ACK + + , + /// Value: The time between PUT and ACK (in nanoseconds). + e_STAT_ACK_TIME + + , + /// Value: Number of NACK messages generated for this queue + e_STAT_NACK + + , + /// Value: Number of CONFIRM messages received by this queue + e_STAT_CONFIRM + + , + /// Value: The time between PUSH and CONFIRM (in nanoseconds). + e_STAT_CONFIRM_TIME + + , + /// Value: Number of messages rejected by this queue (RDA + /// reaching zero) + e_STAT_REJECT + + , + /// Value: The time spent by the message in the queue (in + /// nanoseconds). + e_STAT_QUEUE_TIME + + , + /// Value: Accumulated bytes of all messages ever pushed from + /// the queue + /// Increment: Number of messages ever pushed from the queue + e_STAT_PUSH + + , + /// Value: Accumulated bytes of all messages ever put in the + /// queue + /// Increment: Number of messages ever put in the queue + e_STAT_PUT + + , + /// Value: Accumulated number of messages ever GC'ed in the + /// queue + e_STAT_GC_MSGS + + , + /// Value: Role (Unknown, Primary, Replica, Proxy) + e_STAT_ROLE + + , + /// Value: The configured queue messages capacity + e_CFG_MSGS + + , + /// Value: The configured queue bytes capacity + e_CFG_BYTES + + , + /// Value: Accumulated number of messages in the strong + /// consistency queue expired before receiving quorum + /// Receipts + e_STAT_NO_SC_MSGS + + , + // Value: Current number of GUIDs stored in queue's history + // (does not include messages in the queue) + e_STAT_HISTORY + }; +}; + // ============================================================================ // INLINE DEFINITIONS // ============================================================================ @@ -427,6 +526,193 @@ inline bmqst::StatContext* QueueStatsDomain::statContext() return d_statContext_mp.get(); } +template <> +inline void +QueueStatsDomain::onEvent( + BSLS_ANNOTATION_UNUSED bsls::Types::Int64 value) +{ + BSLS_ASSERT_SAFE(d_statContext_mp && "initialize was not called"); + d_statContext_mp->adjustValue(DomainQueueStats::e_STAT_ACK, 1); +} + +template <> +inline void +QueueStatsDomain::onEvent( + bsls::Types::Int64 value) +{ + BSLS_ASSERT_SAFE(d_statContext_mp && "initialize was not called"); + d_statContext_mp->reportValue(DomainQueueStats::e_STAT_ACK_TIME, value); +} + +template <> +inline void +QueueStatsDomain::onEvent( + BSLS_ANNOTATION_UNUSED bsls::Types::Int64 value) +{ + BSLS_ASSERT_SAFE(d_statContext_mp && "initialize was not called"); + // For NACK, we don't care about the bytes value + d_statContext_mp->adjustValue(DomainQueueStats::e_STAT_NACK, 1); +} + +template <> +inline void +QueueStatsDomain::onEvent( + BSLS_ANNOTATION_UNUSED bsls::Types::Int64 value) +{ + BSLS_ASSERT_SAFE(d_statContext_mp && "initialize was not called"); + // For CONFIRM, we don't care about the bytes value + d_statContext_mp->adjustValue(DomainQueueStats::e_STAT_CONFIRM, 1); +} + +template <> +inline void +QueueStatsDomain::onEvent( + bsls::Types::Int64 value) +{ + BSLS_ASSERT_SAFE(d_statContext_mp && "initialize was not called"); + d_statContext_mp->reportValue(DomainQueueStats::e_STAT_CONFIRM_TIME, + value); +} + +template <> +inline void +QueueStatsDomain::onEvent( + BSLS_ANNOTATION_UNUSED bsls::Types::Int64 value) +{ + BSLS_ASSERT_SAFE(d_statContext_mp && "initialize was not called"); + // For REJECT, we don't care about the bytes value + d_statContext_mp->adjustValue(DomainQueueStats::e_STAT_REJECT, 1); +} + +template <> +inline void +QueueStatsDomain::onEvent( + bsls::Types::Int64 value) +{ + BSLS_ASSERT_SAFE(d_statContext_mp && "initialize was not called"); + d_statContext_mp->reportValue(DomainQueueStats::e_STAT_QUEUE_TIME, value); +} + +template <> +inline void +QueueStatsDomain::onEvent( + bsls::Types::Int64 value) +{ + BSLS_ASSERT_SAFE(d_statContext_mp && "initialize was not called"); + d_statContext_mp->adjustValue(DomainQueueStats::e_STAT_PUSH, value); +} + +template <> +inline void +QueueStatsDomain::onEvent( + bsls::Types::Int64 value) +{ + BSLS_ASSERT_SAFE(d_statContext_mp && "initialize was not called"); + d_statContext_mp->adjustValue(DomainQueueStats::e_STAT_PUT, value); +} + +template <> +inline void +QueueStatsDomain::onEvent( + bsls::Types::Int64 value) +{ + BSLS_ASSERT_SAFE(d_statContext_mp && "initialize was not called"); + d_statContext_mp->adjustValue(DomainQueueStats::e_STAT_BYTES, value); + d_statContext_mp->adjustValue(DomainQueueStats::e_STAT_MESSAGES, 1); + if (!d_subContextsHolder.empty()) { + bsl::list::iterator it = d_subContextsHolder.begin(); + while (it != d_subContextsHolder.end()) { + it->get()->adjustValue(DomainQueueStats::e_STAT_BYTES, value); + it->get()->adjustValue(DomainQueueStats::e_STAT_MESSAGES, 1); + ++it; + } + } +} + +template <> +inline void +QueueStatsDomain::onEvent( + bsls::Types::Int64 value) +{ + BSLS_ASSERT_SAFE(d_statContext_mp && "initialize was not called"); + d_statContext_mp->adjustValue(DomainQueueStats::e_STAT_BYTES, -value); + d_statContext_mp->adjustValue(DomainQueueStats::e_STAT_MESSAGES, -1); +} + +template <> +inline void +QueueStatsDomain::onEvent( + bsls::Types::Int64 value) +{ + BSLS_ASSERT_SAFE(d_statContext_mp && "initialize was not called"); + d_statContext_mp->adjustValue(DomainQueueStats::e_STAT_GC_MSGS, value); +} + +template <> +inline void +QueueStatsDomain::onEvent( + BSLS_ANNOTATION_UNUSED bsls::Types::Int64 value) +{ + BSLS_ASSERT_SAFE(d_statContext_mp && "initialize was not called"); + // NOTE: Setting the value like that will cause weird results if using + // the stat to get rates + d_statContext_mp->setValue(DomainQueueStats::e_STAT_BYTES, 0); + d_statContext_mp->setValue(DomainQueueStats::e_STAT_MESSAGES, 0); + if (!d_subContextsHolder.empty()) { + bsl::list::iterator it = d_subContextsHolder.begin(); + while (it != d_subContextsHolder.end()) { + it->get()->setValue(DomainQueueStats::e_STAT_BYTES, 0); + it->get()->setValue(DomainQueueStats::e_STAT_MESSAGES, 0); + ++it; + } + } +} + +template <> +inline void +QueueStatsDomain::onEvent( + bsls::Types::Int64 value) +{ + BSLS_ASSERT_SAFE(d_statContext_mp && "initialize was not called"); + d_statContext_mp->setValue(DomainQueueStats::e_STAT_ROLE, value); +} + +template <> +inline void +QueueStatsDomain::onEvent( + bsls::Types::Int64 value) +{ + BSLS_ASSERT_SAFE(d_statContext_mp && "initialize was not called"); + d_statContext_mp->setValue(DomainQueueStats::e_CFG_MSGS, value); +} + +template <> +inline void +QueueStatsDomain::onEvent( + bsls::Types::Int64 value) +{ + BSLS_ASSERT_SAFE(d_statContext_mp && "initialize was not called"); + d_statContext_mp->setValue(DomainQueueStats::e_CFG_BYTES, value); +} + +template <> +inline void +QueueStatsDomain::onEvent( + bsls::Types::Int64 value) +{ + BSLS_ASSERT_SAFE(d_statContext_mp && "initialize was not called"); + d_statContext_mp->adjustValue(DomainQueueStats::e_STAT_NO_SC_MSGS, value); +} + +template <> +inline void +QueueStatsDomain::onEvent( + bsls::Types::Int64 value) +{ + BSLS_ASSERT_SAFE(d_statContext_mp && "initialize was not called"); + d_statContext_mp->setValue(DomainQueueStats::e_STAT_HISTORY, value); +} + // ----------------------------- // struct QueueStatsDomain::Role // ----------------------------- diff --git a/src/groups/mqb/mqbstat/mqbstat_queuestats.t.cpp b/src/groups/mqb/mqbstat/mqbstat_queuestats.t.cpp index 65590a150..e06c54d5e 100644 --- a/src/groups/mqb/mqbstat/mqbstat_queuestats.t.cpp +++ b/src/groups/mqb/mqbstat/mqbstat_queuestats.t.cpp @@ -314,25 +314,25 @@ static void test3_queueStatsDomain() queueStatsDomain.setWriterCount(2); // 2 acks : bytes irrelevant - queueStatsDomain.onEvent(QueueStatsDomain::EventType::e_ACK, k_DUMMY); - queueStatsDomain.onEvent(QueueStatsDomain::EventType::e_ACK, k_DUMMY); + queueStatsDomain.onEvent(k_DUMMY); + queueStatsDomain.onEvent(k_DUMMY); // 1 confirm : bytes irrelevant - queueStatsDomain.onEvent(QueueStatsDomain::EventType::e_CONFIRM, k_DUMMY); + queueStatsDomain.onEvent(k_DUMMY); // 1 push : 9 bytes - queueStatsDomain.onEvent(QueueStatsDomain::EventType::e_PUSH, 9); + queueStatsDomain.onEvent(9); // 3 puts : 33 bytes - queueStatsDomain.onEvent(QueueStatsDomain::EventType::e_PUT, 10); - queueStatsDomain.onEvent(QueueStatsDomain::EventType::e_PUT, 11); - queueStatsDomain.onEvent(QueueStatsDomain::EventType::e_PUT, 12); + queueStatsDomain.onEvent(10); + queueStatsDomain.onEvent(11); + queueStatsDomain.onEvent(12); // 1 add message : 15 bytes - queueStatsDomain.onEvent(QueueStatsDomain::EventType::e_ADD_MESSAGE, 15); + queueStatsDomain.onEvent(15); // 1 GUID in history - queueStatsDomain.onEvent(QueueStatsDomain::EventType::e_UPDATE_HISTORY, 1); + queueStatsDomain.onEvent(1); domain->snapshot(); // The following stats are not range based, and therefore always return the @@ -363,29 +363,29 @@ static void test3_queueStatsDomain() queueStatsDomain.setReaderCount(3).setWriterCount(1); // 4 acks : bytes irrelevant - queueStatsDomain.onEvent(QueueStatsDomain::EventType::e_ACK, k_DUMMY); - queueStatsDomain.onEvent(QueueStatsDomain::EventType::e_ACK, k_DUMMY); - queueStatsDomain.onEvent(QueueStatsDomain::EventType::e_ACK, k_DUMMY); - queueStatsDomain.onEvent(QueueStatsDomain::EventType::e_ACK, k_DUMMY); + queueStatsDomain.onEvent(k_DUMMY); + queueStatsDomain.onEvent(k_DUMMY); + queueStatsDomain.onEvent(k_DUMMY); + queueStatsDomain.onEvent(k_DUMMY); // 3 confirms : bytes irrelevant - queueStatsDomain.onEvent(QueueStatsDomain::EventType::e_CONFIRM, k_DUMMY); - queueStatsDomain.onEvent(QueueStatsDomain::EventType::e_CONFIRM, k_DUMMY); - queueStatsDomain.onEvent(QueueStatsDomain::EventType::e_CONFIRM, k_DUMMY); + queueStatsDomain.onEvent(k_DUMMY); + queueStatsDomain.onEvent(k_DUMMY); + queueStatsDomain.onEvent(k_DUMMY); // 1 push : 9 bytes - queueStatsDomain.onEvent(QueueStatsDomain::EventType::e_PUSH, 11); + queueStatsDomain.onEvent(11); // 2 puts : 22 bytes - queueStatsDomain.onEvent(QueueStatsDomain::EventType::e_PUT, 10); - queueStatsDomain.onEvent(QueueStatsDomain::EventType::e_PUT, 12); + queueStatsDomain.onEvent(10); + queueStatsDomain.onEvent(12); // del 1 message - queueStatsDomain.onEvent(QueueStatsDomain::EventType::e_DEL_MESSAGE, 15); + queueStatsDomain.onEvent(15); // 3 GUIDs in history (first 5, then gc results in 3) - queueStatsDomain.onEvent(QueueStatsDomain::EventType::e_UPDATE_HISTORY, 5); - queueStatsDomain.onEvent(QueueStatsDomain::EventType::e_UPDATE_HISTORY, 3); + queueStatsDomain.onEvent(5); + queueStatsDomain.onEvent(3); domain->snapshot(); // The following stats are not range based, and therefore always return the @@ -474,8 +474,8 @@ static void test4_queueStatsDomainContent() } { - obj.onEvent(mqbstat::QueueStatsDomain::EventType::e_ADD_MESSAGE, 3); - obj.onEvent(mqbstat::QueueStatsDomain::EventType::e_ADD_MESSAGE, 5); + obj.onEvent(3); + obj.onEvent(5); sc->snapshot(); @@ -487,8 +487,8 @@ static void test4_queueStatsDomainContent() } { - obj.onEvent(mqbstat::QueueStatsDomain::EventType::e_ADD_MESSAGE, 7); - obj.onEvent(mqbstat::QueueStatsDomain::EventType::e_DEL_MESSAGE, 3); + obj.onEvent(7); + obj.onEvent(3); sc->snapshot(); @@ -500,7 +500,7 @@ static void test4_queueStatsDomainContent() } { - obj.onEvent(mqbstat::QueueStatsDomain::EventType::e_DEL_MESSAGE, 5); + obj.onEvent(5); sc->snapshot();