From f89e62114e787a0f0f7f689347bfd9aee1e4349b Mon Sep 17 00:00:00 2001
From: "W. David Dagenhart" <wdd@fnal.gov>
Date: Thu, 7 Dec 2023 01:00:59 +0100
Subject: [PATCH] Allow a stream to skip a lumi if all events processed

---
 FWCore/Framework/src/EventProcessor.cc        |  7 +-
 .../src/LuminosityBlockProcessingStatus.cc    | 28 ++++++++
 .../src/LuminosityBlockProcessingStatus.h     | 11 ++-
 .../test/modules_2_concurrent_lumis_cfg.py    | 16 ++---
 .../test/stubs/TestGlobalAnalyzers.cc         | 40 +++++++++--
 .../Framework/test/stubs/TestGlobalFilters.cc | 40 +++++++++--
 .../test/stubs/TestGlobalProducers.cc         | 68 ++++++++++++++++---
 .../test/stubs/TestLimitedAnalyzers.cc        | 40 +++++++++--
 .../test/stubs/TestLimitedFilters.cc          | 40 +++++++++--
 .../test/stubs/TestLimitedProducers.cc        | 68 ++++++++++++++++---
 .../Framework/test/testRunLumiCaches_cfg.py   | 12 ++--
 .../test/test_earlyTerminationSignal.sh       |  2 +-
 .../test_exceptionAtGlobalBeginRun_cfg.py     |  8 ++-
 .../Framework/test/test_global_modules_cfg.py | 15 ++--
 .../test/test_limited_modules_cfg.py          | 15 ++--
 .../test/test_TryToContinue_cfg.py            | 14 ++--
 FWCore/Services/plugins/CheckTransitions.cc   | 46 ++++++++++++-
 17 files changed, 399 insertions(+), 71 deletions(-)

diff --git a/FWCore/Framework/src/EventProcessor.cc b/FWCore/Framework/src/EventProcessor.cc
index 08e063b2de550..a161eb4e6ad65 100644
--- a/FWCore/Framework/src/EventProcessor.cc
+++ b/FWCore/Framework/src/EventProcessor.cc
@@ -1668,7 +1668,7 @@ namespace edm {
                                       edm::WaitingTaskHolder iHolder) {
     actReg_->esSyncIOVQueuingSignal_.emit(iSync);
 
-    auto status = std::make_shared<LuminosityBlockProcessingStatus>(preallocations_.numberOfStreams());
+    auto status = std::make_shared<LuminosityBlockProcessingStatus>();
     chain::first([this, &iSync, &status](auto nextTask) {
       espController_->runOrQueueEventSetupForInstanceAsync(iSync,
                                                            nextTask,
@@ -1767,6 +1767,9 @@ namespace edm {
                             streamQueuesInserter_.push(*holder.group(), [this, status, holder, &es]() mutable {
                               for (unsigned int i = 0; i < preallocations_.numberOfStreams(); ++i) {
                                 streamQueues_[i].push(*holder.group(), [this, i, status, holder, &es]() mutable {
+                                  if (!status->shouldStreamStartLumi()) {
+                                    return;
+                                  }
                                   streamQueues_[i].pause();
 
                                   auto& event = principalCache_.eventPrincipal(i);
@@ -1979,6 +1982,7 @@ namespace edm {
       if (streamLumiActive_ > 0) {
         FinalWaitingTask globalWaitTask{taskGroup_};
         assert(streamLumiActive_ == preallocations_.numberOfStreams());
+        streamLumiStatus_[0]->noMoreEventsInLumi();
         streamLumiStatus_[0]->setCleaningUpAfterException(cleaningUpAfterException);
         for (unsigned int i = 0; i < preallocations_.numberOfStreams(); ++i) {
           streamEndLumiAsync(WaitingTaskHolder{taskGroup_, &globalWaitTask}, i);
@@ -2305,6 +2309,7 @@ namespace edm {
           // the stream will stop processing this lumi now
           if (status->eventProcessingState() == LuminosityBlockProcessingStatus::EventProcessingState::kStopLumi) {
             if (not status->haveStartedNextLumiOrEndedRun()) {
+              status->noMoreEventsInLumi();
               status->startNextLumiOrEndRun();
               if (lastTransitionType() == InputSource::ItemType::IsLumi && !iTask.taskHasFailed()) {
                 CMS_SA_ALLOW try {
diff --git a/FWCore/Framework/src/LuminosityBlockProcessingStatus.cc b/FWCore/Framework/src/LuminosityBlockProcessingStatus.cc
index 1b77c83f35e88..c8a03e4aa4dfb 100644
--- a/FWCore/Framework/src/LuminosityBlockProcessingStatus.cc
+++ b/FWCore/Framework/src/LuminosityBlockProcessingStatus.cc
@@ -26,6 +26,34 @@ namespace edm {
     globalEndRunHolder_ = std::move(holder);
   }
 
+  bool LuminosityBlockProcessingStatus::shouldStreamStartLumi() {
+    if (state_ == State::kNoMoreEvents)
+      return false;
+
+    bool changed = false;
+    do {
+      auto expected = State::kRunning;
+      changed = state_.compare_exchange_strong(expected, State::kUpdating);
+      if (expected == State::kNoMoreEvents)
+        return false;
+    } while (changed == false);
+
+    ++nStreamsProcessingLumi_;
+    state_ = State::kRunning;
+    return true;
+  }
+
+  void LuminosityBlockProcessingStatus::noMoreEventsInLumi() {
+    bool changed = false;
+    do {
+      auto expected = State::kRunning;
+      changed = state_.compare_exchange_strong(expected, State::kUpdating);
+      assert(expected != State::kNoMoreEvents);
+    } while (changed == false);
+    nStreamsStillProcessingLumi_.store(nStreamsProcessingLumi_);
+    state_ = State::kNoMoreEvents;
+  }
+
   void LuminosityBlockProcessingStatus::setEndTime() {
     constexpr char kUnset = 0;
     constexpr char kSetting = 1;
diff --git a/FWCore/Framework/src/LuminosityBlockProcessingStatus.h b/FWCore/Framework/src/LuminosityBlockProcessingStatus.h
index b35b4c5258049..ff1397f6bcdce 100644
--- a/FWCore/Framework/src/LuminosityBlockProcessingStatus.h
+++ b/FWCore/Framework/src/LuminosityBlockProcessingStatus.h
@@ -39,7 +39,7 @@ namespace edm {
 
   class LuminosityBlockProcessingStatus {
   public:
-    LuminosityBlockProcessingStatus(unsigned int iNStreams) : nStreamsStillProcessingLumi_(iNStreams) {}
+    LuminosityBlockProcessingStatus() = default;
 
     LuminosityBlockProcessingStatus(LuminosityBlockProcessingStatus const&) = delete;
     LuminosityBlockProcessingStatus const& operator=(LuminosityBlockProcessingStatus const&) = delete;
@@ -70,6 +70,8 @@ namespace edm {
     void setGlobalEndRunHolder(WaitingTaskHolder);
     void globalEndRunHolderDoneWaiting() { globalEndRunHolder_.doneWaiting(std::exception_ptr{}); }
 
+    bool shouldStreamStartLumi();
+    void noMoreEventsInLumi();
     bool streamFinishedLumi() { return 0 == (--nStreamsStillProcessingLumi_); }
 
     //These should only be called while in the InputSource's task queue
@@ -103,10 +105,13 @@ namespace edm {
     std::vector<std::shared_ptr<const EventSetupImpl>> eventSetupImpls_;
     WaitingTaskList endIOVWaitingTasks_;
     edm::WaitingTaskHolder globalEndRunHolder_;
-    std::atomic<unsigned int> nStreamsStillProcessingLumi_{0};  //read/write as streams finish lumi so must be atomic
     edm::Timestamp endTime_{};
-    std::atomic<char> endTimeSetStatus_{0};
+    CMS_THREAD_GUARD(state_) unsigned int nStreamsProcessingLumi_{0};
+    std::atomic<unsigned int> nStreamsStillProcessingLumi_{0};
+    enum class State { kRunning, kUpdating, kNoMoreEvents };
+    std::atomic<State> state_{State::kRunning};
     EventProcessingState eventProcessingState_{EventProcessingState::kProcessing};
+    std::atomic<char> endTimeSetStatus_{0};
     std::atomic<bool> startedNextLumiOrEndedRun_{false};
     bool globalBeginSucceeded_{false};
     bool cleaningUpAfterException_{false};
diff --git a/FWCore/Framework/test/modules_2_concurrent_lumis_cfg.py b/FWCore/Framework/test/modules_2_concurrent_lumis_cfg.py
index 877ef5c29b9b4..c98684d6a2747 100644
--- a/FWCore/Framework/test/modules_2_concurrent_lumis_cfg.py
+++ b/FWCore/Framework/test/modules_2_concurrent_lumis_cfg.py
@@ -15,45 +15,45 @@
                               iterations = cms.uint32(50*1000*20) )
 
 process.LumiSumIntProd = cms.EDProducer("edmtest::global::LumiSummaryIntProducer",
-    transitions = cms.int32(60)
+    transitions = cms.int32(44)
     ,cachevalue = cms.int32(2)
 )
 
 process.LumiSumLumiProd = cms.EDProducer("edmtest::global::LumiSummaryLumiProducer",
-    transitions = cms.int32(70)
+    transitions = cms.int32(34)
     ,cachevalue = cms.int32(2)
 )
 
 process.LumiSumIntFilter = cms.EDFilter("edmtest::global::LumiSummaryIntFilter",
-    transitions = cms.int32(84)
+    transitions = cms.int32(44)
     ,cachevalue = cms.int32(2)
 )
 
 process.LumiSumIntAnalyzer = cms.EDAnalyzer("edmtest::global::LumiSummaryIntAnalyzer",
-    transitions = cms.int32(84)
+    transitions = cms.int32(44)
     ,cachevalue = cms.int32(2)
 )
 
 process.LimitedLumiSumIntProd = cms.EDProducer("edmtest::limited::LumiSummaryIntProducer",
-    transitions = cms.int32(60)
+    transitions = cms.int32(44)
     ,cachevalue = cms.int32(2)
     ,concurrencyLimit = cms.untracked.uint32(1)
 )
 
 process.LimitedLumiSumLumiProd = cms.EDProducer("edmtest::limited::LumiSummaryLumiProducer",
-    transitions = cms.int32(70)
+    transitions = cms.int32(34)
     ,cachevalue = cms.int32(2)
     ,concurrencyLimit = cms.untracked.uint32(1)
 )
 
 process.LimitedLumiSumIntFilter = cms.EDFilter("edmtest::limited::LumiSummaryIntFilter",
-    transitions = cms.int32(84)
+    transitions = cms.int32(44)
     ,cachevalue = cms.int32(2)
     ,concurrencyLimit = cms.untracked.uint32(1)
 )
 
 process.LimitedLumiSumIntAnalyzer = cms.EDAnalyzer("edmtest::limited::LumiSummaryIntAnalyzer",
-    transitions = cms.int32(84)
+    transitions = cms.int32(44)
     ,cachevalue = cms.int32(2)
     ,concurrencyLimit = cms.untracked.uint32(1)
 )
diff --git a/FWCore/Framework/test/stubs/TestGlobalAnalyzers.cc b/FWCore/Framework/test/stubs/TestGlobalAnalyzers.cc
index 753e027e123b2..12511e9499be2 100644
--- a/FWCore/Framework/test/stubs/TestGlobalAnalyzers.cc
+++ b/FWCore/Framework/test/stubs/TestGlobalAnalyzers.cc
@@ -49,7 +49,8 @@ namespace edmtest {
 
     class StreamIntAnalyzer : public edm::global::EDAnalyzer<edm::StreamCache<UnsafeCache>> {
     public:
-      explicit StreamIntAnalyzer(edm::ParameterSet const& p) : trans_(p.getParameter<int>("transitions")) {
+      explicit StreamIntAnalyzer(edm::ParameterSet const& p)
+          : trans_(p.getParameter<int>("transitions")), nLumis_(p.getUntrackedParameter<unsigned int>("nLumis", 1)) {
         bool verbose = p.getUntrackedParameter<bool>("verbose", true);
         callWhenNewProductsRegistered([verbose](edm::BranchDescription const& desc) {
           if (verbose) {
@@ -58,10 +59,15 @@ namespace edmtest {
         });
       }
       const unsigned int trans_;
+      const unsigned int nLumis_;
       mutable std::atomic<unsigned int> m_count{0};
+      mutable std::atomic<unsigned int> m_countStreams{0};
+      mutable std::atomic<unsigned int> m_countStreamBeginLumiTransitions{0};
+      mutable std::atomic<unsigned int> m_countStreamEndLumiTransitions{0};
 
       std::unique_ptr<UnsafeCache> beginStream(edm::StreamID iID) const override {
         ++m_count;
+        ++m_countStreams;
         auto pCache = std::make_unique<UnsafeCache>();
         pCache->value = iID.value();
         return pCache;
@@ -78,7 +84,7 @@ namespace edmtest {
       void streamBeginLuminosityBlock(edm::StreamID iID,
                                       edm::LuminosityBlock const&,
                                       edm::EventSetup const&) const override {
-        ++m_count;
+        ++m_countStreamBeginLumiTransitions;
         if ((streamCache(iID))->value != iID.value()) {
           throw cms::Exception("cache value")
               << "StreamIntAnalyzer cache value " << (streamCache(iID))->value << " but it was supposed to be " << iID;
@@ -96,7 +102,7 @@ namespace edmtest {
       void streamEndLuminosityBlock(edm::StreamID iID,
                                     edm::LuminosityBlock const&,
                                     edm::EventSetup const&) const override {
-        ++m_count;
+        ++m_countStreamEndLumiTransitions;
         if ((streamCache(iID))->value != iID.value()) {
           throw cms::Exception("cache value")
               << "StreamIntAnalyzer cache value " << (streamCache(iID))->value << " but it was supposed to be " << iID;
@@ -124,6 +130,19 @@ namespace edmtest {
           throw cms::Exception("transitions")
               << "StreamIntAnalyzer transitions " << m_count << " but it was supposed to be " << trans_;
         }
+        unsigned int nStreamBeginLumiTransitions = m_countStreamBeginLumiTransitions.load();
+        unsigned int nStreamEndLumiTransitions = m_countStreamEndLumiTransitions.load();
+        unsigned int nStreams = m_countStreams.load();
+        if (nStreamBeginLumiTransitions < nLumis_ || nStreamBeginLumiTransitions > (nLumis_ * nStreams)) {
+          throw cms::Exception("transitions")
+              << "StreamIntAnalyzer stream begin lumi transitions " << nStreamBeginLumiTransitions
+              << " but it was supposed to be between " << nLumis_ << " and " << nLumis_ * nStreams;
+        }
+        if (nStreamEndLumiTransitions != nStreamBeginLumiTransitions) {
+          throw cms::Exception("transitions")
+              << "StreamIntAnalyzer stream end lumi transitions " << nStreamEndLumiTransitions
+              << " does not equal stream begin lumi transitions " << nStreamBeginLumiTransitions;
+        }
       }
     };
 
@@ -260,15 +279,20 @@ namespace edmtest {
       const unsigned int trans_;
       const unsigned int cvalue_;
       mutable std::atomic<unsigned int> m_count{0};
+      mutable std::atomic<unsigned int> m_countLumis{0};
+      mutable std::atomic<unsigned int> m_countStreams{0};
+      mutable std::atomic<unsigned int> m_countStreamLumiTransitions{0};
 
       std::unique_ptr<UnsafeCache> beginStream(edm::StreamID) const override {
         ++m_count;
+        ++m_countStreams;
         return std::make_unique<UnsafeCache>();
       }
 
       std::shared_ptr<UnsafeCache> globalBeginLuminosityBlockSummary(edm::LuminosityBlock const& iLB,
                                                                      edm::EventSetup const&) const override {
         ++m_count;
+        ++m_countLumis;
         auto gCache = std::make_shared<UnsafeCache>();
         gCache->lumi = iLB.luminosityBlockAuxiliary().luminosityBlock();
         return gCache;
@@ -283,7 +307,7 @@ namespace edmtest {
                                            edm::LuminosityBlock const& iLB,
                                            edm::EventSetup const&,
                                            UnsafeCache* gCache) const override {
-        ++m_count;
+        ++m_countStreamLumiTransitions;
         if (gCache->lumi != iLB.luminosityBlockAuxiliary().luminosityBlock()) {
           throw cms::Exception("UnexpectedValue")
               << "streamEndLuminosityBlockSummary unexpected lumi number in Stream " << iID.value();
@@ -310,6 +334,14 @@ namespace edmtest {
           throw cms::Exception("transitions")
               << "LumiSummaryIntAnalyzer transitions " << m_count << " but it was supposed to be " << trans_;
         }
+        unsigned int nStreamLumiTransitions = m_countStreamLumiTransitions.load();
+        unsigned int nLumis = m_countLumis.load();
+        unsigned int nStreams = m_countStreams.load();
+        if (nStreamLumiTransitions < nLumis || nStreamLumiTransitions > (nLumis * nStreams)) {
+          throw cms::Exception("transitions")
+              << "LumiSummaryIntAnalyzer stream lumi transitions " << nStreamLumiTransitions
+              << " but it was supposed to be between " << nLumis << " and " << nLumis * nStreams;
+        }
       }
     };
 
diff --git a/FWCore/Framework/test/stubs/TestGlobalFilters.cc b/FWCore/Framework/test/stubs/TestGlobalFilters.cc
index 50800b415d1ff..7afee51894b2e 100644
--- a/FWCore/Framework/test/stubs/TestGlobalFilters.cc
+++ b/FWCore/Framework/test/stubs/TestGlobalFilters.cc
@@ -55,15 +55,21 @@ namespace edmtest {
 
     class StreamIntFilter : public edm::global::EDFilter<edm::StreamCache<UnsafeCache>> {
     public:
-      explicit StreamIntFilter(edm::ParameterSet const& p) : trans_(p.getParameter<int>("transitions")) {
+      explicit StreamIntFilter(edm::ParameterSet const& p)
+          : trans_(p.getParameter<int>("transitions")), nLumis_(p.getUntrackedParameter<unsigned int>("nLumis", 1)) {
         produces<unsigned int>();
       }
 
       const unsigned int trans_;
+      const unsigned int nLumis_;
       mutable std::atomic<unsigned int> m_count{0};
+      mutable std::atomic<unsigned int> m_countStreams{0};
+      mutable std::atomic<unsigned int> m_countStreamBeginLumiTransitions{0};
+      mutable std::atomic<unsigned int> m_countStreamEndLumiTransitions{0};
 
       std::unique_ptr<UnsafeCache> beginStream(edm::StreamID iID) const override {
         ++m_count;
+        ++m_countStreams;
         auto sCache = std::make_unique<UnsafeCache>();
         ++(sCache->strm);
         sCache->value = iID.value();
@@ -85,7 +91,7 @@ namespace edmtest {
       void streamBeginLuminosityBlock(edm::StreamID iID,
                                       edm::LuminosityBlock const&,
                                       edm::EventSetup const&) const override {
-        ++m_count;
+        ++m_countStreamBeginLumiTransitions;
         auto sCache = streamCache(iID);
         if (sCache->value != iID.value()) {
           throw cms::Exception("cache value") << (streamCache(iID))->value << " but it was supposed to be " << iID;
@@ -114,7 +120,7 @@ namespace edmtest {
       void streamEndLuminosityBlock(edm::StreamID iID,
                                     edm::LuminosityBlock const&,
                                     edm::EventSetup const&) const override {
-        ++m_count;
+        ++m_countStreamEndLumiTransitions;
         auto sCache = streamCache(iID);
         if (sCache->value != iID.value()) {
           throw cms::Exception("cache value") << (streamCache(iID))->value << " but it was supposed to be " << iID;
@@ -157,6 +163,19 @@ namespace edmtest {
           throw cms::Exception("transitions")
               << "StreamIntFilter transitions " << m_count << " but it was supposed to be " << trans_;
         }
+        unsigned int nStreamBeginLumiTransitions = m_countStreamBeginLumiTransitions.load();
+        unsigned int nStreamEndLumiTransitions = m_countStreamEndLumiTransitions.load();
+        unsigned int nStreams = m_countStreams.load();
+        if (nStreamBeginLumiTransitions < nLumis_ || nStreamBeginLumiTransitions > (nLumis_ * nStreams)) {
+          throw cms::Exception("transitions")
+              << "StreamIntFilter stream begin lumi transitions " << nStreamBeginLumiTransitions
+              << " but it was supposed to be between " << nLumis_ << " and " << nLumis_ * nStreams;
+        }
+        if (nStreamEndLumiTransitions != nStreamBeginLumiTransitions) {
+          throw cms::Exception("transitions")
+              << "StreamIntFilter stream end lumi transitions " << nStreamEndLumiTransitions
+              << " does not equal stream begin lumi transitions " << nStreamBeginLumiTransitions;
+        }
       }
     };
 
@@ -364,15 +383,20 @@ namespace edmtest {
       const unsigned int trans_;
       const unsigned int cvalue_;
       mutable std::atomic<unsigned int> m_count{0};
+      mutable std::atomic<unsigned int> m_countLumis{0};
+      mutable std::atomic<unsigned int> m_countStreams{0};
+      mutable std::atomic<unsigned int> m_countStreamLumiTransitions{0};
 
       std::unique_ptr<Cache> beginStream(edm::StreamID) const override {
         ++m_count;
+        ++m_countStreams;
         return std::make_unique<Cache>();
       }
 
       std::shared_ptr<UnsafeCache> globalBeginLuminosityBlockSummary(edm::LuminosityBlock const& iLB,
                                                                      edm::EventSetup const&) const override {
         ++m_count;
+        ++m_countLumis;
         auto gCache = std::make_shared<UnsafeCache>();
         gCache->lumi = iLB.luminosityBlockAuxiliary().luminosityBlock();
         return gCache;
@@ -388,7 +412,7 @@ namespace edmtest {
                                            edm::LuminosityBlock const& iLB,
                                            edm::EventSetup const&,
                                            UnsafeCache* gCache) const override {
-        ++m_count;
+        ++m_countStreamLumiTransitions;
         if (gCache->lumi != iLB.luminosityBlockAuxiliary().luminosityBlock()) {
           throw cms::Exception("out of sequence")
               << "streamEndLuminosityBlockSummary unexpected lumi number in Stream " << iID.value();
@@ -416,6 +440,14 @@ namespace edmtest {
           throw cms::Exception("transitions")
               << "LumiSummaryIntFilter transitions " << m_count << " but it was supposed to be " << trans_;
         }
+        unsigned int nStreamLumiTransitions = m_countStreamLumiTransitions.load();
+        unsigned int nLumis = m_countLumis.load();
+        unsigned int nStreams = m_countStreams.load();
+        if (nStreamLumiTransitions < nLumis || nStreamLumiTransitions > (nLumis * nStreams)) {
+          throw cms::Exception("transitions")
+              << "LumiSummaryIntFilter stream lumi transitions " << nStreamLumiTransitions
+              << " but it was supposed to be between " << nLumis << " and " << nLumis * nStreams;
+        }
       }
     };
 
diff --git a/FWCore/Framework/test/stubs/TestGlobalProducers.cc b/FWCore/Framework/test/stubs/TestGlobalProducers.cc
index c3126c33025a0..84a62963111e8 100644
--- a/FWCore/Framework/test/stubs/TestGlobalProducers.cc
+++ b/FWCore/Framework/test/stubs/TestGlobalProducers.cc
@@ -58,7 +58,8 @@ namespace edmtest {
 
     class StreamIntProducer : public edm::global::EDProducer<edm::StreamCache<UnsafeCache>> {
     public:
-      explicit StreamIntProducer(edm::ParameterSet const& p) : trans_(p.getParameter<int>("transitions")) {
+      explicit StreamIntProducer(edm::ParameterSet const& p)
+          : trans_(p.getParameter<int>("transitions")), nLumis_(p.getUntrackedParameter<unsigned int>("nLumis", 1)) {
         callWhenNewProductsRegistered([](edm::BranchDescription const& desc) {
           std::cout << "global::StreamIntProducer " << desc.moduleLabel() << std::endl;
         });
@@ -66,10 +67,15 @@ namespace edmtest {
       }
 
       const unsigned int trans_;
+      const unsigned int nLumis_;
       mutable std::atomic<unsigned int> m_count{0};
+      mutable std::atomic<unsigned int> m_countStreams{0};
+      mutable std::atomic<unsigned int> m_countStreamBeginLumiTransitions{0};
+      mutable std::atomic<unsigned int> m_countStreamEndLumiTransitions{0};
 
       std::unique_ptr<UnsafeCache> beginStream(edm::StreamID iID) const override {
         ++m_count;
+        ++m_countStreams;
         auto sCache = std::make_unique<UnsafeCache>();
         ++(sCache->strm);
         sCache->value = iID.value();
@@ -81,7 +87,7 @@ namespace edmtest {
         auto sCache = streamCache(iID);
         if (sCache->value != iID.value()) {
           throw cms::Exception("cache value")
-              << "StreamIntAnalyzer cache value " << (streamCache(iID))->value << " but it was supposed to be " << iID;
+              << "StreamIntProducer cache value " << (streamCache(iID))->value << " but it was supposed to be " << iID;
         }
         if (sCache->run != 0 || sCache->lumi != 0 || sCache->work != 0 || sCache->strm != 1) {
           throw cms::Exception("out of sequence") << "streamBeginRun out of sequence in Stream " << iID.value();
@@ -92,7 +98,7 @@ namespace edmtest {
       void streamBeginLuminosityBlock(edm::StreamID iID,
                                       edm::LuminosityBlock const&,
                                       edm::EventSetup const&) const override {
-        ++m_count;
+        ++m_countStreamBeginLumiTransitions;
         auto sCache = streamCache(iID);
         if (sCache->lumi != 0 || sCache->work != 0) {
           throw cms::Exception("out of sequence")
@@ -113,7 +119,7 @@ namespace edmtest {
       void streamEndLuminosityBlock(edm::StreamID iID,
                                     edm::LuminosityBlock const&,
                                     edm::EventSetup const&) const override {
-        ++m_count;
+        ++m_countStreamEndLumiTransitions;
         auto sCache = streamCache(iID);
         --(sCache->lumi);
         sCache->work = 0;
@@ -147,6 +153,19 @@ namespace edmtest {
           throw cms::Exception("transitions")
               << "StreamIntProducer transitions " << m_count << " but it was supposed to be " << trans_;
         }
+        unsigned int nStreamBeginLumiTransitions = m_countStreamBeginLumiTransitions.load();
+        unsigned int nStreamEndLumiTransitions = m_countStreamEndLumiTransitions.load();
+        unsigned int nStreams = m_countStreams.load();
+        if (nStreamBeginLumiTransitions < nLumis_ || nStreamBeginLumiTransitions > (nLumis_ * nStreams)) {
+          throw cms::Exception("transitions")
+              << "StreamIntProducer stream begin lumi transitions " << nStreamBeginLumiTransitions
+              << " but it was supposed to be between " << nLumis_ << " and " << nLumis_ * nStreams;
+        }
+        if (nStreamEndLumiTransitions != nStreamBeginLumiTransitions) {
+          throw cms::Exception("transitions")
+              << "StreamIntProducer stream end lumi transitions " << nStreamEndLumiTransitions
+              << " does not equal stream begin lumi transitions " << nStreamBeginLumiTransitions;
+        }
       }
     };
 
@@ -344,18 +363,27 @@ namespace edmtest {
       const unsigned int trans_;
       const unsigned int cvalue_;
       mutable std::atomic<unsigned int> m_count{0};
+      mutable std::atomic<unsigned int> m_countLumis{0};
+      mutable std::atomic<unsigned int> m_countStreams{0};
+      mutable std::atomic<unsigned int> m_countStreamLumiTransitions{0};
 
-      std::unique_ptr<UnsafeCache> beginStream(edm::StreamID) const override { return std::make_unique<UnsafeCache>(); }
+      std::unique_ptr<UnsafeCache> beginStream(edm::StreamID) const override {
+        ++m_count;
+        ++m_countStreams;
+        return std::make_unique<UnsafeCache>();
+      }
 
       std::shared_ptr<UnsafeCache> globalBeginLuminosityBlockSummary(edm::LuminosityBlock const& iLB,
                                                                      edm::EventSetup const&) const override {
         ++m_count;
+        ++m_countLumis;
         auto gCache = std::make_shared<UnsafeCache>();
         gCache->lumi = iLB.luminosityBlockAuxiliary().luminosityBlock();
         return gCache;
       }
 
       void produce(edm::StreamID iID, edm::Event&, edm::EventSetup const&) const override {
+        ++m_count;
         auto sCache = streamCache(iID);
         ++(sCache->value);
       }
@@ -364,7 +392,7 @@ namespace edmtest {
                                            edm::LuminosityBlock const& iLB,
                                            edm::EventSetup const&,
                                            UnsafeCache* gCache) const override {
-        ++m_count;
+        ++m_countStreamLumiTransitions;
         if (gCache->lumi != iLB.luminosityBlockAuxiliary().luminosityBlock()) {
           throw cms::Exception("UnexpectedValue")
               << "streamEndLuminosityBlockSummary unexpected lumi number in Stream " << iID.value();
@@ -392,6 +420,14 @@ namespace edmtest {
           throw cms::Exception("transitions")
               << "LumiSummaryIntProducer transitions " << m_count << " but it was supposed to be " << trans_;
         }
+        unsigned int nStreamLumiTransitions = m_countStreamLumiTransitions.load();
+        unsigned int nLumis = m_countLumis.load();
+        unsigned int nStreams = m_countStreams.load();
+        if (nStreamLumiTransitions < nLumis || nStreamLumiTransitions > (nLumis * nStreams)) {
+          throw cms::Exception("transitions")
+              << "LumiSummaryIntProducer stream lumi transitions " << nStreamLumiTransitions
+              << " but it was supposed to be between " << nLumis << " and " << nLumis * nStreams;
+        }
       }
     };
 
@@ -406,12 +442,20 @@ namespace edmtest {
       const unsigned int trans_;
       const unsigned int cvalue_;
       mutable std::atomic<unsigned int> m_count{0};
+      mutable std::atomic<unsigned int> m_countLumis{0};
+      mutable std::atomic<unsigned int> m_countStreams{0};
+      mutable std::atomic<unsigned int> m_countStreamLumiTransitions{0};
 
-      std::unique_ptr<UnsafeCache> beginStream(edm::StreamID) const override { return std::make_unique<UnsafeCache>(); }
+      std::unique_ptr<UnsafeCache> beginStream(edm::StreamID) const override {
+        ++m_count;
+        ++m_countStreams;
+        return std::make_unique<UnsafeCache>();
+      }
 
       std::shared_ptr<UnsafeCache> globalBeginLuminosityBlockSummary(edm::LuminosityBlock const& iLB,
                                                                      edm::EventSetup const&) const override {
         ++m_count;
+        ++m_countLumis;
         auto gCache = std::make_shared<UnsafeCache>();
         gCache->lumi = iLB.luminosityBlockAuxiliary().luminosityBlock();
         return gCache;
@@ -426,7 +470,7 @@ namespace edmtest {
                                            edm::LuminosityBlock const& iLB,
                                            edm::EventSetup const&,
                                            UnsafeCache* gCache) const override {
-        ++m_count;
+        ++m_countStreamLumiTransitions;
         if (gCache->lumi != iLB.luminosityBlockAuxiliary().luminosityBlock()) {
           throw cms::Exception("UnexpectedValue")
               << "streamEndLuminosityBlockSummary unexpected lumi number in Stream " << iID.value();
@@ -463,6 +507,14 @@ namespace edmtest {
           throw cms::Exception("transitions")
               << "LumiSummaryLumiProducer transitions " << m_count << " but it was supposed to be " << trans_;
         }
+        unsigned int nStreamLumiTransitions = m_countStreamLumiTransitions.load();
+        unsigned int nLumis = m_countLumis.load();
+        unsigned int nStreams = m_countStreams.load();
+        if (nStreamLumiTransitions < nLumis || nStreamLumiTransitions > (nLumis * nStreams)) {
+          throw cms::Exception("transitions")
+              << "LumiSummaryLumiProducer stream lumi transitions " << nStreamLumiTransitions
+              << " but it was supposed to be between " << nLumis << " and " << nLumis * nStreams;
+        }
       }
     };
 
diff --git a/FWCore/Framework/test/stubs/TestLimitedAnalyzers.cc b/FWCore/Framework/test/stubs/TestLimitedAnalyzers.cc
index 55ac918ad1d92..dedbec5eb7058 100644
--- a/FWCore/Framework/test/stubs/TestLimitedAnalyzers.cc
+++ b/FWCore/Framework/test/stubs/TestLimitedAnalyzers.cc
@@ -52,16 +52,22 @@ namespace edmtest {
       explicit StreamIntAnalyzer(edm::ParameterSet const& p)
           : edm::limited::EDAnalyzerBase(p),
             edm::limited::EDAnalyzer<edm::StreamCache<UnsafeCache>>(p),
-            trans_(p.getParameter<int>("transitions")) {
+            trans_(p.getParameter<int>("transitions")),
+            nLumis_(p.getUntrackedParameter<unsigned int>("nLumis", 1)) {
         callWhenNewProductsRegistered([](edm::BranchDescription const& desc) {
           std::cout << "limited::StreamIntAnalyzer " << desc.moduleLabel() << std::endl;
         });
       }
       const unsigned int trans_;
+      const unsigned int nLumis_;
       mutable std::atomic<unsigned int> m_count{0};
+      mutable std::atomic<unsigned int> m_countStreams{0};
+      mutable std::atomic<unsigned int> m_countStreamBeginLumiTransitions{0};
+      mutable std::atomic<unsigned int> m_countStreamEndLumiTransitions{0};
 
       std::unique_ptr<UnsafeCache> beginStream(edm::StreamID iID) const override {
         ++m_count;
+        ++m_countStreams;
         auto pCache = std::make_unique<UnsafeCache>();
         pCache->value = iID.value();
         return pCache;
@@ -78,7 +84,7 @@ namespace edmtest {
       void streamBeginLuminosityBlock(edm::StreamID iID,
                                       edm::LuminosityBlock const&,
                                       edm::EventSetup const&) const override {
-        ++m_count;
+        ++m_countStreamBeginLumiTransitions;
         if ((streamCache(iID))->value != iID.value()) {
           throw cms::Exception("cache value")
               << "StreamIntAnalyzer cache value " << (streamCache(iID))->value << " but it was supposed to be " << iID;
@@ -96,7 +102,7 @@ namespace edmtest {
       void streamEndLuminosityBlock(edm::StreamID iID,
                                     edm::LuminosityBlock const&,
                                     edm::EventSetup const&) const override {
-        ++m_count;
+        ++m_countStreamEndLumiTransitions;
         if ((streamCache(iID))->value != iID.value()) {
           throw cms::Exception("cache value")
               << "StreamIntAnalyzer cache value " << (streamCache(iID))->value << " but it was supposed to be " << iID;
@@ -124,6 +130,19 @@ namespace edmtest {
           throw cms::Exception("transitions")
               << "StreamIntAnalyzer transitions " << m_count << " but it was supposed to be " << trans_;
         }
+        unsigned int nStreamBeginLumiTransitions = m_countStreamBeginLumiTransitions.load();
+        unsigned int nStreamEndLumiTransitions = m_countStreamEndLumiTransitions.load();
+        unsigned int nStreams = m_countStreams.load();
+        if (nStreamBeginLumiTransitions < nLumis_ || nStreamBeginLumiTransitions > (nLumis_ * nStreams)) {
+          throw cms::Exception("transitions")
+              << "StreamIntAnalyzer stream begin lumi transitions " << nStreamBeginLumiTransitions
+              << " but it was supposed to be between " << nLumis_ << " and " << nLumis_ * nStreams;
+        }
+        if (nStreamEndLumiTransitions != nStreamBeginLumiTransitions) {
+          throw cms::Exception("transitions")
+              << "StreamIntAnalyzer stream end lumi transitions " << nStreamEndLumiTransitions
+              << " does not equal stream begin lumi transitions " << nStreamBeginLumiTransitions;
+        }
       }
     };
 
@@ -272,15 +291,20 @@ namespace edmtest {
       const unsigned int trans_;
       const unsigned int cvalue_;
       mutable std::atomic<unsigned int> m_count{0};
+      mutable std::atomic<unsigned int> m_countLumis{0};
+      mutable std::atomic<unsigned int> m_countStreams{0};
+      mutable std::atomic<unsigned int> m_countStreamLumiTransitions{0};
 
       std::unique_ptr<UnsafeCache> beginStream(edm::StreamID) const override {
         ++m_count;
+        ++m_countStreams;
         return std::make_unique<UnsafeCache>();
       }
 
       std::shared_ptr<UnsafeCache> globalBeginLuminosityBlockSummary(edm::LuminosityBlock const& iLB,
                                                                      edm::EventSetup const&) const override {
         ++m_count;
+        ++m_countLumis;
         auto gCache = std::make_shared<UnsafeCache>();
         gCache->lumi = iLB.luminosityBlockAuxiliary().luminosityBlock();
         return gCache;
@@ -295,7 +319,7 @@ namespace edmtest {
                                            edm::LuminosityBlock const& iLB,
                                            edm::EventSetup const&,
                                            UnsafeCache* gCache) const override {
-        ++m_count;
+        ++m_countStreamLumiTransitions;
         if (gCache->lumi != iLB.luminosityBlockAuxiliary().luminosityBlock()) {
           throw cms::Exception("UnexpectedValue")
               << "streamEndLuminosityBlockSummary unexpected lumi number in Stream " << iID.value();
@@ -322,6 +346,14 @@ namespace edmtest {
           throw cms::Exception("transitions")
               << "LumiSummaryIntAnalyzer transitions " << m_count << " but it was supposed to be " << trans_;
         }
+        unsigned int nStreamLumiTransitions = m_countStreamLumiTransitions.load();
+        unsigned int nLumis = m_countLumis.load();
+        unsigned int nStreams = m_countStreams.load();
+        if (nStreamLumiTransitions < nLumis || nStreamLumiTransitions > (nLumis * nStreams)) {
+          throw cms::Exception("transitions")
+              << "LumiSummaryIntAnalyzer stream lumi transitions " << nStreamLumiTransitions
+              << " but it was supposed to be between " << nLumis << " and " << nLumis * nStreams;
+        }
       }
     };
 
diff --git a/FWCore/Framework/test/stubs/TestLimitedFilters.cc b/FWCore/Framework/test/stubs/TestLimitedFilters.cc
index b5aad0c82510e..ecbf7cbf0c425 100644
--- a/FWCore/Framework/test/stubs/TestLimitedFilters.cc
+++ b/FWCore/Framework/test/stubs/TestLimitedFilters.cc
@@ -59,15 +59,21 @@ namespace edmtest {
       explicit StreamIntFilter(edm::ParameterSet const& p)
           : edm::limited::EDFilterBase(p),
             edm::limited::EDFilter<edm::StreamCache<UnsafeCache>>(p),
-            trans_(p.getParameter<int>("transitions")) {
+            trans_(p.getParameter<int>("transitions")),
+            nLumis_(p.getUntrackedParameter<unsigned int>("nLumis", 1)) {
         produces<unsigned int>();
       }
 
       const unsigned int trans_;
+      const unsigned int nLumis_;
       mutable std::atomic<unsigned int> m_count{0};
+      mutable std::atomic<unsigned int> m_countStreams{0};
+      mutable std::atomic<unsigned int> m_countStreamBeginLumiTransitions{0};
+      mutable std::atomic<unsigned int> m_countStreamEndLumiTransitions{0};
 
       std::unique_ptr<UnsafeCache> beginStream(edm::StreamID iID) const override {
         ++m_count;
+        ++m_countStreams;
         auto sCache = std::make_unique<UnsafeCache>();
         ++(sCache->strm);
         sCache->value = iID.value();
@@ -89,7 +95,7 @@ namespace edmtest {
       void streamBeginLuminosityBlock(edm::StreamID iID,
                                       edm::LuminosityBlock const&,
                                       edm::EventSetup const&) const override {
-        ++m_count;
+        ++m_countStreamBeginLumiTransitions;
         auto sCache = streamCache(iID);
         if (sCache->value != iID.value()) {
           throw cms::Exception("cache value") << (streamCache(iID))->value << " but it was supposed to be " << iID;
@@ -118,7 +124,7 @@ namespace edmtest {
       void streamEndLuminosityBlock(edm::StreamID iID,
                                     edm::LuminosityBlock const&,
                                     edm::EventSetup const&) const override {
-        ++m_count;
+        ++m_countStreamEndLumiTransitions;
         auto sCache = streamCache(iID);
         if (sCache->value != iID.value()) {
           throw cms::Exception("cache value") << (streamCache(iID))->value << " but it was supposed to be " << iID;
@@ -161,6 +167,19 @@ namespace edmtest {
           throw cms::Exception("transitions")
               << "StreamIntFilter transitions " << m_count << " but it was supposed to be " << trans_;
         }
+        unsigned int nStreamBeginLumiTransitions = m_countStreamBeginLumiTransitions.load();
+        unsigned int nStreamEndLumiTransitions = m_countStreamEndLumiTransitions.load();
+        unsigned int nStreams = m_countStreams.load();
+        if (nStreamBeginLumiTransitions < nLumis_ || nStreamBeginLumiTransitions > (nLumis_ * nStreams)) {
+          throw cms::Exception("transitions")
+              << "StreamIntFilter stream begin lumi transitions " << nStreamBeginLumiTransitions
+              << " but it was supposed to be between " << nLumis_ << " and " << nLumis_ * nStreams;
+        }
+        if (nStreamEndLumiTransitions != nStreamBeginLumiTransitions) {
+          throw cms::Exception("transitions")
+              << "StreamIntFilter stream end lumi transitions " << nStreamEndLumiTransitions
+              << " does not equal stream begin lumi transitions " << nStreamBeginLumiTransitions;
+        }
       }
     };
 
@@ -380,15 +399,20 @@ namespace edmtest {
       const unsigned int trans_;
       const unsigned int cvalue_;
       mutable std::atomic<unsigned int> m_count{0};
+      mutable std::atomic<unsigned int> m_countLumis{0};
+      mutable std::atomic<unsigned int> m_countStreams{0};
+      mutable std::atomic<unsigned int> m_countStreamLumiTransitions{0};
 
       std::unique_ptr<Cache> beginStream(edm::StreamID) const override {
         ++m_count;
+        ++m_countStreams;
         return std::make_unique<Cache>();
       }
 
       std::shared_ptr<UnsafeCache> globalBeginLuminosityBlockSummary(edm::LuminosityBlock const& iLB,
                                                                      edm::EventSetup const&) const override {
         ++m_count;
+        ++m_countLumis;
         auto gCache = std::make_shared<UnsafeCache>();
         gCache->lumi = iLB.luminosityBlockAuxiliary().luminosityBlock();
         return gCache;
@@ -404,7 +428,7 @@ namespace edmtest {
                                            edm::LuminosityBlock const& iLB,
                                            edm::EventSetup const&,
                                            UnsafeCache* gCache) const override {
-        ++m_count;
+        ++m_countStreamLumiTransitions;
         if (gCache->lumi != iLB.luminosityBlockAuxiliary().luminosityBlock()) {
           throw cms::Exception("UnexpectedValue")
               << "streamEndLuminosityBlockSummary unexpected lumi number in Stream " << iID.value();
@@ -432,6 +456,14 @@ namespace edmtest {
           throw cms::Exception("transitions")
               << "LumiSummaryIntFilter transitions " << m_count << " but it was supposed to be " << trans_;
         }
+        unsigned int nStreamLumiTransitions = m_countStreamLumiTransitions.load();
+        unsigned int nLumis = m_countLumis.load();
+        unsigned int nStreams = m_countStreams.load();
+        if (nStreamLumiTransitions < nLumis || nStreamLumiTransitions > (nLumis * nStreams)) {
+          throw cms::Exception("transitions")
+              << "LumiSummaryIntFilter stream lumi transitions " << nStreamLumiTransitions
+              << " but it was supposed to be between " << nLumis << " and " << nLumis * nStreams;
+        }
       }
     };
 
diff --git a/FWCore/Framework/test/stubs/TestLimitedProducers.cc b/FWCore/Framework/test/stubs/TestLimitedProducers.cc
index 52a9d734537a3..4d5b050d0076e 100644
--- a/FWCore/Framework/test/stubs/TestLimitedProducers.cc
+++ b/FWCore/Framework/test/stubs/TestLimitedProducers.cc
@@ -59,15 +59,21 @@ namespace edmtest {
       explicit StreamIntProducer(edm::ParameterSet const& p)
           : edm::limited::EDProducerBase(p),
             edm::limited::EDProducer<edm::StreamCache<UnsafeCache>>(p),
-            trans_(p.getParameter<int>("transitions")) {
+            trans_(p.getParameter<int>("transitions")),
+            nLumis_(p.getUntrackedParameter<unsigned int>("nLumis", 1)) {
         produces<unsigned int>();
       }
 
       const unsigned int trans_;
+      const unsigned int nLumis_;
       mutable std::atomic<unsigned int> m_count{0};
+      mutable std::atomic<unsigned int> m_countStreams{0};
+      mutable std::atomic<unsigned int> m_countStreamBeginLumiTransitions{0};
+      mutable std::atomic<unsigned int> m_countStreamEndLumiTransitions{0};
 
       std::unique_ptr<UnsafeCache> beginStream(edm::StreamID iID) const override {
         ++m_count;
+        ++m_countStreams;
         auto sCache = std::make_unique<UnsafeCache>();
         ++(sCache->strm);
         sCache->value = iID.value();
@@ -79,7 +85,7 @@ namespace edmtest {
         auto sCache = streamCache(iID);
         if (sCache->value != iID.value()) {
           throw cms::Exception("cache value")
-              << "StreamIntAnalyzer cache value " << (streamCache(iID))->value << " but it was supposed to be " << iID;
+              << "StreamIntProducer cache value " << (streamCache(iID))->value << " but it was supposed to be " << iID;
         }
         if (sCache->run != 0 || sCache->lumi != 0 || sCache->work != 0 || sCache->strm != 1) {
           throw cms::Exception("out of sequence") << "streamBeginRun out of sequence in Stream " << iID.value();
@@ -90,7 +96,7 @@ namespace edmtest {
       void streamBeginLuminosityBlock(edm::StreamID iID,
                                       edm::LuminosityBlock const&,
                                       edm::EventSetup const&) const override {
-        ++m_count;
+        ++m_countStreamBeginLumiTransitions;
         auto sCache = streamCache(iID);
         if (sCache->lumi != 0 || sCache->work != 0) {
           throw cms::Exception("out of sequence")
@@ -111,7 +117,7 @@ namespace edmtest {
       void streamEndLuminosityBlock(edm::StreamID iID,
                                     edm::LuminosityBlock const&,
                                     edm::EventSetup const&) const override {
-        ++m_count;
+        ++m_countStreamEndLumiTransitions;
         auto sCache = streamCache(iID);
         --(sCache->lumi);
         sCache->work = 0;
@@ -145,6 +151,19 @@ namespace edmtest {
           throw cms::Exception("transitions")
               << "StreamIntProducer transitions " << m_count << " but it was supposed to be " << trans_;
         }
+        unsigned int nStreamBeginLumiTransitions = m_countStreamBeginLumiTransitions.load();
+        unsigned int nStreamEndLumiTransitions = m_countStreamEndLumiTransitions.load();
+        unsigned int nStreams = m_countStreams.load();
+        if (nStreamBeginLumiTransitions < nLumis_ || nStreamBeginLumiTransitions > (nLumis_ * nStreams)) {
+          throw cms::Exception("transitions")
+              << "StreamIntProducer stream begin lumi transitions " << nStreamBeginLumiTransitions
+              << " but it was supposed to be between " << nLumis_ << " and " << nLumis_ * nStreams;
+        }
+        if (nStreamEndLumiTransitions != nStreamBeginLumiTransitions) {
+          throw cms::Exception("transitions")
+              << "StreamIntProducer stream end lumi transitions " << nStreamEndLumiTransitions
+              << " does not equal stream begin lumi transitions " << nStreamBeginLumiTransitions;
+        }
       }
     };
 
@@ -354,18 +373,27 @@ namespace edmtest {
       const unsigned int trans_;
       const unsigned int cvalue_;
       mutable std::atomic<unsigned int> m_count{0};
+      mutable std::atomic<unsigned int> m_countLumis{0};
+      mutable std::atomic<unsigned int> m_countStreams{0};
+      mutable std::atomic<unsigned int> m_countStreamLumiTransitions{0};
 
-      std::unique_ptr<UnsafeCache> beginStream(edm::StreamID) const override { return std::make_unique<UnsafeCache>(); }
+      std::unique_ptr<UnsafeCache> beginStream(edm::StreamID) const override {
+        ++m_count;
+        ++m_countStreams;
+        return std::make_unique<UnsafeCache>();
+      }
 
       std::shared_ptr<UnsafeCache> globalBeginLuminosityBlockSummary(edm::LuminosityBlock const& iLB,
                                                                      edm::EventSetup const&) const override {
         ++m_count;
+        ++m_countLumis;
         auto gCache = std::make_shared<UnsafeCache>();
         gCache->lumi = iLB.luminosityBlockAuxiliary().luminosityBlock();
         return gCache;
       }
 
       void produce(edm::StreamID iID, edm::Event&, edm::EventSetup const&) const override {
+        ++m_count;
         auto sCache = streamCache(iID);
         ++(sCache->value);
       }
@@ -374,7 +402,7 @@ namespace edmtest {
                                            edm::LuminosityBlock const& iLB,
                                            edm::EventSetup const&,
                                            UnsafeCache* gCache) const override {
-        ++m_count;
+        ++m_countStreamLumiTransitions;
         if (gCache->lumi != iLB.luminosityBlockAuxiliary().luminosityBlock()) {
           throw cms::Exception("UnexpectedValue")
               << "streamEndLuminosityBlockSummary unexpected lumi number in Stream " << iID.value();
@@ -402,6 +430,14 @@ namespace edmtest {
           throw cms::Exception("transitions")
               << "LumiSummaryIntProducer transitions " << m_count << " but it was supposed to be " << trans_;
         }
+        unsigned int nStreamLumiTransitions = m_countStreamLumiTransitions.load();
+        unsigned int nLumis = m_countLumis.load();
+        unsigned int nStreams = m_countStreams.load();
+        if (nStreamLumiTransitions < nLumis || nStreamLumiTransitions > (nLumis * nStreams)) {
+          throw cms::Exception("transitions")
+              << "LumiSummaryIntProducer stream lumi transitions " << nStreamLumiTransitions
+              << " but it was supposed to be between " << nLumis << " and " << nLumis * nStreams;
+        }
       }
     };
 
@@ -421,12 +457,20 @@ namespace edmtest {
       const unsigned int trans_;
       const unsigned int cvalue_;
       mutable std::atomic<unsigned int> m_count{0};
+      mutable std::atomic<unsigned int> m_countLumis{0};
+      mutable std::atomic<unsigned int> m_countStreams{0};
+      mutable std::atomic<unsigned int> m_countStreamLumiTransitions{0};
 
-      std::unique_ptr<UnsafeCache> beginStream(edm::StreamID) const override { return std::make_unique<UnsafeCache>(); }
+      std::unique_ptr<UnsafeCache> beginStream(edm::StreamID) const override {
+        ++m_count;
+        ++m_countStreams;
+        return std::make_unique<UnsafeCache>();
+      }
 
       std::shared_ptr<UnsafeCache> globalBeginLuminosityBlockSummary(edm::LuminosityBlock const& iLB,
                                                                      edm::EventSetup const&) const override {
         ++m_count;
+        ++m_countLumis;
         auto gCache = std::make_shared<UnsafeCache>();
         gCache->lumi = iLB.luminosityBlockAuxiliary().luminosityBlock();
         return gCache;
@@ -441,7 +485,7 @@ namespace edmtest {
                                            edm::LuminosityBlock const& iLB,
                                            edm::EventSetup const&,
                                            UnsafeCache* gCache) const override {
-        ++m_count;
+        ++m_countStreamLumiTransitions;
         if (gCache->lumi != iLB.luminosityBlockAuxiliary().luminosityBlock()) {
           throw cms::Exception("UnexpectedValue")
               << "streamEndLuminosityBlockSummary unexpected lumi number in Stream " << iID.value();
@@ -478,6 +522,14 @@ namespace edmtest {
           throw cms::Exception("transitions")
               << "LumiSummaryLumiProducer transitions " << m_count << " but it was supposed to be " << trans_;
         }
+        unsigned int nStreamLumiTransitions = m_countStreamLumiTransitions.load();
+        unsigned int nLumis = m_countLumis.load();
+        unsigned int nStreams = m_countStreams.load();
+        if (nStreamLumiTransitions < nLumis || nStreamLumiTransitions > (nLumis * nStreams)) {
+          throw cms::Exception("transitions")
+              << "LumiSummaryLumiProducer stream lumi transitions " << nStreamLumiTransitions
+              << " but it was supposed to be between " << nLumis << " and " << nLumis * nStreams;
+        }
       }
     };
 
diff --git a/FWCore/Framework/test/testRunLumiCaches_cfg.py b/FWCore/Framework/test/testRunLumiCaches_cfg.py
index b1a794e1569db..c5b799653aaba 100644
--- a/FWCore/Framework/test/testRunLumiCaches_cfg.py
+++ b/FWCore/Framework/test/testRunLumiCaches_cfg.py
@@ -49,7 +49,7 @@
 )
 
 process.globalLumiSumIntProd = cms.EDProducer("edmtest::global::LumiSummaryIntProducer",
-    transitions = cms.int32(2*nLumis+nStreams*nLumis)
+    transitions = cms.int32(nStreams+2*nLumis+nEvents)
     ,cachevalue = cms.int32(nEventsPerLumi)
 )
 
@@ -69,7 +69,7 @@
 )
 
 process.globalLumiSumIntFilt = cms.EDFilter("edmtest::global::LumiSummaryIntFilter",
-    transitions = cms.int32(nStreams+nStreams*nLumis+2*nLumis+nEvents)
+    transitions = cms.int32(nStreams+2*nLumis+nEvents)
     ,cachevalue = cms.int32(nEventsPerLumi)
 )
 
@@ -90,7 +90,7 @@
 )
 
 process.globalLumiSumIntAna = cms.EDAnalyzer("edmtest::global::LumiSummaryIntAnalyzer",
-    transitions = cms.int32(nStreams+nStreams*nLumis+2*nLumis+nEvents)
+    transitions = cms.int32(nStreams+2*nLumis+nEvents)
     ,cachevalue = cms.int32(nEventsPerLumi)
 )
 
@@ -114,7 +114,7 @@
 
 process.limitedLumiSumIntProd = cms.EDProducer("edmtest::limited::LumiSummaryIntProducer",
     concurrencyLimit = cms.untracked.uint32(1),
-    transitions = cms.int32(nStreams*nLumis+2*nLumis)
+    transitions = cms.int32(nStreams+2*nLumis+nEvents)
     ,cachevalue = cms.int32(nEventsPerLumi)
 )
 
@@ -138,7 +138,7 @@
 
 process.limitedLumiSumIntFilt = cms.EDFilter("edmtest::limited::LumiSummaryIntFilter",
     concurrencyLimit = cms.untracked.uint32(1),
-    transitions = cms.int32(nStreams+nStreams*nLumis+2*nLumis+nEvents)
+    transitions = cms.int32(nStreams+2*nLumis+nEvents)
     ,cachevalue = cms.int32(nEventsPerLumi)
 )
 
@@ -163,7 +163,7 @@
 
 process.limitedLumiSumIntAna = cms.EDAnalyzer("edmtest::limited::LumiSummaryIntAnalyzer",
     concurrencyLimit = cms.untracked.uint32(1),
-    transitions = cms.int32(nStreams+nStreams*nLumis+2*nLumis+nEvents)
+    transitions = cms.int32(nStreams+2*nLumis+nEvents)
     ,cachevalue = cms.int32(nEventsPerLumi)
 )
 
diff --git a/FWCore/Framework/test/test_earlyTerminationSignal.sh b/FWCore/Framework/test/test_earlyTerminationSignal.sh
index eccfc14a15c97..c6899452752b7 100755
--- a/FWCore/Framework/test/test_earlyTerminationSignal.sh
+++ b/FWCore/Framework/test/test_earlyTerminationSignal.sh
@@ -22,7 +22,7 @@ echo "running cmsRun test_dependentRunDataAndException_cfg.py"
 (cmsRun ${LOCAL_TEST_DIR}/test_dependentRunDataAndException_cfg.py 2>&1 | grep -q "Intentional 'NotFound' exception for testing purposes") || die "dependent Run data and Exceptions failed" $?
 
 echo "running cmsRun test_exceptionAtGlobalBeginRun_cfg.py"
-(cmsRun ${LOCAL_TEST_DIR}/test_exceptionAtGlobalBeginRun_cfg.py 2>&1 | grep -q -v "An exception of category 'transitions' occurred") || die "exception at globalBeginRun failed" $?
+(cmsRun ${LOCAL_TEST_DIR}/test_exceptionAtGlobalBeginRun_cfg.py 2>&1 | grep "Another exception was caught while endJob was running") && die "test_exceptionAtGlobalBeginRun failed, should not be any endJob exceptions" 1
 
 echo "running cmsRun  test_exceptionInShortLumi_cfg.py"
 cmsRun ${LOCAL_TEST_DIR}/test_exceptionInShortLumi_cfg.py; test_failure "test_exceptionInShortLumi_cfg.py failed" $?
diff --git a/FWCore/Framework/test/test_exceptionAtGlobalBeginRun_cfg.py b/FWCore/Framework/test/test_exceptionAtGlobalBeginRun_cfg.py
index 2ae358d5b2ee4..16289ac17c00e 100644
--- a/FWCore/Framework/test/test_exceptionAtGlobalBeginRun_cfg.py
+++ b/FWCore/Framework/test/test_exceptionAtGlobalBeginRun_cfg.py
@@ -6,7 +6,9 @@
 
 process.fail = cms.EDProducer("edmtest::FailingInRunProducer")
 
-process.tstStream = cms.EDAnalyzer("edmtest::global::StreamIntAnalyzer", transitions=cms.int32(2))
+process.tstStream = cms.EDAnalyzer("edmtest::global::StreamIntAnalyzer",
+                                   transitions=cms.int32(2),
+                                   nLumis = cms.untracked.uint32(0))
 process.tstGlobal = cms.EDAnalyzer("edmtest::global::RunIntAnalyzer",
                                    transitions=cms.int32(2),
                                    cachevalue = cms.int32(0))
@@ -16,7 +18,9 @@
 process.add_(cms.Service("Tracer"))
 
 process2 = cms.Process("Test2")
-process2.tstStreamSub = cms.EDAnalyzer("edmtest::global::StreamIntAnalyzer", transitions=cms.int32(2))
+process2.tstStreamSub = cms.EDAnalyzer("edmtest::global::StreamIntAnalyzer",
+                                       transitions=cms.int32(2),
+                                       nLumis = cms.untracked.uint32(0))
 process2.tstGlobalSub = cms.EDAnalyzer("edmtest::global::RunIntAnalyzer",
                                    transitions=cms.int32(2),
                                    cachevalue = cms.int32(0))
diff --git a/FWCore/Framework/test/test_global_modules_cfg.py b/FWCore/Framework/test/test_global_modules_cfg.py
index f7127453fe4f7..180b4fa616ef1 100644
--- a/FWCore/Framework/test/test_global_modules_cfg.py
+++ b/FWCore/Framework/test/test_global_modules_cfg.py
@@ -28,8 +28,9 @@
 )
 
 process.StreamIntProd = cms.EDProducer("edmtest::global::StreamIntProducer",
-    transitions = cms.int32(int(nEvt+nStreams*(2*(nEvt/nEvtRun)+2*(nEvt/nEvtLumi)+2)))
+    transitions = cms.int32(int(nEvt+nStreams*(2*(nEvt/nEvtRun)+2)))
     ,cachevalue = cms.int32(1)
+    ,nLumis = cms.untracked.uint32(int(nEvt/nEvtLumi))
 )
 
 process.RunIntProd = cms.EDProducer("edmtest::global::RunIntProducer",
@@ -48,7 +49,7 @@
 )
 
 process.LumiSumIntProd = cms.EDProducer("edmtest::global::LumiSummaryIntProducer",
-    transitions = cms.int32(int(nStreams*(nEvt/nEvtLumi)+2*(nEvt/nEvtLumi)))
+    transitions = cms.int32(int(nEvt+nStreams+2*(nEvt/nEvtLumi)))
     ,cachevalue = cms.int32(nEvtLumi)
 )
 
@@ -95,8 +96,9 @@
 )
 
 process.StreamIntAn = cms.EDAnalyzer("edmtest::global::StreamIntAnalyzer",
-    transitions = cms.int32(int(nEvt+nStreams*(2*(nEvt/nEvtRun)+2*(nEvt/nEvtLumi)+2)))
+    transitions = cms.int32(int(nEvt+nStreams*(2*(nEvt/nEvtRun)+2)))
     ,cachevalue = cms.int32(1)
+    ,nLumis = cms.untracked.uint32(int(nEvt/nEvtLumi))
 )
 
 process.RunIntAn= cms.EDAnalyzer("edmtest::global::RunIntAnalyzer",
@@ -117,7 +119,7 @@
 )
 
 process.LumiSumIntAn = cms.EDAnalyzer("edmtest::global::LumiSummaryIntAnalyzer",
-    transitions = cms.int32(int(nEvt+nStreams*((nEvt/nEvtLumi)+1)+2*(nEvt/nEvtLumi)))
+    transitions = cms.int32(int(nEvt+nStreams+2*(nEvt/nEvtLumi)))
     ,cachevalue = cms.int32(nEvtLumi)
 )
 
@@ -128,8 +130,9 @@
 )
 
 process.StreamIntFil = cms.EDFilter("edmtest::global::StreamIntFilter",
-    transitions = cms.int32(int(nEvt+nStreams*(2*(nEvt/nEvtRun)+2*(nEvt/nEvtLumi)+2)))
+    transitions = cms.int32(int(nEvt+nStreams*(2*(nEvt/nEvtRun)+2)))
     ,cachevalue = cms.int32(1)
+    ,nLumis = cms.untracked.uint32(int(nEvt/nEvtLumi))
 )
 
 process.RunIntFil = cms.EDFilter("edmtest::global::RunIntFilter",
@@ -148,7 +151,7 @@
 )
 
 process.LumiSumIntFil = cms.EDFilter("edmtest::global::LumiSummaryIntFilter",
-    transitions = cms.int32(int(nEvt+nStreams*((nEvt/nEvtLumi)+1)+2*(nEvt/nEvtLumi)))
+    transitions = cms.int32(int(nEvt+nStreams+2*(nEvt/nEvtLumi)))
     ,cachevalue = cms.int32(nEvtLumi)
 )
 
diff --git a/FWCore/Framework/test/test_limited_modules_cfg.py b/FWCore/Framework/test/test_limited_modules_cfg.py
index 5212373f9446e..ee140b6e9a7d9 100644
--- a/FWCore/Framework/test/test_limited_modules_cfg.py
+++ b/FWCore/Framework/test/test_limited_modules_cfg.py
@@ -30,8 +30,9 @@
 
 process.StreamIntProd = cms.EDProducer("edmtest::limited::StreamIntProducer",
     concurrencyLimit = cms.untracked.uint32(1),
-    transitions = cms.int32(nEvt+nStreams*(2*int(nEvt/nEvtRun)+2*int(nEvt/nEvtLumi)+2))
+    transitions = cms.int32(nEvt+nStreams*(2*int(nEvt/nEvtRun)+2))
     ,cachevalue = cms.int32(1)
+    ,nLumis = cms.untracked.uint32(int(nEvt/nEvtLumi))
 )
 
 process.RunIntProd = cms.EDProducer("edmtest::limited::RunIntProducer",
@@ -54,7 +55,7 @@
 
 process.LumiSumIntProd = cms.EDProducer("edmtest::limited::LumiSummaryIntProducer",
                                         concurrencyLimit = cms.untracked.uint32(1),
-    transitions = cms.int32(nStreams*int(nEvt/nEvtLumi)+2*int(nEvt/nEvtLumi))
+    transitions = cms.int32(nStreams+nEvt+2*int(nEvt/nEvtLumi))
     ,cachevalue = cms.int32(nEvtLumi)
 )
 
@@ -111,8 +112,9 @@
 
 process.StreamIntAn = cms.EDAnalyzer("edmtest::limited::StreamIntAnalyzer",
                                      concurrencyLimit = cms.untracked.uint32(1),
-    transitions = cms.int32(nEvt+nStreams*(2*int(nEvt/nEvtRun)+2*int(nEvt/nEvtLumi)+2))
+    transitions = cms.int32(nEvt+nStreams*(2*int(nEvt/nEvtRun)+2))
     ,cachevalue = cms.int32(1)
+    ,nLumis = cms.untracked.uint32(int(nEvt/nEvtLumi))
 )
 
 process.RunIntAn= cms.EDAnalyzer("edmtest::limited::RunIntAnalyzer",
@@ -137,7 +139,7 @@
 
 process.LumiSumIntAn = cms.EDAnalyzer("edmtest::limited::LumiSummaryIntAnalyzer",
                                       concurrencyLimit = cms.untracked.uint32(1),
-    transitions = cms.int32(nEvt+nStreams*(int(nEvt/nEvtLumi)+1)+2*int(nEvt/nEvtLumi))
+    transitions = cms.int32(nEvt+nStreams+2*int(nEvt/nEvtLumi))
     ,cachevalue = cms.int32(nEvtLumi)
 )
 
@@ -150,8 +152,9 @@
 
 process.StreamIntFil = cms.EDFilter("edmtest::limited::StreamIntFilter",
                                     concurrencyLimit = cms.untracked.uint32(1),
-    transitions = cms.int32(nEvt+nStreams*(2*int(nEvt/nEvtRun)+2*int(nEvt/nEvtLumi)+2))
+    transitions = cms.int32(nEvt+nStreams*(2*int(nEvt/nEvtRun)+2))
     ,cachevalue = cms.int32(1)
+    ,nLumis = cms.untracked.uint32(int(nEvt/nEvtLumi))
 )
 
 process.RunIntFil = cms.EDFilter("edmtest::limited::RunIntFilter",
@@ -174,7 +177,7 @@
 
 process.LumiSumIntFil = cms.EDFilter("edmtest::limited::LumiSummaryIntFilter",
                                      concurrencyLimit = cms.untracked.uint32(1),
-    transitions = cms.int32(nEvt+nStreams*(int(nEvt/nEvtLumi)+1)+2*int(nEvt/nEvtLumi))
+    transitions = cms.int32(nEvt+nStreams+2*int(nEvt/nEvtLumi))
     ,cachevalue = cms.int32(nEvtLumi)
 )
 
diff --git a/FWCore/Integration/test/test_TryToContinue_cfg.py b/FWCore/Integration/test/test_TryToContinue_cfg.py
index 6d6781c556c38..12d8b7e80e60b 100644
--- a/FWCore/Integration/test/test_TryToContinue_cfg.py
+++ b/FWCore/Integration/test/test_TryToContinue_cfg.py
@@ -25,9 +25,9 @@
 else:
     process.fail = cms.EDProducer("FailingProducer")
 
-process.shouldRun1 = cms.EDAnalyzer("edmtest::global::StreamIntAnalyzer", transitions = cms.int32(6+3), verbose = cms.untracked.bool(False))
-process.shouldRun2 = cms.EDAnalyzer("edmtest::global::StreamIntAnalyzer", transitions = cms.int32(6+3), verbose = cms.untracked.bool(False))
-process.shouldNotRun = cms.EDAnalyzer("edmtest::global::StreamIntAnalyzer", transitions = cms.int32(6), verbose = cms.untracked.bool(False))
+process.shouldRun1 = cms.EDAnalyzer("edmtest::global::StreamIntAnalyzer", transitions = cms.int32(4+3), nLumis = cms.untracked.uint32(1), verbose = cms.untracked.bool(False))
+process.shouldRun2 = cms.EDAnalyzer("edmtest::global::StreamIntAnalyzer", transitions = cms.int32(4+3), nLumis = cms.untracked.uint32(1), verbose = cms.untracked.bool(False))
+process.shouldNotRun = cms.EDAnalyzer("edmtest::global::StreamIntAnalyzer", transitions = cms.int32(4), nLumis = cms.untracked.uint32(1), verbose = cms.untracked.bool(False))
 process.dependentFilter = cms.EDFilter("IntProductFilter",
    label = cms.InputTag("fail"),
    threshold = cms.int32(0),
@@ -52,12 +52,18 @@
     process.shouldRun1.transitions=2
     process.shouldRun2.transitions=2
     process.shouldNotRun.transitions=2
+    process.shouldRun1.nLumis=0
+    process.shouldRun2.nLumis=0
+    process.shouldNotRun.nLumis=0
 
 if args.inLumi:
     process.independentAnalyzer.expectedSum = 0
     process.shouldRun1.transitions=4
     process.shouldRun2.transitions=4
     process.shouldNotRun.transitions=4
+    process.shouldRun1.nLumis=0
+    process.shouldRun2.nLumis=0
+    process.shouldNotRun.nLumis=0
 
 process.seq = cms.Sequence()
 process.t = cms.Task(process.intProd,process.addInts)
@@ -71,5 +77,3 @@
 
 process.errorEndPath = cms.EndPath(process.dependentAnalyzer)
 process.goodEndPath = cms.EndPath(process.independentAnalyzer)
-
-
diff --git a/FWCore/Services/plugins/CheckTransitions.cc b/FWCore/Services/plugins/CheckTransitions.cc
index ab960caeac182..80a38e2f1d66c 100644
--- a/FWCore/Services/plugins/CheckTransitions.cc
+++ b/FWCore/Services/plugins/CheckTransitions.cc
@@ -261,11 +261,45 @@ void CheckTransitions::postEndJob() {
    std::cout <<i.first.m_run<<" "<<i.first.m_lumi<<" "<<i.first.m_event<<" "<<i.second<<std::endl;
    } */
 
+  unsigned int nSkippedStreamLumiTransitions = 0;
+
+  // Use the next two vectors to test that skipped stream lumi transitions
+  // come in matched begin and end pairs.
+  std::vector<std::tuple<Phase, edm::EventID, int>> expectedSkippedStreamEndLumi;
+  std::vector<std::tuple<Phase, edm::EventID, int>> seenSkippedStreamEndLumi;
+
+  // Use the next two sets to test that for every global begin lumi there
+  // is at least one stream begin lumi, even when some stream begin lumi
+  // transitions are skipped.
+  std::set<std::tuple<Phase, edm::EventID, int>> seenGlobalBeginLumi;
+  std::set<std::tuple<Phase, edm::EventID, int>> seenStreamBeginLumi;
+
   auto itOS = orderedSeen.begin();
   for (auto itOE = orderedExpected.begin(); itOE != orderedExpected.end(); ++itOE) {
     if (itOS == orderedSeen.end()) {
       break;
     }
+
+    if (std::get<0>(*itOS) == Phase::kBeginLumi && std::get<2>(*itOS) == -1) {
+      seenGlobalBeginLumi.emplace(*itOS);
+    }
+    if (std::get<0>(*itOS) == Phase::kBeginLumi && (std::get<2>(*itOS) > -1 && std::get<2>(*itOS) < 1000)) {
+      // Note that the third field is falsely filled with the value for a global begin lumi to
+      // make a comparison with seenGlobalBeginLumi easier.
+      seenStreamBeginLumi.emplace(std::get<0>(*itOS), std::get<1>(*itOS), -1);
+    }
+
+    while (*itOE != *itOS && (std::get<0>(*itOE) == Phase::kBeginLumi || std::get<0>(*itOE) == Phase::kEndLumi) &&
+           (std::get<2>(*itOE) > -1 && std::get<2>(*itOE) < 1000)) {
+      ++nSkippedStreamLumiTransitions;
+      if (std::get<0>(*itOE) == Phase::kBeginLumi) {
+        expectedSkippedStreamEndLumi.emplace_back(Phase::kEndLumi, std::get<1>(*itOE), std::get<2>(*itOE));
+      } else {
+        seenSkippedStreamEndLumi.emplace_back(*itOE);
+      }
+
+      ++itOE;
+    }
     if (*itOE != *itOS) {
       auto syncOE = std::get<1>(*itOE);
       auto syncOS = std::get<1>(*itOS);
@@ -276,7 +310,17 @@ void CheckTransitions::postEndJob() {
     ++itOS;
   }
 
-  if (orderedSeen.size() != orderedExpected.size()) {
+  if (seenGlobalBeginLumi != seenStreamBeginLumi) {
+    std::cout << "We didn't see at least one stream begin lumi for every global begin lumi" << std::endl;
+    m_failed = true;
+  }
+
+  if (expectedSkippedStreamEndLumi != seenSkippedStreamEndLumi) {
+    std::cout << "Skipped stream begin and end lumi transitions do not match" << std::endl;
+    m_failed = true;
+  }
+
+  if (orderedSeen.size() + nSkippedStreamLumiTransitions != orderedExpected.size()) {
     std::cout << "Wrong number of transition " << orderedSeen.size() << " " << orderedExpected.size() << std::endl;
     m_failed = true;
     return;