KeyframeSelection: Add debug option to skip sharpness score computation

Add a "skipSharpnessComputation" debug option that allows to compute the
scores with performing the sharpness score computations. All frames will
be assigned a fixed sharpness score of 1.0.

If the smart selection is applied with this option enabled, the selected
frames will be those located in the center of each subset (determined with
the motion scores) as the weights will be applied on constant scores.

This option is useful to determine the impact of the sharpness score
computation on the global processing time.

src/aliceVision/keyframe/KeyframeSelector.cpp

@@ -137,13 +137,13 @@ void KeyframeSelector::processRegular()
 
 void KeyframeSelector::processSmart(const float pxDisplacement, const std::size_t rescaledWidthSharpness,
                                     const std::size_t rescaledWidthFlow, const std::size_t sharpnessWindowSize,
-                                    const std::size_t flowCellSize)
+                                    const std::size_t flowCellSize, const bool skipSharpnessComputation)
 {
     _selectedKeyframes.clear();
     _selectedFrames.clear();
 
     // Step 0: compute all the scores
-    computeScores(rescaledWidthSharpness, rescaledWidthFlow, sharpnessWindowSize, flowCellSize);
+    computeScores(rescaledWidthSharpness, rescaledWidthFlow, sharpnessWindowSize, flowCellSize, skipSharpnessComputation);
 
     // Step 1: determine subsequences based on the motion accumulation
     std::vector<unsigned int> subsequenceLimits;
@@ -273,7 +273,8 @@ void KeyframeSelector::processSmart(const float pxDisplacement, const std::size_
 }
 
 bool KeyframeSelector::computeScores(const std::size_t rescaledWidthSharpness, const std::size_t rescaledWidthFlow,
-                                     const std::size_t sharpnessWindowSize, const std::size_t flowCellSize)
+                                     const std::size_t sharpnessWindowSize, const std::size_t flowCellSize,
+                                     const bool skipSharpnessComputation)
 {
     // Reset the computed scores
     _sharpnessScores.clear();
@@ -330,7 +331,7 @@ bool KeyframeSelector::computeScores(const std::size_t rescaledWidthSharpness, c
     auto ptrFlow = cv::optflow::createOptFlow_DeepFlow();
 
     while (currentFrame < nbFrames) {
-        double minimalSharpness = std::numeric_limits<double>::max();
+        double minimalSharpness = skipSharpnessComputation ? 1.0f : std::numeric_limits<double>::max();
         double minimalFlow = std::numeric_limits<double>::max();
 
         for (std::size_t mediaIndex = 0; mediaIndex < feeds.size(); ++mediaIndex) {
@@ -349,26 +350,29 @@ bool KeyframeSelector::computeScores(const std::size_t rescaledWidthSharpness, c
              *   - otherwise (feed not correctly moved to the next frame), throw a runtime error exception as something
              *     is wrong with the video
              */
-            try {
-                currentMatSharpness = readImage(feed, rescaledWidthSharpness);  // Read image for sharpness and rescale it if requested
-            } catch (const std::invalid_argument& ex) {
-                // currentFrame + 1 = currently evaluated frame with indexing starting at 1, for display reasons
-                // currentFrame + 2 = next frame to evaluate with indexing starting at 1, for display reasons
-                ALICEVISION_LOG_WARNING("Invalid or missing frame " << currentFrame + 1
-                                        << ", attempting to read frame " << currentFrame + 2 << ".");
-                bool success = feed.goToFrame(++currentFrame);
-                if (success) {
-                    // Will throw an exception if next frame is also invalid
+            if (!skipSharpnessComputation) {
+                try {
+                    // Read image for sharpness and rescale it if requested
                     currentMatSharpness = readImage(feed, rescaledWidthSharpness);
-                    // If no exception has been thrown, push dummy scores for the frame that was skipped
-                    _sharpnessScores.push_back(-1.f);
-                    _flowScores.push_back(-1.f);
-                } else
-                    ALICEVISION_THROW_ERROR("Could not go to frame " << currentFrame + 1
-                                            << " either. The feed might be corrupted.");
+                } catch (const std::invalid_argument& ex) {
+                    // currentFrame + 1 = currently evaluated frame with indexing starting at 1, for display reasons
+                    // currentFrame + 2 = next frame to evaluate with indexing starting at 1, for display reasons
+                    ALICEVISION_LOG_WARNING("Invalid or missing frame " << currentFrame + 1
+                                            << ", attempting to read frame " << currentFrame + 2 << ".");
+                    bool success = feed.goToFrame(++currentFrame);
+                    if (success) {
+                        // Will throw an exception if next frame is also invalid
+                        currentMatSharpness = readImage(feed, rescaledWidthSharpness);
+                        // If no exception has been thrown, push dummy scores for the frame that was skipped
+                        _sharpnessScores.push_back(-1.f);
+                        _flowScores.push_back(-1.f);
+                    } else
+                        ALICEVISION_THROW_ERROR("Could not go to frame " << currentFrame + 1
+                                                << " either. The feed might be corrupted.");
+                }
             }
 
-            if (rescaledWidthSharpness == rescaledWidthFlow) {
+            if (rescaledWidthSharpness == rescaledWidthFlow && !skipSharpnessComputation) {
                 currentMatFlow = currentMatSharpness;
             } else {
                 currentMatFlow = readImage(feed, rescaledWidthFlow);
@@ -380,8 +384,10 @@ bool KeyframeSelector::computeScores(const std::size_t rescaledWidthSharpness, c
             }
 
             // Compute sharpness
-            const double sharpness = computeSharpness(currentMatSharpness, sharpnessWindowSize);
-            minimalSharpness = std::min(minimalSharpness, sharpness);
+            if (!skipSharpnessComputation) {
+                const double sharpness = computeSharpness(currentMatSharpness, sharpnessWindowSize);
+                minimalSharpness = std::min(minimalSharpness, sharpness);
+            }
 
             // Compute optical flow
             if (currentFrame > 0) {

src/aliceVision/keyframe/KeyframeSelector.hpp

@@ -82,10 +82,12 @@ class KeyframeSelector
      * @param[in] sharpnessWindowSize the size of the sliding window used to compute sharpness scores, in pixels
      * @param[in] flowCellSize the size of the cells within a frame that are used to compute the optical flow scores,
      *            in pixels
+     * @param[in] skipSharpnessComputation if true, the sharpness score computations will not be performed and a fixed
+     *            sharpness score will be given to all the input frames
      */
     void processSmart(const float pxDisplacement, const std::size_t rescaledWidthSharpness,
                       const std::size_t rescaledWidthFlow, const std::size_t sharpnessWindowSize,
-                      const std::size_t flowCellSize);
+                      const std::size_t flowCellSize, const bool skipSharpnessComputation = false);
 
     /**
      * @brief Compute the sharpness and optical flow scores for the input media paths
@@ -96,10 +98,13 @@ class KeyframeSelector
      * @param[in] sharpnessWindowSize the size of the sliding window used to compute sharpness scores, in pixels
      * @param[in] flowCellSize the size of the cells within a frame that are used to compute the optical flow scores,
      *            in pixels
+     * @param[in] skipSharpnessComputation if true, the sharpness score computations will not be performed and a fixed
+     *            sharpness score will be given to all the input frames
      * @return true if the scores have been successfully computed for all frames, false otherwise
      */
     bool computeScores(const std::size_t rescaledWidthSharpness, const std::size_t rescaledWidthFlow,
-                       const std::size_t sharpnessWindowSize, const std::size_t flowCellSize);
+                       const std::size_t sharpnessWindowSize, const std::size_t flowCellSize,
+                       const bool skipSharpnessComputation);
 
     /**
      * @brief Write the selected keyframes in the output folder

src/aliceVision/keyframe/README.md

@@ -86,7 +86,9 @@ The weights aim at favouring the selection of keyframes that are as temporally f
 
 Debug options specific to the smart selection method are available:
 - Export scores to CSV: the sharpness and motion scores for all the frames are written to a CSV file;
-- Visualise the optical flow: the computed motion vectors are, for each frame, visualised with HSV images that are written as PNG images.
+- Visualise the optical flow: the computed motion vectors are, for each frame, visualised with HSV images that are written as PNG images;
+- Skip the sharpess score computations: the motion scores are computed normally, but all the sharpness score computations are skipped and replaced by a fixed value (1.0), which allows to assess the impact of the sharpness score computations (and, by extension, of the motion scores) on the global processing time;
+- Skip the frame selection: the scores are computed normally (the sharpness scores can be skipped) but will not be used to perform the final selection. This is mainly useful to determine the processing time solely dedicated to the score computations or, combined with the CSV export export, to evaluate the quality of the scoring without needing to go through the complete selection process.
 
 
 ## API 
@@ -107,14 +109,16 @@ void processSmart(const float pxDisplacement,
                   const std::size_t rescaledWidthSharpness,
                   const std::size_t rescaledWidthFlow,
                   const std::size_t sharpnessWindowSize,
-                  const std::size_t flowCellSize);
+                  const std::size_t flowCellSize,
+                  const bool skipSharpnessComputation = false);
 ```
 - Score computation
 ```cpp
 bool computeScores(const std::size_t rescaledWidthSharpness,
                    const std::size_t rescaledWidthFlow,
                    const std::size_t sharpnessWindowSize,
-                   const std::size_t flowCellSize);
+                   const std::size_t flowCellSize,
+                   const bool skipSharpnessComputation);
 ```
 - Write selected keyframes
 ```cpp

src/software/utils/main_keyframeSelection.cpp

@@ -61,6 +61,7 @@ int aliceVision_main(int argc, char** argv)
     bool skipSelection = false;             // only compute the scores and do not proceed with the selection
     bool exportFlowVisualisation = false;   // export optical flow visualisation for all the frames
     bool flowVisualisationOnly = false;     // export optical flow visualisation for all the frames but do not compute scores
+    bool skipSharpnessComputation = false;  // skip sharpness score computations
 
     po::options_description inputParams("Required parameters");
     inputParams.add_options()
@@ -139,7 +140,10 @@ int aliceVision_main(int argc, char** argv)
         ("exportFlowVisualisation", po::value<bool>(&exportFlowVisualisation)->default_value(exportFlowVisualisation),
             "For all frames, export the optical flow visualisation in HSV as PNG images.")
         ("flowVisualisationOnly", po::value<bool>(&flowVisualisationOnly)->default_value(flowVisualisationOnly),
-            "Export the optical flow visualisation in HSV as PNG images for all frames but do not compute scores.");
+            "Export the optical flow visualisation in HSV as PNG images for all frames but do not compute scores.")
+        ("skipSharpnessComputation", po::value<bool>(&skipSharpnessComputation)->default_value(skipSharpnessComputation),
+            "Skip the computations for the sharpness score of each frame. A fixed sharpness score of 1.0 will be "
+            "assigned to each frame.");
 
     aliceVision::CmdLine cmdline("This program is used to extract keyframes from single camera or a camera rig.\n"
                                 "AliceVision keyframeSelection");
@@ -223,7 +227,8 @@ int aliceVision_main(int argc, char** argv)
     }
 
     if (skipSelection) {
-        selector.computeScores(rescaledWidthSharp, rescaledWidthFlow, sharpnessWindowSize, flowCellSize);
+        selector.computeScores(rescaledWidthSharp, rescaledWidthFlow, sharpnessWindowSize, flowCellSize,
+                               skipSharpnessComputation);
         if (exportScores)
             selector.exportScoresToFile(csvFilename);  // Frames have not been selected, ignore 'exportSelectedFrames'
         if (exportFlowVisualisation)
@@ -234,7 +239,8 @@ int aliceVision_main(int argc, char** argv)
 
     // Process media paths with regular or smart method
     if (useSmartSelection)
-        selector.processSmart(pxDisplacement, rescaledWidthSharp, rescaledWidthFlow, sharpnessWindowSize, flowCellSize);
+        selector.processSmart(pxDisplacement, rescaledWidthSharp, rescaledWidthFlow, sharpnessWindowSize, flowCellSize,
+                              skipSharpnessComputation);
     else
         selector.processRegular();