add checker for "decoding compressed data"

CMakeLists.txt

@@ -5,7 +5,7 @@
 cmake_minimum_required(VERSION 3.15)
 
 project(CZICheck 
-      VERSION 0.1.1
+      VERSION 0.1.2
       HOMEPAGE_URL "https://github.com/ZEISS/czicheck"
       DESCRIPTION "CZICheck is a validator for CZI-documents")
 

CZICheck/CMakeLists.txt

@@ -68,6 +68,8 @@ set(CZICHECKSRCFILES
 "checkers/checkerXmlBasicMetadataValidation.cpp"
 "checkers/checkerOverlappingScenes.h"
 "checkers/checkerOverlappingScenes.cpp"
+"checkers/checkerSubBlkBitmapValid.h"
+"checkers/checkerSubBlkBitmapValid.cpp"
 checkerfactory.cpp
 checkerfactory.h
 checks.h
@@ -158,6 +160,8 @@ if (CZICHECK_BUILD_TESTS)
           -r positive_plane_start_index.czi=DATA{${CMAKE_CURRENT_SOURCE_DIR}/../Test/CZICheckSamples/positive_plane_start_index.czi}
           -r sparse_planes.czi=DATA{${CMAKE_CURRENT_SOURCE_DIR}/../Test/CZICheckSamples/sparse_planes.czi}
           -r overlapping_scenes.czi=DATA{${CMAKE_CURRENT_SOURCE_DIR}/../Test/CZICheckSamples/overlapping_scenes.czi}
+          -r jpgxrcompressed_inconsistent_size.czi=DATA{${CMAKE_CURRENT_SOURCE_DIR}/../Test/CZICheckSamples/jpgxrcompressed_inconsistent_size.czi}
+          -r jpgxrcompressed_inconsistent_pixeltype.czi=DATA{${CMAKE_CURRENT_SOURCE_DIR}/../Test/CZICheckSamples/jpgxrcompressed_inconsistent_pixeltype.czi}
     )
 
     # Add a build target to populate the real data.

CZICheck/checkerfactory.cpp

@@ -19,6 +19,7 @@
 #include "checkers/checkerMissingMindex.h"
 #include "checkers/checkerXmlBasicMetadataValidation.h"
 #include "checkers/checkerOverlappingScenes.h"
+#include "checkers/checkerSubBlkBitmapValid.h"
 
 using namespace std;
 
@@ -35,7 +36,7 @@ struct classEntry
     /// A short name identifying the checker class. This string has to be unique.
     const string shortname;
 
-    /// A flag to indicate that the checker won't be executed by default, but it has to be explicitely opted in for.
+    /// A flag to indicate that the checker won't be executed by default, but it has to be explicitly opted in for.
     bool isOptIn;
 
     /// A function pointer which creates a new instance of the respective checker class.
@@ -85,7 +86,7 @@ static const classEntry classesList[] =
 {
     MakeEntry<CCheckConsistentCoordinates>(),
     MakeEntry<CCheckSubBlkDirPositions>(),
-    MakeEntry<CCheckSubBlkSegmentsValid>(),
+    MakeEntry<CCheckSubBlkSegmentsValid>(true), // we make this "opt-in" because "CCheckSubBlkBitmapValid" includes the same check (and is more extensive)
     MakeEntry<CCheckDuplicateCoordinates>(),
     MakeEntry<CCheckBenabled>(),
     MakeEntry<CCheckSamePixeltypePerChannel>(),
@@ -97,6 +98,7 @@ static const classEntry classesList[] =
     MakeEntry<CCheckXmlMetadataXsdValidation>(true),
 #endif
     MakeEntry<CCheckOverlappingScenesOnLayer0>(),
+    MakeEntry<CCheckSubBlkBitmapValid>(),
 };
 
 /*static*/std::unique_ptr<IChecker> CCheckerFactory::CreateChecker(
@@ -113,7 +115,7 @@ static const classEntry classesList[] =
         }
     }
 
-    return unique_ptr<IChecker>();
+    return {};
 }
 
 /*static*/const std::string& CCheckerFactory::GetCheckerDisplayName(CZIChecks check_type)

CZICheck/checkers/checkerOverlappingScenes.h

@@ -9,7 +9,7 @@
 #include <map>
 #include "checkerbase.h"
 
-/// This checker is about checking whether scenes are overlapping (on pyramid-layer-0).
+/// This checker is about checking whether scenes are overlapping (on pyramid-layer 0).
 class CCheckOverlappingScenesOnLayer0 : public IChecker, CCheckerBase
 {
 public:

CZICheck/checkers/checkerSubBlkBitmapValid.cpp

@@ -0,0 +1,81 @@
+// SPDX-FileCopyrightText: 2024 Carl Zeiss Microscopy GmbH
+//
+// SPDX-License-Identifier: MIT
+
+#include "checkerSubBlkBitmapValid.h"
+#include <exception>
+#include <sstream>
+#include <memory>
+
+using namespace libCZI;
+using namespace std;
+
+/*static*/const char* CCheckSubBlkBitmapValid::kDisplayName = "SubBlock-Segments in SubBlockDirectory are valid and valid content";
+/*static*/const char* CCheckSubBlkBitmapValid::kShortName = "subblkbitmapvalid";
+
+CCheckSubBlkBitmapValid::CCheckSubBlkBitmapValid(
+    const std::shared_ptr<libCZI::ICZIReader>& reader,
+    CResultGatherer& result_gatherer,
+    const CheckerCreateInfo& additional_info) :
+    CCheckerBase(reader, result_gatherer, additional_info)
+{
+}
+
+void CCheckSubBlkBitmapValid::RunCheck()
+{
+    this->result_gatherer_.StartCheck(CCheckSubBlkBitmapValid::kCheckType);
+
+    this->reader_->EnumerateSubBlocks(
+        [this](int index, const SubBlockInfo& info)->bool
+        {
+            try
+            {
+                auto sub_block = this->reader_->ReadSubBlock(index);
+                const auto compression_mode = sub_block->GetSubBlockInfo().GetCompressionMode();
+                if (compression_mode != CompressionMode::Invalid)
+                {
+                    // According to documentation, for a subblock with a compression mode which is *not* supported by
+                    //  libCZI, we'd be getting CompressionMode::Invalid here. So, if we get a valid compression mode,
+                    //  then we can rightfully expect that the subblock can be decoded, or that we can get a bitmap here
+                    try
+                    {
+                        auto bitmap = sub_block->CreateBitmap();
+                    }
+                    catch (exception& exception)
+                    {
+                        CResultGatherer::Finding finding(CCheckSubBlkBitmapValid::kCheckType);
+                        finding.severity = CResultGatherer::Severity::Fatal;
+                        stringstream ss;
+                        ss << "Error decoding subblock #" << index << " with compression \"" << Utils::CompressionModeToInformalString(compression_mode) << "\"";
+                        finding.information = ss.str();
+                        finding.details = exception.what();
+                        this->result_gatherer_.ReportFinding(finding);
+                    }
+                }
+                else
+                {
+                    CResultGatherer::Finding finding(CCheckSubBlkBitmapValid::kCheckType);
+                    finding.severity = CResultGatherer::Severity::Info;
+                    stringstream ss;
+                    ss << "Subblock #" << index << " has a non-standard compression mode (" << sub_block->GetSubBlockInfo().compressionModeRaw << ")";
+                    finding.information = ss.str();
+                    this->result_gatherer_.ReportFinding(finding);
+                }
+            }
+            catch (exception& exception)
+            {
+                CResultGatherer::Finding finding(CCheckSubBlkBitmapValid::kCheckType);
+                finding.severity = CResultGatherer::Severity::Fatal;
+                stringstream ss;
+                ss << "Error reading subblock #" << index;
+                finding.information = ss.str();
+                finding.details = exception.what();
+                this->result_gatherer_.ReportFinding(finding);
+            }
+
+            return true;
+        });
+
+    this->result_gatherer_.FinishCheck(CCheckSubBlkBitmapValid::kCheckType);
+}
+

CZICheck/checkers/checkerSubBlkBitmapValid.h

@@ -0,0 +1,24 @@
+// SPDX-FileCopyrightText: 2024 Carl Zeiss Microscopy GmbH
+//
+// SPDX-License-Identifier: MIT
+
+#pragma once
+
+#include "checkerbase.h"
+#include <memory>
+
+/// This checker reads all the segments pointed to in the subblock-directory
+/// and decodes the subblock-content.
+class CCheckSubBlkBitmapValid : public IChecker, CCheckerBase
+{
+public:
+    static const CZIChecks kCheckType = CZIChecks::CheckSubBlockBitmapValid;
+    static const char* kDisplayName;
+    static const char* kShortName;
+
+    CCheckSubBlkBitmapValid(
+        const std::shared_ptr<libCZI::ICZIReader>& reader,
+        CResultGatherer& result_gatherer,
+        const CheckerCreateInfo& additional_info);
+    void RunCheck() override;
+};

CZICheck/checks.h

@@ -47,7 +47,10 @@ enum class CZIChecks
     /// It is checked whether subblocks within different scenes are overlapping (on pyramid-layer 0).
     CCheckOverlappingScenesOnLayer0,
 
+    /// All subblocks are read AND their content is decoded and checked for consistency.
+    CheckSubBlockBitmapValid,
+
     ConsistentMIndex,   ///< To be done, not yet implemented.
 
-    AttachmentDirectoryPositionsWithinRagne ///< To be done, not yet implemented.
+    AttachmentDirectoryPositionsWithinRange ///< To be done, not yet implemented.
 };

CZICheck/cmdlineoptions.cpp

@@ -58,16 +58,16 @@ CCmdLineOptions::ParseResult CCmdLineOptions::Parse(int argc, char** argv)
         ChecksValidator()
         {
             this->func_ = [](const std::string& str) -> string
-            {
-                string error_message;
-                const bool parsed_ok = CCmdLineOptions::ParseChecksArgument(str, nullptr, &error_message);
-                if (!parsed_ok)
                 {
-                    throw CLI::ValidationError(error_message);
-                }
+                    string error_message;
+                    const bool parsed_ok = CCmdLineOptions::ParseChecksArgument(str, nullptr, &error_message);
+                    if (!parsed_ok)
+                    {
+                        throw CLI::ValidationError(error_message);
+                    }
 
-                return {};
-            };
+                    return {};
+                };
         }
     };
 
@@ -77,16 +77,16 @@ CCmdLineOptions::ParseResult CCmdLineOptions::Parse(int argc, char** argv)
         PrintDetailsValidator()
         {
             this->func_ = [](const std::string& str) -> string
-            {
-                string error_message;
-                const bool parsed_ok = CCmdLineOptions::ParsePrintDetailsArgument(str, nullptr, &error_message);
-                if (!parsed_ok)
                 {
-                    throw CLI::ValidationError(error_message);
-                }
+                    string error_message;
+                    const bool parsed_ok = CCmdLineOptions::ParsePrintDetailsArgument(str, nullptr, &error_message);
+                    if (!parsed_ok)
+                    {
+                        throw CLI::ValidationError(error_message);
+                    }
 
-                return {};
-            };
+                    return {};
+                };
         }
     };
 
@@ -185,15 +185,20 @@ CCmdLineOptions::ParseResult CCmdLineOptions::Parse(int argc, char** argv)
 /*static*/std::string CCmdLineOptions::GetCheckerListHelpText()
 {
     ostringstream string_stream;
-    string_stream << "Available checkers:" << endl;
+    string_stream << "Available checkers (checkers enabled with the default set are marked with '*'):" << endl;
     CCheckerFactory::EnumerateCheckers(
         [&](const CCheckerFactory::CheckersInfo& checkerInfo)->bool
         {
-            string_stream << "\"" << checkerInfo.shortName << "\" -> " << checkerInfo.displayName;
             if (checkerInfo.isOptIn)
             {
-                string_stream << " [opt-in]";
+                string_stream << "  ";
             }
+            else
+            {
+                string_stream << "* ";
+            }
+
+            string_stream << "\"" << checkerInfo.shortName << "\" -> " << checkerInfo.displayName;
 
             string_stream << endl;
             return true;
@@ -224,10 +229,10 @@ CCmdLineOptions::ParseResult CCmdLineOptions::Parse(int argc, char** argv)
     vector<string> tokenized_items{ it, {} };
     tokenized_items.erase(
         std::remove_if(tokenized_items.begin(), tokenized_items.end(),
-            [](const string& s) -> bool
-            {
-                return s.empty();
-            }),
+        [](const string& s) -> bool
+        {
+            return s.empty();
+        }),
         tokenized_items.end());
 
     if (tokenized_items.empty())
@@ -345,6 +350,9 @@ CCmdLineOptions::ParseResult CCmdLineOptions::Parse(int argc, char** argv)
         checks_enabled->clear();
         checks_enabled->reserve(checks_to_run.size());
         copy(checks_to_run.begin(), checks_to_run.end(), back_inserter(*checks_enabled));
+
+        // Sort the vector by the numerical value of the enum items
+        std::sort(checks_enabled->begin(), checks_enabled->end());
     }
 
     return true;

CZICheck/test/CZICheckRunTests.py

@@ -131,7 +131,8 @@ def compare_result_of_test_to_knowngood(result: str, knowngood: str):
 
 def check_file(cmdline_parameters: Parameters, czi_filename: str, expected_result_file: str, expected_returncode: int):
     cmdlineargs = [cmdline_parameters.get_fully_qualified_czicheck_executable(), '-s',
-                   cmdline_parameters.build_fully_qualified_czi_filename(czi_filename)]
+                   cmdline_parameters.build_fully_qualified_czi_filename(czi_filename),
+                   '-c','all']
     print(f"test {czi_filename}")
 
     output = subprocess.run(cmdlineargs, capture_output=True, check=False, universal_newlines=True)

Test/CZICheckSamples/TestCasesLists.txt

@@ -20,9 +20,11 @@ layer_0_subblocks_with_no_m_index.czi,1,layer_0_subblocks_with_no_m_index.txt
 negative_plane_start_index.czi,1,negative_plane_start_index.txt
 positive_plane_start_index.czi,1,positive_plane_start_index.txt
 sparse_planes.czi,1,sparse_planes.txt
-pixeltype_mismatch_between_metadata_and_subblocks.czi,1,pixeltype_mismatch_between_metadata_and_subblocks.txt
+pixeltype_mismatch_between_metadata_and_subblocks.czi,2,pixeltype_mismatch_between_metadata_and_subblocks.txt
 
 # The following czi file must not exist. If the input file is not accessible, CZICheck should return error code 5.
 # Note that this does not compare the output to stdout - '*' is a special value instruction to skip this test.
 inaccessible_file.czi,5,*
-overlapping_scenes.czi,1,overlapping_scenes.txt
+overlapping_scenes.czi,2,overlapping_scenes.txt
+jpgxrcompressed_inconsistent_size.czi,2,jpgxrcompressed_inconsistent_size.txt
+jpgxrcompressed_inconsistent_pixeltype.czi,2,jpgxrcompressed_inconsistent_pixeltype.txt

Test/CZICheckSamples/differentpixeltypeinchannel.txt

@@ -1,6 +1,6 @@
-Test "check subblock's coordinates for 'consistent dimensions'" : OK
 Test "SubBlock-Segment in SubBlockDirectory within file" : OK
 Test "SubBlock-Segments in SubBlockDirectory are valid" : OK
+Test "check subblock's coordinates for 'consistent dimensions'" : OK
 Test "check subblock's coordinates being unique" : OK
 Test "check whether the document uses the deprecated 'B-index'" : OK
 Test "check that the subblocks of a channel have the same pixeltype" :
@@ -12,7 +12,11 @@ Test "check if all subblocks have the M index" : OK
 Test "Basic semantic checks of the XML-metadata" :
   Could not read metadata-segment
  WARN
+Test "validate the XML-metadata against XSD-schema" :
+  Could not read metadata-segment
+ WARN
 Test "check if subblocks at pyramid-layer 0 of different scenes are overlapping" : OK
+Test "SubBlock-Segments in SubBlockDirectory are valid and valid content" : OK
 
 
 Result: With Warnings

Test/CZICheckSamples/duplicate_coordinates.txt

@@ -1,6 +1,6 @@
-Test "check subblock's coordinates for 'consistent dimensions'" : OK
 Test "SubBlock-Segment in SubBlockDirectory within file" : OK
 Test "SubBlock-Segments in SubBlockDirectory are valid" : OK
+Test "check subblock's coordinates for 'consistent dimensions'" : OK
 Test "check subblock's coordinates being unique" :
   duplicate subblock #0 and # 2 : "C0T0 M=0"
  FAIL
@@ -12,7 +12,11 @@ Test "check if all subblocks have the M index" : OK
 Test "Basic semantic checks of the XML-metadata" :
   Could not read metadata-segment
  WARN
+Test "validate the XML-metadata against XSD-schema" :
+  Could not read metadata-segment
+ WARN
 Test "check if subblocks at pyramid-layer 0 of different scenes are overlapping" : OK
+Test "SubBlock-Segments in SubBlockDirectory are valid and valid content" : OK
 
 
 Result: Errors Detected

Test/CZICheckSamples/inconsistent_coordinates.txt

@@ -1,8 +1,8 @@
+Test "SubBlock-Segment in SubBlockDirectory within file" : OK
+Test "SubBlock-Segments in SubBlockDirectory are valid" : OK
 Test "check subblock's coordinates for 'consistent dimensions'" :
   subblock #1 has dimensions "Z,C,T", whereas "C,T" was expected.
  FAIL
-Test "SubBlock-Segment in SubBlockDirectory within file" : OK
-Test "SubBlock-Segments in SubBlockDirectory are valid" : OK
 Test "check subblock's coordinates being unique" : OK
 Test "check whether the document uses the deprecated 'B-index'" : OK
 Test "check that the subblocks of a channel have the same pixeltype" : OK
@@ -12,7 +12,11 @@ Test "check if all subblocks have the M index" : OK
 Test "Basic semantic checks of the XML-metadata" :
   Could not read metadata-segment
  WARN
+Test "validate the XML-metadata against XSD-schema" :
+  Could not read metadata-segment
+ WARN
 Test "check if subblocks at pyramid-layer 0 of different scenes are overlapping" : OK
+Test "SubBlock-Segments in SubBlockDirectory are valid and valid content" : OK
 
 
 Result: Errors Detected

Test/CZICheckSamples/jpgxrcompressed_inconsistent_pixeltype.czi.md5

@@ -0,0 +1 @@
+04818550ad6b14ec79dab773f34b8a6b