release v0.6 (#244)

Big-Life-Lab · Dec 3, 2024 · bed697f · bed697f
2 parents c4d185d + 873bf8c
commit bed697f
Show file tree

Hide file tree

Showing 57 changed files with 2,099 additions and 2,432 deletions.
diff --git a/.github/workflows/test-code.yml b/.github/workflows/test-code.yml
@@ -13,6 +13,6 @@ jobs:
         python-version: '3.10'
         cache: 'pip' # caching pip dependencies
     - run: pip install -r requirements.txt
-    - run: pip install -r tools/requirements.txt
     - run: pip install -r tests/requirements.txt
+    - run: pip install .
     - run: python -m unittest discover tests
diff --git a/.github/workflows/test-docs.yml b/.github/workflows/test-docs.yml
@@ -15,6 +15,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/setup-python@v2
+      with:
+        python-version: '3.8'
     - uses: nikeee/setup-pandoc@v1
     - uses: quarto-dev/quarto-actions/setup@v2.0.3
     - uses: actions/checkout@v3
@@ -25,7 +27,7 @@ jobs:
         pip install -r requirements.txt
         pip install -r docs/requirements.txt
     - name: Render Quarto docs
-      run: python tools/render_quarto_docs.py
+      run: python src/odm_validation/tools/render_quarto_docs.py
     - name: Build and Commit
       uses: sphinx-notes/pages@2.1 # requires checkout with `fetch-depth: 0`
       with:

diff --git a/.gitignore b/.gitignore
@@ -1,9 +1,11 @@
-# temporary
-__pycache__/
+# temp dirs
+__pycache__
+
+# temp files
 *.log
 
 # python `venv`
-.venv/
+/.env/
 
 # python `pyenv`
 .python-version

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,33 @@
 # Changelog
 
+## [0.6.0] - 2024-12-02
+
+### General
+
+- Added CLI command `odm-validate`
+- Fixed documentation generation
+- Fixed issue where empty values triggered rules "less-than-min-length" and
+  "invalid_category"
+- Fixed json serialization of summarized report
+- Fixed misc bugs and errors
+
+### API
+
+- Added verbosity parameter to `_validate_data_ext`
+
+### Validation schemas
+
+- Added `anyof` with `empty` for fields with `allowed` or `minlength`
+- Fixed boolean-set values
+
+#### v2.0.0
+
+- Removed tables:
+    - languages
+    - parts
+    - sets
+    - translations
+
 ## [0.5.0] - 2024-01-03
 
 ### General

diff --git a/assets/specs/module-functions/schema-1.yml b/assets/specs/module-functions/schema-1.yml
@@ -7,9 +7,11 @@ schema:
       schema:
         SiteID:
           required: true
-          allowed:
-          - Ottawa Site
-          - Montreal Site
+          anyof:
+          - allowed:
+            - Ottawa Site
+            - Montreal Site
+            empty: true
           meta:
           - ruleID: missing_mandatory_column
             meta:

diff --git a/assets/specs/module-functions/schema-2.yml b/assets/specs/module-functions/schema-2.yml
@@ -6,12 +6,14 @@ schema:
       type: dict
       schema:
         collection:
-          allowed:
-          - comp3h
-          - comp8h
-          - flowPr
-          - comp3
-          - comp3dep
+          anyof:
+          - allowed:
+            - comp3h
+            - comp8h
+            - flowPr
+            - comp3
+            - comp3dep
+            empty: true
           meta:
           - ruleID: invalid_category
             meta:

diff --git a/assets/specs/module-functions/schema-additions-1.json b/assets/specs/module-functions/schema-additions-1.json
@@ -1,7 +1,12 @@
 {
     "Site": {
         "SiteID": {
-            "allowed": ["Ottawa Site", "Montreal Site"]
+            "anyof": [
+                {
+                    "allowed": ["Ottawa Site", "Montreal Site"],
+                    "empty": true
+                }
+            ]
         }
     }
 }
diff --git a/assets/specs/module-functions/schema-additions-2.json b/assets/specs/module-functions/schema-additions-2.json
@@ -1,7 +1,11 @@
 {
     "samples": {
         "collection": {
-            "allowed": ["comp3", "comp3dep"]
+            "anyof": [
+                {
+                    "allowed": ["comp3", "comp3dep"]
+                }
+            ]
         }
     }
 }
diff --git a/assets/validation-rules/empty/dataset-allowed.csv b/assets/validation-rules/empty/dataset-allowed.csv
@@ -0,0 +1,6 @@
+siteID,allowedField
+1,
+2,""
+3,"   "
+4,NA
+5,a
diff --git a/assets/validation-rules/empty/dataset-minlength.csv b/assets/validation-rules/empty/dataset-minlength.csv
@@ -0,0 +1,7 @@
+siteID,minlengthField
+1,
+2,""
+3,"   "
+4,NA
+5,x
+6,xxxxx
diff --git a/assets/validation-rules/empty/error-report-allowed.json b/assets/validation-rules/empty/error-report-allowed.json
@@ -0,0 +1,66 @@
+{
+    "warnings": [
+        {
+            "columnName": "allowedField",
+            "invalidValue": "",
+            "message": "missing_values_found rule triggered in table sites, column allowedField, row(s) 1: Empty string found",
+            "row": {"allowedField": "", "siteID": "1"},
+            "rowNumber": 1,
+            "tableName": "sites",
+            "validationRuleFields": [],
+            "warningType": "missing_values_found"
+        },
+        {
+            "columnName": "allowedField",
+            "invalidValue": "",
+            "message": "missing_values_found rule triggered in table sites, column allowedField, row(s) 2: Empty string found",
+            "row": {"allowedField": "", "siteID": "2"},
+            "rowNumber": 2,
+            "tableName": "sites",
+            "validationRuleFields": [],
+            "warningType": "missing_values_found"
+        },
+        {
+            "columnName": "allowedField",
+            "invalidValue": "   ",
+            "message": "missing_values_found rule triggered in table sites, column allowedField, row(s) 3: Missing value \"   \"",
+            "row": {"allowedField": "   ", "siteID": "3"},
+            "rowNumber": 3,
+            "tableName": "sites",
+            "validationRuleFields": [],
+            "warningType": "missing_values_found"
+        },
+        {
+            "columnName": "allowedField",
+            "invalidValue": "NA",
+            "message": "missing_values_found rule triggered in table sites, column allowedField, row(s) 4: Missing value \"NA\"",
+            "row": {"allowedField": "NA", "siteID": "4"},
+            "rowNumber": 4,
+            "tableName": "sites",
+            "validationRuleFields": [],
+            "warningType": "missing_values_found"
+        }
+    ],
+    "errors": [
+        {
+            "columnName": "allowedField",
+            "errorType": "invalid_category",
+            "invalidValue": "   ",
+            "message": "invalid_category rule violated in table sites, column allowedField, row(s) 3: Invalid category \"   \"",
+            "row": {"allowedField": "   ", "siteID": "3"},
+            "rowNumber": 3,
+            "tableName": "sites",
+            "validationRuleFields": []
+        },
+        {
+            "columnName": "allowedField",
+            "errorType": "invalid_category",
+            "invalidValue": "NA",
+            "message": "invalid_category rule violated in table sites, column allowedField, row(s) 4: Invalid category \"NA\"",
+            "row": {"allowedField": "NA", "siteID": "4"},
+            "rowNumber": 4,
+            "tableName": "sites",
+            "validationRuleFields": []
+        }
+    ]
+}
diff --git a/assets/validation-rules/empty/error-report-minlength.json b/assets/validation-rules/empty/error-report-minlength.json
@@ -0,0 +1,75 @@
+{
+    "warnings": [
+        {
+            "warningType": "missing_values_found",
+            "tableName": "sites",
+            "columnName": "minlengthField",
+            "validationRuleFields": [],
+            "message": "missing_values_found rule triggered in table sites, column minlengthField, row(s) 1: Empty string found",
+            "rowNumber": 1,
+            "row": {"siteID": "1", "minlengthField": ""},
+            "invalidValue": ""
+        },
+        {
+            "warningType": "missing_values_found",
+            "tableName": "sites",
+            "columnName": "minlengthField",
+            "validationRuleFields": [],
+            "message": "missing_values_found rule triggered in table sites, column minlengthField, row(s) 2: Empty string found",
+            "rowNumber": 2,
+            "row": {"siteID": "2", "minlengthField": ""}, "invalidValue": ""
+        },
+        {
+            "warningType": "missing_values_found",
+            "tableName": "sites",
+            "columnName": "minlengthField",
+            "validationRuleFields": [],
+            "message": "missing_values_found rule triggered in table sites, column minlengthField, row(s) 3: Missing value \"   \"",
+            "rowNumber": 3,
+            "row": {"siteID": "3", "minlengthField": "   "},
+            "invalidValue": "   "
+        },
+        {
+            "warningType": "missing_values_found",
+            "tableName": "sites",
+            "columnName": "minlengthField",
+            "validationRuleFields": [],
+            "message": "missing_values_found rule triggered in table sites, column minlengthField, row(s) 4: Missing value \"NA\"",
+            "rowNumber": 4,
+            "row": {"siteID": "4", "minlengthField": "NA"},
+            "invalidValue": "NA"
+        }
+    ],
+    "errors": [
+        {
+            "errorType": "less_than_min_length",
+            "tableName": "sites",
+            "columnName": "minlengthField",
+            "validationRuleFields": [],
+            "message": "less_than_min_length rule violated in table sites, column minlengthField, row(s) 3: Value \"   \" (of length 3) is less than the min length of \"5\"",
+            "rowNumber": 3,
+            "row": {"siteID": "3", "minlengthField": "   "},
+            "invalidValue": "   "
+        },
+        {
+            "errorType": "less_than_min_length",
+            "tableName": "sites",
+            "columnName": "minlengthField",
+            "validationRuleFields": [],
+            "message": "less_than_min_length rule violated in table sites, column minlengthField, row(s) 4: Value \"NA\" (of length 2) is less than the min length of \"5\"",
+            "rowNumber": 4,
+            "row": {"siteID": "4", "minlengthField": "NA"},
+            "invalidValue": "NA"
+        },
+        {
+            "errorType": "less_than_min_length",
+            "tableName": "sites",
+            "columnName": "minlengthField",
+            "validationRuleFields": [],
+            "message": "less_than_min_length rule violated in table sites, column minlengthField, row(s) 5: Value \"x\" (of length 1) is less than the min length of \"5\"",
+            "rowNumber" : 5,
+            "row": {"siteID": "5", "minlengthField": "x"},
+            "invalidValue": "x"
+        }
+    ]
+}
diff --git a/assets/validation-rules/empty/schema-v2-allowed.yml b/assets/validation-rules/empty/schema-v2-allowed.yml
@@ -0,0 +1,16 @@
+schemaVersion: 2.0.0
+schema:
+  sites:
+    type: list
+    schema:
+      type: dict
+      schema:
+        allowedField:
+          anyof:
+          - allowed:
+            - a
+            - b
+            empty: true
+          emptyTrimmed: false
+          forbidden:
+          - NA
diff --git a/assets/validation-rules/empty/schema-v2-minlength.yml b/assets/validation-rules/empty/schema-v2-minlength.yml
@@ -0,0 +1,14 @@
+schemaVersion: 2.0.0
+schema:
+  sites:
+    type: list
+    schema:
+      type: dict
+      schema:
+        minlengthField:
+          anyof:
+          - minlength: 5
+            empty: true
+          emptyTrimmed: false
+          forbidden:
+          - NA
diff --git a/assets/validation-rules/invalid-category/schema-v1.yml b/assets/validation-rules/invalid-category/schema-v1.yml
@@ -7,12 +7,14 @@ schema:
       type: dict
       schema:
         Collection:
-          allowed:
-          - Comp3h
-          - Comp8h
-          - FlowPr
-          - FlowRatePr
-          - other
+          anyof:
+          - allowed:
+            - Comp3h
+            - Comp8h
+            - FlowPr
+            - FlowRatePr
+            - other
+            empty: true
           meta:
           - ruleID: invalid_category
             meta:
@@ -47,11 +49,13 @@ schema:
       type: dict
       schema:
         type:
-          allowed:
+          anyof:
+          - allowed:
             - other
             - someOldCat1
             - wwtpMuC
             - wwtpMuS
+            empty: true
           meta:
           - ruleID: invalid_category
             meta:
@@ -86,10 +90,12 @@ schema:
       type: dict
       schema:
         type:
-          allowed:
-          - other
-          - wqCOD
-          - wwCOD
+          anyof:
+          - allowed:
+            - other
+            - wqCOD
+            - wwCOD
+            empty: true
           meta:
           - ruleID: invalid_category
             meta:

diff --git a/assets/validation-rules/invalid-category/schema-v2.yml b/assets/validation-rules/invalid-category/schema-v2.yml
@@ -6,10 +6,12 @@ schema:
       type: dict
       schema:
         coll:
-          allowed:
-          - comp3h
-          - comp8h
-          - flowPr
+          anyof:
+          - allowed:
+            - comp3h
+            - comp8h
+            - flowPr
+            empty: True
           meta:
           - ruleID: invalid_category
             meta: