Upgrade data-access to v0.1.9 (#67)

climatepolicyradar · Oct 16, 2023 · 1733cc3 · 1733cc3
1 parent 06c31dd
commit 1733cc3
Show file tree

Hide file tree

Showing 39 changed files with 1,081 additions and 754 deletions.
diff --git a/integration_tests/data/pipeline_in/execution_data/123456.json b/integration_tests/data/pipeline_in/execution_data/123456.json
@@ -1,3 +1,3 @@
 {
-    "input_dir_path": "s3://pipbucket123123123/input/2022-11-01T21.53.26.945831"
+  "input_dir_path": "s3://pipbucket123123123/input/2022-11-01T21.53.26.945831"
 }
diff --git a/...tion_tests/data/pipeline_in/ingest_unit_test_embeddings_input/TESTCCLW.executive.1.1.json b/...tion_tests/data/pipeline_in/ingest_unit_test_embeddings_input/TESTCCLW.executive.1.1.json
@@ -29,4 +29,4 @@
     ]
   },
   "pdf_data": null
-}
+}
diff --git a/...tion_tests/data/pipeline_in/ingest_unit_test_embeddings_input/TESTCCLW.executive.2.2.json b/...tion_tests/data/pipeline_in/ingest_unit_test_embeddings_input/TESTCCLW.executive.2.2.json
@@ -6,27 +6,29 @@
   "document_cdn_object": null,
   "document_content_type": "text/html",
   "document_md5_sum": null,
-  "document_metadata": {"publication_ts": "2021-12-25T00:00:00"},
+  "document_metadata": {
+    "publication_ts": "2021-12-25T00:00:00"
+  },
   "document_slug": "fake_slug",
   "languages": [
-        "en"
-    ],
-    "translated": false,
-    "html_data": {
-        "detected_title": "One Stop Shop Service",
-        "detected_date": null,
-        "has_valid_text": true,
-        "text_blocks": [
-            {
-                "text": [
-                    "Why use a One Stop Shop"
-                ],
-                "text_block_id": "b0",
-                "language": "en",
-                "type": "Text",
-                "type_confidence": 1.0
-            }
-        ]
-    },
-    "pdf_data": null
+    "en"
+  ],
+  "translated": false,
+  "html_data": {
+    "detected_title": "One Stop Shop Service",
+    "detected_date": null,
+    "has_valid_text": true,
+    "text_blocks": [
+      {
+        "text": [
+          "Why use a One Stop Shop"
+        ],
+        "text_block_id": "b0",
+        "language": "en",
+        "type": "Text",
+        "type_confidence": 1.0
+      }
+    ]
+  },
+  "pdf_data": null
 }
diff --git a/...tion_tests/data/pipeline_in/ingest_unit_test_embeddings_input/TESTCCLW.executive.3.3.json b/...tion_tests/data/pipeline_in/ingest_unit_test_embeddings_input/TESTCCLW.executive.3.3.json
@@ -29,4 +29,4 @@
     ]
   },
   "pdf_data": null
-}
+}
diff --git a/...tion_tests/data/pipeline_in/ingest_unit_test_embeddings_input/TESTCCLW.executive.4.4.json b/...tion_tests/data/pipeline_in/ingest_unit_test_embeddings_input/TESTCCLW.executive.4.4.json
@@ -29,4 +29,4 @@
     ]
   },
   "pdf_data": null
-}
+}
diff --git a/...tion_tests/data/pipeline_in/ingest_unit_test_embeddings_input/TESTCCLW.executive.5.5.json b/...tion_tests/data/pipeline_in/ingest_unit_test_embeddings_input/TESTCCLW.executive.5.5.json
@@ -29,4 +29,4 @@
     ]
   },
   "pdf_data": null
-}
+}
diff --git a/...gration_tests/data/pipeline_in/ingest_unit_test_indexer_input/TESTCCLW.executive.1.1.json b/...gration_tests/data/pipeline_in/ingest_unit_test_indexer_input/TESTCCLW.executive.1.1.json
@@ -29,4 +29,4 @@
     ]
   },
   "pdf_data": null
-}
+}
diff --git a/...gration_tests/data/pipeline_in/ingest_unit_test_indexer_input/TESTCCLW.executive.2.2.json b/...gration_tests/data/pipeline_in/ingest_unit_test_indexer_input/TESTCCLW.executive.2.2.json
@@ -6,7 +6,9 @@
   "document_cdn_object": null,
   "document_content_type": "text/html",
   "document_md5_sum": null,
-  "document_metadata": {"publication_ts": "2021-12-25T00:00:00"},
+  "document_metadata": {
+    "publication_ts": "2021-12-25T00:00:00"
+  },
   "document_slug": "fake_slug",
   "languages": [
     "en"
@@ -29,4 +31,4 @@
     ]
   },
   "pdf_data": null
-}
+}
diff --git a/...gration_tests/data/pipeline_in/ingest_unit_test_indexer_input/TESTCCLW.executive.3.3.json b/...gration_tests/data/pipeline_in/ingest_unit_test_indexer_input/TESTCCLW.executive.3.3.json
@@ -29,4 +29,4 @@
     ]
   },
   "pdf_data": null
-}
+}
diff --git a/...gration_tests/data/pipeline_in/ingest_unit_test_indexer_input/TESTCCLW.executive.4.4.json b/...gration_tests/data/pipeline_in/ingest_unit_test_indexer_input/TESTCCLW.executive.4.4.json
@@ -29,4 +29,4 @@
     ]
   },
   "pdf_data": null
-}
+}
diff --git a/integration_tests/data/pipeline_in/ingest_unit_test_parser_input/TESTCCLW.executive.2.2.json b/integration_tests/data/pipeline_in/ingest_unit_test_parser_input/TESTCCLW.executive.2.2.json
@@ -6,6 +6,8 @@
   "document_cdn_object": null,
   "document_content_type": "text/html",
   "document_md5_sum": null,
-  "document_metadata": {"publication_ts": "2021-12-25T00:00:00"},
+  "document_metadata": {
+    "publication_ts": "2021-12-25T00:00:00"
+  },
   "document_slug": "fake_slug"
 }
-Original file line number
+Diff line change
@@ Expand Up / @@ -29,4 +29,4 @@ @@
         ]
       },
       "pdf_data": null
-    }
+    }