HMRC-371: Updating the training to use the 2025 CN codes

data_sources/search_references.py

@@ -8,9 +8,7 @@
 
 
 class SearchReferencesDataSource(DataSource):
-    SEARCH_REFS_API_URL = (
-        "https://staging.trade-tariff.service.gov.uk/api/v2/search_references"
-    )
+    SEARCH_REFS_API_URL = "https://trade-tariff.service.gov.uk/api/v2/search_references"
     DEFAULT_PATH = "reference_data/search_references.json"
 
     def __init__(
@@ -56,9 +54,9 @@ def commodities(self):
         commodities = {}
         for entry in json_entries:
             if entry["attributes"]["referenced_class"] in ["Commodity", "Subheading"]:
-                commodities[entry["attributes"]["negated_title"].strip().lower()] = (
-                    entry["attributes"]["goods_nomenclature_item_id"]
-                )
+                commodities[
+                    entry["attributes"]["negated_title"].strip().lower()
+                ] = entry["attributes"]["goods_nomenclature_item_id"]
 
         self._commodities = commodities
         return commodities

search-config.toml

@@ -1,12 +1,12 @@
 name = "fpo-search-model-generator"
-version = "1.8.0"
+version = "1.9.0"
 learning_rate = 0.0011
 max_epochs = 4
 model_batch_size = 1024
 embedding_batch_size = 250
 vague_terms_data_file = "reference_data/vague_terms.csv"
 extra_data_file = "reference_data/extra_references.csv"
-cn_data_file = "reference_data/CN2024_SelfText_EN_DE_FR.csv"
+cn_data_file = "reference_data/CN2025_SelfText_EN_DE_FR.csv"
 exact_english_terms = "reference_data/exact_english_terms.txt"
 partial_english_terms = "reference_data/partial_english_terms.txt"
 partial_non_english_terms = "reference_data/partial_non_english_terms.txt"

tests/training/cleaners/test_map_2024_to_2025_codes.py

@@ -0,0 +1,39 @@
+import logging
+from typing import List, Tuple
+import unittest
+
+from training.cleaners.map_2024_to_2025_codes import Map2024CodesTo2025Codes
+
+logger = logging.getLogger()
+logger.addHandler(logging.StreamHandler())
+
+filter = Map2024CodesTo2025Codes()
+
+
+class TestMap2024To2025Codes(unittest.TestCase):
+    EXAMPLES: List[Tuple] = [
+        (
+            ("8521109510", "10 digit changed"),
+            ("85211000", "10 digit changed"),
+        ),
+        (
+            ("85211095", "8 digit changed"),
+            ("85211000", "8 digit changed"),
+        ),
+        (
+            ("1234567890", "unchanged"),
+            ("1234567890", "unchanged"),
+        ),
+    ]
+
+    def test_filter(self):
+        for example, expected in TestMap2024To2025Codes.EXAMPLES:
+            example_subheading, example_description = example
+            expected_subheading, expected_description = expected
+
+            actual_subheading, actual_description, _meta = filter.filter(
+                example_subheading, example_description
+            )
+
+            self.assertEqual(actual_subheading, expected_subheading)
+            self.assertEqual(actual_description, expected_description)