raise IndexError when transforming unknown tokens

requires Cython from master branch since unordered_map::at is broken in release version

.github/workflows/pythonpackage.yml

@@ -36,7 +36,7 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
-        python -m pip install setuptools cython numpy
+        python -m pip install setuptools Cython@git+https://github.com/cython/cython.git@c25c87d71107e634162302f7f61a119eff539a48 numpy
         python -m pip install -r requirements-test.txt
     - name: Build
       run: |
@@ -63,7 +63,7 @@ jobs:
         python-version: 3.8
     - name: Install dependencies
       run: |
-        python -m pip install setuptools wheel cython cibuildwheel==2.3.1
+        python -m pip install setuptools wheel Cython@git+https://github.com/cython/cython.git@c25c87d71107e634162302f7f61a119eff539a48 cibuildwheel==2.3.1
     - name: Build wheels
       run: |
         python -m cibuildwheel --output-dir wheelhouse
@@ -88,7 +88,7 @@ jobs:
         python-version: 3.8
     - name: Install dependencies
       run: |
-        python -m pip install setuptools cython numpy
+        python -m pip install setuptools Cython@git+https://github.com/cython/cython.git@c25c87d71107e634162302f7f61a119eff539a48 numpy
     - name: Build dists
       run: |
         python setup.py sdist

.pre-commit-config.yaml

@@ -3,8 +3,8 @@ repos:
     rev: 'v4.0.1'
     hooks:
       - id: check-json
-      - id: check-yaml
       - id: check-toml
+      - id: check-yaml
       - id: check-case-conflict
       - id: check-added-large-files
       - id: debug-statements
@@ -14,6 +14,11 @@ repos:
       - id: trailing-whitespace
         args: ["--markdown-linebreak-ext=md"]
       - id: end-of-file-fixer
+  - repo: https://github.com/asottile/pyupgrade
+    rev: 'v2.29.1'
+    hooks:
+      - id: pyupgrade
+        args: ["--py36-plus"]
   - repo: https://github.com/psf/black
     rev: '21.10b0'
     hooks:

encoders/cyfuncs.pyx

@@ -42,7 +42,7 @@ cdef class BytesLabelEncoder:
 		cdef size_t[::1] out = view.array(shape=(len(seq), ), itemsize=sizeof(size_t), format="Q")
 
 		for i, item in enumerate(seq):
-			out[i] = self._labels[item]
+			out[i] = self._labels.at(item)
 
 		return np.asarray(out)
 
@@ -118,7 +118,7 @@ cdef class StringLabelEncoder:
 			if not isinstance(item, str):
 				raise TypeError(f"expected bytes, {type(item)} found")
 
-			out[i] = self._labels[PyUnicodeSmartPtr(<PyObject *>item)]
+			out[i] = self._labels.at(PyUnicodeSmartPtr(<PyObject *>item))
 
 		return np.asarray(out)
 

pyproject.toml

@@ -1,5 +1,5 @@
 [build-system]
-requires = ["setuptools", "wheel", "Cython", "numpy"]
+requires = ["setuptools", "wheel", "Cython@git+https://github.com/cython/cython.git@c25c87d71107e634162302f7f61a119eff539a48", "numpy"]
 
 [tool.black]
 line-length = 120

setup.py

@@ -48,7 +48,7 @@
     "warn.unused_result": True,
 }
 
-with open("readme.md", "r", encoding="utf-8") as fr:
+with open("readme.md", encoding="utf-8") as fr:
     long_description = fr.read()
 
 if __name__ == "__main__":

test.py

@@ -32,6 +32,14 @@ def test_bytes_inv(self):
         result = le.inverse_transform(encoded)
         np.testing.assert_array_equal(result, np.array(stuff))
 
+    def test_bytes_transform_fail(self):
+        stuff = [b"asd", b"qwe", b"zxc"] * 2
+
+        le = BytesLabelEncoder()
+        le.finalize()
+        with self.assertRaises(IndexError):
+            le.transform(stuff)
+
     def test_bytes_typeerror(self):
 
         with self.assertRaises(TypeError):
@@ -71,6 +79,14 @@ def test_str_inv(self):
         result = le.inverse_transform(encoded)
         np.testing.assert_array_equal(result, np.array(stuff))
 
+    def test_str_transform_fail(self):
+        stuff = ["asü", "😀", "zxä"] * 2
+
+        le = StringLabelEncoder()
+        le.finalize()
+        with self.assertRaises(IndexError):
+            le.transform(stuff)
+
     def test_str_typeerror(self):
 
         with self.assertRaises(TypeError):