check for hash collisions

Author: jreback

pandas/indexes/multi.py

@@ -2181,7 +2181,9 @@ def equals(self, other):
             return True
 
         if not isinstance(other, Index):
-            return False
+            if not isinstance(other, tuple):
+                return False
+            other = Index([other])
 
         if not isinstance(other, MultiIndex):
             return array_equivalent(self._values,

pandas/src/hashtable_class_helper.pxi.in

@@ -859,24 +859,50 @@ cdef class MultiIndexHashTable(HashTable):
                                        sizeof(size_t) + # vals
                                        sizeof(uint32_t)) # flags
 
+    def _check_for_collisions(self, int64_t[:] locs, object mi):
+        # validate that the locs map to the actual values
+        # provided in the mi
+        # we can only check if we *don't* have any missing values
+        # :<
+        cdef:
+            ndarray[int64_t] alocs
+
+        alocs = np.asarray(locs)
+        if (alocs!=-1).all():
+
+            result = self.mi.take(locs)
+            if not result.equals(mi):
+                raise ValueError("hash collision alert")
+
     def __contains__(self, object key):
         cdef:
             khiter_t k
             uint64_t value
 
         value = self.mi._hashed_indexing_key(key)
         k = kh_get_uint64(self.table, value)
-        return k != self.table.n_buckets
+        if k != self.table.n_buckets:
+            loc = self.table.vals[k]
+            locs = np.array([loc], dtype=np.int64)
+            self._check_for_collisions(locs, key)
+            return True
+
+        return False
 
     cpdef get_item(self, object key):
         cdef:
             khiter_t k
             uint64_t value
+            int64_t[:] locs
+            Py_ssize_t loc
 
         value = self.mi._hashed_indexing_key(key)
         k = kh_get_uint64(self.table, value)
         if k != self.table.n_buckets:
-            return self.table.vals[k]
+            loc = self.table.vals[k]
+            locs = np.array([loc], dtype=np.int64)
+            self._check_for_collisions(locs, key)
+            return loc
         else:
             raise KeyError(key)
 
@@ -927,6 +953,7 @@ cdef class MultiIndexHashTable(HashTable):
                 else:
                     locs[i] = -1
 
+        self._check_for_collisions(locs, mi)
         return np.asarray(locs)
 
     def unique(self, object mi):

pandas/tests/indexes/test_multi.py

@@ -1317,6 +1317,18 @@ def test_get_indexer_nearest(self):
         with tm.assertRaises(NotImplementedError):
             midx.get_indexer(['a'], method='pad', tolerance=2)
 
+    def test_hash_collisions(self):
+        # non-smoke test that we don't get hash collisions
+
+        index = MultiIndex.from_product([np.arange(1000), np.arange(1000)],
+                                        names=['one','two'])
+        result = index.get_indexer(index.values)
+        self.assert_numpy_array_equal(result, np.arange(len(index), dtype='int64'))
+
+        for i in [0, 1, len(index)-2, len(index)-1]:
+            result = index.get_loc(index[i])
+            self.assertEqual(result, i)
+
     def test_format(self):
         self.index.format()
         self.index[:0].format()