Merge pull request #53 from jponf/master

Add 64bit Min Sketch

.travis.yml

@@ -4,7 +4,7 @@ python:
   - "3.4"
   - "3.5"
   - "3.6"
-  - "3.7"
+  - "3.7.13"
 
 install:
   - pip install -r requirements.txt

bounter/count_min_sketch.py

@@ -7,9 +7,15 @@
 # This code is distributed under the terms and conditions
 # from the MIT License (MIT).
 
+import enum
 import bounter_cmsc as cmsc
 
 
+class CellSize(enum.Enum):
+    BITS_32 = 32
+    BITS_64 = 64
+
+
 class CountMinSketch(object):
     """
     Data structure used to estimate frequencies of elements in massive data sets with fixed memory footprint.
@@ -28,9 +34,9 @@ class CountMinSketch(object):
     To calculate memory footprint:
         ( width * depth * cell_size ) + HLL size
         Cell size is
-           - 4B for default counting
-           - 2B for log1024 counting
-           - 1B for log8 counting
+            - 4B for default counting
+            - 2B for log1024 counting
+            - 1B for log8 counting
         HLL size is 64 KB
     Memory usage example:
         width 2^25 (33 554 432), depth 8, log1024 (2B) has 2^(25 + 3 + 1) + 64 KB = 512.06 MB
@@ -47,7 +53,14 @@ class CountMinSketch(object):
         counting as the collision bias will already be minimal.
     """
 
-    def __init__(self, size_mb=64, width=None, depth=None, log_counting=None):
+    def __init__(
+        self,
+        size_mb=64,
+        width=None,
+        depth=None,
+        log_counting=None,
+        cell_size=CellSize.BITS_32,
+    ):
         """
         Initialize the Count-Min Sketch structure with the given parameters
 
@@ -64,33 +77,40 @@ def __init__(self, size_mb=64, width=None, depth=None, log_counting=None):
                 The more, the better, should be very large, preferably in the same order of magnitude as the cardinality
                 of the counted set.
             log_counting (int): Use logarithmic approximate counter value for reduced bucket size:
-                - None (default): 4B, no counter error
+                - None (default): 4B or 8B according to `cell_size`, no counter error
                 - 1024: 2B, value approximation error ~2% for values larger than 2048
                 - 8: 1B, value approximation error ~30% for values larger than 16
+            cell_size (CellSize): Size of the cells when `log_counting` is None.
         """
 
-        cell_size = CountMinSketch.cell_size(log_counting)
-        self.cell_size_v = cell_size
+        cell_bytes = CountMinSketch.cell_size(cell_size, log_counting)
+        self.cell_size_v = cell_bytes
 
         if size_mb is None or not isinstance(size_mb, int):
-            raise ValueError("size_mb must be an integer representing the maximum size of the structure in MB")
+            raise ValueError(
+                "size_mb must be an integer representing the maximum size of the structure in MB"
+            )
 
         if width is None and depth is None:
-            self.width = 1 << (size_mb * (2 ** 20) // (cell_size * 8 * 2)).bit_length()
-            self.depth = (size_mb * (2 ** 20)) // (self.width * cell_size)
+            self.width = 1 << (size_mb * (2**20) // (cell_bytes * 8 * 2)).bit_length()
+            self.depth = (size_mb * (2**20)) // (self.width * cell_bytes)
         elif width is None:
             self.depth = depth
-            avail_width = (size_mb * (2 ** 20)) // (depth * cell_size)
+            avail_width = (size_mb * (2**20)) // (depth * cell_bytes)
             self.width = 1 << (avail_width.bit_length() - 1)
             if not self.width:
-                raise ValueError("Requested depth is too large for maximum memory size.")
+                raise ValueError(
+                    "Requested depth is too large for maximum memory size."
+                )
         elif depth is None:
             if width != 1 << (width.bit_length() - 1):
                 raise ValueError("Requested width must be a power of 2.")
             self.width = width
-            self.depth = (size_mb * (2 ** 20)) // (width * cell_size)
+            self.depth = (size_mb * (2**20)) // (width * cell_bytes)
             if not self.depth:
-                raise ValueError("Requested width is too large for maximum memory size.")
+                raise ValueError(
+                    "Requested width is too large for maximum memory size."
+                )
         else:
             if width != 1 << (width.bit_length() - 1):
                 raise ValueError("Requested width must be a power of 2.")
@@ -102,20 +122,38 @@ def __init__(self, size_mb=64, width=None, depth=None, log_counting=None):
         elif log_counting == 1024:
             self.cms = cmsc.CMS_Log1024(width=self.width, depth=self.depth)
         elif log_counting is None:
-            self.cms = cmsc.CMS_Conservative(width=self.width, depth=self.depth)
+            if cell_size == CellSize.BITS_32:
+                self.cms = cmsc.CMS_Conservative(width=self.width, depth=self.depth)
+            elif cell_size == CellSize.BITS_64:
+                self.cms = cmsc.CMS64_Conservative(width=self.width, depth=self.depth)
+            else:
+                raise ValueError(
+                    "Unsupported parameter cell_size=%s. Use CellSize.BITS_32 or CellSize.BITS_64."
+                    % (cell_size)
+                )
         else:
-            raise ValueError("Unsupported parameter log_counting=%s. Use None, 8, or 1024." % log_counting)
+            raise ValueError(
+                "Unsupported parameter log_counting=%s. Use None, 8, or 1024."
+                % (log_counting)
+            )
 
         # optimize calls by directly binding to C implementation
         self.increment = self.cms.increment
 
     @staticmethod
-    def cell_size(log_counting=None):
+    def cell_size(cell_size, log_counting=None):
         if log_counting == 8:
             return 1
         if log_counting == 1024:
             return 2
-        return 4
+        if log_counting is None:
+            if cell_size == CellSize.BITS_32:
+                return 4
+            if cell_size == CellSize.BITS_64:
+                return 8
+        raise ValueError(
+            "cell_size must be one of [BITS32, BITS64] and log_counting one of [None, 8, 1024]"
+        )
 
     @staticmethod
     def table_size(width, depth=4, log_counting=None):
@@ -192,4 +230,6 @@ def __init__(self):
         super(CardinalityEstimator, self).__init__(width=1, depth=1)
 
     def __getitem__(self, key):
-        raise NotImplementedError("Individual item counting is not supported for cardinality estimator!")
+        raise NotImplementedError(
+            "Individual item counting is not supported for cardinality estimator!"
+        )

cbounter/cms64_conservative.h

@@ -0,0 +1,36 @@
+//-----------------------------------------------------------------------------
+// Author: Josep Pon Farreny <jponfarreny@gmail.com.com>
+// Copyright (C) 2017 Rare Technologies
+//
+// This code is distributed under the terms and conditions
+// from the MIT License (MIT).
+
+#ifndef _CMS64_CONSERVATIE_H_
+#define _CMS64_CONSERVATIE_H_
+
+#define CMS_TYPE CMS64_Conservative
+#define CMS_TYPE_STRING "CMS64_Conservative"
+#define CMS_CELL_TYPE uint64_t
+
+#include "cms_common.h"
+
+static inline int CMS_VARIANT(should_inc)(CMS_CELL_TYPE value)
+{
+    return 1;
+}
+
+static inline long long CMS_VARIANT(decode)(CMS_CELL_TYPE value)
+{
+    return value;
+}
+
+static inline CMS_CELL_TYPE CMS_VARIANT(_merge_value)(CMS_CELL_TYPE v1, CMS_CELL_TYPE v2, uint32_t merge_seed)
+{
+    return v1 + v2;
+}
+
+#undef CMS_TYPE
+#undef CMS_TYPE_STRING
+#undef CMS_CELL_TYPE
+
+#endif /* _CMS64_CONSERVATIE_H_ */

cbounter/cms_cmodule.c

@@ -18,7 +18,8 @@ static inline uint32_t rand_32b()
     return r;
 }
 
-#include "cms_conservative.c"
+#include "cms_conservative.h"
+#include "cms64_conservative.h"
 #include "cms_log8.c"
 #include "cms_log1024.c"
 #include <time.h>
@@ -50,6 +51,7 @@ PyMODINIT_FUNC initbounter_cmsc(void)
 {
     PyObject* m;
     if (PyType_Ready(&CMS_ConservativeType) < 0
+        || PyType_Ready(&CMS64_ConservativeType) < 0
         || PyType_Ready(&CMS_Log8Type) < 0
         || PyType_Ready(&CMS_Log1024Type) < 0) {
 
@@ -76,6 +78,9 @@ PyMODINIT_FUNC initbounter_cmsc(void)
     Py_INCREF(&CMS_ConservativeType);
     PyModule_AddObject(m, "CMS_Conservative", (PyObject *)&CMS_ConservativeType);
 
+    Py_INCREF(&CMS64_ConservativeType);
+    PyModule_AddObject(m, "CMS64_Conservative", (PyObject *)&CMS64_ConservativeType);
+
     srand(time(NULL));
 
     Py_INCREF(&CMS_Log8Type);

cbounter/cms_common.c → cbounter/cms_common.h

@@ -136,8 +136,9 @@ CMS_VARIANT(_increment_obj)(CMS_TYPE *self, char *data, Py_ssize_t dataLength, l
     }
 
     CMS_CELL_TYPE result = min_value;
-    for (; increment > 0; increment--)
+    for (; increment > 0; increment--) {
         result += CMS_VARIANT(should_inc)(result);
+    }
 
     if (result > min_value)
     {

cbounter/cms_conservative.c → cbounter/cms_conservative.h

@@ -5,11 +5,14 @@
 // This code is distributed under the terms and conditions
 // from the MIT License (MIT).
 
+#ifndef _CMS_CONSERVATIE_H_
+#define _CMS_CONSERVATIE_H_
+
 #define CMS_TYPE CMS_Conservative
 #define CMS_TYPE_STRING "CMS_Conservative"
 #define CMS_CELL_TYPE uint32_t
 
-#include "cms_common.c"
+#include "cms_common.h"
 
 static inline int CMS_VARIANT(should_inc)(CMS_CELL_TYPE value)
 {
@@ -21,7 +24,13 @@ static inline long long CMS_VARIANT(decode)(CMS_CELL_TYPE value)
     return value;
 }
 
-static inline CMS_CELL_TYPE CMS_VARIANT(_merge_value) (CMS_CELL_TYPE v1, CMS_CELL_TYPE v2, uint32_t merge_seed)
+static inline CMS_CELL_TYPE CMS_VARIANT(_merge_value)(CMS_CELL_TYPE v1, CMS_CELL_TYPE v2, uint32_t merge_seed)
 {
     return v1 + v2;
 }
+
+#undef CMS_TYPE
+#undef CMS_TYPE_STRING
+#undef CMS_CELL_TYPE
+
+#endif /* _CMS_CONSERVATIE_H_ */

cbounter/cms_log1024.c

@@ -9,15 +9,16 @@
 #define CMS_TYPE_STRING "CMS_Log1024"
 #define CMS_CELL_TYPE uint16_t
 
-#include "cms_common.c"
+#include "cms_common.h"
 
 static inline int CMS_VARIANT(should_inc)(CMS_CELL_TYPE value)
 {
     if (value >= 2048)
     {
         uint8_t shift = 33 - (value >> 10);
         uint32_t mask = 0xFFFFFFFF >> shift;
-        if (mask & rand_32b()) return 0;
+        if (mask & rand_32b())
+            return 0;
     }
     return 1;
 }
@@ -30,7 +31,7 @@ static inline long long CMS_VARIANT(decode)(CMS_CELL_TYPE value)
         return (1024 + (value & 1023)) << ((value >> 10) - 1);
 }
 
-static inline CMS_CELL_TYPE CMS_VARIANT(_merge_value) (CMS_CELL_TYPE v1, CMS_CELL_TYPE v2, uint32_t merge_seed)
+static inline CMS_CELL_TYPE CMS_VARIANT(_merge_value)(CMS_CELL_TYPE v1, CMS_CELL_TYPE v2, uint32_t merge_seed)
 {
     long long decoded = CMS_VARIANT(decode)(v1);
     decoded += CMS_VARIANT(decode)(v2);
@@ -55,8 +56,12 @@ static inline CMS_CELL_TYPE CMS_VARIANT(_merge_value) (CMS_CELL_TYPE v1, CMS_CEL
     uint32_t mask = 0xFFFFFFFF >> shift;
 
     uint32_t r;
-    MurmurHash3_x86_32  ((void *) &decoded, 8, merge_seed, (void *) &r);
+    MurmurHash3_x86_32((void *)&decoded, 8, merge_seed, (void *)&r);
     uint32_t remainder = mask & decoded;
 
     return (log_result << 10) + (h & 1023) + ((mask & r) < remainder);
 }
+
+#undef CMS_TYPE
+#undef CMS_TYPE_STRING
+#undef CMS_CELL_TYPE

cbounter/cms_log8.c

@@ -9,15 +9,16 @@
 #define CMS_TYPE_STRING "CMS_Log8"
 #define CMS_CELL_TYPE uint8_t
 
-#include "cms_common.c"
+#include "cms_common.h"
 
 static inline int CMS_VARIANT(should_inc)(CMS_CELL_TYPE value)
 {
     if (value >= 16)
     {
         uint8_t shift = 33 - (value >> 3);
         uint32_t mask = 0xFFFFFFFF >> shift;
-        if (mask & rand_32b()) return 0;
+        if (mask & rand_32b())
+            return 0;
     }
     return 1;
 }
@@ -32,7 +33,7 @@ static inline long long CMS_VARIANT(decode)(CMS_CELL_TYPE value)
 
 #include <stdio.h>
 
-static inline CMS_CELL_TYPE CMS_VARIANT(_merge_value) (CMS_CELL_TYPE v1, CMS_CELL_TYPE v2, uint32_t merge_seed)
+static inline CMS_CELL_TYPE CMS_VARIANT(_merge_value)(CMS_CELL_TYPE v1, CMS_CELL_TYPE v2, uint32_t merge_seed)
 {
     long long decoded = CMS_VARIANT(decode)(v1);
     decoded += CMS_VARIANT(decode)(v2);
@@ -57,8 +58,12 @@ static inline CMS_CELL_TYPE CMS_VARIANT(_merge_value) (CMS_CELL_TYPE v1, CMS_CEL
     uint32_t mask = 0xFFFFFFFF >> shift;
 
     uint32_t r;
-    MurmurHash3_x86_32  ((void *) &decoded, 8, merge_seed, (void *) &r);
+    MurmurHash3_x86_32((void *)&decoded, 8, merge_seed, (void *)&r);
     uint32_t remainder = mask & decoded;
 
     return (log_result << 3) + (h & 7) + ((mask & r) < remainder);
 }
+
+#undef CMS_TYPE
+#undef CMS_TYPE_STRING
+#undef CMS_CELL_TYPE