Merge branch 'fix-issue-77-small-segment-count'

Author: lemire

lib/xor_singleheader

@@ -54,6 +54,7 @@
 
 typedef struct binary_fuse8_s {
   uint64_t Seed;
+  uint32_t Size;
   uint32_t SegmentLength;
   uint32_t SegmentLengthMask;
   uint32_t SegmentCount;
@@ -64,6 +65,7 @@
 
 typedef struct binary_fuse16_s {
   uint64_t Seed;
+  uint32_t Size;
   uint32_t SegmentLength;
   uint32_t SegmentLengthMask;
   uint32_t SegmentCount;

pyfusefilter/ffibuild.py

@@ -31,7 +31,8 @@ def __init__(self, size_or_data):
                 raise MemoryError("Unable to allocate memory for filter")
             else:
                 data = list(map(lambda x: c_ulonglong((hash(x))).value, data))
-                lib.xor8_buffered_populate(data, len(data), self.__filter)
+                if not lib.xor8_buffered_populate(data, len(data), self.__filter):
+                    raise ValueError("Unable to populate the filter")
 
     def __repr__(self):
         return "Xor8 object with size(in bytes):{}".format(self.size_in_bytes())
@@ -125,7 +126,8 @@ def __init__(self, size_or_data):
                 raise MemoryError("Unable to allocate memory for filter")
             else:
                 data = list(map(lambda x: c_ulonglong((hash(x))).value, data))
-                lib.xor16_buffered_populate(data, len(data), self.__filter)
+                if not lib.xor16_buffered_populate(data, len(data), self.__filter):
+                    raise ValueError("Unable to populate the filter")
 
     def __repr__(self):
         """
@@ -224,7 +226,8 @@ def __init__(self, size_or_data):
                 raise MemoryError("Unable to allocate memory for filter")
             else:
                 data = list(map(lambda x: c_ulonglong((hash(x))).value, data))
-                lib.binary_fuse8_populate(data, len(data), self.__filter)
+                if not lib.binary_fuse8_populate(data, len(data), self.__filter):
+                    raise ValueError("Unable to populate the filter")
 
     def __repr__(self):
         """
@@ -257,11 +260,15 @@ def __del__(self):
     def populate(self, data: list):
         """
         Set the data of the filter. You may use this method to add data after
-        allocating the filter with an integer size. The sizes should match.
-        You can reuse a filter with new data (i.e., call populate several times)
-        as long as the size remains a constant.
+        allocating the filter with an integer size. If the number of items
+        differs from the allocated size, the filter is reallocated.
+        You can reuse a filter with new data (i.e., call populate several times).
         """
         data = list(map(lambda x: c_ulonglong((hash(x))).value, data))
+        if len(data) != self.__filter.Size:
+            lib.binary_fuse8_free(self.__filter)
+            if not lib.binary_fuse8_allocate(len(data), self.__filter):
+                raise MemoryError("Unable to allocate memory for filter")
         return lib.binary_fuse8_populate(data, len(data), self.__filter)
 
     def contains(self, item):
@@ -323,7 +330,8 @@ def __init__(self, size_or_data):
                 raise MemoryError("Unable to allocate memory for filter")
             else:
                 data = list(map(lambda x: c_ulonglong((hash(x))).value, data))
-                lib.binary_fuse16_populate(data, len(data), self.__filter)
+                if not lib.binary_fuse16_populate(data, len(data), self.__filter):
+                    raise ValueError("Unable to populate the filter")
 
     def __repr__(self):
         """
@@ -356,11 +364,15 @@ def __del__(self):
     def populate(self, data: list):
         """
         Set the data of the filter. You may use this method to add data after
-        allocating the filter with an integer size. The sizes should match.
-        You can reuse a filter with new data (i.e., call populate several times)
-        as long as the size remains a constant.
+        allocating the filter with an integer size. If the number of items
+        differs from the allocated size, the filter is reallocated.
+        You can reuse a filter with new data (i.e., call populate several times).
         """
         data = list(map(lambda x: c_ulonglong((hash(x))).value, data))
+        if len(data) != self.__filter.Size:
+            lib.binary_fuse16_free(self.__filter)
+            if not lib.binary_fuse16_allocate(len(data), self.__filter):
+                raise MemoryError("Unable to allocate memory for filter")
         return lib.binary_fuse16_populate(data, len(data), self.__filter)
 
     def contains(self, item):

pyfusefilter/pyfusefilter.py

@@ -3,7 +3,7 @@ py-modules = []
 # cffi-modules handled in setup.py
 [project]
 name = "pyfusefilter"
-version = "1.2.1"
+version = "1.3.0"
 description = "Python bindings for C implementation of xor and fuse filters"
 authors = [
     {name = "Amey Narkhede", email = "ameynarkhede02@gmail.com"},
@@ -43,7 +43,7 @@ before-all = "apk add libffi-dev"
 # pdoc configuration
 name = "pyfusefilter"
 description = "Python bindings for C implementation of xor and fuse filters"
-version = "1.2.1"
+version = "1.3.0"
 author = "Amey Narkhede & Daniel Lemire"
 author_email = "daniel@lemire.me"
 url = "https://github.com/FastFilter/pyfusefilter"

pyproject.toml

@@ -3,7 +3,7 @@
 
 setup(
     name="pyfusefilter",
-    version="1.2.1",
+    version="1.3.0",
     description="Python bindings for C implementation of xorfilter",
     long_description=open("README.md", "r", encoding='utf-8').read(),
     long_description_content_type="text/markdown",

setup.py

@@ -61,3 +61,12 @@ def test_fuse16_serialize():
         assert recover_xor_filter.contains(i)
 
     os.remove(serialized_filter)
+
+def test_fuse16_few_segments():
+    # https://github.com/FastFilter/xor_singleheader/issues/77
+    for size in (3551, 11504, 11521, 12372, 13224, 37454):
+        for _ in range(5):
+            keys = sample(range(1 << 62), size)
+            filter = Fuse16(keys)
+            for key in keys:
+                assert key in filter

tests/test_fuse16.py

@@ -60,3 +60,12 @@ def test_fuse8_serialize():
         assert recover_xor_filter.contains(i)
 
     os.remove(serialized_filter)
+
+def test_fuse8_few_segments():
+    # https://github.com/FastFilter/xor_singleheader/issues/77
+    for size in (3551, 11504, 11521, 12372, 13224, 37454):
+        for _ in range(5):
+            keys = sample(range(1 << 62), size)
+            filter = Fuse8(keys)
+            for key in keys:
+                assert key in filter