BSONVector - updated treatment of ignored bits to match spec.

Author: caseyclements

bson/binary.py

@@ -14,6 +14,7 @@
 from __future__ import annotations
 
 import struct
+import warnings
 from enum import Enum
 from typing import TYPE_CHECKING, Any, Optional, Sequence, Tuple, Type, Union, overload
 from uuid import UUID
@@ -471,6 +472,10 @@ def from_vector(
 
         metadata = struct.pack("<sB", dtype.value, padding)
         data = struct.pack(f"<{len(vector)}{format_str}", *vector)  # type: ignore
+        if padding and len(vector) and not (data[-1] & ((1 << padding) - 1)) == 0:
+            raise ValueError(
+                "Vector has a padding P, but bits in the final byte lower than P are non-zero. They must be zero."
+            )
         return cls(metadata + data, subtype=VECTOR_SUBTYPE)
 
     def as_vector(self) -> BinaryVector:
@@ -522,6 +527,12 @@ def as_vector(self) -> BinaryVector:
             dtype_format = "B"
             format_string = f"<{n_values}{dtype_format}"
             unpacked_uint8s = list(struct.unpack_from(format_string, self, position))
+            if padding and n_values and unpacked_uint8s[-1] & (1 << padding) - 1 != 0:
+                warnings.warn(
+                    "Vector has a padding P, but bits in the final byte lower than P are non-zero. In the next major version, they must be zero.",
+                    DeprecationWarning,
+                    stacklevel=2,
+                )
             return BinaryVector(unpacked_uint8s, dtype, padding)
 
         else:

test/bson_binary_vector/packed_bit.json

@@ -29,6 +29,15 @@
       "padding": 3,
       "canonical_bson": "1600000005766563746F7200040000000910037F0800"
     },
+    {
+      "description": "PACKED_BIT with inconsistent padding",
+      "valid": false,
+      "vector": [127, 7],
+      "dtype_hex": "0x10",
+      "dtype_alias": "PACKED_BIT",
+      "padding": 3,
+      "canonical_bson": "1600000005766563746F7200040000000910037F0700"
+    },
     {
       "description": "Empty Vector PACKED_BIT",
       "valid": true,

test/test_bson.py

@@ -739,7 +739,7 @@ def test_vector(self):
         """Tests of subtype 9"""
         # We start with valid cases, across the 3 dtypes implemented.
         # Work with a simple vector that can be interpreted as int8, float32, or ubyte
-        list_vector = [127, 7]
+        list_vector = [127, 8]
         # As INT8, vector has length 2
         binary_vector = Binary.from_vector(list_vector, BinaryVectorDtype.INT8)
         vector = binary_vector.as_vector()
@@ -764,18 +764,18 @@ def test_vector(self):
         uncompressed = ""
         for val in list_vector:
             uncompressed += format(val, "08b")
-        assert uncompressed[:-padding] == "0111111100000"
+        assert uncompressed[:-padding] == "0111111100001"
 
         # It is worthwhile explicitly showing the values encoded to BSON
         padded_doc = {"padded_vec": padded_vec}
         assert (
             encode(padded_doc)
-            == b"\x1a\x00\x00\x00\x05padded_vec\x00\x04\x00\x00\x00\t\x10\x03\x7f\x07\x00"
+            == b"\x1a\x00\x00\x00\x05padded_vec\x00\x04\x00\x00\x00\t\x10\x03\x7f\x08\x00"
         )
         # and dumped to json
         assert (
             json_util.dumps(padded_doc)
-            == '{"padded_vec": {"$binary": {"base64": "EAN/Bw==", "subType": "09"}}}'
+            == '{"padded_vec": {"$binary": {"base64": "EAN/CA==", "subType": "09"}}}'
         )
 
         # FLOAT32 is also implemented
@@ -791,8 +791,15 @@ def test_vector(self):
             else:
                 self.fail("Failed to raise an exception.")
 
-        # Test form of Binary.from_vector(BinaryVector)
+        # Test one must pass zeros for all ignored bits
+        try:
+            Binary.from_vector([255], BinaryVectorDtype.PACKED_BIT, padding=7)
+        except Exception as exc:
+            self.assertIsInstance(exc, ValueError)
+        else:
+            self.fail("Failed to raise an exception.")
 
+        # Test form of Binary.from_vector(BinaryVector)
         assert padded_vec == Binary.from_vector(
             BinaryVector(list_vector, BinaryVectorDtype.PACKED_BIT, padding)
         )