perf(storage): use google_crc32c.value for checksums (#16719)

zhixiangli · web-flow · commit c6461a42850f · 2026-04-22T07:13:10.000+08:00
Updates _ReadResumptionStrategy to use google_crc32c.value(data) instead
of manually converting Checksum(data).digest() to an integer.

Updated unit tests to mock google_crc32c.value accordingly.

Test code

```
import timeit
import os
import google_crc32c
from google_crc32c import Checksum

def method1(data):
    return int.from_bytes(Checksum(data).digest(), "big")

def method2(data):
    return google_crc32c.value(data)

def benchmark():
    # Testing larger sizes and more iterations
    # 1KB, 1MB, 10MB, 100MB
    data_sizes = [1024, 1024 * 1024, 10 * 1024 * 1024, 100 * 1024 * 1024] 
    
    for size in data_sizes:
        data = os.urandom(size)
        print(f"\nData size: {size / (1024*1024):.2f} MB" if size &gt;= 1024*1024 else f"\nData size: {size / 1024:.2f} KB")
        
        # Correctness check
        res1 = method1(data)
        res2 = method2(data)
        assert res1 == res2, f"Failed for size {size}: {res1} != {res2}"
        print("Assertion passed: results are equal.")
        
        # Increase iterations for more stable results, with a minimum of 1000
        if size &lt;= 1024:
            number = 100000
        elif size &lt;= 1024 * 1024:
            number = 10000
        else:
            number = 1000
        
        t1 = timeit.timeit(lambda: method1(data), number=number)
        t2 = timeit.timeit(lambda: method2(data), number=number)
        
        avg1 = t1 / number
        avg2 = t2 / number
        
        print(f"Method 1 (Checksum(data).digest()): {t1:.6f} s total ({avg1:.8f} s/call)")
        print(f"Method 2 (google_crc32c.value(data)): {t2:.6f} s total ({avg2:.8f} s/call)")
        print(f"Improvement: {(t1 - t2) / t1 * 100:.2f}%")

if __name__ == "__main__":
    print(f"google_crc32c implementation: {google_crc32c.implementation}")
    benchmark()
```

Output
```
google_crc32c implementation: c

Data size: 1.00 KB
Assertion passed: results are equal.
Method 1 (Checksum(data).digest()): 0.046088 s total (0.00000046 s/call)
Method 2 (google_crc32c.value(data)): 0.016062 s total (0.00000016 s/call)
Improvement: 65.15%

Data size: 1.00 MB
Assertion passed: results are equal.
Method 1 (Checksum(data).digest()): 0.464044 s total (0.00004640 s/call)
Method 2 (google_crc32c.value(data)): 0.439121 s total (0.00004391 s/call)
Improvement: 5.37%

Data size: 10.00 MB
Assertion passed: results are equal.
Method 1 (Checksum(data).digest()): 0.450953 s total (0.00045095 s/call)
Method 2 (google_crc32c.value(data)): 0.445793 s total (0.00044579 s/call)
Improvement: 1.14%

Data size: 100.00 MB
Assertion passed: results are equal.
Method 1 (Checksum(data).digest()): 6.287893 s total (0.00628789 s/call)
Method 2 (google_crc32c.value(data)): 6.095833 s total (0.00609583 s/call)
Improvement: 3.05%

```
diff --git a/packages/google-cloud-storage/google/cloud/storage/asyncio/retry/reads_resumption_strategy.py b/packages/google-cloud-storage/google/cloud/storage/asyncio/retry/reads_resumption_strategy.py
@@ -15,7 +15,7 @@
 import logging
 from typing import IO, Any, Dict, List
 
-from google_crc32c import Checksum
+import google_crc32c
 
 from google.cloud import _storage_v2 as storage_v2
 from google.cloud.storage.asyncio.retry._helpers import (
@@ -127,7 +127,7 @@ def update_state_from_response(
 
             if checksummed_data.HasField("crc32c"):
                 server_checksum = checksummed_data.crc32c
-                client_checksum = int.from_bytes(Checksum(data).digest(), "big")
+                client_checksum = google_crc32c.value(data)
                 if server_checksum != client_checksum:
                     raise DataCorruption(
                         response,
diff --git a/packages/google-cloud-storage/tests/unit/asyncio/test_async_multi_range_downloader.py b/packages/google-cloud-storage/tests/unit/asyncio/test_async_multi_range_downloader.py
@@ -324,10 +324,10 @@ def test_init_raises_if_crc32c_c_extension_is_missing(self, mock_google_crc32c):
         )
 
     @pytest.mark.asyncio
-    @mock.patch("google.cloud.storage.asyncio.retry.reads_resumption_strategy.Checksum")
-    async def test_download_ranges_raises_on_checksum_mismatch(
-        self, mock_checksum_class
-    ):
+    @mock.patch(
+        "google.cloud.storage.asyncio.retry.reads_resumption_strategy.google_crc32c.value"
+    )
+    async def test_download_ranges_raises_on_checksum_mismatch(self, mock_crc32c_value):
         from google.cloud.storage.asyncio._stream_multiplexer import _StreamMultiplexer
         from google.cloud.storage.asyncio.async_multi_range_downloader import (
             AsyncMultiRangeDownloader,
@@ -340,8 +340,7 @@ async def test_download_ranges_raises_on_checksum_mismatch(
 
         test_data = b"some-data"
         server_checksum = 12345
-        mock_checksum_instance = mock_checksum_class.return_value
-        mock_checksum_instance.digest.return_value = (54321).to_bytes(4, "big")
+        mock_crc32c_value.return_value = 54321
 
         mock_response = _storage_v2.BidiReadObjectResponse(
             object_data_ranges=[
@@ -372,7 +371,7 @@ async def test_download_ranges_raises_on_checksum_mismatch(
                 await mrd.download_ranges([(0, len(test_data), BytesIO())])
 
         assert "Checksum mismatch" in str(exc_info.value)
-        mock_checksum_class.assert_called_once_with(test_data)
+        mock_crc32c_value.assert_called_once_with(test_data)
 
     @mock.patch(
         "google.cloud.storage.asyncio.async_multi_range_downloader.AsyncMultiRangeDownloader.open",
@@ -575,18 +574,11 @@ async def staged_recv():
         # Act
         buffer = BytesIO()
 
-        # Patch Checksum where it is likely used (reads_resumption_strategy or similar),
-        # but actually if we use google_crc32c directly, we should patch that or provide valid CRC.
-        # Since we can't reliably predict where Checksum is imported/used without more digging,
-        # let's provide a valid CRC for b"data".
-        # Checksum(b"data").digest() -> needs to match crc32c=123.
-        # But we can't force b"data" to have crc=123.
-        # So we MUST patch Checksum.
-        # It is used in google.cloud.storage.asyncio.retry.reads_resumption_strategy
+        # Patch google_crc32c.value where it is used in reads_resumption_strategy
         with mock.patch(
-            "google.cloud.storage.asyncio.retry.reads_resumption_strategy.Checksum"
-        ) as mock_chk:
-            mock_chk.return_value.digest.return_value = (123).to_bytes(4, "big")
+            "google.cloud.storage.asyncio.retry.reads_resumption_strategy.google_crc32c.value"
+        ) as mock_crc_value:
+            mock_crc_value.return_value = 123
             await mock_mrd.download_ranges([(0, 4, buffer)])
 
         # Assert
