fix: retry cancelled errors (#1235)

daniel-sanche · web-flow · commit e3fd5d866830 · 2025-12-04T11:19:30.000-08:00
There's an internal race condition where when an rpc hits the timeout
limit, it sometimes receives a DEADLINE_EXCEEDED, but sometimes receives
a CANCELLED error. This PR marks CANCELLED as retryable, so this
situation will always eventually reach a DEADLINE_EXCEEDED state.

This will fix the flake currently seen in the conformance tests
diff --git a/google/cloud/bigtable/data/_async/client.py b/google/cloud/bigtable/data/_async/client.py
@@ -59,6 +59,7 @@
 from google.api_core.exceptions import DeadlineExceeded
 from google.api_core.exceptions import ServiceUnavailable
 from google.api_core.exceptions import Aborted
+from google.api_core.exceptions import Cancelled
 from google.protobuf.message import Message
 from google.protobuf.internal.enum_type_wrapper import EnumTypeWrapper
 
@@ -938,6 +939,7 @@ def __init__(
             DeadlineExceeded,
             ServiceUnavailable,
             Aborted,
+            Cancelled,
         ),
         default_mutate_rows_retryable_errors: Sequence[type[Exception]] = (
             DeadlineExceeded,
diff --git a/google/cloud/bigtable/data/_sync_autogen/client.py b/google/cloud/bigtable/data/_sync_autogen/client.py
@@ -49,6 +49,7 @@
 from google.api_core.exceptions import DeadlineExceeded
 from google.api_core.exceptions import ServiceUnavailable
 from google.api_core.exceptions import Aborted
+from google.api_core.exceptions import Cancelled
 from google.protobuf.message import Message
 from google.protobuf.internal.enum_type_wrapper import EnumTypeWrapper
 import google.auth.credentials
@@ -729,6 +730,7 @@ def __init__(
             DeadlineExceeded,
             ServiceUnavailable,
             Aborted,
+            Cancelled,
         ),
         default_mutate_rows_retryable_errors: Sequence[type[Exception]] = (
             DeadlineExceeded,
diff --git a/tests/unit/data/_async/test_client.py b/tests/unit/data/_async/test_client.py
@@ -1334,6 +1334,7 @@ def test_table_ctor_sync(self):
                     core_exceptions.DeadlineExceeded,
                     core_exceptions.ServiceUnavailable,
                     core_exceptions.Aborted,
+                    core_exceptions.Cancelled,
                 ],
             ),
             (
@@ -1832,7 +1833,6 @@ async def test_read_rows_retryable_error(self, exc_type):
     @pytest.mark.parametrize(
         "exc_type",
         [
-            core_exceptions.Cancelled,
             core_exceptions.PreconditionFailed,
             core_exceptions.NotFound,
             core_exceptions.PermissionDenied,
diff --git a/tests/unit/data/_async/test_mutations_batcher.py b/tests/unit/data/_async/test_mutations_batcher.py
@@ -1169,6 +1169,7 @@ def test__add_exceptions(self, limit, in_e, start_e, end_e):
                     core_exceptions.DeadlineExceeded,
                     core_exceptions.ServiceUnavailable,
                     core_exceptions.Aborted,
+                    core_exceptions.Cancelled,
                 ],
             ),
             (
diff --git a/tests/unit/data/_sync_autogen/test_client.py b/tests/unit/data/_sync_autogen/test_client.py
@@ -1063,6 +1063,7 @@ def test_ctor_invalid_timeout_values(self):
                     core_exceptions.DeadlineExceeded,
                     core_exceptions.ServiceUnavailable,
                     core_exceptions.Aborted,
+                    core_exceptions.Cancelled,
                 ],
             ),
             (
@@ -1507,7 +1508,6 @@ def test_read_rows_retryable_error(self, exc_type):
     @pytest.mark.parametrize(
         "exc_type",
         [
-            core_exceptions.Cancelled,
             core_exceptions.PreconditionFailed,
             core_exceptions.NotFound,
             core_exceptions.PermissionDenied,
diff --git a/tests/unit/data/_sync_autogen/test_mutations_batcher.py b/tests/unit/data/_sync_autogen/test_mutations_batcher.py
@@ -1021,6 +1021,7 @@ def test__add_exceptions(self, limit, in_e, start_e, end_e):
                     core_exceptions.DeadlineExceeded,
                     core_exceptions.ServiceUnavailable,
                     core_exceptions.Aborted,
+                    core_exceptions.Cancelled,
                 ],
             ),
             (

Original file line number	Diff line number	Diff line change
`@@ -1169,6 +1169,7 @@ def test__add_exceptions(self, limit, in_e, start_e, end_e):`
`1169`	`1169`	`core_exceptions.DeadlineExceeded,`
`1170`	`1170`	`core_exceptions.ServiceUnavailable,`
`1171`	`1171`	`core_exceptions.Aborted,`
	`1172`	`+ core_exceptions.Cancelled,`
`1172`	`1173`	`],`
`1173`	`1174`	`),`
`1174`	`1175`	`(`
Original file line number	Diff line number	Diff line change
`@@ -1021,6 +1021,7 @@ def test__add_exceptions(self, limit, in_e, start_e, end_e):`
`1021`	`1021`	`core_exceptions.DeadlineExceeded,`
`1022`	`1022`	`core_exceptions.ServiceUnavailable,`
`1023`	`1023`	`core_exceptions.Aborted,`
	`1024`	`+ core_exceptions.Cancelled,`
`1024`	`1025`	`],`
`1025`	`1026`	`),`
`1026`	`1027`	`(`