fix!: do not count downtime towards crawler_runtime after migration o… · apify/crawlee-python@fbab2a8 · GitHub
Skip to content

Commit fbab2a8

Browse files
committed
fix!: do not count downtime towards crawler_runtime after migration or resurrection
The `crawler_runtime` statistic no longer counts the downtime between the runs of a resumed crawler, and it accumulates correctly across any number of persist/resume cycles: - The first periodic statistics log reports exactly the runtime accumulated so far: zero for a fresh start, the previously persisted total for a resumed run. - The accumulated runtime is stored in the new `accumulated_runtime` field, persisted under `crawlerRuntimeMillis`. Every persist writes the live total, so a state persisted mid-run (migration, abort) carries the runtime up to that moment and a resumed run continues from the last checkpoint. - The timestamps of the previous run are cleared before the new run starts being measured, so they cannot leak the downtime into the runtime, and a backwards clock jump during a run cannot push the runtime below the accumulated total. BREAKING CHANGE: `StatisticsState.crawler_runtime_for_serialization` is removed; the `crawler_runtime` property is backed by the new `accumulated_runtime` field. `crawlerRuntimeMillis` is serialized as a number of milliseconds instead of an ISO 8601 duration string. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01VMHwDx87jV4rnu5LkbpNxy
1 parent 758e546 commit fbab2a8

4 files changed

Lines changed: 203 additions & 22 deletions

File tree

‎docs/upgrading/upgrading_to_v2.md‎

Lines changed: 4 additions & 0 deletions

‎src/crawlee/statistics/_models.py‎

Lines changed: 30 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,15 @@
55
from datetime import UTC, datetime, timedelta
66
from typing import TYPE_CHECKING, Annotated, Any
77

8-
from pydantic import BaseModel, ConfigDict, Field, PlainSerializer, PlainValidator, computed_field
8+
from pydantic import (
9+
BaseModel,
10+
ConfigDict,
11+
Field,
12+
PlainSerializer,
13+
PlainValidator,
14+
computed_field,
15+
field_serializer,
16+
)
917
from pydantic.alias_generators import to_camel
1018
from typing_extensions import override
1119

@@ -100,25 +108,31 @@ class StatisticsState(BaseModel):
100108
),
101109
] = {}
102110

103-
# Used to track the crawler runtime, that had already been persisted. This is the runtime from previous runs.
104-
_runtime_offset: Annotated[timedelta, Field(exclude=True)] = timedelta()
111+
accumulated_runtime: Annotated[timedelta_ms, Field(alias='crawlerRuntimeMillis')] = timedelta()
112+
"""The runtime accumulated by the previous runs of the crawler, excluding the run in progress.
105113
106-
def model_post_init(self, /, __context: Any) -> None:
107-
self._runtime_offset = self.crawler_runtime or self._runtime_offset
114+
When serialized, the runtime of the run in progress is included, so that a persisted state always carries
115+
the total runtime accumulated so far. Do not assign to it: it is restored from the persisted state, and
116+
serialization derives the live total from the timestamps.
117+
"""
108118

109119
@property
110120
def crawler_runtime(self) -> timedelta:
111-
if self.crawler_last_started_at:
112-
finished_at = self.crawler_finished_at or datetime.now(UTC)
113-
return self._runtime_offset + finished_at - self.crawler_last_started_at
114-
return self._runtime_offset
115-
116-
@computed_field(alias='crawlerRuntimeMillis')
117-
def crawler_runtime_for_serialization(self) -> timedelta:
118-
if self.crawler_last_started_at:
119-
finished_at = self.crawler_finished_at or datetime.now(UTC)
120-
return self._runtime_offset + finished_at - self.crawler_last_started_at
121-
return self._runtime_offset
121+
"""The total runtime of the crawler, accumulated over all its runs, excluding the downtime between them.
122+
123+
The run in progress is measured by the `crawler_last_started_at` and `crawler_finished_at` timestamps.
124+
In a loaded state, the timestamps describe the previous run, whose runtime `accumulated_runtime` already
125+
contains, so they must be reset before a new run is measured. `Statistics` resets them on enter, before
126+
the runtime is first read.
127+
"""
128+
if self.crawler_last_started_at is None:
129+
return self.accumulated_runtime
130+
finished_at = self.crawler_finished_at or datetime.now(UTC)
131+
return self.accumulated_runtime + max(timedelta(), finished_at - self.crawler_last_started_at)
132+
133+
@field_serializer('accumulated_runtime')
134+
def _serialize_accumulated_runtime(self, _value: timedelta) -> float:
135+
return round(self.crawler_runtime.total_seconds() * 1000)
122136

123137
@computed_field(alias='requestTotalDurationMillis', return_type=timedelta_ms)
124138
@property

‎src/crawlee/statistics/_statistics.py‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -169,10 +169,15 @@ async def __aenter__(self) -> Self:
169169
raise RuntimeError(f'The {self.__class__.__name__} is already active.')
170170

171171
await self._state.initialize()
172-
# Reset `crawler_finished_at` to indicate a new run in progress.
172+
173+
# The restored timestamps belong to the previous run and must not measure this one: left in place,
174+
# they would count the downtime between the runs (e.g. after a migration or resurrection) towards
175+
# the runtime. Clear them, so that the initial periodic log reports exactly the runtime accumulated
176+
# by the previous runs (zero for a fresh start).
177+
self.state.crawler_last_started_at = None
173178
self.state.crawler_finished_at = None
174179

175-
# Start periodic logging and let it print initial state before activation.
180+
# Start periodic logging and let it print the restored state before the new run starts being measured.
176181
self._periodic_logger.start()
177182
await asyncio.sleep(0.01)
178183
self._active = True
@@ -195,10 +200,7 @@ async def __aexit__(
195200
if not self._active:
196201
raise RuntimeError(f'The {self.__class__.__name__} is not active.')
197202

198-
if not self.state.crawler_last_started_at:
199-
raise RuntimeError('Statistics.state.crawler_last_started_at not set.')
200-
201-
# Stop logging and deactivate the statistics to prevent further changes to crawler_runtime
203+
# Stop logging; the finish timestamp freezes the measured run before the final persist.
202204
await self._periodic_logger.stop()
203205
self.state.crawler_finished_at = datetime.now(UTC)
204206
self._active = False

‎tests/unit/_statistics/test_persistence.py‎

Lines changed: 161 additions & 0 deletions

0 commit comments

Comments
 (0)