66import time
77import uuid
88import weakref
9- from collections .abc import Callable , Iterable
9+ from collections .abc import Callable , Iterable , Mapping
1010from dataclasses import dataclass
1111from enum import Enum
1212from io import BytesIO
1818from pydantic import BaseModel
1919
2020from comfy import utils
21+ from comfy .model_management import (
22+ InterruptProcessingException ,
23+ interrupt_current_processing ,
24+ throw_exception_if_processing_interrupted ,
25+ )
2126from comfy_api .latest import IO
2227from server import PromptServer
2328
@@ -65,7 +70,6 @@ class _RequestConfig:
6570 retry_backoff : float
6671 wait_label : str = "Waiting"
6772 monitor_progress : bool = True
68- estimated_total : int | None = None
6973 final_label_on_success : str | None = "Completed"
7074 progress_origin_ts : float | None = None
7175 price_extractor : Callable [[dict [str , Any ]], float | None ] | None = None
@@ -80,6 +84,8 @@ class _PollUIState:
8084 is_queued : bool = True
8185 price : float | None = None
8286 estimated_duration : int | None = None
87+ estimated_p90 : int | None = None
88+ estimate_includes_queue : bool = False
8389 base_processing_elapsed : float = 0.0 # sum of completed active intervals
8490 active_since : float | None = None # start time of current active interval (None if queued)
8591
@@ -91,8 +97,22 @@ class _PollUIState:
9197"""Proxy response header with the actual cost in Comfy credits. When present on any successful proxied response,
9298it takes precedence over ``price_extractor``."""
9399
100+ ESTIMATED_DURATION_HEADER = "X-Comfy-Estimated-Duration-Seconds"
101+ ESTIMATED_DURATION_P90_HEADER = "X-Comfy-Estimated-Duration-P90-Seconds"
102+ ESTIMATE_SOURCE_HEADER = "X-Comfy-Estimate-Source"
103+
94104_credits_used_by_execution : "weakref.WeakKeyDictionary[type, float]" = weakref .WeakKeyDictionary ()
95105"""Last PRICE_CREDITS_HEADER value per node execution, keyed by the node's per-execution class clone."""
106+
107+
108+ @dataclass
109+ class _ServerEstimate :
110+ p50_seconds : int
111+ p90_seconds : int | None
112+ source : str | None
113+
114+
115+ _server_estimates_by_execution : "weakref.WeakKeyDictionary[type, _ServerEstimate]" = weakref .WeakKeyDictionary ()
96116COMPLETED_STATUSES = ["succeeded" , "succeed" , "success" , "completed" , "finished" , "done" , "complete" ]
97117FAILED_STATUSES = ["cancelled" , "canceled" , "canceling" , "fail" , "failed" , "error" ]
98118QUEUED_STATUSES = ["created" , "queued" , "queueing" , "submitted" , "initializing" , "wait" , "in_queue" ]
@@ -117,6 +137,32 @@ def _get_remembered_credits_used(node_cls: type[IO.ComfyNode]) -> float | None:
117137 return _credits_used_by_execution .get (node_cls )
118138
119139
140+ def _parse_estimate_seconds (header_value : str | None ) -> int | None :
141+ if not header_value :
142+ return None
143+ try :
144+ seconds = int (header_value .strip ())
145+ except ValueError :
146+ logging .debug ("Ignoring malformed estimate header value: %r" , header_value )
147+ return None
148+ if not 1 <= seconds <= 86400 :
149+ logging .debug ("Ignoring out-of-range estimate header value: %r" , header_value )
150+ return None
151+ return seconds
152+
153+
154+ def _maybe_remember_server_estimate (node_cls : type [IO .ComfyNode ], headers : Mapping [str , str ]) -> None :
155+ p50 = _parse_estimate_seconds (headers .get (ESTIMATED_DURATION_HEADER ))
156+ if p50 is None :
157+ return
158+ p90 = _parse_estimate_seconds (headers .get (ESTIMATED_DURATION_P90_HEADER ))
159+ if p90 is not None and p90 < p50 :
160+ p90 = p50
161+ source = headers .get (ESTIMATE_SOURCE_HEADER ) or None
162+ _server_estimates_by_execution [node_cls ] = _ServerEstimate (p50 , p90 , source )
163+ logging .debug ("Server duration estimate: p50=%ss, p90=%s, source=%s" , p50 , p90 , source )
164+
165+
120166async def sync_op (
121167 cls : type [IO .ComfyNode ],
122168 endpoint : ApiEndpoint ,
@@ -132,7 +178,6 @@ async def sync_op(
132178 retry_delay : float = 1.0 ,
133179 retry_backoff : float = 2.0 ,
134180 wait_label : str = "Waiting for server" ,
135- estimated_duration : int | None = None ,
136181 final_label_on_success : str | None = "Completed" ,
137182 progress_origin_ts : float | None = None ,
138183 monitor_progress : bool = True ,
@@ -152,7 +197,6 @@ async def sync_op(
152197 retry_delay = retry_delay ,
153198 retry_backoff = retry_backoff ,
154199 wait_label = wait_label ,
155- estimated_duration = estimated_duration ,
156200 as_binary = False ,
157201 final_label_on_success = final_label_on_success ,
158202 progress_origin_ts = progress_origin_ts ,
@@ -228,7 +272,6 @@ async def sync_op_raw(
228272 retry_delay : float = 1.0 ,
229273 retry_backoff : float = 2.0 ,
230274 wait_label : str = "Waiting for server" ,
231- estimated_duration : int | None = None ,
232275 as_binary : bool = False ,
233276 final_label_on_success : str | None = "Completed" ,
234277 progress_origin_ts : float | None = None ,
@@ -261,7 +304,6 @@ async def sync_op_raw(
261304 retry_backoff = retry_backoff ,
262305 wait_label = wait_label ,
263306 monitor_progress = monitor_progress ,
264- estimated_total = estimated_duration ,
265307 final_label_on_success = final_label_on_success ,
266308 progress_origin_ts = progress_origin_ts ,
267309 price_extractor = price_extractor ,
@@ -311,11 +353,22 @@ async def poll_op_raw(
311353 progress_bar = utils .ProgressBar (100 ) if progress_extractor else None
312354 last_progress : int | None = None
313355
314- state = _PollUIState (started = started , estimated_duration = estimated_duration )
356+ server_estimate = _server_estimates_by_execution .pop (cls , None )
357+ if server_estimate is not None :
358+ state = _PollUIState (
359+ started = started ,
360+ estimated_duration = server_estimate .p50_seconds ,
361+ estimated_p90 = server_estimate .p90_seconds ,
362+ estimate_includes_queue = True ,
363+ )
364+ else :
365+ state = _PollUIState (started = started , estimated_duration = estimated_duration )
366+ estimate_bar = utils .ProgressBar (100 ) if progress_bar is None and server_estimate is not None else None
315367 stop_ticker = asyncio .Event ()
316368
317369 async def _ticker ():
318370 """Emit a UI update every second while polling is in progress."""
371+ last_estimate_pct = - 1
319372 try :
320373 while not stop_ticker .is_set ():
321374 if is_processing_interrupted ():
@@ -324,6 +377,15 @@ async def _ticker():
324377 proc_elapsed = state .base_processing_elapsed + (
325378 (now - state .active_since ) if state .active_since is not None else 0.0
326379 )
380+ if estimate_bar is not None :
381+ pct = _estimate_progress_pct (now - state .started , state .estimated_duration , state .estimated_p90 )
382+ if pct != last_estimate_pct :
383+ try :
384+ estimate_bar .update_absolute (pct , total = 100 )
385+ except InterruptProcessingException :
386+ interrupt_current_processing ()
387+ break
388+ last_estimate_pct = pct
327389 _display_time_progress (
328390 cls ,
329391 status = state .status_label ,
@@ -333,6 +395,8 @@ async def _ticker():
333395 is_queued = state .is_queued ,
334396 processing_elapsed_seconds = int (proc_elapsed ),
335397 extra_text = extra_text ,
398+ estimated_p90 = state .estimated_p90 ,
399+ estimate_includes_queue = state .estimate_includes_queue ,
336400 )
337401 await asyncio .sleep (1.0 )
338402 except Exception as exc :
@@ -351,7 +415,6 @@ async def _ticker():
351415 retry_delay = retry_delay_per_poll ,
352416 retry_backoff = retry_backoff_per_poll ,
353417 wait_label = "Checking" ,
354- estimated_duration = None ,
355418 as_binary = False ,
356419 final_label_on_success = None ,
357420 monitor_progress = False ,
@@ -367,7 +430,6 @@ async def _ticker():
367430 timeout = cancel_timeout ,
368431 max_retries = 0 ,
369432 wait_label = "Cancelling task" ,
370- estimated_duration = None ,
371433 as_binary = False ,
372434 final_label_on_success = None ,
373435 monitor_progress = False ,
@@ -381,15 +443,26 @@ async def _ticker():
381443 status = None
382444
383445 if price_extractor :
384- new_price = price_extractor (resp_json )
446+ try :
447+ new_price = price_extractor (resp_json )
448+ except Exception as e :
449+ logging .error ("Price extraction failed: %s" , e )
450+ new_price = None
385451 if new_price is not None :
386452 state .price = new_price
387453
388454 if progress_extractor :
389- new_progress = progress_extractor (resp_json )
455+ try :
456+ new_progress = progress_extractor (resp_json )
457+ except Exception as e :
458+ logging .error ("Progress extraction failed: %s" , e )
459+ new_progress = None
390460 if new_progress is not None and last_progress != new_progress :
391- progress_bar .update_absolute (new_progress , total = 100 )
392- last_progress = new_progress
461+ try :
462+ progress_bar .update_absolute (new_progress , total = 100 )
463+ last_progress = new_progress
464+ except InterruptProcessingException :
465+ interrupt_current_processing ()
393466
394467 now_ts = time .monotonic ()
395468 is_queued = status in queued_states
@@ -414,12 +487,14 @@ async def _ticker():
414487
415488 if progress_bar and last_progress != 100 :
416489 progress_bar .update_absolute (100 , total = 100 )
490+ if estimate_bar is not None :
491+ estimate_bar .update_absolute (100 , total = 100 )
417492
418493 _display_time_progress (
419494 cls ,
420495 status = status if status else "Completed" ,
421496 elapsed_seconds = int (now_ts - started ),
422- estimated_total = estimated_duration ,
497+ estimated_total = None ,
423498 price = state .price ,
424499 is_queued = False ,
425500 processing_elapsed_seconds = int (state .base_processing_elapsed ),
@@ -428,12 +503,13 @@ async def _ticker():
428503 return resp_json
429504
430505 if status in failed_states :
506+ throw_exception_if_processing_interrupted ()
431507 msg = f"Task failed: { json .dumps (resp_json )} "
432508 logging .error (msg )
433509 raise Exception (msg )
434510
435511 try :
436- await sleep_with_interrupt (poll_interval , cls , None , None , None )
512+ await sleep_with_interrupt (poll_interval , cls , None , None )
437513 except ProcessingInterrupted :
438514 if cancel_endpoint :
439515 with contextlib .suppress (Exception ):
@@ -443,7 +519,6 @@ async def _ticker():
443519 timeout = cancel_timeout ,
444520 max_retries = 0 ,
445521 wait_label = "Cancelling task" ,
446- estimated_duration = None ,
447522 as_binary = False ,
448523 final_label_on_success = None ,
449524 monitor_progress = False ,
@@ -503,17 +578,37 @@ def _display_time_progress(
503578 is_queued : bool | None = None ,
504579 processing_elapsed_seconds : int | None = None ,
505580 extra_text : str | None = None ,
581+ estimated_p90 : int | None = None ,
582+ estimate_includes_queue : bool = False ,
506583) -> None :
507- if estimated_total is not None and estimated_total > 0 and is_queued is False :
508- pe = processing_elapsed_seconds if processing_elapsed_seconds is not None else elapsed_seconds
509- remaining = max (0 , int (estimated_total ) - int (pe ))
510- time_line = f"Time elapsed: { int (elapsed_seconds )} s (~{ remaining } s remaining)"
511- else :
512- time_line = f"Time elapsed: { int (elapsed_seconds )} s"
584+ time_line = f"Time elapsed: { int (elapsed_seconds )} s"
585+ if estimated_total is not None and estimated_total > 0 :
586+ if estimate_includes_queue :
587+ if elapsed_seconds < estimated_total :
588+ time_line += f" (~{ int (estimated_total ) - int (elapsed_seconds )} s remaining)"
589+ elif estimated_p90 is not None and elapsed_seconds < estimated_p90 :
590+ time_line += " (should finish soon)"
591+ else :
592+ time_line += " (taking longer than usual)"
593+ elif is_queued is False :
594+ pe = processing_elapsed_seconds if processing_elapsed_seconds is not None else elapsed_seconds
595+ remaining = max (0 , int (estimated_total ) - int (pe ))
596+ time_line += f" (~{ remaining } s remaining)"
513597 text = f"{ time_line } \n \n { extra_text } " if extra_text else time_line
514598 _display_text (node_cls , text , status = status , price = price )
515599
516600
601+ def _estimate_progress_pct (elapsed_seconds : float , p50_seconds : int | None , p90_seconds : int | None ) -> int :
602+ if not p50_seconds or p50_seconds <= 0 or elapsed_seconds <= 0 :
603+ return 0
604+ if elapsed_seconds < p50_seconds :
605+ return int (90.0 * elapsed_seconds / p50_seconds )
606+ horizon = p90_seconds if p90_seconds is not None and p90_seconds > p50_seconds else p50_seconds
607+ if elapsed_seconds >= horizon :
608+ return 95
609+ return 90 + int (5.0 * (elapsed_seconds - p50_seconds ) / (horizon - p50_seconds ))
610+
611+
517612async def _diagnose_connectivity () -> dict [str , bool ]:
518613 """Best-effort connectivity diagnostics to distinguish local vs. server issues."""
519614 results = {
@@ -694,9 +789,7 @@ async def _monitor(stop_evt: asyncio.Event, start_ts: float):
694789 if is_processing_interrupted ():
695790 return
696791 if cfg .monitor_progress :
697- _display_time_progress (
698- cfg .node_cls , cfg .wait_label , int (time .monotonic () - start_ts ), cfg .estimated_total
699- )
792+ _display_time_progress (cfg .node_cls , cfg .wait_label , int (time .monotonic () - start_ts ))
700793 await asyncio .sleep (1.0 )
701794 except asyncio .CancelledError :
702795 return # normal shutdown
@@ -851,7 +944,6 @@ async def _monitor(stop_evt: asyncio.Event, start_ts: float):
851944 cfg .node_cls ,
852945 cfg .wait_label if cfg .monitor_progress else None ,
853946 start_time if cfg .monitor_progress else None ,
854- cfg .estimated_total ,
855947 display_callback = _display_time_progress if cfg .monitor_progress else None ,
856948 )
857949 continue
@@ -878,13 +970,12 @@ async def _monitor(stop_evt: asyncio.Event, start_ts: float):
878970 if is_processing_interrupted ():
879971 raise ProcessingInterrupted ("Task cancelled" )
880972 if cfg .monitor_progress :
881- _display_time_progress (
882- cfg .node_cls , cfg .wait_label , int (now - start_time ), cfg .estimated_total
883- )
973+ _display_time_progress (cfg .node_cls , cfg .wait_label , int (now - start_time ))
884974 bytes_payload = bytes (buff )
885975 resp_headers = {k .lower (): v for k , v in resp .headers .items ()}
886976 if is_comfy_api_request :
887977 _maybe_remember_credits_used (cfg .node_cls , resp .headers .get (PRICE_CREDITS_HEADER ))
978+ _maybe_remember_server_estimate (cfg .node_cls , resp .headers )
888979 if cfg .price_extractor :
889980 with contextlib .suppress (Exception ):
890981 extracted_price = cfg .price_extractor (resp_headers )
@@ -914,6 +1005,7 @@ async def _monitor(stop_evt: asyncio.Event, start_ts: float):
9141005 response_content_to_log = payload if isinstance (payload , dict ) else text
9151006 if is_comfy_api_request :
9161007 _maybe_remember_credits_used (cfg .node_cls , resp .headers .get (PRICE_CREDITS_HEADER ))
1008+ _maybe_remember_server_estimate (cfg .node_cls , resp .headers )
9171009 with contextlib .suppress (Exception ):
9181010 extracted_price = cfg .price_extractor (payload ) if cfg .price_extractor else None
9191011 operation_succeeded = True
@@ -956,7 +1048,6 @@ async def _monitor(stop_evt: asyncio.Event, start_ts: float):
9561048 cfg .node_cls ,
9571049 cfg .wait_label if cfg .monitor_progress else None ,
9581050 start_time if cfg .monitor_progress else None ,
959- cfg .estimated_total ,
9601051 display_callback = _display_time_progress if cfg .monitor_progress else None ,
9611052 )
9621053 delay *= cfg .retry_backoff
@@ -1007,7 +1098,7 @@ async def _monitor(stop_evt: asyncio.Event, start_ts: float):
10071098 if final_elapsed_seconds is not None
10081099 else int (time .monotonic () - start_time )
10091100 ),
1010- estimated_total = cfg . estimated_total ,
1101+ estimated_total = None ,
10111102 price = extracted_price ,
10121103 is_queued = False ,
10131104 processing_elapsed_seconds = final_elapsed_seconds ,
0 commit comments