2121import subprocess
2222import sys
2323import tempfile
24+ from concurrent .futures import ThreadPoolExecutor , as_completed
2425from pathlib import Path
2526from typing import Any , NamedTuple , NoReturn
2627from urllib .parse import quote
3031REPOSITORY = "openai/codex"
3132RELEASE_METADATA_NAME = "release.json"
3233INSTALLER_NAMES = ("install.sh" , "install.ps1" )
34+ MAX_UPLOAD_WORKERS = 8
3335# Keep this pattern in sync with release-tag validation in
3436# .github/workflows/rust-release.yml.
3537VERSION_RE = re .compile (
3840)
3941CRC64_RE = re .compile (r"^[A-Za-z0-9+/]{11}=$" )
4042SHA256_RE = re .compile (r"^sha256:([0-9a-f]{64})$" )
43+ MISSING_OBJECT_RE = re .compile (r"\((?:404|NoSuchKey|NotFound)\)" )
4144
4245
4346class PublishError (RuntimeError ):
@@ -65,7 +68,7 @@ def run_command(args: list[str]) -> str:
6568 return result .stdout or ""
6669
6770
68- def download_assets (tag : str , directory : Path ) -> list [ReleaseAsset ]:
71+ def get_release_metadata (tag : str , directory : Path , stage : str ) -> list [ReleaseAsset ]:
6972 try :
7073 metadata = json .loads (
7174 run_command (
@@ -83,21 +86,9 @@ def download_assets(tag: str, directory: Path) -> list[ReleaseAsset]:
8386 ]
8487 )
8588 )
86- run_command (
87- [
88- "gh" ,
89- "release" ,
90- "download" ,
91- tag ,
92- "--repo" ,
93- REPOSITORY ,
94- "--dir" ,
95- str (directory ),
96- ]
97- )
9889 except (OSError , subprocess .CalledProcessError ) as error :
9990 raise PublishError (
100- f"GitHub release download failed for { tag } : { error } "
91+ f"GitHub release metadata request failed for { tag } : { error } "
10192 ) from error
10293 except json .JSONDecodeError as error :
10394 raise PublishError (
@@ -112,6 +103,9 @@ def download_assets(tag: str, directory: Path) -> list[ReleaseAsset]:
112103 raise PublishError (
113104 f"GitHub returned invalid release metadata for { tag } : { asset !r} "
114105 )
106+ # DotSlash runs concurrently, so defer its in-progress assets to finalization.
107+ if stage == "assets" and asset .get ("state" ) != "uploaded" :
108+ continue
115109 name = asset .get ("name" )
116110 size = asset .get ("size" )
117111 digest = asset .get ("digest" )
@@ -131,14 +125,37 @@ def download_assets(tag: str, directory: Path) -> list[ReleaseAsset]:
131125 )
132126 expected [name ] = ReleaseAsset (directory / name , size , match .group (1 ))
133127
134- assets = sorted (directory .iterdir (), key = lambda path : path .name )
135- if not assets :
128+ if not expected :
136129 raise PublishError (f"GitHub Release { tag } has no assets" )
137- if any (not path .is_file () for path in assets ) or {
138- path .name for path in assets
139- } != set (expected ):
130+ return sorted (expected .values (), key = lambda asset : asset .path .name )
131+
132+
133+ def download_assets (tag : str , directory : Path , assets : list [ReleaseAsset ]) -> None :
134+ if not assets :
135+ return
136+
137+ command = [
138+ "gh" ,
139+ "release" ,
140+ "download" ,
141+ tag ,
142+ "--repo" ,
143+ REPOSITORY ,
144+ "--dir" ,
145+ str (directory ),
146+ ]
147+ for asset in assets :
148+ command .extend (["--pattern" , asset .path .name ])
149+
150+ try :
151+ run_command (command )
152+ except (OSError , subprocess .CalledProcessError ) as error :
153+ raise PublishError (
154+ f"GitHub release download failed for { tag } : { error } "
155+ ) from error
156+
157+ if any (not asset .path .is_file () for asset in assets ):
140158 raise PublishError ("GitHub returned invalid release assets" )
141- return [expected [path .name ] for path in assets ]
142159
143160
144161def stream_digest (source : Any ) -> tuple [int , str ]:
@@ -150,6 +167,17 @@ def stream_digest(source: Any) -> tuple[int, str]:
150167 return size , digest .hexdigest ()
151168
152169
170+ def validate_asset (asset : ReleaseAsset ) -> None :
171+ with asset .path .open ("rb" ) as source :
172+ size , sha256 = stream_digest (source )
173+ if size != asset .size or sha256 != asset .sha256 :
174+ raise PublishError (
175+ f"GitHub asset mismatch for { asset .path .name } : expected "
176+ f"size={ asset .size } sha256={ asset .sha256 } , got "
177+ f"size={ size } sha256={ sha256 } "
178+ )
179+
180+
153181def raise_s3 (
154182 action : str , key : str , error : Exception , detail : str | None = None
155183) -> NoReturn :
@@ -246,6 +274,7 @@ def publish_installers(endpoint: str, tag: str, assets: list[ReleaseAsset]) -> N
246274 )
247275 for name in INSTALLER_NAMES :
248276 asset = installers [name ]
277+ validate_asset (asset )
249278 installer_key = f"{ PREFIX } /{ name } "
250279 put_object (endpoint , installer_key , asset .path , asset .sha256 , extra_args = [])
251280 verify_remote (endpoint , installer_key , asset .size , asset .sha256 )
@@ -256,11 +285,92 @@ def publish_installers(endpoint: str, tag: str, assets: list[ReleaseAsset]) -> N
256285 )
257286
258287
288+ def publish_asset (
289+ endpoint : str , version : str , asset : ReleaseAsset , * , verify : bool = True
290+ ) -> dict [str , Any ]:
291+ validate_asset (asset )
292+ key = f"{ PREFIX } /releases/{ version } /{ asset .path .name } "
293+ put_object (endpoint , key , asset .path , asset .sha256 , extra_args = ["--no-overwrite" ])
294+ if verify :
295+ verify_remote (endpoint , key , asset .size , asset .sha256 )
296+ status = "published and verified" if verify else "published"
297+ print (
298+ f"{ status } s3://{ BUCKET } /{ key } size={ asset .size } sha256={ asset .sha256 } " ,
299+ file = sys .stderr ,
300+ )
301+ return {
302+ "key" : key ,
303+ "name" : asset .path .name ,
304+ "sha256" : asset .sha256 ,
305+ "size" : asset .size ,
306+ }
307+
308+
309+ def publish_assets (
310+ endpoint : str , version : str , assets : list [ReleaseAsset ], * , verify : bool = True
311+ ) -> list [dict [str , Any ]]:
312+ if not assets :
313+ return []
314+
315+ published = {}
316+ with ThreadPoolExecutor (
317+ max_workers = min (MAX_UPLOAD_WORKERS , len (assets ))
318+ ) as executor :
319+ futures = {
320+ executor .submit (
321+ publish_asset , endpoint , version , asset , verify = verify
322+ ): asset
323+ for asset in assets
324+ }
325+ for future in as_completed (futures ):
326+ asset = futures [future ]
327+ published [asset .path .name ] = future .result ()
328+ return [published [asset .path .name ] for asset in assets ]
329+
330+
331+ def find_published_assets (
332+ endpoint : str , version : str , assets : list [ReleaseAsset ]
333+ ) -> dict [str , dict [str , Any ]]:
334+ published = {}
335+ with ThreadPoolExecutor (
336+ max_workers = min (MAX_UPLOAD_WORKERS , len (assets ))
337+ ) as executor :
338+ futures = {
339+ executor .submit (
340+ verify_remote ,
341+ endpoint ,
342+ f"{ PREFIX } /releases/{ version } /{ asset .path .name } " ,
343+ asset .size ,
344+ asset .sha256 ,
345+ ): asset
346+ for asset in assets
347+ }
348+ for future in as_completed (futures ):
349+ asset = futures [future ]
350+ try :
351+ future .result ()
352+ except PublishError as error :
353+ cause = error .__cause__
354+ if isinstance (cause , subprocess .CalledProcessError ) and (
355+ MISSING_OBJECT_RE .search (cause .stderr or "" )
356+ ):
357+ continue
358+ raise
359+ published [asset .path .name ] = {
360+ "key" : f"{ PREFIX } /releases/{ version } /{ asset .path .name } " ,
361+ "name" : asset .path .name ,
362+ "sha256" : asset .sha256 ,
363+ "size" : asset .size ,
364+ }
365+ return published
366+
367+
259368def parse_args () -> argparse .Namespace :
260369 parser = argparse .ArgumentParser (description = __doc__ )
261370 parser .add_argument ("--tag" , required = True )
262371 parser .add_argument ("--make-latest" , choices = ("true" , "false" ), required = True )
263372 parser .add_argument ("--prerelease" , choices = ("true" , "false" ), required = True )
373+ parser .add_argument ("--stage" , choices = ("assets" , "finalize" ), required = True )
264374 return parser .parse_args ()
265375
266376
@@ -276,50 +386,62 @@ def main() -> int:
276386 version = args .tag .removeprefix ("rust-v" )
277387 if args .tag == version or not VERSION_RE .fullmatch (version ):
278388 raise PublishError (f"invalid rust release tag: { args .tag } " )
279- published = []
280- metadata_assets = []
281389 with tempfile .TemporaryDirectory () as temp_dir :
282390 assets_directory = Path (temp_dir ) / "assets"
283391 assets_directory .mkdir ()
284- assets = download_assets (args .tag , assets_directory )
285- for asset in assets :
286- with asset .path .open ("rb" ) as source :
287- size , sha256 = stream_digest (source )
288- if size != asset .size or sha256 != asset .sha256 :
289- raise PublishError (
290- f"GitHub asset mismatch for { asset .path .name } : expected "
291- f"size={ asset .size } sha256={ asset .sha256 } , got "
292- f"size={ size } sha256={ sha256 } "
293- )
294- for asset in assets :
295- path = asset .path
296- size = asset .size
297- sha256 = asset .sha256
298- key = f"{ PREFIX } /releases/{ version } /{ path .name } "
299- put_object (endpoint , key , path , sha256 , extra_args = ["--no-overwrite" ])
300- verify_remote (endpoint , key , size , sha256 )
392+ assets = get_release_metadata (args .tag , assets_directory , args .stage )
393+
394+ if args .stage == "assets" :
395+ download_assets (args .tag , assets_directory , assets )
396+ published = publish_assets (endpoint , version , assets , verify = False )
301397 print (
302- f"published and verified s3://{ BUCKET } /{ key } "
303- f"size={ size } sha256={ sha256 } " ,
304- file = sys .stderr ,
305- )
306- published .append (
307- {
308- "key" : key ,
309- "sha256" : sha256 ,
310- "size" : size ,
311- }
398+ json .dumps (
399+ {
400+ "assetCount" : len (published ),
401+ "assets" : published ,
402+ "releasePrefix" : f"{ PREFIX } /releases/{ version } /" ,
403+ "stage" : args .stage ,
404+ "tag" : args .tag ,
405+ "version" : version ,
406+ },
407+ sort_keys = True ,
408+ )
312409 )
313- metadata_assets .append (
314- {
315- "name" : path .name ,
316- "digest" : f"sha256:{ sha256 } " ,
317- "browser_download_url" : (
318- f"https://releases.openai.com/{ PREFIX } /releases/"
319- f"{ version } /{ quote (path .name , safe = '' )} "
320- ),
321- }
410+ return 0
411+
412+ previously_published = find_published_assets (endpoint , version , assets )
413+ remaining = [
414+ asset for asset in assets if asset .path .name not in previously_published
415+ ]
416+ required_downloads = list (remaining )
417+ if args .prerelease == "false" :
418+ required_downloads .extend (
419+ asset
420+ for asset in assets
421+ if asset .path .name in INSTALLER_NAMES
422+ and asset .path .name in previously_published
322423 )
424+ download_assets (args .tag , assets_directory , required_downloads )
425+ additionally_published = {
426+ asset ["name" ]: asset
427+ for asset in publish_assets (endpoint , version , remaining )
428+ }
429+ published = [
430+ previously_published .get (asset .path .name )
431+ or additionally_published [asset .path .name ]
432+ for asset in assets
433+ ]
434+ metadata_assets = [
435+ {
436+ "name" : asset .path .name ,
437+ "digest" : f"sha256:{ asset .sha256 } " ,
438+ "browser_download_url" : (
439+ f"https://releases.openai.com/{ PREFIX } /releases/"
440+ f"{ version } /{ quote (asset .path .name , safe = '' )} "
441+ ),
442+ }
443+ for asset in assets
444+ ]
323445
324446 metadata_path = Path (temp_dir ) / RELEASE_METADATA_NAME
325447 metadata_path .write_text (
@@ -393,6 +515,7 @@ def main() -> int:
393515 "size" : metadata_size ,
394516 },
395517 "releasePrefix" : f"{ PREFIX } /releases/{ version } /" ,
518+ "stage" : args .stage ,
396519 "tag" : args .tag ,
397520 "version" : version ,
398521 },
0 commit comments