gh-155292: Skip updating unicodedata with mismatched interpreter by encukou · Pull Request #157066 · python/cpython · GitHub
Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 32 additions & 5 deletions Tools/unicode/makeunicodedata.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,9 @@
SCRIPT = os.path.normpath(sys.argv[0])
VERSION = "3.3"

# Local cache location
DATA_DIR = os.path.join('Tools', 'unicode', 'data')

# The Unicode Database
# --------------------
# When changing UCD version please update
Expand Down Expand Up @@ -816,10 +819,31 @@ def makeunicodename(unicode, trace):
def makestringprep():
FILE = "Lib/stringprep.py"

RFC_LOCAL = os.path.join(DATA_DIR, "rfc3454.txt")
RFC_URL = "https://www.rfc-editor.org/rfc/rfc3454.txt"

print("--- Preparing", FILE, "...")

# mkstringprep expects a local copy of RFC 3454. Download it now.
# (stringprep is used for URL handling, and if it's not matched
# with the compiled unicodedata, downloads would fail.)
if not os.path.exists(RFC_LOCAL):
download_data(RFC_LOCAL, RFC_URL)

MKSTRINGPREP = "Tools/unicode/mkstringprep.py"

# mkstringprep needs to be run with a Python version that has "its"
# unicode data, since it uses str.lower() and similar.
import unicodedata
if unicodedata.unidata_version != UNIDATA_VERSION:
print()
print("!! Skipping mkstringprep -- mismatched Unicode version !!")
print()
print("Please compile CPython with the updated Unicode database,")
print("then use that interpreter to run:")
print(f" python {MKSTRINGPREP} > {FILE}")
return

with open(FILE, "w") as f:
f.truncate()
subprocess.check_call([sys.executable, MKSTRINGPREP], stdout=f)
Expand Down Expand Up @@ -929,26 +953,29 @@ class Difference(Exception):pass
normalization_changes))


DATA_DIR = os.path.join('Tools', 'unicode', 'data')

def open_data(template, version):
local = os.path.join(DATA_DIR, template % ('-'+version,))
if not os.path.exists(local):
import urllib.request
if version == '3.2.0':
# irregular url structure
url = ('https://www.unicode.org/Public/3.2-Update/'+template) % ('-'+version,)
else:
url = ('https://www.unicode.org/Public/%s/ucd/'+template) % (version, '')
os.makedirs(os.path.dirname(local), exist_ok=True)
urllib.request.urlretrieve(url, filename=local)
download_data(local, url)
if local.endswith('.txt'):
return open(local, encoding='utf-8')
else:
# Unihan.zip
return open(local, 'rb')


def download_data(local, url):
import urllib.request
os.makedirs(os.path.dirname(local), exist_ok=True)
print(f'Downloading {url} to {local}')
urllib.request.urlretrieve(url, filename=local)


def expand_range(char_range: str) -> Iterator[int]:
'''
Parses ranges of code points, as described in UAX #44:
Expand Down
14 changes: 1 addition & 13 deletions Tools/unicode/mkstringprep.py
Loading