makeunicodedata.py

#
# (re)generate unicode property and type databases
#
# This script converts Unicode database files to Modules/unicodedata_db.h,
# Modules/unicodename_db.h, and Objects/unicodetype_db.h
#
# history:
# 2000-09-24 fl created (based on bits and pieces from unidb)
# 2000-09-25 fl merged tim's splitbin fixes, separate decomposition table
# 2000-09-25 fl added character type table
# 2000-09-26 fl added LINEBREAK, DECIMAL, and DIGIT flags/fields (2.0)
# 2000-11-03 fl expand first/last ranges
# 2001-01-19 fl added character name tables (2.1)
# 2001-01-21 fl added decomp compression; dynamic phrasebook threshold
# 2002-09-11 wd use string methods
# 2002-10-18 mvl update to Unicode 3.2
# 2002-10-22 mvl generate NFC tables
# 2002-11-24 mvl expand all ranges, sort names version-independently
# 2002-11-25 mvl add UNIDATA_VERSION
# 2004-05-29 perky add east asian width information
# 2006-03-10 mvl update to Unicode 4.1; add UCD 3.2 delta
# 2008-06-11 gb add PRINTABLE_MASK for Atsuo Ishimoto's ascii() patch
# 2011-10-21 ezio add support for name aliases and named sequences
# 2012-01 benjamin add full case mappings
#
# written by Fredrik Lundh (fredrik@pythonware.com)
#

importdataclasses
importos
importsys
importzipfile

fromfunctoolsimportpartial
fromtextwrapimportdedent
fromtypingimportIterator, List, Optional, Set, Tuple

SCRIPT=os.path.normpath(sys.argv[0])
VERSION="3.3"

# The Unicode Database
# --------------------
# When changing UCD version please update
# * Doc/library/stdtypes.rst, and
# * Doc/library/unicodedata.rst
# * Doc/reference/lexical_analysis.rst (two occurrences)
UNIDATA_VERSION="15.1.0"
UNICODE_DATA="UnicodeData%s.txt"
COMPOSITION_EXCLUSIONS="CompositionExclusions%s.txt"
EASTASIAN_WIDTH="EastAsianWidth%s.txt"
UNIHAN="Unihan%s.zip"
DERIVED_CORE_PROPERTIES="DerivedCoreProperties%s.txt"
DERIVEDNORMALIZATION_PROPS="DerivedNormalizationProps%s.txt"
LINE_BREAK="LineBreak%s.txt"
NAME_ALIASES="NameAliases%s.txt"
NAMED_SEQUENCES="NamedSequences%s.txt"
SPECIAL_CASING="SpecialCasing%s.txt"
CASE_FOLDING="CaseFolding%s.txt"

# Private Use Areas -- in planes 1, 15, 16
PUA_1=range(0xE000, 0xF900)
PUA_15=range(0xF0000, 0xFFFFE)
PUA_16=range(0x100000, 0x10FFFE)

# we use this ranges of PUA_15 to store name aliases and named sequences
NAME_ALIASES_START=0xF0000
NAMED_SEQUENCES_START=0xF0200

old_versions= ["3.2.0"]

CATEGORY_NAMES= [ "Cn", "Lu", "Ll", "Lt", "Mn", "Mc", "Me", "Nd",
"Nl", "No", "Zs", "Zl", "Zp", "Cc", "Cf", "Cs", "Co", "Cn", "Lm",
"Lo", "Pc", "Pd", "Ps", "Pe", "Pi", "Pf", "Po", "Sm", "Sc", "Sk",
"So" ]

BIDIRECTIONAL_NAMES= [ "", "L", "LRE", "LRO", "R", "AL", "RLE", "RLO",
"PDF", "EN", "ES", "ET", "AN", "CS", "NSM", "BN", "B", "S", "WS",
"ON", "LRI", "RLI", "FSI", "PDI" ]

# "N" needs to be the first entry, see the comment in makeunicodedata
EASTASIANWIDTH_NAMES= [ "N", "H", "W", "Na", "A", "F" ]

MANDATORY_LINE_BREAKS= [ "BK", "CR", "LF", "NL" ]

# note: should match definitions in Objects/unicodectype.c
ALPHA_MASK=0x01
DECIMAL_MASK=0x02
DIGIT_MASK=0x04
LOWER_MASK=0x08
LINEBREAK_MASK=0x10
SPACE_MASK=0x20
TITLE_MASK=0x40
UPPER_MASK=0x80
XID_START_MASK=0x100
XID_CONTINUE_MASK=0x200
PRINTABLE_MASK=0x400
NUMERIC_MASK=0x800
CASE_IGNORABLE_MASK=0x1000
CASED_MASK=0x2000
EXTENDED_CASE_MASK=0x4000

# these ranges need to match unicodedata.c:is_unified_ideograph
cjk_ranges= [
 ('3400', '4DBF'), # CJK Ideograph Extension A CJK
 ('4E00', '9FFF'), # CJK Ideograph
 ('20000', '2A6DF'), # CJK Ideograph Extension B
 ('2A700', '2B739'), # CJK Ideograph Extension C
 ('2B740', '2B81D'), # CJK Ideograph Extension D
 ('2B820', '2CEA1'), # CJK Ideograph Extension E
 ('2CEB0', '2EBE0'), # CJK Ideograph Extension F
 ('2EBF0', '2EE5D'), # CJK Ideograph Extension I
 ('30000', '3134A'), # CJK Ideograph Extension G
 ('31350', '323AF'), # CJK Ideograph Extension H
]


defmaketables(trace=0):

print("--- Reading", UNICODE_DATA%"", "...")

unicode=UnicodeData(UNIDATA_VERSION)

print(len(list(filter(None, unicode.table))), "characters")

forversioninold_versions:
print("--- Reading", UNICODE_DATA% ("-"+version), "...")
old_unicode=UnicodeData(version, cjk_check=False)
print(len(list(filter(None, old_unicode.table))), "characters")
merge_old_version(version, unicode, old_unicode)

makeunicodename(unicode, trace)
makeunicodedata(unicode, trace)
makeunicodetype(unicode, trace)


# --------------------------------------------------------------------
# unicode character properties

defmakeunicodedata(unicode, trace):

# the default value of east_asian_width is "N", for unassigned code points
# not mentioned in EastAsianWidth.txt
# in addition there are some reserved but unassigned code points in CJK
# ranges that are classified as "W". code points in private use areas
# have a width of "A". both of these have entries in
# EastAsianWidth.txt
# see https://unicode.org/reports/tr11/#Unassigned
assertEASTASIANWIDTH_NAMES[0] =="N"
dummy= (0, 0, 0, 0, 0, 0)
table= [dummy]
cache= {0: dummy}
index= [0] *len(unicode.chars)

FILE="Modules/unicodedata_db.h"

print("--- Preparing", FILE, "...")

# 1) database properties

forcharinunicode.chars:
record=unicode.table[char]
ifrecord:
# extract database properties
category=CATEGORY_NAMES.index(record.general_category)
combining=int(record.canonical_combining_class)
bidirectional=BIDIRECTIONAL_NAMES.index(record.bidi_class)
mirrored=record.bidi_mirrored=="Y"
eastasianwidth=EASTASIANWIDTH_NAMES.index(record.east_asian_width)
normalizationquickcheck=record.quick_check
item= (
category, combining, bidirectional, mirrored, eastasianwidth,
normalizationquickcheck
 )
elifunicode.widths[char] isnotNone:
# an unassigned but reserved character, with a known
# east_asian_width
eastasianwidth=EASTASIANWIDTH_NAMES.index(unicode.widths[char])
item= (0, 0, 0, 0, eastasianwidth, 0)
else:
continue

# add entry to index and item tables
i=cache.get(item)
ifiisNone:
cache[item] =i=len(table)
table.append(item)
index[char] =i

# 2) decomposition data

decomp_data_cache= {}
decomp_data= [0]
decomp_prefix= [""]
decomp_index= [0] *len(unicode.chars)
decomp_size=0

comp_pairs= []
comp_first= [None] *len(unicode.chars)
comp_last= [None] *len(unicode.chars)

forcharinunicode.chars:
record=unicode.table[char]
ifrecord:
ifrecord.decomposition_type:
decomp=record.decomposition_type.split()
iflen(decomp) >19:
raiseException("character %x has a decomposition too large for nfd_nfkd"%char)
# prefix
ifdecomp[0][0] =="<":
prefix=decomp.pop(0)
else:
prefix=""
try:
i=decomp_prefix.index(prefix)
exceptValueError:
i=len(decomp_prefix)
decomp_prefix.append(prefix)
prefix=i
assertprefix<256
# content
decomp= [prefix+ (len(decomp)<<8)] + [int(s, 16) forsindecomp]
# Collect NFC pairs
ifnotprefixandlen(decomp) ==3and \
charnotinunicode.exclusionsand \
unicode.table[decomp[1]].canonical_combining_class=="0":
p, l, r=decomp
comp_first[l] =1
comp_last[r] =1
comp_pairs.append((l,r,char))
key=tuple(decomp)
i=decomp_data_cache.get(key, -1)
ifi==-1:
i=len(decomp_data)
decomp_data.extend(decomp)
decomp_size=decomp_size+len(decomp) *2
decomp_data_cache[key] =i
else:
assertdecomp_data[i:i+len(decomp)] ==decomp
else:
i=0
decomp_index[char] =i

f=l=0
comp_first_ranges= []
comp_last_ranges= []
prev_f=prev_l=None
foriinunicode.chars:
ifcomp_first[i] isnotNone:
comp_first[i] =f
f+=1
ifprev_fisNone:
prev_f= (i,i)
elifprev_f[1]+1==i:
prev_f=prev_f[0],i
else:
comp_first_ranges.append(prev_f)
prev_f= (i,i)
ifcomp_last[i] isnotNone:
comp_last[i] =l
l+=1
ifprev_lisNone:
prev_l= (i,i)
elifprev_l[1]+1==i:
prev_l=prev_l[0],i
else:
comp_last_ranges.append(prev_l)
prev_l= (i,i)
comp_first_ranges.append(prev_f)
comp_last_ranges.append(prev_l)
total_first=f
total_last=l

comp_data= [0]*(total_first*total_last)
forf,l,charincomp_pairs:
f=comp_first[f]
l=comp_last[l]
comp_data[f*total_last+l] =char

print(len(table), "unique properties")
print(len(decomp_prefix), "unique decomposition prefixes")
print(len(decomp_data), "unique decomposition entries:", end=' ')
print(decomp_size, "bytes")
print(total_first, "first characters in NFC")
print(total_last, "last characters in NFC")
print(len(comp_pairs), "NFC pairs")

print("--- Writing", FILE, "...")

withopen(FILE, "w") asfp:
fprint=partial(print, file=fp)

fprint("/* this file was generated by %s %s */"% (SCRIPT, VERSION))
fprint()
fprint('#define UNIDATA_VERSION "%s"'%UNIDATA_VERSION)
fprint("/* a list of unique database records */")
fprint("const _PyUnicode_DatabaseRecord _PyUnicode_Database_Records[] = {")
foritemintable:
fprint(" {%d, %d, %d, %d, %d, %d},"%item)
fprint("};")
fprint()

fprint("/* Reindexing of NFC first characters. */")
fprint("#define TOTAL_FIRST",total_first)
fprint("#define TOTAL_LAST",total_last)
fprint("struct reindex{int start;short count,index;};")
fprint("static struct reindex nfc_first[] = {")
forstart,endincomp_first_ranges:
fprint(" { %d, %d, %d},"% (start,end-start,comp_first[start]))
fprint(" {0,0,0}")
fprint("};\n")
fprint("static struct reindex nfc_last[] = {")
forstart,endincomp_last_ranges:
fprint(" { %d, %d, %d},"% (start,end-start,comp_last[start]))
fprint(" {0,0,0}")
fprint("};\n")

# FIXME: <fl> the following tables could be made static, and
# the support code moved into unicodedatabase.c

fprint("/* string literals */")
fprint("const char *_PyUnicode_CategoryNames[] = {")
fornameinCATEGORY_NAMES:
fprint(" \"%s\","%name)
fprint(" NULL")
fprint("};")

fprint("const char *_PyUnicode_BidirectionalNames[] = {")
fornameinBIDIRECTIONAL_NAMES:
fprint(" \"%s\","%name)
fprint(" NULL")
fprint("};")

fprint("const char *_PyUnicode_EastAsianWidthNames[] = {")
fornameinEASTASIANWIDTH_NAMES:
fprint(" \"%s\","%name)
fprint(" NULL")
fprint("};")

fprint("static const char *decomp_prefix[] = {")
fornameindecomp_prefix:
fprint(" \"%s\","%name)
fprint(" NULL")
fprint("};")

# split record index table
index1, index2, shift=splitbins(index, trace)

fprint("/* index tables for the database records */")
fprint("#define SHIFT", shift)
Array("index1", index1).dump(fp, trace)
Array("index2", index2).dump(fp, trace)

# split decomposition index table
index1, index2, shift=splitbins(decomp_index, trace)

fprint("/* decomposition data */")
Array("decomp_data", decomp_data).dump(fp, trace)

fprint("/* index tables for the decomposition data */")
fprint("#define DECOMP_SHIFT", shift)
Array("decomp_index1", index1).dump(fp, trace)
Array("decomp_index2", index2).dump(fp, trace)

index, index2, shift=splitbins(comp_data, trace)
fprint("/* NFC pairs */")
fprint("#define COMP_SHIFT", shift)
Array("comp_index", index).dump(fp, trace)
Array("comp_data", index2).dump(fp, trace)

# Generate delta tables for old versions
forversion, table, normalizationinunicode.changed:
cversion=version.replace(".","_")
records= [table[0]]
cache= {table[0]:0}
index= [0] *len(table)
fori, recordinenumerate(table):
try:
index[i] =cache[record]
exceptKeyError:
index[i] =cache[record] =len(records)
records.append(record)
index1, index2, shift=splitbins(index, trace)
fprint("static const change_record change_records_%s[] = {"%cversion)
forrecordinrecords:
fprint(" { %s },"%", ".join(map(str,record)))
fprint("};")
Array("changes_%s_index"%cversion, index1).dump(fp, trace)
Array("changes_%s_data"%cversion, index2).dump(fp, trace)
fprint("static const change_record* get_change_%s(Py_UCS4 n)"%cversion)
fprint("{")
fprint(" int index;")
fprint(" if (n >= 0x110000) index = 0;")
fprint(" else {")
fprint(" index = changes_%s_index[n>>%d];"% (cversion, shift))
fprint(" index = changes_%s_data[(index<<%d)+(n & %d)];"% \
 (cversion, shift, ((1<<shift)-1)))
fprint(" }")
fprint(" return change_records_%s+index;"%cversion)
fprint("}\n")
fprint("static Py_UCS4 normalization_%s(Py_UCS4 n)"%cversion)
fprint("{")
fprint(" switch(n) {")
fork, vinnormalization:
fprint(" case %s: return 0x%s;"% (hex(k), v))
fprint(" default: return 0;")
fprint(" }\n}\n")


# --------------------------------------------------------------------
# unicode character type tables

defmakeunicodetype(unicode, trace):

FILE="Objects/unicodetype_db.h"

print("--- Preparing", FILE, "...")

# extract unicode types
dummy= (0, 0, 0, 0, 0, 0)
table= [dummy]
cache= {dummy: 0}
index= [0] *len(unicode.chars)
numeric= {}
spaces= []
linebreaks= []
extra_casing= []

forcharinunicode.chars:
record=unicode.table[char]
ifrecord:
# extract database properties
category=record.general_category
bidirectional=record.bidi_class
properties=record.binary_properties
flags=0
ifcategoryin ["Lm", "Lt", "Lu", "Ll", "Lo"]:
flags|=ALPHA_MASK
if"Lowercase"inproperties:
flags|=LOWER_MASK
if'Line_Break'inpropertiesorbidirectional=="B":
flags|=LINEBREAK_MASK
linebreaks.append(char)
ifcategory=="Zs"orbidirectionalin ("WS", "B", "S"):
flags|=SPACE_MASK
spaces.append(char)
ifcategory=="Lt":
flags|=TITLE_MASK
if"Uppercase"inproperties:
flags|=UPPER_MASK
ifchar==ord(" ") orcategory[0] notin ("C", "Z"):
flags|=PRINTABLE_MASK
if"XID_Start"inproperties:
flags|=XID_START_MASK
if"XID_Continue"inproperties:
flags|=XID_CONTINUE_MASK
if"Cased"inproperties:
flags|=CASED_MASK
if"Case_Ignorable"inproperties:
flags|=CASE_IGNORABLE_MASK
sc=unicode.special_casing.get(char)
cf=unicode.case_folding.get(char, [char])
ifrecord.simple_uppercase_mapping:
upper=int(record.simple_uppercase_mapping, 16)
else:
upper=char
ifrecord.simple_lowercase_mapping:
lower=int(record.simple_lowercase_mapping, 16)
else:
lower=char
ifrecord.simple_titlecase_mapping:
title=int(record.simple_titlecase_mapping, 16)
else:
title=upper
ifscisNoneandcf!= [lower]:
sc= ([lower], [title], [upper])
ifscisNone:
ifupper==lower==title:
upper=lower=title=0
else:
upper=upper-char
lower=lower-char
title=title-char
assert (abs(upper) <=2147483647and
abs(lower) <=2147483647and
abs(title) <=2147483647)
else:
# This happens either when some character maps to more than one
# character in uppercase, lowercase, or titlecase or the
# casefolded version of the character is different from the
# lowercase. The extra characters are stored in a different
# array.
flags|=EXTENDED_CASE_MASK
lower=len(extra_casing) | (len(sc[0]) <<24)
extra_casing.extend(sc[0])
ifcf!=sc[0]:
lower|=len(cf) <<20
extra_casing.extend(cf)
upper=len(extra_casing) | (len(sc[2]) <<24)
extra_casing.extend(sc[2])
# Title is probably equal to upper.
ifsc[1] ==sc[2]:
title=upper
else:
title=len(extra_casing) | (len(sc[1]) <<24)
extra_casing.extend(sc[1])
# decimal digit, integer digit
decimal=0
ifrecord.decomposition_mapping:
flags|=DECIMAL_MASK
decimal=int(record.decomposition_mapping)
digit=0
ifrecord.numeric_type:
flags|=DIGIT_MASK
digit=int(record.numeric_type)
ifrecord.numeric_value:
flags|=NUMERIC_MASK
numeric.setdefault(record.numeric_value, []).append(char)
item= (
upper, lower, title, decimal, digit, flags
 )
# add entry to index and item tables
i=cache.get(item)
ifiisNone:
cache[item] =i=len(table)
table.append(item)
index[char] =i

print(len(table), "unique character type entries")
print(sum(map(len, numeric.values())), "numeric code points")
print(len(spaces), "whitespace code points")
print(len(linebreaks), "linebreak code points")
print(len(extra_casing), "extended case array")

print("--- Writing", FILE, "...")

withopen(FILE, "w") asfp:
fprint=partial(print, file=fp)

fprint("/* this file was generated by %s %s */"% (SCRIPT, VERSION))
fprint()
fprint("/* a list of unique character type descriptors */")
fprint("const _PyUnicode_TypeRecord _PyUnicode_TypeRecords[] = {")
foritemintable:
fprint(" {%d, %d, %d, %d, %d, %d},"%item)
fprint("};")
fprint()

fprint("/* extended case mappings */")
fprint()
fprint("const Py_UCS4 _PyUnicode_ExtendedCase[] = {")
forcinextra_casing:
fprint(" %d,"%c)
fprint("};")
fprint()

# split decomposition index table
index1, index2, shift=splitbins(index, trace)

fprint("/* type indexes */")
fprint("#define SHIFT", shift)
Array("index1", index1).dump(fp, trace)
Array("index2", index2).dump(fp, trace)

# Generate code for _PyUnicode_ToNumeric()
numeric_items=sorted(numeric.items())
fprint('/* Returns the numeric value as double for Unicode characters')
fprint(' * having this property, -1.0 otherwise.')
fprint(' */')
fprint('double _PyUnicode_ToNumeric(Py_UCS4 ch)')
fprint('{')
fprint(' switch (ch) {')
forvalue, codepointsinnumeric_items:
# Turn text into float literals
parts=value.split('/')
parts= [repr(float(part)) forpartinparts]
value='/'.join(parts)

codepoints.sort()
forcodepointincodepoints:
fprint(' case 0x%04X:'% (codepoint,))
fprint(' return (double) %s;'% (value,))
fprint(' }')
fprint(' return -1.0;')
fprint('}')
fprint()

# Generate code for _PyUnicode_IsWhitespace()
fprint("/* Returns 1 for Unicode characters having the bidirectional")
fprint(" * type 'WS', 'B' or 'S' or the category 'Zs', 0 otherwise.")
fprint(" */")
fprint('int _PyUnicode_IsWhitespace(const Py_UCS4 ch)')
fprint('{')
fprint(' switch (ch) {')

forcodepointinsorted(spaces):
fprint(' case 0x%04X:'% (codepoint,))
fprint(' return 1;')

fprint(' }')
fprint(' return 0;')
fprint('}')
fprint()

# Generate code for _PyUnicode_IsLinebreak()
fprint("/* Returns 1 for Unicode characters having the line break")
fprint(" * property 'BK', 'CR', 'LF' or 'NL' or having bidirectional")
fprint(" * type 'B', 0 otherwise.")
fprint(" */")
fprint('int _PyUnicode_IsLinebreak(const Py_UCS4 ch)')
fprint('{')
fprint(' switch (ch) {')
forcodepointinsorted(linebreaks):
fprint(' case 0x%04X:'% (codepoint,))
fprint(' return 1;')

fprint(' }')
fprint(' return 0;')
fprint('}')
fprint()


# --------------------------------------------------------------------
# unicode name database

defmakeunicodename(unicode, trace):
fromdawgimportbuild_compression_dawg

FILE="Modules/unicodename_db.h"

print("--- Preparing", FILE, "...")

# unicode name hash table

# extract names
data= []
forcharinunicode.chars:
record=unicode.table[char]
ifrecord:
name=record.name.strip()
ifnameandname[0] !="<":
data.append((name, char))

print("--- Writing", FILE, "...")

withopen(FILE, "w") asfp:
fprint=partial(print, file=fp)

fprint("/* this file was generated by %s %s */"% (SCRIPT, VERSION))
fprint()
fprint("#define NAME_MAXLEN", 256)
assertmax(len(x) forxindata) <256
fprint()

fprint("/* name->code dictionary */")
packed_dawg, pos_to_codepoint=build_compression_dawg(data)
notfound=len(pos_to_codepoint)
inverse_list= [notfound] *len(unicode.chars)
forpos, codepointinenumerate(pos_to_codepoint):
inverse_list[codepoint] =pos
Array("packed_name_dawg", list(packed_dawg)).dump(fp, trace)
Array("dawg_pos_to_codepoint", pos_to_codepoint).dump(fp, trace)
index1, index2, shift=splitbins(inverse_list, trace)
fprint("#define DAWG_CODEPOINT_TO_POS_SHIFT", shift)
fprint("#define DAWG_CODEPOINT_TO_POS_NOTFOUND", notfound)
Array("dawg_codepoint_to_pos_index1", index1).dump(fp, trace)
Array("dawg_codepoint_to_pos_index2", index2).dump(fp, trace)

fprint()
fprint('static const unsigned int aliases_start = %#x;'%
NAME_ALIASES_START)
fprint('static const unsigned int aliases_end = %#x;'%
 (NAME_ALIASES_START+len(unicode.aliases)))

fprint('static const unsigned int name_aliases[] = {')
forname, codepointinunicode.aliases:
fprint(' 0x%04X,'%codepoint)
fprint('};')

# In Unicode 6.0.0, the sequences contain at most 4 BMP chars,
# so we are using Py_UCS2 seq[4]. This needs to be updated if longer
# sequences or sequences with non-BMP chars are added.
# unicodedata_lookup should be adapted too.
fprint(dedent("""
 typedef struct NamedSequence {
 int seqlen;
 Py_UCS2 seq[4];
 } named_sequence;
 """))

fprint('static const unsigned int named_sequences_start = %#x;'%
NAMED_SEQUENCES_START)
fprint('static const unsigned int named_sequences_end = %#x;'%
 (NAMED_SEQUENCES_START+len(unicode.named_sequences)))

fprint('static const named_sequence named_sequences[] = {')
forname, sequenceinunicode.named_sequences:
seq_str=', '.join('0x%04X'%cpforcpinsequence)
fprint(' {%d, {%s}},'% (len(sequence), seq_str))
fprint('};')


defmerge_old_version(version, new, old):
# Changes to exclusion file not implemented yet
ifold.exclusions!=new.exclusions:
raiseNotImplementedError("exclusions differ")

# In these change records, 0xFF means "no change"
bidir_changes= [0xFF]*0x110000
category_changes= [0xFF]*0x110000
decimal_changes= [0xFF]*0x110000
mirrored_changes= [0xFF]*0x110000
east_asian_width_changes= [0xFF]*0x110000
# In numeric data, 0 means "no change",
# -1 means "did not have a numeric value
numeric_changes= [0] *0x110000
# normalization_changes is a list of key-value pairs
normalization_changes= []
foriinrange(0x110000):
ifnew.table[i] isNone:
# Characters unassigned in the new version ought to
# be unassigned in the old one
assertold.table[i] isNone
continue
# check characters unassigned in the old version
ifold.table[i] isNone:
# category 0 is "unassigned"
category_changes[i] =0
continue
# check characters that differ
ifold.table[i] !=new.table[i]:
fork, fieldinenumerate(dataclasses.fields(UcdRecord)):
value=getattr(old.table[i], field.name)
new_value=getattr(new.table[i], field.name)
ifvalue!=new_value:
ifk==1andiinPUA_15:
# the name is not set in the old.table, but in the
# new.table we are using it for aliases and named seq
assertvalue==''
elifk==2:
category_changes[i] =CATEGORY_NAMES.index(value)
elifk==4:
bidir_changes[i] =BIDIRECTIONAL_NAMES.index(value)
elifk==5:
# We assume that all normalization changes are in 1:1 mappings
assert" "notinvalue
normalization_changes.append((i, value))
elifk==6:
# we only support changes where the old value is a single digit
assertvaluein"0123456789"
decimal_changes[i] =int(value)
elifk==8:
# Since 0 encodes "no change", the old value is better not 0
ifnotvalue:
numeric_changes[i] =-1
else:
numeric_changes[i] =float(value)
assertnumeric_changes[i] notin (0, -1)
elifk==9:
ifvalue=='Y':
mirrored_changes[i] ='1'
else:
mirrored_changes[i] ='0'
elifk==11:
# change to ISO comment, ignore
pass
elifk==12:
# change to simple uppercase mapping; ignore
pass
elifk==13:
# change to simple lowercase mapping; ignore
pass
elifk==14:
# change to simple titlecase mapping; ignore
pass
elifk==15:
# change to east asian width
east_asian_width_changes[i] =EASTASIANWIDTH_NAMES.index(value)
elifk==16:
# derived property changes; not yet
pass
elifk==17:
# normalization quickchecks are not performed
# for older versions
pass
else:
classDifference(Exception):pass
raiseDifference(hex(i), k, old.table[i], new.table[i])
new.changed.append((version, list(zip(bidir_changes, category_changes,
decimal_changes, mirrored_changes,
east_asian_width_changes,
numeric_changes)),
normalization_changes))


DATA_DIR=os.path.join('Tools', 'unicode', 'data')

defopen_data(template, version):
local=os.path.join(DATA_DIR, template% ('-'+version,))
ifnotos.path.exists(local):
importurllib.request
ifversion=='3.2.0':
# irregular url structure
url= ('https://www.unicode.org/Public/3.2-Update/'+template) % ('-'+version,)
else:
url= ('https://www.unicode.org/Public/%s/ucd/'+template) % (version, '')
os.makedirs(DATA_DIR, exist_ok=True)
urllib.request.urlretrieve(url, filename=local)
iflocal.endswith('.txt'):
returnopen(local, encoding='utf-8')
else:
# Unihan.zip
returnopen(local, 'rb')


defexpand_range(char_range: str) ->Iterator[int]:
'''
 Parses ranges of code points, as described in UAX #44:
 https://www.unicode.org/reports/tr44/#Code_Point_Ranges
 '''
if'..'inchar_range:
first, last= [int(c, 16) forcinchar_range.split('..')]
else:
first=last=int(char_range, 16)
forcharinrange(first, last+1):
yieldchar


classUcdFile:
'''
 A file in the standard format of the UCD.

 See: https://www.unicode.org/reports/tr44/#Format_Conventions

 Note that, as described there, the Unihan data files have their
 own separate format.
 '''

def__init__(self, template: str, version: str) ->None:
self.template=template
self.version=version

defrecords(self) ->Iterator[List[str]]:
withopen_data(self.template, self.version) asfile:
forlineinfile:
line=line.split('#', 1)[0].strip()
ifnotline:
continue
yield [field.strip() forfieldinline.split(';')]

def__iter__(self) ->Iterator[List[str]]:
returnself.records()

defexpanded(self) ->Iterator[Tuple[int, List[str]]]:
forrecordinself.records():
char_range, rest=record[0], record[1:]
forcharinexpand_range(char_range):
yieldchar, rest


@dataclasses.dataclass
classUcdRecord:
# 15 fields from UnicodeData.txt . See:
# https://www.unicode.org/reports/tr44/#UnicodeData.txt
codepoint: str
name: str
general_category: str
canonical_combining_class: str
bidi_class: str
decomposition_type: str
decomposition_mapping: str
numeric_type: str
numeric_value: str
bidi_mirrored: str
unicode_1_name: str# obsolete
iso_comment: str# obsolete
simple_uppercase_mapping: str
simple_lowercase_mapping: str
simple_titlecase_mapping: str

# https://www.unicode.org/reports/tr44/#EastAsianWidth.txt
east_asian_width: Optional[str]

# Binary properties, as a set of those that are true.
# Taken from multiple files:
# https://www.unicode.org/reports/tr44/#DerivedCoreProperties.txt
# https://www.unicode.org/reports/tr44/#LineBreak.txt
binary_properties: Set[str]

# The Quick_Check properties related to normalization:
# https://www.unicode.org/reports/tr44/#Decompositions_and_Normalization
# We store them as a bitmask.
quick_check: int


deffrom_row(row: List[str]) ->UcdRecord:
returnUcdRecord(*row, None, set(), 0)


# --------------------------------------------------------------------
# the following support code is taken from the unidb utilities
# Copyright (c) 1999-2000 by Secret Labs AB

# load a unicode-data file from disk

classUnicodeData:
# table: List[Optional[UcdRecord]] # index is codepoint; None means unassigned

def__init__(self, version, cjk_check=True):
self.changed= []
table= [None] *0x110000
forsinUcdFile(UNICODE_DATA, version):
char=int(s[0], 16)
table[char] =from_row(s)

cjk_ranges_found= []

# expand first-last ranges
field=None
foriinrange(0, 0x110000):
# The file UnicodeData.txt has its own distinct way of
# expressing ranges. See:
# https://www.unicode.org/reports/tr44/#Code_Point_Ranges
s=table[i]
ifs:
ifs.name[-6:] =="First>":
s.name=""
field=dataclasses.astuple(s)[:15]
elifs.name[-5:] =="Last>":
ifs.name.startswith("<CJK Ideograph"):
cjk_ranges_found.append((field[0],
s.codepoint))
s.name=""
field=None
eliffield:
table[i] =from_row(('%X'%i,) +field[1:])
ifcjk_checkandcjk_ranges!=cjk_ranges_found:
raiseValueError("CJK ranges deviate: have %r"%cjk_ranges_found)

# public attributes
self.filename=UNICODE_DATA%''
self.table=table
self.chars=list(range(0x110000)) # unicode 3.2

# check for name aliases and named sequences, see #12753
# aliases and named sequences are not in 3.2.0
ifversion!='3.2.0':
self.aliases= []
# store aliases in the Private Use Area 15, in range U+F0000..U+F00FF,
# in order to take advantage of the compression and lookup
# algorithms used for the other characters
pua_index=NAME_ALIASES_START
forchar, name, abbrevinUcdFile(NAME_ALIASES, version):
char=int(char, 16)
self.aliases.append((name, char))
# also store the name in the PUA 1
self.table[pua_index].name=name
pua_index+=1
assertpua_index-NAME_ALIASES_START==len(self.aliases)

self.named_sequences= []
# store named sequences in the PUA 1, in range U+F0100..,
# in order to take advantage of the compression and lookup
# algorithms used for the other characters.

assertpua_index<NAMED_SEQUENCES_START
pua_index=NAMED_SEQUENCES_START
forname, charsinUcdFile(NAMED_SEQUENCES, version):
chars=tuple(int(char, 16) forcharinchars.split())
# check that the structure defined in makeunicodename is OK
assert2<=len(chars) <=4, "change the Py_UCS2 array size"
assertall(c<=0xFFFFforcinchars), ("use Py_UCS4 in "
"the NamedSequence struct and in unicodedata_lookup")
self.named_sequences.append((name, chars))
# also store these in the PUA 1
self.table[pua_index].name=name
pua_index+=1
assertpua_index-NAMED_SEQUENCES_START==len(self.named_sequences)

self.exclusions= {}
forchar, inUcdFile(COMPOSITION_EXCLUSIONS, version):
char=int(char, 16)
self.exclusions[char] =1

widths= [None] *0x110000
forchar, (width,) inUcdFile(EASTASIAN_WIDTH, version).expanded():
widths[char] =width

foriinrange(0, 0x110000):
iftable[i] isnotNone:
table[i].east_asian_width=widths[i]
self.widths=widths

forchar, (propname, *propinfo) inUcdFile(DERIVED_CORE_PROPERTIES, version).expanded():
ifpropinfo:
# this is not a binary property, ignore it
continue

iftable[char]:
# Some properties (e.g. Default_Ignorable_Code_Point)
# apply to unassigned code points; ignore them