[gen-ucd-table.py] black

This commit is contained in:
Behdad Esfahbod committed 2025-05-12 15:27:51 -06:00
1 parent d5c4b1eed7
commit 083ca198d2
2 files changed
+116 -71

No files matched your search

+114 -69
View File
@@ -12,27 +12,28 @@ import packTab.ucdxml
import sys, re
import logging
logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO)
if len(sys.argv) > 1 and sys.argv[1] == '--rust':
logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.INFO)
if len(sys.argv) > 1 and sys.argv[1] == "--rust":
del sys.argv[1]
logging.info('Generating Rust code...')
logging.info("Generating Rust code...")
language = "rust"
else:
logging.info('Generating C code...')
logging.info("Generating C code...")
language = "c"
language = packTab.languages[language]
if len (sys.argv) not in (2, 3):
sys.exit (__doc__)
if len(sys.argv) not in (2, 3):
sys.exit(__doc__)
logging.info('Loading UCDXML...')
logging.info("Loading UCDXML...")
ucdxml = packTab.ucdxml.load_ucdxml(sys.argv[1])
ucd = packTab.ucdxml.ucdxml_get_repertoire(ucdxml)
hb_script_list_h = 'hb-script-list.h' if len (sys.argv) < 3 else sys.argv[2]
hb_script_list_h = "hb-script-list.h" if len(sys.argv) < 3 else sys.argv[2]
logging.info('Preparing data tables...')
logging.info("Preparing data tables...")
# This is how the data is encoded:
@@ -46,41 +47,54 @@ logging.info('Preparing data tables...')
# Composition & Decomposition (dm) are encoded elaborately,
# as discussed below.
gc = [u['gc'] for u in ucd]
ccc = [int(u['ccc']) for u in ucd]
bmg = [int(v, 16) - int(u) if v else 0 for u,v in enumerate(u['bmg'] for u in ucd)]
sc = [u['sc'] for u in ucd]
gc = [u["gc"] for u in ucd]
ccc = [int(u["ccc"]) for u in ucd]
bmg = [int(v, 16) - int(u) if v else 0 for u, v in enumerate(u["bmg"] for u in ucd)]
sc = [u["sc"] for u in ucd]
# Prepare Compose / Decompose data
#
# This code is very dense. See hb_ucd_compose() / hb_ucd_decompose() for the logic.
dm = {i:tuple(int(v, 16) for v in u['dm'].split()) for i,u in enumerate(ucd)
if u['dm'] != '#' and u['dt'] == 'can' and not (0xAC00 <= i < 0xAC00+11172)}
ce = {i for i,u in enumerate(ucd) if u['Comp_Ex'] == 'Y'}
dm = {
i: tuple(int(v, 16) for v in u["dm"].split())
for i, u in enumerate(ucd)
if u["dm"] != "#" and u["dt"] == "can" and not (0xAC00 <= i < 0xAC00 + 11172)
}
ce = {i for i, u in enumerate(ucd) if u["Comp_Ex"] == "Y"}
assert not any(v for v in dm.values() if len(v) not in (1,2))
assert not any(v for v in dm.values() if len(v) not in (1, 2))
dm1 = sorted(set(v for v in dm.values() if len(v) == 1))
assert all((v[0] >> 16) in (0,2) for v in dm1)
dm1_p0_array = ['0x%04Xu' % (v[0] & 0xFFFF) for v in dm1 if (v[0] >> 16) == 0]
dm1_p2_array = ['0x%04Xu' % (v[0] & 0xFFFF) for v in dm1 if (v[0] >> 16) == 2]
dm1_order = {v:i+1 for i,v in enumerate(dm1)}
assert all((v[0] >> 16) in (0, 2) for v in dm1)
dm1_p0_array = ["0x%04Xu" % (v[0] & 0xFFFF) for v in dm1 if (v[0] >> 16) == 0]
dm1_p2_array = ["0x%04Xu" % (v[0] & 0xFFFF) for v in dm1 if (v[0] >> 16) == 2]
dm1_order = {v: i + 1 for i, v in enumerate(dm1)}
dm2 = sorted((v+(i if i not in ce and not ccc[i] else 0,), v)
for i,v in dm.items() if len(v) == 2)
dm2 = sorted(
(v + (i if i not in ce and not ccc[i] else 0,), v)
for i, v in dm.items()
if len(v) == 2
)
filt = lambda v: ((v[0] & 0xFFFFF800) == 0x0000 and
(v[1] & 0xFFFFFF80) == 0x0300 and
(v[2] & 0xFFF0C000) == 0x0000)
filt = lambda v: (
(v[0] & 0xFFFFF800) == 0x0000
and (v[1] & 0xFFFFFF80) == 0x0300
and (v[2] & 0xFFF0C000) == 0x0000
)
dm2_u32_array = [v for v in dm2 if filt(v[0])]
dm2_u64_array = [v for v in dm2 if not filt(v[0])]
assert dm2_u32_array + dm2_u64_array == dm2
dm2_u32_array = ["HB_CODEPOINT_ENCODE3_11_7_14 (0x%04Xu, 0x%04Xu, 0x%04Xu)" % v[0] for v in dm2_u32_array]
dm2_u64_array = ["HB_CODEPOINT_ENCODE3 (0x%04Xu, 0x%04Xu, 0x%04Xu)" % v[0] for v in dm2_u64_array]
dm2_u32_array = [
"HB_CODEPOINT_ENCODE3_11_7_14 (0x%04Xu, 0x%04Xu, 0x%04Xu)" % v[0]
for v in dm2_u32_array
]
dm2_u64_array = [
"HB_CODEPOINT_ENCODE3 (0x%04Xu, 0x%04Xu, 0x%04Xu)" % v[0] for v in dm2_u64_array
]
l = 1 + len(dm1_p0_array) + len(dm1_p2_array)
dm2_order = {v[1]:i+l for i,v in enumerate(dm2)}
dm2_order = {v[1]: i + l for i, v in enumerate(dm2)}
dm_order = {None: 0}
dm_order.update(dm1_order)
@@ -90,9 +104,40 @@ dm_order.update(dm2_order)
# Prepare General_Category / Script mapping arrays
gc_order = dict()
for i,v in enumerate(('Cc', 'Cf', 'Cn', 'Co', 'Cs', 'Ll', 'Lm', 'Lo', 'Lt', 'Lu',
'Mc', 'Me', 'Mn', 'Nd', 'Nl', 'No', 'Pc', 'Pd', 'Pe', 'Pf',
'Pi', 'Po', 'Ps', 'Sc', 'Sk', 'Sm', 'So', 'Zl', 'Zp', 'Zs',)):
for i, v in enumerate(
(
"Cc",
"Cf",
"Cn",
"Co",
"Cs",
"Ll",
"Lm",
"Lo",
"Lt",
"Lu",
"Mc",
"Me",
"Mn",
"Nd",
"Nl",
"No",
"Pc",
"Pd",
"Pe",
"Pf",
"Pi",
"Po",
"Ps",
"Sc",
"Sk",
"Sm",
"So",
"Zl",
"Zp",
"Zs",
)
):
gc_order[i] = v
gc_order[v] = i
@@ -100,10 +145,11 @@ sc_order = dict()
sc_array = []
sc_re = re.compile(r"\b(HB_SCRIPT_[_A-Z]*).*HB_TAG [(]'(.)','(.)','(.)','(.)'[)]")
for line in open(hb_script_list_h):
m = sc_re.search (line)
if not m: continue
m = sc_re.search(line)
if not m:
continue
name = m.group(1)
tag = ''.join(m.group(i) for i in range(2, 6))
tag = "".join(m.group(i) for i in range(2, 6))
i = len(sc_array)
sc_order[tag] = i
sc_order[i] = tag
@@ -112,17 +158,17 @@ for line in open(hb_script_list_h):
# Write out main data
DEFAULT = 'DEFAULT'
COMPACT = 'COMPACT'
SLOPPY = 'SLOPPY'
DEFAULT = "DEFAULT"
COMPACT = "COMPACT"
SLOPPY = "SLOPPY"
compression_level = {
DEFAULT: 5,
COMPACT: 9,
SLOPPY: 9,
SLOPPY: 9,
}
logging.info('Generating output...')
logging.info("Generating output...")
print("/* == Start of generated table == */")
print("/*")
print(" * The following table is generated by running:")
@@ -148,20 +194,20 @@ uint16_t = language.type_name("u16")
uint32_t = language.type_name("u32")
uint64_t = language.type_name("u64")
code = packTab.Code('_hb_ucd')
sc_array, _ = code.addArray('hb_script_t', 'sc_map', sc_array)
dm1_p0_array, _ = code.addArray(uint16_t, 'dm1_p0_map', dm1_p0_array)
dm1_p2_array, _ = code.addArray(uint16_t, 'dm1_p2_map', dm1_p2_array)
dm2_u32_array, _ = code.addArray(uint32_t, 'dm2_u32_map', dm2_u32_array)
dm2_u64_array, _ = code.addArray(uint64_t, 'dm2_u64_map', dm2_u64_array)
code = packTab.Code("_hb_ucd")
sc_array, _ = code.addArray("hb_script_t", "sc_map", sc_array)
dm1_p0_array, _ = code.addArray(uint16_t, "dm1_p0_map", dm1_p0_array)
dm1_p2_array, _ = code.addArray(uint16_t, "dm1_p2_map", dm1_p2_array)
dm2_u32_array, _ = code.addArray(uint32_t, "dm2_u32_map", dm2_u32_array)
dm2_u64_array, _ = code.addArray(uint64_t, "dm2_u64_map", dm2_u64_array)
code.print_code(linkage=language.private_function_linkage, language=language)
datasets = [
('gc', gc, 'Cn', gc_order),
('ccc', ccc, 0, None),
('bmg', bmg, 0, None),
('sc', sc, 'Zzzz', sc_order),
('dm', dm, None, dm_order),
("gc", gc, "Cn", gc_order),
("ccc", ccc, 0, None),
("bmg", bmg, 0, None),
("sc", sc, "Zzzz", sc_order),
("dm", dm, None, dm_order),
]
@@ -169,12 +215,12 @@ datasets = [
modes = {}
if language.name == "c":
modes[DEFAULT] = '#ifndef HB_OPTIMIZE_SIZE'
modes[COMPACT] = '#elif !defined(HB_NO_UCD_UNASSIGNED)'
modes[SLOPPY] = '#else'
modes[None] = '#endif'
modes[DEFAULT] = "#ifndef HB_OPTIMIZE_SIZE"
modes[COMPACT] = "#elif !defined(HB_NO_UCD_UNASSIGNED)"
modes[SLOPPY] = "#else"
modes[None] = "#endif"
else:
modes[DEFAULT] = ''
modes[DEFAULT] = ""
for step, text in modes.items():
print()
@@ -185,38 +231,37 @@ for step, text in modes.items():
continue
compression = compression_level[step]
logging.info(' Compression=%d:' % compression)
logging.info(" Compression=%d:" % compression)
if step == SLOPPY:
for i in range(len(gc)):
if (i % 128) and gc[i] == 'Cn':
if (i % 128) and gc[i] == "Cn":
gc[i] = gc[i - 1]
for i in range(len(gc) - 2, -1, -1):
if ((i + 1) % 128) and gc[i] == 'Cn':
if ((i + 1) % 128) and gc[i] == "Cn":
gc[i] = gc[i + 1]
for i in range(len(sc)):
if (i % 128) and sc[i] == 'Zzzz':
if (i % 128) and sc[i] == "Zzzz":
sc[i] = sc[i - 1]
for i in range(len(sc) - 2, -1, -1):
if ((i + 1) % 128) and sc[i] == 'Zzzz':
if ((i + 1) % 128) and sc[i] == "Zzzz":
sc[i] = sc[i + 1]
code = packTab.Code("_hb_ucd")
code = packTab.Code('_hb_ucd')
for name,data,default,mapping in datasets:
sol = packTab.pack_table(data, default, mapping=mapping, compression=compression)
logging.info(' Dataset=%-8s FullCost=%d' % (name, sol.fullCost))
for name, data, default, mapping in datasets:
sol = packTab.pack_table(
data, default, mapping=mapping, compression=compression
)
logging.info(" Dataset=%-8s FullCost=%d" % (name, sol.fullCost))
sol.genCode(code, name)
code.print_code(linkage=language.private_function_linkage, language=language)
print()
if language.name == "c":
print()
print("#endif /* HB_UCD_TABLE_HH */")
print()
print("/* == End of generated table == */")
logging.info('Done.')
logging.info("Done.")
+2 -2
View File
@@ -2,7 +2,7 @@
/*
* The following table is generated by running:
*
* ./gen-ucd-table.py ucd.nounihan.grouped.xml
* ./gen-ucd-table.py ucd.nounihan.grouped.xml hb-script-list.h
*
* on file with this description: Unicode 16.0.0
*/
@@ -5849,8 +5849,8 @@ static inline uint16_t _hb_ucd_dm (unsigned u)
return u<195102 ? _hb_ucd_u16[1656+(((_hb_ucd_u8[12834+(((_hb_ucd_u8[12452+(u>>4>>5)])<<5)+((u>>4)&31))])<<4)+((u)&15))] : 0;
}
#endif
#endif
#endif /* HB_UCD_TABLE_HH */