mirror of
https://github.com/storytold/harfbuzz.git
synced 2026-11-22 15:31:05 +01:00
[gen-ucd-table.py] black
This commit is contained in:
1 parent
d5c4b1eed7
commit
083ca198d2
2 files changed
+116
-71
No files matched your search
+114
-69
@@ -12,27 +12,28 @@ import packTab.ucdxml
|
||||
|
||||
import sys, re
|
||||
import logging
|
||||
logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO)
|
||||
|
||||
if len(sys.argv) > 1 and sys.argv[1] == '--rust':
|
||||
logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.INFO)
|
||||
|
||||
if len(sys.argv) > 1 and sys.argv[1] == "--rust":
|
||||
del sys.argv[1]
|
||||
logging.info('Generating Rust code...')
|
||||
logging.info("Generating Rust code...")
|
||||
language = "rust"
|
||||
else:
|
||||
logging.info('Generating C code...')
|
||||
logging.info("Generating C code...")
|
||||
language = "c"
|
||||
language = packTab.languages[language]
|
||||
|
||||
if len (sys.argv) not in (2, 3):
|
||||
sys.exit (__doc__)
|
||||
if len(sys.argv) not in (2, 3):
|
||||
sys.exit(__doc__)
|
||||
|
||||
logging.info('Loading UCDXML...')
|
||||
logging.info("Loading UCDXML...")
|
||||
ucdxml = packTab.ucdxml.load_ucdxml(sys.argv[1])
|
||||
ucd = packTab.ucdxml.ucdxml_get_repertoire(ucdxml)
|
||||
|
||||
hb_script_list_h = 'hb-script-list.h' if len (sys.argv) < 3 else sys.argv[2]
|
||||
hb_script_list_h = "hb-script-list.h" if len(sys.argv) < 3 else sys.argv[2]
|
||||
|
||||
logging.info('Preparing data tables...')
|
||||
logging.info("Preparing data tables...")
|
||||
|
||||
|
||||
# This is how the data is encoded:
|
||||
@@ -46,41 +47,54 @@ logging.info('Preparing data tables...')
|
||||
# Composition & Decomposition (dm) are encoded elaborately,
|
||||
# as discussed below.
|
||||
|
||||
gc = [u['gc'] for u in ucd]
|
||||
ccc = [int(u['ccc']) for u in ucd]
|
||||
bmg = [int(v, 16) - int(u) if v else 0 for u,v in enumerate(u['bmg'] for u in ucd)]
|
||||
sc = [u['sc'] for u in ucd]
|
||||
gc = [u["gc"] for u in ucd]
|
||||
ccc = [int(u["ccc"]) for u in ucd]
|
||||
bmg = [int(v, 16) - int(u) if v else 0 for u, v in enumerate(u["bmg"] for u in ucd)]
|
||||
sc = [u["sc"] for u in ucd]
|
||||
|
||||
|
||||
# Prepare Compose / Decompose data
|
||||
#
|
||||
# This code is very dense. See hb_ucd_compose() / hb_ucd_decompose() for the logic.
|
||||
|
||||
dm = {i:tuple(int(v, 16) for v in u['dm'].split()) for i,u in enumerate(ucd)
|
||||
if u['dm'] != '#' and u['dt'] == 'can' and not (0xAC00 <= i < 0xAC00+11172)}
|
||||
ce = {i for i,u in enumerate(ucd) if u['Comp_Ex'] == 'Y'}
|
||||
dm = {
|
||||
i: tuple(int(v, 16) for v in u["dm"].split())
|
||||
for i, u in enumerate(ucd)
|
||||
if u["dm"] != "#" and u["dt"] == "can" and not (0xAC00 <= i < 0xAC00 + 11172)
|
||||
}
|
||||
ce = {i for i, u in enumerate(ucd) if u["Comp_Ex"] == "Y"}
|
||||
|
||||
assert not any(v for v in dm.values() if len(v) not in (1,2))
|
||||
assert not any(v for v in dm.values() if len(v) not in (1, 2))
|
||||
dm1 = sorted(set(v for v in dm.values() if len(v) == 1))
|
||||
assert all((v[0] >> 16) in (0,2) for v in dm1)
|
||||
dm1_p0_array = ['0x%04Xu' % (v[0] & 0xFFFF) for v in dm1 if (v[0] >> 16) == 0]
|
||||
dm1_p2_array = ['0x%04Xu' % (v[0] & 0xFFFF) for v in dm1 if (v[0] >> 16) == 2]
|
||||
dm1_order = {v:i+1 for i,v in enumerate(dm1)}
|
||||
assert all((v[0] >> 16) in (0, 2) for v in dm1)
|
||||
dm1_p0_array = ["0x%04Xu" % (v[0] & 0xFFFF) for v in dm1 if (v[0] >> 16) == 0]
|
||||
dm1_p2_array = ["0x%04Xu" % (v[0] & 0xFFFF) for v in dm1 if (v[0] >> 16) == 2]
|
||||
dm1_order = {v: i + 1 for i, v in enumerate(dm1)}
|
||||
|
||||
dm2 = sorted((v+(i if i not in ce and not ccc[i] else 0,), v)
|
||||
for i,v in dm.items() if len(v) == 2)
|
||||
dm2 = sorted(
|
||||
(v + (i if i not in ce and not ccc[i] else 0,), v)
|
||||
for i, v in dm.items()
|
||||
if len(v) == 2
|
||||
)
|
||||
|
||||
filt = lambda v: ((v[0] & 0xFFFFF800) == 0x0000 and
|
||||
(v[1] & 0xFFFFFF80) == 0x0300 and
|
||||
(v[2] & 0xFFF0C000) == 0x0000)
|
||||
filt = lambda v: (
|
||||
(v[0] & 0xFFFFF800) == 0x0000
|
||||
and (v[1] & 0xFFFFFF80) == 0x0300
|
||||
and (v[2] & 0xFFF0C000) == 0x0000
|
||||
)
|
||||
dm2_u32_array = [v for v in dm2 if filt(v[0])]
|
||||
dm2_u64_array = [v for v in dm2 if not filt(v[0])]
|
||||
assert dm2_u32_array + dm2_u64_array == dm2
|
||||
dm2_u32_array = ["HB_CODEPOINT_ENCODE3_11_7_14 (0x%04Xu, 0x%04Xu, 0x%04Xu)" % v[0] for v in dm2_u32_array]
|
||||
dm2_u64_array = ["HB_CODEPOINT_ENCODE3 (0x%04Xu, 0x%04Xu, 0x%04Xu)" % v[0] for v in dm2_u64_array]
|
||||
dm2_u32_array = [
|
||||
"HB_CODEPOINT_ENCODE3_11_7_14 (0x%04Xu, 0x%04Xu, 0x%04Xu)" % v[0]
|
||||
for v in dm2_u32_array
|
||||
]
|
||||
dm2_u64_array = [
|
||||
"HB_CODEPOINT_ENCODE3 (0x%04Xu, 0x%04Xu, 0x%04Xu)" % v[0] for v in dm2_u64_array
|
||||
]
|
||||
|
||||
l = 1 + len(dm1_p0_array) + len(dm1_p2_array)
|
||||
dm2_order = {v[1]:i+l for i,v in enumerate(dm2)}
|
||||
dm2_order = {v[1]: i + l for i, v in enumerate(dm2)}
|
||||
|
||||
dm_order = {None: 0}
|
||||
dm_order.update(dm1_order)
|
||||
@@ -90,9 +104,40 @@ dm_order.update(dm2_order)
|
||||
# Prepare General_Category / Script mapping arrays
|
||||
|
||||
gc_order = dict()
|
||||
for i,v in enumerate(('Cc', 'Cf', 'Cn', 'Co', 'Cs', 'Ll', 'Lm', 'Lo', 'Lt', 'Lu',
|
||||
'Mc', 'Me', 'Mn', 'Nd', 'Nl', 'No', 'Pc', 'Pd', 'Pe', 'Pf',
|
||||
'Pi', 'Po', 'Ps', 'Sc', 'Sk', 'Sm', 'So', 'Zl', 'Zp', 'Zs',)):
|
||||
for i, v in enumerate(
|
||||
(
|
||||
"Cc",
|
||||
"Cf",
|
||||
"Cn",
|
||||
"Co",
|
||||
"Cs",
|
||||
"Ll",
|
||||
"Lm",
|
||||
"Lo",
|
||||
"Lt",
|
||||
"Lu",
|
||||
"Mc",
|
||||
"Me",
|
||||
"Mn",
|
||||
"Nd",
|
||||
"Nl",
|
||||
"No",
|
||||
"Pc",
|
||||
"Pd",
|
||||
"Pe",
|
||||
"Pf",
|
||||
"Pi",
|
||||
"Po",
|
||||
"Ps",
|
||||
"Sc",
|
||||
"Sk",
|
||||
"Sm",
|
||||
"So",
|
||||
"Zl",
|
||||
"Zp",
|
||||
"Zs",
|
||||
)
|
||||
):
|
||||
gc_order[i] = v
|
||||
gc_order[v] = i
|
||||
|
||||
@@ -100,10 +145,11 @@ sc_order = dict()
|
||||
sc_array = []
|
||||
sc_re = re.compile(r"\b(HB_SCRIPT_[_A-Z]*).*HB_TAG [(]'(.)','(.)','(.)','(.)'[)]")
|
||||
for line in open(hb_script_list_h):
|
||||
m = sc_re.search (line)
|
||||
if not m: continue
|
||||
m = sc_re.search(line)
|
||||
if not m:
|
||||
continue
|
||||
name = m.group(1)
|
||||
tag = ''.join(m.group(i) for i in range(2, 6))
|
||||
tag = "".join(m.group(i) for i in range(2, 6))
|
||||
i = len(sc_array)
|
||||
sc_order[tag] = i
|
||||
sc_order[i] = tag
|
||||
@@ -112,17 +158,17 @@ for line in open(hb_script_list_h):
|
||||
|
||||
# Write out main data
|
||||
|
||||
DEFAULT = 'DEFAULT'
|
||||
COMPACT = 'COMPACT'
|
||||
SLOPPY = 'SLOPPY'
|
||||
DEFAULT = "DEFAULT"
|
||||
COMPACT = "COMPACT"
|
||||
SLOPPY = "SLOPPY"
|
||||
|
||||
compression_level = {
|
||||
DEFAULT: 5,
|
||||
COMPACT: 9,
|
||||
SLOPPY: 9,
|
||||
SLOPPY: 9,
|
||||
}
|
||||
|
||||
logging.info('Generating output...')
|
||||
logging.info("Generating output...")
|
||||
print("/* == Start of generated table == */")
|
||||
print("/*")
|
||||
print(" * The following table is generated by running:")
|
||||
@@ -148,20 +194,20 @@ uint16_t = language.type_name("u16")
|
||||
uint32_t = language.type_name("u32")
|
||||
uint64_t = language.type_name("u64")
|
||||
|
||||
code = packTab.Code('_hb_ucd')
|
||||
sc_array, _ = code.addArray('hb_script_t', 'sc_map', sc_array)
|
||||
dm1_p0_array, _ = code.addArray(uint16_t, 'dm1_p0_map', dm1_p0_array)
|
||||
dm1_p2_array, _ = code.addArray(uint16_t, 'dm1_p2_map', dm1_p2_array)
|
||||
dm2_u32_array, _ = code.addArray(uint32_t, 'dm2_u32_map', dm2_u32_array)
|
||||
dm2_u64_array, _ = code.addArray(uint64_t, 'dm2_u64_map', dm2_u64_array)
|
||||
code = packTab.Code("_hb_ucd")
|
||||
sc_array, _ = code.addArray("hb_script_t", "sc_map", sc_array)
|
||||
dm1_p0_array, _ = code.addArray(uint16_t, "dm1_p0_map", dm1_p0_array)
|
||||
dm1_p2_array, _ = code.addArray(uint16_t, "dm1_p2_map", dm1_p2_array)
|
||||
dm2_u32_array, _ = code.addArray(uint32_t, "dm2_u32_map", dm2_u32_array)
|
||||
dm2_u64_array, _ = code.addArray(uint64_t, "dm2_u64_map", dm2_u64_array)
|
||||
code.print_code(linkage=language.private_function_linkage, language=language)
|
||||
|
||||
datasets = [
|
||||
('gc', gc, 'Cn', gc_order),
|
||||
('ccc', ccc, 0, None),
|
||||
('bmg', bmg, 0, None),
|
||||
('sc', sc, 'Zzzz', sc_order),
|
||||
('dm', dm, None, dm_order),
|
||||
("gc", gc, "Cn", gc_order),
|
||||
("ccc", ccc, 0, None),
|
||||
("bmg", bmg, 0, None),
|
||||
("sc", sc, "Zzzz", sc_order),
|
||||
("dm", dm, None, dm_order),
|
||||
]
|
||||
|
||||
|
||||
@@ -169,12 +215,12 @@ datasets = [
|
||||
|
||||
modes = {}
|
||||
if language.name == "c":
|
||||
modes[DEFAULT] = '#ifndef HB_OPTIMIZE_SIZE'
|
||||
modes[COMPACT] = '#elif !defined(HB_NO_UCD_UNASSIGNED)'
|
||||
modes[SLOPPY] = '#else'
|
||||
modes[None] = '#endif'
|
||||
modes[DEFAULT] = "#ifndef HB_OPTIMIZE_SIZE"
|
||||
modes[COMPACT] = "#elif !defined(HB_NO_UCD_UNASSIGNED)"
|
||||
modes[SLOPPY] = "#else"
|
||||
modes[None] = "#endif"
|
||||
else:
|
||||
modes[DEFAULT] = ''
|
||||
modes[DEFAULT] = ""
|
||||
|
||||
for step, text in modes.items():
|
||||
print()
|
||||
@@ -185,38 +231,37 @@ for step, text in modes.items():
|
||||
continue
|
||||
|
||||
compression = compression_level[step]
|
||||
logging.info(' Compression=%d:' % compression)
|
||||
logging.info(" Compression=%d:" % compression)
|
||||
|
||||
if step == SLOPPY:
|
||||
for i in range(len(gc)):
|
||||
if (i % 128) and gc[i] == 'Cn':
|
||||
if (i % 128) and gc[i] == "Cn":
|
||||
gc[i] = gc[i - 1]
|
||||
for i in range(len(gc) - 2, -1, -1):
|
||||
if ((i + 1) % 128) and gc[i] == 'Cn':
|
||||
if ((i + 1) % 128) and gc[i] == "Cn":
|
||||
gc[i] = gc[i + 1]
|
||||
for i in range(len(sc)):
|
||||
if (i % 128) and sc[i] == 'Zzzz':
|
||||
if (i % 128) and sc[i] == "Zzzz":
|
||||
sc[i] = sc[i - 1]
|
||||
for i in range(len(sc) - 2, -1, -1):
|
||||
if ((i + 1) % 128) and sc[i] == 'Zzzz':
|
||||
if ((i + 1) % 128) and sc[i] == "Zzzz":
|
||||
sc[i] = sc[i + 1]
|
||||
|
||||
code = packTab.Code("_hb_ucd")
|
||||
|
||||
code = packTab.Code('_hb_ucd')
|
||||
|
||||
for name,data,default,mapping in datasets:
|
||||
sol = packTab.pack_table(data, default, mapping=mapping, compression=compression)
|
||||
logging.info(' Dataset=%-8s FullCost=%d' % (name, sol.fullCost))
|
||||
for name, data, default, mapping in datasets:
|
||||
sol = packTab.pack_table(
|
||||
data, default, mapping=mapping, compression=compression
|
||||
)
|
||||
logging.info(" Dataset=%-8s FullCost=%d" % (name, sol.fullCost))
|
||||
sol.genCode(code, name)
|
||||
|
||||
code.print_code(linkage=language.private_function_linkage, language=language)
|
||||
|
||||
print()
|
||||
|
||||
|
||||
if language.name == "c":
|
||||
print()
|
||||
print("#endif /* HB_UCD_TABLE_HH */")
|
||||
print()
|
||||
print("/* == End of generated table == */")
|
||||
logging.info('Done.')
|
||||
logging.info("Done.")
|
||||
+2
-2
@@ -2,7 +2,7 @@
|
||||
/*
|
||||
* The following table is generated by running:
|
||||
*
|
||||
* ./gen-ucd-table.py ucd.nounihan.grouped.xml
|
||||
* ./gen-ucd-table.py ucd.nounihan.grouped.xml hb-script-list.h
|
||||
*
|
||||
* on file with this description: Unicode 16.0.0
|
||||
*/
|
||||
@@ -5849,8 +5849,8 @@ static inline uint16_t _hb_ucd_dm (unsigned u)
|
||||
return u<195102 ? _hb_ucd_u16[1656+(((_hb_ucd_u8[12834+(((_hb_ucd_u8[12452+(u>>4>>5)])<<5)+((u>>4)&31))])<<4)+((u)&15))] : 0;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
||||
#endif /* HB_UCD_TABLE_HH */
|
||||
|
||||
|
||||
Reference in new issue
Block a user