mirror of
https://github.com/storytold/harfbuzz.git
synced 2026-11-21 23:11:04 +01:00
293 lines
7.5 KiB
Python
Executable File
293 lines
7.5 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
"""usage: ./gen-ucd-table [--rust] ucd.nounihan.grouped.xml [/path/to/hb-script-list.h]
|
|
|
|
Input file:
|
|
* https://unicode.org/Public/UCD/latest/ucdxml/ucd.nounihan.grouped.zip
|
|
"""
|
|
|
|
# https://github.com/harfbuzz/packtab
|
|
import packTab
|
|
import packTab.ucdxml
|
|
|
|
import sys, re
|
|
import logging
|
|
|
|
logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.INFO)
|
|
|
|
if len(sys.argv) > 1 and sys.argv[1] == "--rust":
|
|
del sys.argv[1]
|
|
logging.info("Generating Rust code...")
|
|
language = "rust"
|
|
else:
|
|
logging.info("Generating C code...")
|
|
language = "c"
|
|
language = packTab.languages[language]
|
|
|
|
if len(sys.argv) not in (2, 3):
|
|
sys.exit(__doc__)
|
|
|
|
logging.info("Loading UCDXML...")
|
|
ucdxml = packTab.ucdxml.load_ucdxml(sys.argv[1])
|
|
ucd = packTab.ucdxml.ucdxml_get_repertoire(ucdxml)
|
|
|
|
hb_script_list_h = "hb-script-list.h" if len(sys.argv) < 3 else sys.argv[2]
|
|
|
|
logging.info("Preparing data tables...")
|
|
|
|
|
|
# This is how the data is encoded:
|
|
#
|
|
# General_Category (gc), Canonical_Combining_Class (ccc),
|
|
# and Script (sc) are encoded as integers.
|
|
#
|
|
# Mirroring character (bmg) is encoded as difference from
|
|
# the original character.
|
|
#
|
|
# Composition & Decomposition (dm) are encoded elaborately,
|
|
# as discussed below.
|
|
|
|
gc = [u["gc"] for u in ucd]
|
|
ccc = [int(u["ccc"]) for u in ucd]
|
|
bmg = [int(v, 16) - int(u) if v else 0 for u, v in enumerate(u["bmg"] for u in ucd)]
|
|
sc = [u["sc"] for u in ucd]
|
|
|
|
|
|
# Prepare Compose / Decompose data
|
|
#
|
|
# This code is very dense. See hb_ucd_compose() / hb_ucd_decompose() for the logic.
|
|
|
|
dm = {
|
|
i: tuple(int(v, 16) for v in u["dm"].split())
|
|
for i, u in enumerate(ucd)
|
|
if u["dm"] != "#" and u["dt"] == "can" and not (0xAC00 <= i < 0xAC00 + 11172)
|
|
}
|
|
ce = {i for i, u in enumerate(ucd) if u["Comp_Ex"] == "Y"}
|
|
|
|
assert not any(v for v in dm.values() if len(v) not in (1, 2))
|
|
dm1 = sorted(set(v for v in dm.values() if len(v) == 1))
|
|
assert all((v[0] >> 16) in (0, 2) for v in dm1)
|
|
dm1_p0_array = ["0x%04X" % (v[0] & 0xFFFF) for v in dm1 if (v[0] >> 16) == 0]
|
|
dm1_p2_array = ["0x%04X" % (v[0] & 0xFFFF) for v in dm1 if (v[0] >> 16) == 2]
|
|
dm1_order = {v: i + 1 for i, v in enumerate(dm1)}
|
|
|
|
dm2 = sorted(
|
|
(v + (i if i not in ce and not ccc[i] else 0,), v)
|
|
for i, v in dm.items()
|
|
if len(v) == 2
|
|
)
|
|
|
|
filt = lambda v: (
|
|
(v[0] & 0xFFFFF800) == 0x0000
|
|
and (v[1] & 0xFFFFFF80) == 0x0300
|
|
and (v[2] & 0xFFF0C000) == 0x0000
|
|
)
|
|
dm2_u32_array = [v for v in dm2 if filt(v[0])]
|
|
dm2_u64_array = [v for v in dm2 if not filt(v[0])]
|
|
assert dm2_u32_array + dm2_u64_array == dm2
|
|
dm2_u32_array = [
|
|
"HB_CODEPOINT_ENCODE3_11_7_14 (0x%04X, 0x%04X, 0x%04X)" % v[0]
|
|
for v in dm2_u32_array
|
|
]
|
|
dm2_u64_array = [
|
|
"HB_CODEPOINT_ENCODE3 (0x%04X, 0x%04X, 0x%04X)" % v[0] for v in dm2_u64_array
|
|
]
|
|
|
|
l = 1 + len(dm1_p0_array) + len(dm1_p2_array)
|
|
dm2_order = {v[1]: i + l for i, v in enumerate(dm2)}
|
|
|
|
dm_order = {None: 0}
|
|
dm_order.update(dm1_order)
|
|
dm_order.update(dm2_order)
|
|
|
|
|
|
# Prepare General_Category / Script mapping arrays
|
|
|
|
gc_order = dict()
|
|
for i, v in enumerate(
|
|
(
|
|
"Cc",
|
|
"Cf",
|
|
"Cn",
|
|
"Co",
|
|
"Cs",
|
|
"Ll",
|
|
"Lm",
|
|
"Lo",
|
|
"Lt",
|
|
"Lu",
|
|
"Mc",
|
|
"Me",
|
|
"Mn",
|
|
"Nd",
|
|
"Nl",
|
|
"No",
|
|
"Pc",
|
|
"Pd",
|
|
"Pe",
|
|
"Pf",
|
|
"Pi",
|
|
"Po",
|
|
"Ps",
|
|
"Sc",
|
|
"Sk",
|
|
"Sm",
|
|
"So",
|
|
"Zl",
|
|
"Zp",
|
|
"Zs",
|
|
)
|
|
):
|
|
gc_order[i] = v
|
|
gc_order[v] = i
|
|
|
|
sc_order = dict()
|
|
sc_array = []
|
|
sc_re = re.compile(r"\b(HB_SCRIPT_[_A-Z]*).*HB_TAG [(]'(.)','(.)','(.)','(.)'[)]")
|
|
for line in open(hb_script_list_h):
|
|
m = sc_re.search(line)
|
|
if not m:
|
|
continue
|
|
name = m.group(1)
|
|
tag = "".join(m.group(i) for i in range(2, 6))
|
|
i = len(sc_array)
|
|
sc_order[tag] = i
|
|
sc_order[i] = tag
|
|
if language.name == "rust":
|
|
name = name.replace("HB_SCRIPT_", "script::")
|
|
sc_array.append(name)
|
|
|
|
|
|
# Write out main data
|
|
|
|
DEFAULT = "DEFAULT"
|
|
COMPACT = "COMPACT"
|
|
SLOPPY = "SLOPPY"
|
|
|
|
compression_level = {
|
|
DEFAULT: 3,
|
|
COMPACT: 9,
|
|
SLOPPY: 9,
|
|
}
|
|
|
|
logging.info("Generating output...")
|
|
print("/* == Start of generated table == */")
|
|
print("/*")
|
|
print(" * The following table is generated by running:")
|
|
print(" *")
|
|
print(
|
|
" * ./gen-ucd-table.py %sucd.nounihan.grouped.xml hb-script-list.h"
|
|
% (("--%s " % language.name) if language.name != "c" else "")
|
|
)
|
|
print(" *")
|
|
print(" * on file with this description:", ucdxml.description)
|
|
print(" */")
|
|
print()
|
|
if language.name == "c":
|
|
print("#ifndef HB_UCD_TABLE_HH")
|
|
print("#define HB_UCD_TABLE_HH")
|
|
print()
|
|
print('#include "hb.hh"')
|
|
print()
|
|
elif language.name == "rust":
|
|
print("pub(crate) mod ucd {")
|
|
print()
|
|
print("#![allow(unused_parens)]")
|
|
print("#![allow(clippy::unnecessary_cast, clippy::unreadable_literal, clippy::double_parens)]")
|
|
print()
|
|
print("use crate::hb::algs::{HB_CODEPOINT_ENCODE3, HB_CODEPOINT_ENCODE3_11_7_14};")
|
|
print("use crate::hb::common::script;")
|
|
print("use crate::hb::common::Script as hb_script_t;")
|
|
print()
|
|
else:
|
|
assert False, "Unknown language: %s" % language.name
|
|
|
|
# Write mapping data
|
|
|
|
uint16_t = language.type_name("u16")
|
|
uint32_t = language.type_name("u32")
|
|
uint64_t = language.type_name("u64")
|
|
|
|
if language.name == "c":
|
|
private = True
|
|
elif language.name == "rust":
|
|
private = False
|
|
else:
|
|
assert False, "Unknown language: %s" % language.name
|
|
|
|
code = packTab.Code("_hb_ucd")
|
|
sc_array, _ = code.addArray("hb_script_t", "sc_map", sc_array)
|
|
dm1_p0_array, _ = code.addArray(uint16_t, "dm1_p0_map", dm1_p0_array)
|
|
dm1_p2_array, _ = code.addArray(uint16_t, "dm1_p2_map", dm1_p2_array)
|
|
dm2_u32_array, _ = code.addArray(uint32_t, "dm2_u32_map", dm2_u32_array)
|
|
dm2_u64_array, _ = code.addArray(uint64_t, "dm2_u64_map", dm2_u64_array)
|
|
code.print_code(language=language, private=private)
|
|
|
|
datasets = [
|
|
("gc", gc, "Cn", gc_order),
|
|
("ccc", ccc, 0, None),
|
|
("bmg", bmg, 0, None),
|
|
("sc", sc, "Zzzz", sc_order),
|
|
("dm", dm, None, dm_order),
|
|
]
|
|
|
|
|
|
# Write main data
|
|
|
|
modes = {}
|
|
if language.name == "c":
|
|
modes[DEFAULT] = "#ifndef HB_OPTIMIZE_SIZE"
|
|
modes[COMPACT] = "#elif !defined(HB_NO_UCD_UNASSIGNED)"
|
|
modes[SLOPPY] = "#else"
|
|
modes[None] = "#endif"
|
|
else:
|
|
modes[DEFAULT] = ""
|
|
|
|
for step, text in modes.items():
|
|
print()
|
|
if text:
|
|
print(text)
|
|
print()
|
|
if step is None:
|
|
continue
|
|
|
|
compression = compression_level[step]
|
|
logging.info(" Compression=%d:" % compression)
|
|
|
|
if step == SLOPPY:
|
|
for i in range(len(gc)):
|
|
if (i % 128) and gc[i] == "Cn":
|
|
gc[i] = gc[i - 1]
|
|
for i in range(len(gc) - 2, -1, -1):
|
|
if ((i + 1) % 128) and gc[i] == "Cn":
|
|
gc[i] = gc[i + 1]
|
|
for i in range(len(sc)):
|
|
if (i % 128) and sc[i] == "Zzzz":
|
|
sc[i] = sc[i - 1]
|
|
for i in range(len(sc) - 2, -1, -1):
|
|
if ((i + 1) % 128) and sc[i] == "Zzzz":
|
|
sc[i] = sc[i + 1]
|
|
|
|
code = packTab.Code("_hb_ucd")
|
|
|
|
for name, data, default, mapping in datasets:
|
|
sol = packTab.pack_table(
|
|
data, default, mapping=mapping, compression=compression
|
|
)
|
|
logging.info(" Dataset=%-8s FullCost=%d" % (name, sol.fullCost))
|
|
sol.genCode(code, name, private=private, language=language)
|
|
|
|
code.print_code(language=language)
|
|
|
|
print()
|
|
|
|
if language.name == "c":
|
|
print("#endif /* HB_UCD_TABLE_HH */")
|
|
elif language.name == "rust":
|
|
print("}")
|
|
else:
|
|
assert False, "Unknown language: %s" % language.name
|
|
print()
|
|
print("/* == End of generated table == */")
|
|
logging.info("Done.")
|