From 083ca198d29dbcc66bb6f108eb45dd79fb60d3e0 Mon Sep 17 00:00:00 2001 From: Behdad Esfahbod Date: Mon, 12 May 2025 15:25:14 -0600 Subject: [PATCH] [gen-ucd-table.py] black --- src/gen-ucd-table.py | 183 +++++++++++++++++++++++++++---------------- src/hb-ucd-table.hh | 4 +- 2 files changed, 116 insertions(+), 71 deletions(-) diff --git a/src/gen-ucd-table.py b/src/gen-ucd-table.py index 428a8db56..3b57e04b9 100755 --- a/src/gen-ucd-table.py +++ b/src/gen-ucd-table.py @@ -12,27 +12,28 @@ import packTab.ucdxml import sys, re import logging -logging.basicConfig(format='%(levelname)s: %(message)s', level=logging.INFO) -if len(sys.argv) > 1 and sys.argv[1] == '--rust': +logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.INFO) + +if len(sys.argv) > 1 and sys.argv[1] == "--rust": del sys.argv[1] - logging.info('Generating Rust code...') + logging.info("Generating Rust code...") language = "rust" else: - logging.info('Generating C code...') + logging.info("Generating C code...") language = "c" language = packTab.languages[language] -if len (sys.argv) not in (2, 3): - sys.exit (__doc__) +if len(sys.argv) not in (2, 3): + sys.exit(__doc__) -logging.info('Loading UCDXML...') +logging.info("Loading UCDXML...") ucdxml = packTab.ucdxml.load_ucdxml(sys.argv[1]) ucd = packTab.ucdxml.ucdxml_get_repertoire(ucdxml) -hb_script_list_h = 'hb-script-list.h' if len (sys.argv) < 3 else sys.argv[2] +hb_script_list_h = "hb-script-list.h" if len(sys.argv) < 3 else sys.argv[2] -logging.info('Preparing data tables...') +logging.info("Preparing data tables...") # This is how the data is encoded: @@ -46,41 +47,54 @@ logging.info('Preparing data tables...') # Composition & Decomposition (dm) are encoded elaborately, # as discussed below. -gc = [u['gc'] for u in ucd] -ccc = [int(u['ccc']) for u in ucd] -bmg = [int(v, 16) - int(u) if v else 0 for u,v in enumerate(u['bmg'] for u in ucd)] -sc = [u['sc'] for u in ucd] +gc = [u["gc"] for u in ucd] +ccc = [int(u["ccc"]) for u in ucd] +bmg = [int(v, 16) - int(u) if v else 0 for u, v in enumerate(u["bmg"] for u in ucd)] +sc = [u["sc"] for u in ucd] # Prepare Compose / Decompose data # # This code is very dense. See hb_ucd_compose() / hb_ucd_decompose() for the logic. -dm = {i:tuple(int(v, 16) for v in u['dm'].split()) for i,u in enumerate(ucd) - if u['dm'] != '#' and u['dt'] == 'can' and not (0xAC00 <= i < 0xAC00+11172)} -ce = {i for i,u in enumerate(ucd) if u['Comp_Ex'] == 'Y'} +dm = { + i: tuple(int(v, 16) for v in u["dm"].split()) + for i, u in enumerate(ucd) + if u["dm"] != "#" and u["dt"] == "can" and not (0xAC00 <= i < 0xAC00 + 11172) +} +ce = {i for i, u in enumerate(ucd) if u["Comp_Ex"] == "Y"} -assert not any(v for v in dm.values() if len(v) not in (1,2)) +assert not any(v for v in dm.values() if len(v) not in (1, 2)) dm1 = sorted(set(v for v in dm.values() if len(v) == 1)) -assert all((v[0] >> 16) in (0,2) for v in dm1) -dm1_p0_array = ['0x%04Xu' % (v[0] & 0xFFFF) for v in dm1 if (v[0] >> 16) == 0] -dm1_p2_array = ['0x%04Xu' % (v[0] & 0xFFFF) for v in dm1 if (v[0] >> 16) == 2] -dm1_order = {v:i+1 for i,v in enumerate(dm1)} +assert all((v[0] >> 16) in (0, 2) for v in dm1) +dm1_p0_array = ["0x%04Xu" % (v[0] & 0xFFFF) for v in dm1 if (v[0] >> 16) == 0] +dm1_p2_array = ["0x%04Xu" % (v[0] & 0xFFFF) for v in dm1 if (v[0] >> 16) == 2] +dm1_order = {v: i + 1 for i, v in enumerate(dm1)} -dm2 = sorted((v+(i if i not in ce and not ccc[i] else 0,), v) - for i,v in dm.items() if len(v) == 2) +dm2 = sorted( + (v + (i if i not in ce and not ccc[i] else 0,), v) + for i, v in dm.items() + if len(v) == 2 +) -filt = lambda v: ((v[0] & 0xFFFFF800) == 0x0000 and - (v[1] & 0xFFFFFF80) == 0x0300 and - (v[2] & 0xFFF0C000) == 0x0000) +filt = lambda v: ( + (v[0] & 0xFFFFF800) == 0x0000 + and (v[1] & 0xFFFFFF80) == 0x0300 + and (v[2] & 0xFFF0C000) == 0x0000 +) dm2_u32_array = [v for v in dm2 if filt(v[0])] dm2_u64_array = [v for v in dm2 if not filt(v[0])] assert dm2_u32_array + dm2_u64_array == dm2 -dm2_u32_array = ["HB_CODEPOINT_ENCODE3_11_7_14 (0x%04Xu, 0x%04Xu, 0x%04Xu)" % v[0] for v in dm2_u32_array] -dm2_u64_array = ["HB_CODEPOINT_ENCODE3 (0x%04Xu, 0x%04Xu, 0x%04Xu)" % v[0] for v in dm2_u64_array] +dm2_u32_array = [ + "HB_CODEPOINT_ENCODE3_11_7_14 (0x%04Xu, 0x%04Xu, 0x%04Xu)" % v[0] + for v in dm2_u32_array +] +dm2_u64_array = [ + "HB_CODEPOINT_ENCODE3 (0x%04Xu, 0x%04Xu, 0x%04Xu)" % v[0] for v in dm2_u64_array +] l = 1 + len(dm1_p0_array) + len(dm1_p2_array) -dm2_order = {v[1]:i+l for i,v in enumerate(dm2)} +dm2_order = {v[1]: i + l for i, v in enumerate(dm2)} dm_order = {None: 0} dm_order.update(dm1_order) @@ -90,9 +104,40 @@ dm_order.update(dm2_order) # Prepare General_Category / Script mapping arrays gc_order = dict() -for i,v in enumerate(('Cc', 'Cf', 'Cn', 'Co', 'Cs', 'Ll', 'Lm', 'Lo', 'Lt', 'Lu', - 'Mc', 'Me', 'Mn', 'Nd', 'Nl', 'No', 'Pc', 'Pd', 'Pe', 'Pf', - 'Pi', 'Po', 'Ps', 'Sc', 'Sk', 'Sm', 'So', 'Zl', 'Zp', 'Zs',)): +for i, v in enumerate( + ( + "Cc", + "Cf", + "Cn", + "Co", + "Cs", + "Ll", + "Lm", + "Lo", + "Lt", + "Lu", + "Mc", + "Me", + "Mn", + "Nd", + "Nl", + "No", + "Pc", + "Pd", + "Pe", + "Pf", + "Pi", + "Po", + "Ps", + "Sc", + "Sk", + "Sm", + "So", + "Zl", + "Zp", + "Zs", + ) +): gc_order[i] = v gc_order[v] = i @@ -100,10 +145,11 @@ sc_order = dict() sc_array = [] sc_re = re.compile(r"\b(HB_SCRIPT_[_A-Z]*).*HB_TAG [(]'(.)','(.)','(.)','(.)'[)]") for line in open(hb_script_list_h): - m = sc_re.search (line) - if not m: continue + m = sc_re.search(line) + if not m: + continue name = m.group(1) - tag = ''.join(m.group(i) for i in range(2, 6)) + tag = "".join(m.group(i) for i in range(2, 6)) i = len(sc_array) sc_order[tag] = i sc_order[i] = tag @@ -112,17 +158,17 @@ for line in open(hb_script_list_h): # Write out main data -DEFAULT = 'DEFAULT' -COMPACT = 'COMPACT' -SLOPPY = 'SLOPPY' +DEFAULT = "DEFAULT" +COMPACT = "COMPACT" +SLOPPY = "SLOPPY" compression_level = { DEFAULT: 5, COMPACT: 9, - SLOPPY: 9, + SLOPPY: 9, } -logging.info('Generating output...') +logging.info("Generating output...") print("/* == Start of generated table == */") print("/*") print(" * The following table is generated by running:") @@ -148,20 +194,20 @@ uint16_t = language.type_name("u16") uint32_t = language.type_name("u32") uint64_t = language.type_name("u64") -code = packTab.Code('_hb_ucd') -sc_array, _ = code.addArray('hb_script_t', 'sc_map', sc_array) -dm1_p0_array, _ = code.addArray(uint16_t, 'dm1_p0_map', dm1_p0_array) -dm1_p2_array, _ = code.addArray(uint16_t, 'dm1_p2_map', dm1_p2_array) -dm2_u32_array, _ = code.addArray(uint32_t, 'dm2_u32_map', dm2_u32_array) -dm2_u64_array, _ = code.addArray(uint64_t, 'dm2_u64_map', dm2_u64_array) +code = packTab.Code("_hb_ucd") +sc_array, _ = code.addArray("hb_script_t", "sc_map", sc_array) +dm1_p0_array, _ = code.addArray(uint16_t, "dm1_p0_map", dm1_p0_array) +dm1_p2_array, _ = code.addArray(uint16_t, "dm1_p2_map", dm1_p2_array) +dm2_u32_array, _ = code.addArray(uint32_t, "dm2_u32_map", dm2_u32_array) +dm2_u64_array, _ = code.addArray(uint64_t, "dm2_u64_map", dm2_u64_array) code.print_code(linkage=language.private_function_linkage, language=language) datasets = [ - ('gc', gc, 'Cn', gc_order), - ('ccc', ccc, 0, None), - ('bmg', bmg, 0, None), - ('sc', sc, 'Zzzz', sc_order), - ('dm', dm, None, dm_order), + ("gc", gc, "Cn", gc_order), + ("ccc", ccc, 0, None), + ("bmg", bmg, 0, None), + ("sc", sc, "Zzzz", sc_order), + ("dm", dm, None, dm_order), ] @@ -169,12 +215,12 @@ datasets = [ modes = {} if language.name == "c": - modes[DEFAULT] = '#ifndef HB_OPTIMIZE_SIZE' - modes[COMPACT] = '#elif !defined(HB_NO_UCD_UNASSIGNED)' - modes[SLOPPY] = '#else' - modes[None] = '#endif' + modes[DEFAULT] = "#ifndef HB_OPTIMIZE_SIZE" + modes[COMPACT] = "#elif !defined(HB_NO_UCD_UNASSIGNED)" + modes[SLOPPY] = "#else" + modes[None] = "#endif" else: - modes[DEFAULT] = '' + modes[DEFAULT] = "" for step, text in modes.items(): print() @@ -185,38 +231,37 @@ for step, text in modes.items(): continue compression = compression_level[step] - logging.info(' Compression=%d:' % compression) + logging.info(" Compression=%d:" % compression) if step == SLOPPY: for i in range(len(gc)): - if (i % 128) and gc[i] == 'Cn': + if (i % 128) and gc[i] == "Cn": gc[i] = gc[i - 1] for i in range(len(gc) - 2, -1, -1): - if ((i + 1) % 128) and gc[i] == 'Cn': + if ((i + 1) % 128) and gc[i] == "Cn": gc[i] = gc[i + 1] for i in range(len(sc)): - if (i % 128) and sc[i] == 'Zzzz': + if (i % 128) and sc[i] == "Zzzz": sc[i] = sc[i - 1] for i in range(len(sc) - 2, -1, -1): - if ((i + 1) % 128) and sc[i] == 'Zzzz': + if ((i + 1) % 128) and sc[i] == "Zzzz": sc[i] = sc[i + 1] + code = packTab.Code("_hb_ucd") - code = packTab.Code('_hb_ucd') - - for name,data,default,mapping in datasets: - sol = packTab.pack_table(data, default, mapping=mapping, compression=compression) - logging.info(' Dataset=%-8s FullCost=%d' % (name, sol.fullCost)) + for name, data, default, mapping in datasets: + sol = packTab.pack_table( + data, default, mapping=mapping, compression=compression + ) + logging.info(" Dataset=%-8s FullCost=%d" % (name, sol.fullCost)) sol.genCode(code, name) code.print_code(linkage=language.private_function_linkage, language=language) print() - if language.name == "c": - print() print("#endif /* HB_UCD_TABLE_HH */") print() print("/* == End of generated table == */") -logging.info('Done.') +logging.info("Done.") diff --git a/src/hb-ucd-table.hh b/src/hb-ucd-table.hh index 16b973d55..52535a250 100644 --- a/src/hb-ucd-table.hh +++ b/src/hb-ucd-table.hh @@ -2,7 +2,7 @@ /* * The following table is generated by running: * - * ./gen-ucd-table.py ucd.nounihan.grouped.xml + * ./gen-ucd-table.py ucd.nounihan.grouped.xml hb-script-list.h * * on file with this description: Unicode 16.0.0 */ @@ -5849,8 +5849,8 @@ static inline uint16_t _hb_ucd_dm (unsigned u) return u<195102 ? _hb_ucd_u16[1656+(((_hb_ucd_u8[12834+(((_hb_ucd_u8[12452+(u>>4>>5)])<<5)+((u>>4)&31))])<<4)+((u)&15))] : 0; } -#endif +#endif #endif /* HB_UCD_TABLE_HH */