#!/usr/bin/env python3
#
# Copyright 2011-2022 The Rust Project Developers. See the COPYRIGHT
# file at the top-level directory of this distribution and at
# http://rust-lang.org/COPYRIGHT.
#
# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
# option. This file may not be copied, modified, or distributed
# except according to those terms.
# This script uses the following Unicode tables:
#
# - DerivedCoreProperties.txt
# - EastAsianWidth.txt
# - HangulSyllableType.txt
# - NormalizationTest.txt (for tests only)
# - PropList.txt
# - ReadMe.txt
# - Scripts.txt
# - UnicodeData.txt
# - emoji/emoji-data.txt
# - emoji/emoji-variation-sequences.txt
# - extracted/DerivedGeneralCategory.txt
#
# Since this should not require frequent updates, we just store this
# out-of-line and check the generated module into git.
import enum
import math
import operator
import os
import re
import sys
import urllib.request
from collections import defaultdict
from itertools import batched
from typing import Callable
UNICODE_VERSION = "15.1.0"
"""The version of the Unicode data files to download."""
NUM_CODEPOINTS = 0x110000
"""An upper bound for which `range(0, NUM_CODEPOINTS)` contains Unicode's codespace."""
MAX_CODEPOINT_BITS = math.ceil(math.log2(NUM_CODEPOINTS - 1))
"""The maximum number of bits required to represent a Unicode codepoint."""
class OffsetType(enum.IntEnum):
"""Represents the data type of a lookup table's offsets. Each variant's value represents the
number of bits required to represent that variant's type."""
U2 = 2
"""Offsets are 2-bit unsigned integers, packed four-per-byte."""
U4 = 4
"""Offsets are 4-bit unsigned integers, packed two-per-byte."""
U8 = 8
"""Each offset is a single byte (u8)."""
TABLE_CFGS = [
(13, MAX_CODEPOINT_BITS, OffsetType.U8),
(6, 13, OffsetType.U8),
(0, 6, OffsetType.U2),
]
"""Represents the format of each level of the multi-level lookup table.
A level's entry is of the form `(low_bit, cap_bit, offset_type)`.
This means that every sub-table in that level is indexed by bits `low_bit..cap_bit` of the
codepoint and those tables offsets are stored according to `offset_type`.
If this is edited, you must ensure that `emit_module` reflects your changes."""
MODULE_PATH = "../src/tables.rs"
"""The path of the emitted Rust module (relative to the working directory)"""
Codepoint = int
BitPos = int
def fetch_open(filename: str, local_prefix: str = ""):
"""Opens `filename` and return its corresponding file object. If `filename` isn't on disk,
fetches it from `https://www.unicode.org/Public/`. Exits with code 1 on failure.
"""
basename = os.path.basename(filename)
localname = os.path.join(local_prefix, basename)
if not os.path.exists(localname):
urllib.request.urlretrieve(
f"https://www.unicode.org/Public/{UNICODE_VERSION}/ucd/{filename}",
localname,
)
try:
return open(localname, encoding="utf-8")
except OSError:
sys.stderr.write(f"cannot load {localname}")
sys.exit(1)
def load_unicode_version() -> tuple[int, int, int]:
"""Returns the current Unicode version by fetching and processing `ReadMe.txt`."""
with fetch_open("ReadMe.txt") as readme:
pattern = r"for Version (\d+)\.(\d+)\.(\d+) of the Unicode"
return tuple(map(int, re.search(pattern, readme.read()).groups()))
def load_property(filename: str, pattern: str, action: Callable[[int], None]):
with fetch_open(filename) as properties:
single = re.compile(rf"^([0-9A-F]+)\s*;\s*{pattern}\s+")
multiple = re.compile(rf"^([0-9A-F]+)\.\.([0-9A-F]+)\s*;\s*{pattern}\s+")
for line in properties.readlines():
raw_data = None # (low, high)
if match := single.match(line):
raw_data = (match.group(1), match.group(1))
elif match := multiple.match(line):
raw_data = (match.group(1), match.group(2))
else:
continue
low = int(raw_data[0], 16)
high = int(raw_data[1], 16)
for cp in range(low, high + 1):
action(cp)
class EffectiveWidth(enum.IntEnum):
"""Represents the width of a Unicode character. All East Asian Width classes resolve into
either `EffectiveWidth.NARROW`, `EffectiveWidth.WIDE`, or `EffectiveWidth.AMBIGUOUS`.
"""
ZERO = 0
""" Zero columns wide. """
NARROW = 1
""" One column wide. """
WIDE = 2
""" Two columns wide. """
AMBIGUOUS = 3
""" Two columns wide in a CJK context. One column wide in all other contexts. """
def load_east_asian_widths() -> list[EffectiveWidth]:
"""Return a list of effective widths, indexed by codepoint.
Widths are determined by fetching and parsing `EastAsianWidth.txt`.
`Neutral`, `Narrow`, and `Halfwidth` characters are assigned `EffectiveWidth.NARROW`.
`Wide` and `Fullwidth` characters are assigned `EffectiveWidth.WIDE`.
`Ambiguous` characters are assigned `EffectiveWidth.AMBIGUOUS`."""
with fetch_open("EastAsianWidth.txt") as eaw:
# matches a width assignment for a single codepoint, i.e. "1F336;N # ..."
single = re.compile(r"^([0-9A-F]+)\s*;\s*(\w+) +# (\w+)")
# matches a width assignment for a range of codepoints, i.e. "3001..3003;W # ..."
multiple = re.compile(r"^([0-9A-F]+)\.\.([0-9A-F]+)\s*;\s*(\w+) +# (\w+)")
# map between width category code and condensed width
width_codes = {
**{c: EffectiveWidth.NARROW for c in ["N", "Na", "H"]},
**{c: EffectiveWidth.WIDE for c in ["W", "F"]},
"A": EffectiveWidth.AMBIGUOUS,
}
width_map = []
current = 0
for line in eaw.readlines():
raw_data = None # (low, high, width)
if match := single.match(line):
raw_data = (match.group(1), match.group(1), match.group(2))
elif match := multiple.match(line):
raw_data = (match.group(1), match.group(2), match.group(3))
else:
continue
low = int(raw_data[0], 16)
high = int(raw_data[1], 16)
width = width_codes[raw_data[2]]
assert current <= high
while current <= high:
# Some codepoints don't fall into any of the ranges in EastAsianWidth.txt.
# All such codepoints are implicitly given Neural width (resolves to narrow)
width_map.append(EffectiveWidth.NARROW if current < low else width)
current += 1
while len(width_map) < NUM_CODEPOINTS:
# Catch any leftover codepoints and assign them implicit Neutral/narrow width.
width_map.append(EffectiveWidth.NARROW)
# Characters from alphabetic scripts are narrow
load_property(
"Scripts.txt",
r"(?:Latin|Greek|Cyrillic)",
lambda cp: (
operator.setitem(width_map, cp, EffectiveWidth.NARROW)
if width_map[cp] == EffectiveWidth.AMBIGUOUS
and not (0x2160 <= cp <= 0x217F) # Roman numerals remain ambiguous
else None
),
)
# Ambiguous `Modifier_Symbol`s are narrow
load_property(
"extracted/DerivedGeneralCategory.txt",
"Sk",
lambda cp: (
operator.setitem(width_map, cp, EffectiveWidth.NARROW)
if width_map[cp] == EffectiveWidth.AMBIGUOUS
else None
),
)
# GREEK ANO TELEIA: NFC decomposes to U+00B7 MIDDLE DOT
width_map[0x0387] = EffectiveWidth.AMBIGUOUS
# Canonical equivalence for symbols with stroke
with fetch_open("UnicodeData.txt") as udata:
single = re.compile(r"([0-9A-Z]+);.*?;.*?;.*?;.*?;([0-9A-Z]+) 0338;")
for line in udata.readlines():
if match := single.match(line):
composed = int(match.group(1), 16)
decomposed = int(match.group(2), 16)
if width_map[decomposed] == EffectiveWidth.AMBIGUOUS:
width_map[composed] = EffectiveWidth.AMBIGUOUS
return width_map
def load_zero_widths() -> list[bool]:
"""Returns a list `l` where `l[c]` is true if codepoint `c` is considered a zero-width
character. `c` is considered a zero-width character if
- it has the `Default_Ignorable_Code_Point` property (determined from `DerivedCoreProperties.txt`),
- or if it has the `Grapheme_Extend` property (determined from `DerivedCoreProperties.txt`),
- or if it one of eight characters that should be `Grapheme_Extend` but aren't due to a Unicode spec bug,
- or if it has a `Hangul_Syllable_Type` of `Vowel_Jamo` or `Trailing_Jamo` (determined from `HangulSyllableType.txt`).
"""
zw_map = [False] * NUM_CODEPOINTS
# `Default_Ignorable_Code_Point`s also have 0 width:
# https://www.unicode.org/faq/unsup_char.html#3
# https://www.unicode.org/versions/Unicode15.1.0/ch05.pdf#G40095
#
# `Grapheme_Extend` includes characters with general category `Mn` or `Me`,
# as well as a few `Mc` characters that need to be included so that
# canonically equivalent sequences have the same width.
load_property(
"DerivedCoreProperties.txt",
r"(?:Default_Ignorable_Code_Point|Grapheme_Extend)",
lambda cp: operator.setitem(zw_map, cp, True),
)
# Unicode spec bug: these should be `Grapheme_Cluster_Break=Extend`,
# as they canonically decompose to two characters with this property,
# but they aren't.
for c in [0x0CC0, 0x0CC7, 0x0CC8, 0x0CCA, 0x0CCB, 0x1B3B, 0x1B3D, 0x1B43]:
zw_map[c] = True
# Treat `Hangul_Syllable_Type`s of `Vowel_Jamo` and `Trailing_Jamo`
# as zero-width. This matches the behavior of glibc `wcwidth`.
#
# Decomposed Hangul characters consist of 3 parts: a `Leading_Jamo`,
# a `Vowel_Jamo`, and an optional `Trailing_Jamo`. Together these combine
# into a single wide grapheme. So we treat vowel and trailing jamo as
# 0-width, such that only the width of the leading jamo is counted
# and the resulting grapheme has width 2.
#
# (See the Unicode Standard sections 3.12 and 18.6 for more on Hangul)
load_property(
"HangulSyllableType.txt",
r"(?:V|T)",
lambda cp: operator.setitem(zw_map, cp, True),
)
# Syriac abbreviation mark:
# Zero-width `Prepended_Concatenation_Mark`
zw_map[0x070F] = True
# Some Arabic Prepended_Concatenation_Mark`s
# https://www.unicode.org/versions/Unicode15.0.0/ch09.pdf#G27820
zw_map[0x0605] = True
zw_map[0x0890] = True
zw_map[0x0891] = True
zw_map[0x08E2] = True
# HANGUL CHOSEONG FILLER
# U+115F is a `Default_Ignorable_Code_Point`, and therefore would normally have
# zero width. However, the expected usage is to combine it with vowel or trailing jamo
# (which are considered 0-width on their own) to form a composed Hangul syllable with
# width 2. Therefore, we treat it as having width 2.
zw_map[0x115F] = False
# DEVANAGARI CARET
# https://www.unicode.org/versions/Unicode15.0.0/ch12.pdf#G667447
zw_map[0xA8FA] = True
return zw_map
class Bucket:
"""A bucket contains a group of codepoints and an ordered width list. If one bucket's width
list overlaps with another's width list, those buckets can be merged via `try_extend`.
"""
def __init__(self):
"""Creates an empty bucket."""
self.entry_set = set()
self.widths = []
def append(self, codepoint: Codepoint, width: EffectiveWidth):
"""Adds a codepoint/width pair to the bucket, and appends `width` to the width list."""
self.entry_set.add((codepoint, width))
self.widths.append(width)
def try_extend(self, attempt: "Bucket") -> bool:
"""If either `self` or `attempt`'s width list starts with the other bucket's width list,
set `self`'s width list to the longer of the two, add all of `attempt`'s codepoints
into `self`, and return `True`. Otherwise, return `False`."""
(less, more) = (self.widths, attempt.widths)
if len(self.widths) > len(attempt.widths):
(less, more) = (attempt.widths, self.widths)
if less != more[: len(less)]:
return False
self.entry_set |= attempt.entry_set
self.widths = more
return True
def entries(self) -> list[tuple[Codepoint, EffectiveWidth]]:
"""Return a list of the codepoint/width pairs in this bucket, sorted by codepoint."""
result = list(self.entry_set)
result.sort()
return result
def width(self) -> EffectiveWidth | None:
"""If all codepoints in this bucket have the same width, return that width; otherwise,
return `None`."""
if len(self.widths) == 0:
return None
potential_width = self.widths[0]
for width in self.widths[1:]:
if potential_width != width:
return None
return potential_width
def make_buckets(entries, low_bit: BitPos, cap_bit: BitPos) -> list[Bucket]:
"""Partitions the `(Codepoint, EffectiveWidth)` tuples in `entries` into `Bucket`s. All
codepoints with identical bits from `low_bit` to `cap_bit` (exclusive) are placed in the
same bucket. Returns a list of the buckets in increasing order of those bits."""
num_bits = cap_bit - low_bit
assert num_bits > 0
buckets = [Bucket() for _ in range(0, 2**num_bits)]
mask = (1 << num_bits) - 1
for codepoint, width in entries:
buckets[(codepoint >> low_bit) & mask].append(codepoint, width)
return buckets
class Table:
"""Represents a lookup table. Each table contains a certain number of subtables; each
subtable is indexed by a contiguous bit range of the codepoint and contains a list
of `2**(number of bits in bit range)` entries. (The bit range is the same for all subtables.)
Typically, tables contain a list of buckets of codepoints. Bucket `i`'s codepoints should
be indexed by sub-table `i` in the next-level lookup table. The entries of this table are
indexes into the bucket list (~= indexes into the sub-tables of the next-level table.) The
key to compression is that two different buckets in two different sub-tables may have the
same width list, which means that they can be merged into the same bucket.
If no bucket contains two codepoints with different widths, calling `indices_to_widths` will
discard the buckets and convert the entries into `EffectiveWidth` values."""
def __init__(
self, entry_groups, low_bit: BitPos, cap_bit: BitPos, offset_type: OffsetType
):
"""Create a lookup table with a sub-table for each `(Codepoint, EffectiveWidth)` iterator
in `entry_groups`. Each sub-table is indexed by codepoint bits in `low_bit..cap_bit`,
and each table entry is represented in the format specified by `offset_type`. Asserts
that this table is actually representable with `offset_type`."""
self.low_bit = low_bit
self.cap_bit = cap_bit
self.offset_type = offset_type
self.entries = []
self.indexed = []
buckets = []
for entries in entry_groups:
buckets.extend(make_buckets(entries, self.low_bit, self.cap_bit))
for bucket in buckets:
for i, existing in enumerate(self.indexed):
if existing.try_extend(bucket):
self.entries.append(i)
break
else:
self.entries.append(len(self.indexed))
self.indexed.append(bucket)
# Validate offset type
for index in self.entries:
assert index < (1 << int(self.offset_type))
def indices_to_widths(self):
"""Destructively converts the indices in this table to the `EffectiveWidth` values of
their buckets. Assumes that no bucket contains codepoints with different widths.
"""
self.entries = list(map(lambda i: int(self.indexed[i].width()), self.entries))
del self.indexed
def buckets(self):
"""Returns an iterator over this table's buckets."""
return self.indexed
def to_bytes(self) -> list[int]:
"""Returns this table's entries as a list of bytes. The bytes are formatted according to
the `OffsetType` which the table was created with, converting any `EffectiveWidth` entries
to their enum variant's integer value. For example, with `OffsetType.U2`, each byte will
contain four packed 2-bit entries."""
entries_per_byte = 8 // int(self.offset_type)
byte_array = []
for i in range(0, len(self.entries), entries_per_byte):
byte = 0
for j in range(0, entries_per_byte):
byte |= self.entries[i + j] << (j * int(self.offset_type))
byte_array.append(byte)
return byte_array
def make_tables(
table_cfgs: list[tuple[BitPos, BitPos, OffsetType]], entries
) -> list[Table]:
"""Creates a table for each configuration in `table_cfgs`, with the first config corresponding
to the top-level lookup table, the second config corresponding to the second-level lookup
table, and so forth. `entries` is an iterator over the `(Codepoint, EffectiveWidth)` pairs
to include in the top-level table."""
tables = []
entry_groups = [entries]
for low_bit, cap_bit, offset_type in table_cfgs:
table = Table(entry_groups, low_bit, cap_bit, offset_type)
entry_groups = map(lambda bucket: bucket.entries(), table.buckets())
tables.append(table)
return tables
def load_emoji_presentation_sequences() -> list[int]:
"""Outputs a list of character ranages, corresponding to all the valid characters for starting
an emoji presentation sequence."""
with fetch_open("emoji/emoji-variation-sequences.txt") as sequences:
# Match all emoji presentation sequences
# (one codepoint followed by U+FE0F, and labeled "emoji style")
sequence = re.compile(r"^([0-9A-F]+)\s+FE0F\s*;\s*emoji style")
codepoints = []
for line in sequences.readlines():
if match := sequence.match(line):
cp = int(match.group(1), 16)
codepoints.append(cp)
return codepoints
def load_text_presentation_sequences() -> list[int]:
"""Outputs a list of character ranages, corresponding to all the valid characters
whose widths change with a text presentation sequence."""
text_presentation_seq_codepoints = set()
with fetch_open("emoji/emoji-variation-sequences.txt") as sequences:
# Match all text presentation sequences
# (one codepoint followed by U+FE0E, and labeled "text style")
sequence = re.compile(r"^([0-9A-F]+)\s+FE0E\s*;\s*text style")
for line in sequences.readlines():
if match := sequence.match(line):
cp = int(match.group(1), 16)
text_presentation_seq_codepoints.add(cp)
default_emoji_codepoints = set()
load_property(
"emoji/emoji-data.txt",
"Emoji_Presentation",
lambda cp: default_emoji_codepoints.add(cp),
)
codepoints = []
for cp in text_presentation_seq_codepoints.intersection(default_emoji_codepoints):
# "Enclosed Ideographic Supplement" block;
# wide even in text presentation
if not cp in range(0x1F200, 0x1F300):
codepoints.append(cp)
codepoints.sort()
return codepoints
def make_presentation_sequence_table(
seqs: list[Codepoint],
width_map: list[EffectiveWidth],
spurious_false: set[EffectiveWidth],
spurious_true: set[EffectiveWidth],
) -> tuple[list[tuple[int, int]], list[list[int]]]:
"""Generates 2-level lookup table for whether a codepoint might start an emoji variation sequence.
The first level is a match on all but the 10 LSB, the second level is a 1024-bit bitmap for those 10 LSB.
"""
prefixes_dict = defaultdict(set)
for cp in seqs:
prefixes_dict[cp >> 10].add(cp & 0x3FF)
for k in list(prefixes_dict.keys()):
if all(
map(
lambda cp: width_map[(k << 10) | cp] in spurious_false,
prefixes_dict[k],
)
):
del prefixes_dict[k]
msbs: list[int] = list(prefixes_dict.keys())
for cp, width in enumerate(width_map):
if width in spurious_true and (cp >> 10) in msbs:
prefixes_dict[cp >> 10].add(cp & 0x3FF)
leaves: list[list[int]] = []
for cps in prefixes_dict.values():
leaf = [0] * 128
for cp in cps:
idx_in_leaf, bit_shift = divmod(cp, 8)
leaf[idx_in_leaf] |= 1 << bit_shift
leaves.append(leaf)
indexes = [(msb, index) for (index, msb) in enumerate(msbs)]
# Cull duplicate leaves
i = 0
while i < len(leaves):
first_idx = leaves.index(leaves[i])
if first_idx == i:
i += 1
else:
for j in range(0, len(indexes)):
if indexes[j][1] == i:
indexes[j] = (indexes[j][0], first_idx)
elif indexes[j][1] > i:
indexes[j] = (indexes[j][0], indexes[j][1] - 1)
leaves.pop(i)
return (indexes, leaves)
def emit_module(
out_name: str,
unicode_version: tuple[int, int, int],
tables: list[Table],
emoji_presentation_table: tuple[list[tuple[int, int]], list[list[int]]],
text_presentation_table: tuple[list[tuple[int, int]], list[list[int]]],
):
"""Outputs a Rust module to `out_name` using table data from `tables`.
If `TABLE_CFGS` is edited, you may need to edit the included code for `lookup_width`.
"""
if os.path.exists(out_name):
os.remove(out_name)
with open(out_name, "w", newline="\n", encoding="utf-8") as module:
module.write(
"""// Copyright 2012-2022 The Rust Project Developers. See the COPYRIGHT
// file at the top-level directory of this distribution and at
// http://rust-lang.org/COPYRIGHT.
//
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
// option. This file may not be copied, modified, or distributed
// except according to those terms.
// NOTE: The following code was generated by "scripts/unicode.py", do not edit directly
"""
)
module.write(
f"""
/// The version of [Unicode](http://www.unicode.org/)
/// that this version of unicode-width is based on.
pub const UNICODE_VERSION: (u8, u8, u8) = {unicode_version};
"""
)
module.write(
"""
pub mod charwidth {
/// Returns the [UAX #11](https://www.unicode.org/reports/tr11/) based width of `c` by
/// consulting a multi-level lookup table.
/// If `is_cjk == true`, ambiguous width characters are treated as double width; otherwise,
/// they're treated as single width.
///
/// # Maintenance
/// The tables themselves are autogenerated but this function is hardcoded. You should have
/// nothing to worry about if you re-run `unicode.py` (for example, when updating Unicode.)
/// However, if you change the *actual structure* of the lookup tables (perhaps by editing the
/// `TABLE_CFGS` global in `unicode.py`) you must ensure that this code reflects those changes.
#[inline]
pub fn lookup_width(c: char, is_cjk: bool) -> usize {
let cp = c as usize;
let t1_offset = TABLES_0.0[cp >> 13 & 0xFF];
// Each sub-table in TABLES_1 is 7 bits, and each stored entry is a byte,
// so each sub-table is 128 bytes in size.
// (Sub-tables are selected using the computed offset from the previous table.)
let t2_offset = TABLES_1.0[128 * usize::from(t1_offset) + (cp >> 6 & 0x7F)];
// Each sub-table in TABLES_2 is 6 bits, but each stored entry is 2 bits.
// This is accomplished by packing four stored entries into one byte.
// So each sub-table is 2**(6-2) == 16 bytes in size.
// Since this is the last table, each entry represents an encoded width.
let packed_widths = TABLES_2.0[16 * usize::from(t2_offset) + (cp >> 2 & 0xF)];
// Extract the packed width
let width = packed_widths >> (2 * (cp & 0b11)) & 0b11;
// A width of 3 signifies that the codepoint is ambiguous width.
if width == 3 {
if is_cjk {
2
} else {
1
}
} else {
width.into()
}
}
"""
)
emoji_presentation_idx, emoji_presentation_leaves = emoji_presentation_table
text_presentation_idx, text_presentation_leaves = text_presentation_table
module.write(
"""
/// Whether this character forms an [emoji presentation sequence]
/// (https://www.unicode.org/reports/tr51/#def_emoji_presentation_sequence)
/// when followed by `'\\u{FEOF}'`.
/// Emoji presentation sequences are considered to have width 2.
/// This may spuriously return `true` or `false` for characters that are always wide.
#[inline]
pub fn starts_emoji_presentation_seq(c: char) -> bool {
let cp: u32 = c.into();
// First level of lookup uses all but 10 LSB
let top_bits = cp >> 10;
let idx_of_leaf: usize = match top_bits {
"""
)
for msbs, i in emoji_presentation_idx:
module.write(f" {msbs} => {i},\n")
module.write(
""" _ => return false,
};
// Extract the 3-9th (0-indexed) least significant bits of `cp`,
// and use them to index into `leaf_row`.
let idx_within_leaf = usize::try_from((cp >> 3) & 0x7F).unwrap();
let leaf_byte = EMOJI_PRESENTATION_LEAVES.0[idx_of_leaf][idx_within_leaf];
// Use the 3 LSB of `cp` to index into `leaf_byte`.
((leaf_byte >> (cp & 7)) & 1) == 1
}
"""
)
module.write(
"""
/// Returns `true` iff `c` has default emoji presentation, but forms a [text presentation sequence]
/// (https://www.unicode.org/reports/tr51/#def_text_presentation_sequence)
/// when followed by `'\\u{FEOE}'`, and is not ideographic.
/// Such sequences are considered to have width 1.
///
/// This may spuriously return `true` for characters of narrow or ambiguous width.
#[inline]
pub fn starts_non_ideographic_text_presentation_seq(c: char) -> bool {
let cp: u32 = c.into();
// First level of lookup uses all but 10 LSB
let top_bits = cp >> 10;
let idx_of_leaf: usize = match top_bits {
"""
)
for msbs, i in text_presentation_idx:
module.write(f" {msbs} => {i},\n")
module.write(
""" _ => return false,
};
// Extract the 3-9th (0-indexed) least significant bits of `cp`,
// and use them to index into `leaf_row`.
let idx_within_leaf = usize::try_from((cp >> 3) & 0x7F).unwrap();
let leaf_byte = TEXT_PRESENTATION_LEAVES.0[idx_of_leaf][idx_within_leaf];
// Use the 3 LSB of `cp` to index into `leaf_byte`.
((leaf_byte >> (cp & 7)) & 1) == 1
}
#[repr(align(128))]
struct Align128<T>(T);
#[repr(align(16))]
struct Align16<T>(T);
"""
)
subtable_count = 1
for i, table in enumerate(tables):
new_subtable_count = len(table.buckets())
if i == len(tables) - 1:
table.indices_to_widths() # for the last table, indices == widths
align = 16
else:
align = 128
byte_array = table.to_bytes()
module.write(
f"""
/// Autogenerated. {subtable_count} sub-table(s). Consult [`lookup_width`] for layout info.
static TABLES_{i}: Align{align}<[u8; {len(byte_array)}]> = Align{align}(["""
)
for j, byte in enumerate(byte_array):
# Add line breaks for every 15th entry (chosen to match what rustfmt does)
if j % 15 == 0:
module.write("\n ")
module.write(f" 0x{byte:02X},")
module.write("\n ]);\n")
subtable_count = new_subtable_count
# emoji table
module.write(
f"""
/// Array of 1024-bit bitmaps. Index into the correct bitmap with the 10 LSB of your codepoint
/// to get whether it can start an emoji presentation sequence.
static EMOJI_PRESENTATION_LEAVES: Align128<[[u8; 128]; {len(emoji_presentation_leaves)}]> = Align128([
"""
)
for leaf in emoji_presentation_leaves:
module.write(" [\n")
for row in batched(leaf, 14):
module.write(" ")
for entry in row:
module.write(f" 0x{entry:02X},")
module.write("\n")
module.write(" ],\n")
module.write(" ]);\n")
# text table
module.write(
f"""
/// Array of 1024-bit bitmaps. Index into the correct bitmap with the 10 LSB of your codepoint
/// to get whether it can start a text presentation sequence.
static TEXT_PRESENTATION_LEAVES: Align128<[[u8; 128]; {len(text_presentation_leaves)}]> = Align128([
"""
)
for leaf in text_presentation_leaves:
module.write(" [\n")
for row in batched(leaf, 14):
module.write(" ")
for entry in row:
module.write(f" 0x{entry:02X},")
module.write("\n")
module.write(" ],\n")
module.write(" ]);\n")
module.write("}\n")
def main(module_path: str):
"""Obtain character data from the latest version of Unicode, transform it into a multi-level
lookup table for character width, and write a Rust module utilizing that table to
`module_filename`.
See `lib.rs` for documentation of the exact width rules.
"""
version = load_unicode_version()
print(f"Generating module for Unicode {version[0]}.{version[1]}.{version[2]}")
eaw_map = load_east_asian_widths()
zw_map = load_zero_widths()
# Characters marked as zero-width in zw_map should be zero-width in the final map
width_map = list(
map(lambda x: EffectiveWidth.ZERO if x[1] else x[0], zip(eaw_map, zw_map))
)
tables = make_tables(TABLE_CFGS, enumerate(width_map))
emoji_presentations = load_emoji_presentation_sequences()
emoji_presentation_table = make_presentation_sequence_table(
emoji_presentations, width_map, {EffectiveWidth.WIDE}, {EffectiveWidth.WIDE}
)
text_presentations = load_text_presentation_sequences()
text_presentation_table = make_presentation_sequence_table(
text_presentations,
width_map,
set(),
{EffectiveWidth.NARROW, EffectiveWidth.AMBIGUOUS},
)
# Download normalization test file for use by tests
fetch_open("NormalizationTest.txt", "../tests/")
print("------------------------")
total_size = 0
for i, table in enumerate(tables):
size_bytes = len(table.to_bytes())
print(f"Table {i} size: {size_bytes} bytes")
total_size += size_bytes
for s, table in [
("Emoji", emoji_presentation_table),
("Text", text_presentation_table),
]:
index_size = len(table[0]) * 4
print(f"{s} presentation index size: {index_size} bytes")
total_size += index_size
leaves_size = len(table[1]) * len(table[1][0])
print(f"{s} presentation leaves size: {leaves_size} bytes")
total_size += leaves_size
print("------------------------")
print(f" Total size: {total_size} bytes")
emit_module(
module_path, version, tables, emoji_presentation_table, text_presentation_table
)
print(f'Wrote to "{module_path}"')
if __name__ == "__main__":
main(MODULE_PATH)
Codebase Browser
chromium