| #!/usr/bin/env python3 |
| # SPDX-License-Identifier: GPL-2.0 |
| # |
| # Leverage Python's unicodedata module to generate ucs_width_table.h |
| |
| import unicodedata |
| import sys |
| import argparse |
| |
| # This script's file name |
| from pathlib import Path |
| this_file = Path(__file__).name |
| |
| # Default output file name |
| DEFAULT_OUT_FILE = "ucs_width_table.h" |
| |
| # --- Global Constants for Width Assignments --- |
| |
| # Known zero-width characters |
| KNOWN_ZERO_WIDTH = ( |
| 0x200B, # ZERO WIDTH SPACE |
| 0x200C, # ZERO WIDTH NON-JOINER |
| 0x200D, # ZERO WIDTH JOINER |
| 0x2060, # WORD JOINER |
| 0xFEFF # ZERO WIDTH NO-BREAK SPACE (BOM) |
| ) |
| |
| # Zero-width emoji modifiers and components |
| # NOTE: Some of these characters would normally be single-width according to |
| # East Asian Width properties, but we deliberately override them to be |
| # zero-width because they function as modifiers in emoji sequences. |
| EMOJI_ZERO_WIDTH = [ |
| # Skin tone modifiers |
| (0x1F3FB, 0x1F3FF), # Emoji modifiers (skin tones) |
| |
| # Variation selectors (note: VS16 is treated specially in vt.c) |
| (0xFE00, 0xFE0F), # Variation Selectors 1-16 |
| |
| # Gender and hair style modifiers |
| # These would be single-width by Unicode properties, but are zero-width |
| # when part of emoji |
| (0x2640, 0x2640), # Female sign |
| (0x2642, 0x2642), # Male sign |
| (0x26A7, 0x26A7), # Transgender symbol |
| (0x1F9B0, 0x1F9B3), # Hair components (red, curly, white, bald) |
| |
| # Tag characters |
| (0xE0020, 0xE007E), # Tags |
| ] |
| |
| # Regional indicators (flag components) |
| REGIONAL_INDICATORS = (0x1F1E6, 0x1F1FF) # Regional indicator symbols A-Z |
| |
| # Double-width emoji ranges |
| # |
| # Many emoji characters are classified as single-width according to Unicode |
| # Standard Annex #11 East Asian Width property (N or Neutral), but we |
| # deliberately override them to be double-width. References: |
| # 1. Unicode Technical Standard #51: Unicode Emoji |
| # (https://www.unicode.org/reports/tr51/) |
| # 2. Principle of "emoji presentation" in WHATWG CSS Text specification |
| # (https://drafts.csswg.org/css-text-3/#character-properties) |
| # 3. Terminal emulator implementations (iTerm2, Windows Terminal, etc.) which |
| # universally render emoji as double-width characters regardless of their |
| # Unicode EAW property |
| # 4. W3C Work Item: Requirements for Japanese Text Layout - Section 3.8.1 |
| # Emoji width (https://www.w3.org/TR/jlreq/) |
| EMOJI_RANGES = [ |
| (0x1F000, 0x1F02F), # Mahjong Tiles (EAW: N, but displayed as double-width) |
| (0x1F0A0, 0x1F0FF), # Playing Cards (EAW: N, but displayed as double-width) |
| (0x1F300, 0x1F5FF), # Miscellaneous Symbols and Pictographs |
| (0x1F600, 0x1F64F), # Emoticons |
| (0x1F680, 0x1F6FF), # Transport and Map Symbols |
| (0x1F700, 0x1F77F), # Alchemical Symbols |
| (0x1F780, 0x1F7FF), # Geometric Shapes Extended |
| (0x1F800, 0x1F8FF), # Supplemental Arrows-C |
| (0x1F900, 0x1F9FF), # Supplemental Symbols and Pictographs |
| (0x1FA00, 0x1FA6F), # Chess Symbols |
| (0x1FA70, 0x1FAFF), # Symbols and Pictographs Extended-A |
| ] |
| |
| def create_width_tables(): |
| """ |
| Creates Unicode character width tables and returns the data structures. |
| |
| Returns: |
| tuple: (zero_width_ranges, double_width_ranges) |
| """ |
| |
| # Width data mapping |
| width_map = {} # Maps code points to width (0, 1, 2) |
| |
| # Mark emoji modifiers as zero-width |
| for start, end in EMOJI_ZERO_WIDTH: |
| for cp in range(start, end + 1): |
| width_map[cp] = 0 |
| |
| # Mark all regional indicators as single-width as they are usually paired |
| # providing a combined width of 2 when displayed together. |
| start, end = REGIONAL_INDICATORS |
| for cp in range(start, end + 1): |
| width_map[cp] = 1 |
| |
| # Process all assigned Unicode code points (Basic Multilingual Plane + |
| # Supplementary Planes) Range 0x0 to 0x10FFFF (the full Unicode range) |
| for block_start in range(0, 0x110000, 0x1000): |
| block_end = block_start + 0x1000 |
| for cp in range(block_start, block_end): |
| try: |
| char = chr(cp) |
| |
| # Skip if already processed |
| if cp in width_map: |
| continue |
| |
| # Check for combining marks and a format characters |
| category = unicodedata.category(char) |
| |
| # Combining marks |
| if category.startswith('M'): |
| width_map[cp] = 0 |
| continue |
| |
| # Format characters |
| # Since we have no support for bidirectional text, all format |
| # characters (category Cf) can be treated with width 0 (zero) |
| # for simplicity, as they don't need to occupy visual space |
| # in a non-bidirectional text environment. |
| if category == 'Cf': |
| width_map[cp] = 0 |
| continue |
| |
| # Known zero-width characters |
| if cp in KNOWN_ZERO_WIDTH: |
| width_map[cp] = 0 |
| continue |
| |
| # Use East Asian Width property |
| eaw = unicodedata.east_asian_width(char) |
| if eaw in ('F', 'W'): # Fullwidth or Wide |
| width_map[cp] = 2 |
| elif eaw in ('Na', 'H', 'N', 'A'): # Narrow, Halfwidth, Neutral, Ambiguous |
| width_map[cp] = 1 |
| else: |
| # Default to single-width for unknown |
| width_map[cp] = 1 |
| |
| except (ValueError, OverflowError): |
| # Skip invalid code points |
| continue |
| |
| # Process Emoji - generally double-width |
| for start, end in EMOJI_RANGES: |
| for cp in range(start, end + 1): |
| if cp not in width_map or width_map[cp] != 0: # Don't override zero-width |
| try: |
| char = chr(cp) |
| width_map[cp] = 2 |
| except (ValueError, OverflowError): |
| continue |
| |
| # Optimize to create range tables |
| def ranges_optimize(width_data, target_width): |
| points = sorted([cp for cp, width in width_data.items() if width == target_width]) |
| if not points: |
| return [] |
| |
| # Group consecutive code points into ranges |
| ranges = [] |
| start = points[0] |
| prev = start |
| |
| for cp in points[1:]: |
| if cp > prev + 1: |
| ranges.append((start, prev)) |
| start = cp |
| prev = cp |
| |
| # Add the last range |
| ranges.append((start, prev)) |
| return ranges |
| |
| # Extract ranges for each width |
| zero_width_ranges = ranges_optimize(width_map, 0) |
| double_width_ranges = ranges_optimize(width_map, 2) |
| |
| return zero_width_ranges, double_width_ranges |
| |
| def write_tables(zero_width_ranges, double_width_ranges, out_file=DEFAULT_OUT_FILE): |
| """ |
| Write the generated tables to C header file. |
| |
| Args: |
| zero_width_ranges: List of (start, end) ranges for zero-width characters |
| double_width_ranges: List of (start, end) ranges for double-width characters |
| out_file: Output file name (default: DEFAULT_OUT_FILE) |
| """ |
| |
| # Function to split ranges into BMP (16-bit) and non-BMP (above 16-bit) |
| def split_ranges_by_size(ranges): |
| bmp_ranges = [] |
| non_bmp_ranges = [] |
| |
| for start, end in ranges: |
| if end <= 0xFFFF: |
| bmp_ranges.append((start, end)) |
| elif start > 0xFFFF: |
| non_bmp_ranges.append((start, end)) |
| else: |
| # Split the range at 0xFFFF |
| bmp_ranges.append((start, 0xFFFF)) |
| non_bmp_ranges.append((0x10000, end)) |
| |
| return bmp_ranges, non_bmp_ranges |
| |
| # Split ranges into BMP and non-BMP |
| zero_width_bmp, zero_width_non_bmp = split_ranges_by_size(zero_width_ranges) |
| double_width_bmp, double_width_non_bmp = split_ranges_by_size(double_width_ranges) |
| |
| # Function to generate code point description comments |
| def get_code_point_comment(start, end): |
| try: |
| start_char_desc = unicodedata.name(chr(start)) |
| if start == end: |
| return f"/* {start_char_desc} */" |
| else: |
| end_char_desc = unicodedata.name(chr(end)) |
| return f"/* {start_char_desc} - {end_char_desc} */" |
| except: |
| if start == end: |
| return f"/* U+{start:04X} */" |
| else: |
| return f"/* U+{start:04X} - U+{end:04X} */" |
| |
| # Generate C tables |
| with open(out_file, 'w') as f: |
| f.write(f"""\ |
| /* SPDX-License-Identifier: GPL-2.0 */ |
| /* |
| * {out_file} - Unicode character width |
| * |
| * Auto-generated by {this_file} |
| * |
| * Unicode Version: {unicodedata.unidata_version} |
| */ |
| |
| /* Zero-width character ranges (BMP - Basic Multilingual Plane, U+0000 to U+FFFF) */ |
| static const struct ucs_interval16 ucs_zero_width_bmp_ranges[] = {{ |
| """) |
| |
| for start, end in zero_width_bmp: |
| comment = get_code_point_comment(start, end) |
| f.write(f"\t{{ 0x{start:04X}, 0x{end:04X} }}, {comment}\n") |
| |
| f.write("""\ |
| }; |
| |
| /* Zero-width character ranges (non-BMP, U+10000 and above) */ |
| static const struct ucs_interval32 ucs_zero_width_non_bmp_ranges[] = { |
| """) |
| |
| for start, end in zero_width_non_bmp: |
| comment = get_code_point_comment(start, end) |
| f.write(f"\t{{ 0x{start:05X}, 0x{end:05X} }}, {comment}\n") |
| |
| f.write("""\ |
| }; |
| |
| /* Double-width character ranges (BMP - Basic Multilingual Plane, U+0000 to U+FFFF) */ |
| static const struct ucs_interval16 ucs_double_width_bmp_ranges[] = { |
| """) |
| |
| for start, end in double_width_bmp: |
| comment = get_code_point_comment(start, end) |
| f.write(f"\t{{ 0x{start:04X}, 0x{end:04X} }}, {comment}\n") |
| |
| f.write("""\ |
| }; |
| |
| /* Double-width character ranges (non-BMP, U+10000 and above) */ |
| static const struct ucs_interval32 ucs_double_width_non_bmp_ranges[] = { |
| """) |
| |
| for start, end in double_width_non_bmp: |
| comment = get_code_point_comment(start, end) |
| f.write(f"\t{{ 0x{start:05X}, 0x{end:05X} }}, {comment}\n") |
| |
| f.write("};\n") |
| |
| if __name__ == "__main__": |
| # Parse command line arguments |
| parser = argparse.ArgumentParser(description="Generate Unicode width tables") |
| parser.add_argument("-o", "--output", dest="output_file", default=DEFAULT_OUT_FILE, |
| help=f"Output file name (default: {DEFAULT_OUT_FILE})") |
| args = parser.parse_args() |
| |
| # Write tables to header file |
| zero_width_ranges, double_width_ranges = create_width_tables() |
| write_tables(zero_width_ranges, double_width_ranges, out_file=args.output_file) |
| |
| # Print summary |
| zero_width_count = sum(end - start + 1 for start, end in zero_width_ranges) |
| double_width_count = sum(end - start + 1 for start, end in double_width_ranges) |
| print(f"Generated {args.output_file} with:") |
| print(f"- {len(zero_width_ranges)} zero-width ranges covering ~{zero_width_count} code points") |
| print(f"- {len(double_width_ranges)} double-width ranges covering ~{double_width_count} code points") |
| print(f"- Unicode Version: {unicodedata.unidata_version}") |
