diff options
Diffstat (limited to 'intl/unicharutil/tools/genUnicodePropertyData.pl')
| -rwxr-xr-x | intl/unicharutil/tools/genUnicodePropertyData.pl | 434 |
1 files changed, 14 insertions, 420 deletions
diff --git a/intl/unicharutil/tools/genUnicodePropertyData.pl b/intl/unicharutil/tools/genUnicodePropertyData.pl index e17b6cb76a..8c7437f82d 100755 --- a/intl/unicharutil/tools/genUnicodePropertyData.pl +++ b/intl/unicharutil/tools/genUnicodePropertyData.pl @@ -9,6 +9,10 @@ # read from the Unicode Character Database and compiled into multi-level arrays # for efficient lookup. # +# Note that for most properties, we now rely on ICU; this tool and the tables +# it generates are used only for a couple of properties not readily exposed +# via ICU APIs. +# # To regenerate the tables in nsUnicodePropertyData.cpp: # # (1) Download the current Unicode data files from @@ -17,13 +21,6 @@ # # NB: not all the files are actually needed; currently, we require # - UnicodeData.txt -# - Scripts.txt -# - BidiMirroring.txt -# - BidiBrackets.txt -# - HangulSyllableType.txt -# - LineBreak.txt -# - EastAsianWidth.txt -# - DerivedCoreProperties.txt # - ReadMe.txt (to record version/date of the UCD) # - Unihan_Variants.txt (from Unihan.zip) # though this may change if we find a need for additional properties. @@ -44,7 +41,6 @@ # (2) Run this tool using a command line of the form # # perl genUnicodePropertyData.pl \ -# /path/to/harfbuzz/src \ # /path/to/icu/common/unicode \ # /path/to/UCD-directory # @@ -58,17 +54,15 @@ use strict; use List::Util qw(first); -if ($#ARGV != 2) { +if ($#ARGV != 1) { print <<__EOT; # Run this tool using a command line of the form # # perl genUnicodePropertyData.pl \\ -# /path/to/harfbuzz/src \\ # /path/to/icu/common/unicode \\ # /path/to/UCD-directory # -# where harfbuzz/src is the directory containing harfbuzz .cc and .hh files, -# icu/common/unicode is the directory containing ICU 'common' public headers, +# where icu/common/unicode is the directory containing ICU 'common' headers, # and UCD-directory is a directory containing the current Unicode Character # Database files (UnicodeData.txt, etc), available from # http://www.unicode.org/Public/UNIDATA/, with additional resources as @@ -84,35 +78,11 @@ __EOT exit 0; } -my $HARFBUZZ = $ARGV[0]; -my $ICU = $ARGV[1]; -my $UNICODE = $ARGV[2]; - -# load HB_Category constants - -my $cc = -1; -my %catCode; +my $ICU = $ARGV[0]; +my $UNICODE = $ARGV[1]; -sub readHarfBuzzHeader -{ - my $file = shift; - open FH, "< $HARFBUZZ/$file" or die "can't open harfbuzz header $HARFBUZZ/$file\n"; - while (<FH>) { - if (m/HB_UNICODE_GENERAL_CATEGORY_([A-Z_]+)/) { - $cc++; - $catCode{$1} = $cc; - } - } - close FH; -} - -&readHarfBuzzHeader("hb-unicode.h"); - -die "didn't find HarfBuzz category codes\n" if $cc == -1; - -my %scriptCode; -my @scriptCodeToTag; my @scriptCodeToName; +my @idtype; my $sc = -1; @@ -129,8 +99,6 @@ sub readIcuHeader s/SIGN_WRITING/SIGNWRITING/; if (m|USCRIPT_([A-Z_]+)\s*=\s*([0-9]+),\s*/\*\s*([A-Z][a-z]{3})\s*\*/|) { $sc = $2; - $scriptCode{$1} = $sc; - $scriptCodeToTag[$sc] = $3; $scriptCodeToName[$sc] = $1; } } @@ -170,32 +138,6 @@ my %mappedIdType = ( "Allowed" => 1 ); -my %bidicategoryCode = ( - "L" => 0, # Left-to-Right - "R" => 1, # Right-to-Left - "EN" => 2, # European Number - "ES" => 3, # European Number Separator - "ET" => 4, # European Number Terminator - "AN" => 5, # Arabic Number - "CS" => 6, # Common Number Separator - "B" => 7, # Paragraph Separator - "S" => 8, # Segment Separator - "WS" => 9, # Whitespace - "ON" => 10, # Other Neutrals - "LRE" => 11, # Left-to-Right Embedding - "LRO" => 12, # Left-to-Right Override - "AL" => 13, # Right-to-Left Arabic - "RLE" => 14, # Right-to-Left Embedding - "RLO" => 15, # Right-to-Left Override - "PDF" => 16, # Pop Directional Format - "NSM" => 17, # Non-Spacing Mark - "BN" => 18, # Boundary Neutral - "FSI" => 19, # First Strong Isolate - "LRI" => 20, # Left-to-Right Isolate - "RLI" => 21, # Right-to-left Isolate - "PDI" => 22 # Pop Direcitonal Isolate -); - my %verticalOrientationCode = ( 'U' => 0, # U - Upright, the same orientation as in the code charts 'R' => 1, # R - Rotated 90 degrees clockwise compared to the code charts @@ -203,141 +145,18 @@ my %verticalOrientationCode = ( 'Tr' => 3 # Tr - Transformed typographically, with fallback to Rotated ); -my %lineBreakCode = ( # ordering matches ICU's ULineBreak enum - "XX" => 0, - "AI" => 1, - "AL" => 2, - "B2" => 3, - "BA" => 4, - "BB" => 5, - "BK" => 6, - "CB" => 7, - "CL" => 8, - "CM" => 9, - "CR" => 10, - "EX" => 11, - "GL" => 12, - "HY" => 13, - "ID" => 14, - "IN" => 15, - "IS" => 16, - "LF" => 17, - "NS" => 18, - "NU" => 19, - "OP" => 20, - "PO" => 21, - "PR" => 22, - "QU" => 23, - "SA" => 24, - "SG" => 25, - "SP" => 26, - "SY" => 27, - "ZW" => 28, - "NL" => 29, - "WJ" => 30, - "H2" => 31, - "H3" => 32, - "JL" => 33, - "JT" => 34, - "JV" => 35, - "CP" => 36, - "CJ" => 37, - "HL" => 38, - "RI" => 39, - "EB" => 40, - "EM" => 41, - "ZWJ" => 42 -); - -my %eastAsianWidthCode = ( - "N" => 0, - "A" => 1, - "H" => 2, - "W" => 3, - "F" => 4, - "Na" => 5 -); - # initialize default properties -my @script; -my @category; -my @combining; -my @mirror; -my @pairedBracketType; -my @hangul; -my @casemap; -my @idtype; -my @numericvalue; my @hanVariant; -my @bidicategory; my @fullWidth; my @fullWidthInverse; my @verticalOrientation; -my @lineBreak; -my @eastAsianWidthFWH; -my @defaultIgnorable; for (my $i = 0; $i < 0x110000; ++$i) { - $script[$i] = $scriptCode{"UNKNOWN"}; - $category[$i] = $catCode{"UNASSIGNED"}; - $combining[$i] = 0; - $pairedBracketType[$i] = 0; - $casemap[$i] = 0; - $idtype[$i] = $mappedIdType{'Restricted'}; - $numericvalue[$i] = -1; $hanVariant[$i] = 0; - $bidicategory[$i] = $bidicategoryCode{"L"}; $fullWidth[$i] = 0; $fullWidthInverse[$i] = 0; $verticalOrientation[$i] = 1; # default for unlisted codepoints is 'R' - $lineBreak[$i] = $lineBreakCode{"XX"}; - $eastAsianWidthFWH[$i] = 0; - $defaultIgnorable[$i] = 0; } -# blocks where the default for bidi category is not L -for my $i (0x0600..0x07BF, 0x08A0..0x08FF, 0xFB50..0xFDCF, 0xFDF0..0xFDFF, 0xFE70..0xFEFF, 0x1EE00..0x0001EEFF) { - $bidicategory[$i] = $bidicategoryCode{"AL"}; -} -for my $i (0x0590..0x05FF, 0x07C0..0x089F, 0xFB1D..0xFB4F, 0x00010800..0x00010FFF, 0x0001E800..0x0001EDFF, 0x0001EF00..0x0001EFFF) { - $bidicategory[$i] = $bidicategoryCode{"R"}; -} -for my $i (0x20A0..0x20CF) { - $bidicategory[$i] = $bidicategoryCode{"ET"}; -} - -my %ucd2hb = ( -'Cc' => 'CONTROL', -'Cf' => 'FORMAT', -'Cn' => 'UNASSIGNED', -'Co' => 'PRIVATE_USE', -'Cs' => 'SURROGATE', -'Ll' => 'LOWERCASE_LETTER', -'Lm' => 'MODIFIER_LETTER', -'Lo' => 'OTHER_LETTER', -'Lt' => 'TITLECASE_LETTER', -'Lu' => 'UPPERCASE_LETTER', -'Mc' => 'SPACING_MARK', -'Me' => 'ENCLOSING_MARK', -'Mn' => 'NON_SPACING_MARK', -'Nd' => 'DECIMAL_NUMBER', -'Nl' => 'LETTER_NUMBER', -'No' => 'OTHER_NUMBER', -'Pc' => 'CONNECT_PUNCTUATION', -'Pd' => 'DASH_PUNCTUATION', -'Pe' => 'CLOSE_PUNCTUATION', -'Pf' => 'FINAL_PUNCTUATION', -'Pi' => 'INITIAL_PUNCTUATION', -'Po' => 'OTHER_PUNCTUATION', -'Ps' => 'OPEN_PUNCTUATION', -'Sc' => 'CURRENCY_SYMBOL', -'Sk' => 'MODIFIER_SYMBOL', -'Sm' => 'MATH_SYMBOL', -'So' => 'OTHER_SYMBOL', -'Zl' => 'LINE_SEPARATOR', -'Zp' => 'PARAGRAPH_SEPARATOR', -'Zs' => 'SPACE_SEPARATOR' -); - # read ReadMe.txt my @versionInfo; open FH, "< $UNICODE/ReadMe.txt" or die "can't open Unicode ReadMe.txt file\n"; @@ -347,12 +166,6 @@ while (<FH>) { } close FH; -my $kTitleToUpper = 0x80000000; -my $kUpperToLower = 0x40000000; -my $kLowerToTitle = 0x20000000; -my $kLowerToUpper = 0x10000000; -my $kCaseMapCharMask = 0x001fffff; - # read UnicodeData.txt open FH, "< $UNICODE/UnicodeData.txt" or die "can't open UCD file UnicodeData.txt\n"; while (<FH>) { @@ -365,12 +178,6 @@ while (<FH>) { if ($fields[1] =~ /Last/) { my $last = hex "0x$fields[0]"; do { - $category[$first] = $catCode{$ucd2hb{$fields[2]}}; - $combining[$first] = $fields[3]; - $bidicategory[$first] = $bidicategoryCode{$fields[4]}; - unless (length($fields[7]) == 0) { - $numericvalue[$first] = $fields[7]; - } if ($fields[1] =~ /CJK/) { @hanVariant[$first] = 3; } @@ -381,33 +188,6 @@ while (<FH>) { } } else { my $usv = hex "0x$fields[0]"; - $category[$usv] = $catCode{$ucd2hb{$fields[2]}}; - $combining[$usv] = $fields[3]; - my $upper = hex $fields[12]; - my $lower = hex $fields[13]; - my $title = hex $fields[14]; - # we only store one mapping for each character, - # but also record what kind of mapping it is - if ($upper && $lower) { - $casemap[$usv] |= $kTitleToUpper; - $casemap[$usv] |= ($usv ^ $upper); - } - elsif ($lower) { - $casemap[$usv] |= $kUpperToLower; - $casemap[$usv] |= ($usv ^ $lower); - } - elsif ($title && ($title != $upper)) { - $casemap[$usv] |= $kLowerToTitle; - $casemap[$usv] |= ($usv ^ $title); - } - elsif ($upper) { - $casemap[$usv] |= $kLowerToUpper; - $casemap[$usv] |= ($usv ^ $upper); - } - $bidicategory[$usv] = $bidicategoryCode{$fields[4]}; - unless (length($fields[7]) == 0) { - $numericvalue[$usv] = $fields[7]; - } if ($fields[1] =~ /CJK/) { @hanVariant[$usv] = 3; } @@ -427,180 +207,6 @@ while (<FH>) { } close FH; -# read Scripts.txt -open FH, "< $UNICODE/Scripts.txt" or die "can't open UCD file Scripts.txt\n"; -push @versionInfo, ""; -while (<FH>) { - chomp; - push @versionInfo, $_; - last if /Date:/; -} -while (<FH>) { - if (m/([0-9A-F]{4,6})(?:\.\.([0-9A-F]{4,6}))*\s+;\s+([^ ]+)/) { - my $script = uc($3); - unless (exists $scriptCode{$script}) { - warn "unknown ICU script $script"; - $scriptCode{$script} = $scriptCode{"UNKNOWN"}; - } - $script = $scriptCode{$script}; - my $script = $scriptCode{$script}; - my $start = hex "0x$1"; - my $end = (defined $2) ? hex "0x$2" : $start; - for (my $i = $start; $i <= $end; ++$i) { - $script[$i] = $script; - } - } -} -close FH; - -# read BidiMirroring.txt -my @offsets = (); -push @offsets, 0; - -open FH, "< $UNICODE/BidiMirroring.txt" or die "can't open UCD file BidiMirroring.txt\n"; -push @versionInfo, ""; -while (<FH>) { - chomp; - push @versionInfo, $_; - last if /Date:/; -} -while (<FH>) { - s/#.*//; - if (m/([0-9A-F]{4,6});\s*([0-9A-F]{4,6})/) { - my $mirrorOffset = hex("0x$2") - hex("0x$1"); - my $offsetIndex = first { $offsets[$_] eq $mirrorOffset } 0..$#offsets; - if ($offsetIndex == undef) { - die "too many offset codes\n" if scalar @offsets == 31; - push @offsets, $mirrorOffset; - $offsetIndex = $#offsets; - } - $mirror[hex "0x$1"] = $offsetIndex; - } -} -close FH; - -# read BidiBrackets.txt -my %pairedBracketTypeCode = ( - 'N' => 0, - 'O' => 1, - 'C' => 2 -); -open FH, "< $UNICODE/BidiBrackets.txt" or die "can't open UCD file BidiBrackets.txt\n"; -push @versionInfo, ""; -while (<FH>) { - chomp; - push @versionInfo, $_; - last if /Date:/; -} -while (<FH>) { - s/#.*//; - if (m/([0-9A-F]{4,6});\s*([0-9A-F]{4,6});\s*(.)/) { - my $mirroredChar = $offsets[$mirror[hex "0x$1"]] + hex "0x$1"; - die "bidi bracket does not match mirrored char\n" unless $mirroredChar == hex "0x$2"; - my $pbt = uc($3); - warn "unknown Bidi Bracket type" unless exists $pairedBracketTypeCode{$pbt}; - $pairedBracketType[hex "0x$1"] = $pairedBracketTypeCode{$pbt}; - } -} -close FH; - -# read HangulSyllableType.txt -my %hangulType = ( - 'L' => 0x01, - 'V' => 0x02, - 'T' => 0x04, - 'LV' => 0x03, - 'LVT' => 0x07 -); -open FH, "< $UNICODE/HangulSyllableType.txt" or die "can't open UCD file HangulSyllableType.txt\n"; -push @versionInfo, ""; -while (<FH>) { - chomp; - push @versionInfo, $_; - last if /Date:/; -} -while (<FH>) { - s/#.*//; - if (m/([0-9A-F]{4,6})(?:\.\.([0-9A-F]{4,6}))*\s*;\s*([^ ]+)/) { - my $hangul = uc($3); - warn "unknown Hangul syllable type" unless exists $hangulType{$hangul}; - $hangul = $hangulType{$hangul}; - my $start = hex "0x$1"; - my $end = (defined $2) ? hex "0x$2" : $start; - for (my $i = $start; $i <= $end; ++$i) { - $hangul[$i] = $hangul; - } - } -} -close FH; - -# read LineBreak.txt -open FH, "< $UNICODE/LineBreak.txt" or die "can't open UCD file LineBreak.txt\n"; -push @versionInfo, ""; -while (<FH>) { - chomp; - push @versionInfo, $_; - last if /Date:/; -} -while (<FH>) { - s/#.*//; - if (m/([0-9A-F]{4,6})(?:\.\.([0-9A-F]{4,6}))*\s*;\s*([^ ]+)/) { - my $lb = uc($3); - warn "unknown LineBreak class" unless exists $lineBreakCode{$lb}; - $lb = $lineBreakCode{$lb}; - my $start = hex "0x$1"; - my $end = (defined $2) ? hex "0x$2" : $start; - for (my $i = $start; $i <= $end; ++$i) { - $lineBreak[$i] = $lb; - } - } -} -close FH; - -# read EastAsianWidth.txt -open FH, "< $UNICODE/EastAsianWidth.txt" or die "can't open UCD file EastAsianWidth.txt\n"; -push @versionInfo, ""; -while (<FH>) { - chomp; - push @versionInfo, $_; - last if /Date:/; -} -while (<FH>) { - s/#.*//; - if (m/([0-9A-F]{4,6})(?:\.\.([0-9A-F]{4,6}))*\s*;\s*([^ ]+)/) { - my $start = hex "0x$1"; - my $end = (defined $2) ? hex "0x$2" : $start; - my $eaw = $3; - warn "unknown EastAsianWidth class" unless exists $eastAsianWidthCode{$eaw}; - my $isFWH = ($eaw =~ m/^[FWH]$/) ? 1 : 0; - for (my $i = $start; $i <= $end; ++$i) { - $eastAsianWidthFWH[$i] = $isFWH; - } - } -} -close FH; - -# read DerivedCoreProperties.txt (for Default-Ignorables) -open FH, "< $UNICODE/DerivedCoreProperties.txt" or die "can't open UCD file DerivedCoreProperties.txt\n"; -push @versionInfo, ""; - -while (<FH>) { - chomp; - push @versionInfo, $_; - last if /Date:/; -} -while (<FH>) { - s/#.*//; - if (m/([0-9A-F]{4,6})(?:\.\.([0-9A-F]{4,6}))*\s*;\s*Default_Ignorable_Code_Point/) { - my $start = hex "0x$1"; - my $end = (defined $2) ? hex "0x$2" : $start; - for (my $i = $start; $i <= $end; ++$i) { - $defaultIgnorable[$i] = 1; - } - } -} -close FH; - # read IdentifierStatus.txt open FH, "< $UNICODE/security/IdentifierStatus.txt" or die "can't open UCD file IdentifierStatus.txt\n"; push @versionInfo, ""; @@ -759,8 +365,7 @@ struct nsCharProps2 { unsigned char mIdType:2; }; |; -&genTables("", "", - "CharProp2", $type, "nsCharProps2", 9, 7, \&sprintCharProps2, 16, 1, 1); +&genTables("CharProp2", $type, "nsCharProps2", 9, 7, \&sprintCharProps2, 16, 1, 1); print HEADER "#pragma pack()\n\n"; @@ -776,42 +381,32 @@ sub sprintHanVariants return sprintf("0x%02x,", $val); } ## Han Variant data currently unused but may be needed in future, see bug 857481 -## &genTables("", "", "HanVariant", "", "uint8_t", 9, 7, \&sprintHanVariants, 2, 1, 4); +## &genTables("HanVariant", "", "uint8_t", 9, 7, \&sprintHanVariants, 2, 1, 4); sub sprintFullWidth { my $usv = shift; return sprintf("0x%04x,", $fullWidth[$usv]); } -&genTables("", "", "FullWidth", "", "uint16_t", 10, 6, \&sprintFullWidth, 0, 2, 1); +&genTables("FullWidth", "", "uint16_t", 10, 6, \&sprintFullWidth, 0, 2, 1); sub sprintFullWidthInverse { my $usv = shift; return sprintf("0x%04x,", $fullWidthInverse[$usv]); } -&genTables("", "", "FullWidthInverse", "", "uint16_t", 10, 6, \&sprintFullWidthInverse, 0, 2, 1); +&genTables("FullWidthInverse", "", "uint16_t", 10, 6, \&sprintFullWidthInverse, 0, 2, 1); print STDERR "Total data = $totalData\n"; -printf DATA_TABLES "const uint32_t kTitleToUpper = 0x%08x;\n", $kTitleToUpper; -printf DATA_TABLES "const uint32_t kUpperToLower = 0x%08x;\n", $kUpperToLower; -printf DATA_TABLES "const uint32_t kLowerToTitle = 0x%08x;\n", $kLowerToTitle; -printf DATA_TABLES "const uint32_t kLowerToUpper = 0x%08x;\n", $kLowerToUpper; -printf DATA_TABLES "const uint32_t kCaseMapCharMask = 0x%08x;\n\n", $kCaseMapCharMask; - sub genTables { - my ($guardBegin, $guardEnd, - $prefix, $typedef, $type, $indexBits, $charBits, $func, $maxPlane, $bytesPerEntry, $charsPerEntry) = @_; + my ($prefix, $typedef, $type, $indexBits, $charBits, $func, $maxPlane, $bytesPerEntry, $charsPerEntry) = @_; if ($typedef ne '') { - print HEADER "$guardBegin\n"; print HEADER "$typedef\n"; - print HEADER "$guardEnd\n\n"; } - print DATA_TABLES "\n$guardBegin\n"; print DATA_TABLES "#define k${prefix}MaxPlane $maxPlane\n"; print DATA_TABLES "#define k${prefix}IndexBits $indexBits\n"; print DATA_TABLES "#define k${prefix}CharBits $charBits\n"; @@ -880,7 +475,6 @@ sub genTables print DATA_TABLES $i < $#char ? "},\n" : "}\n"; } print DATA_TABLES "};\n"; - print DATA_TABLES "$guardEnd\n"; my $dataSize = $pmCount * $indexLen * $pmBits/8 + $chCount * $pageLen * $bytesPerEntry + |