Breakiterator patch for ICU 4.6
word.txt : Chinese/Japanese segmentation rules, Hebrew-script-specific handling of U+0022, and
splitting of FQDN into labels at '.'.
line.txt : Incorporated line_he and minor changes in CL, OP and ID definitions.
brklocal.mk : build file changes to drop unnecessary brkitr rule files (e.g. word_ja.txt,
line_he.txt)
BUG=61514
TEST=NONE
Review URL: http://codereview.chromium.org/6349014
git-svn-id: http://src.chromium.org/svn/trunk/deps/third_party/icu46@72730 4ff67af0-8c30-449e-8e8b-ad334ec8d88c
diff --git a/patches/brkitr.patch b/patches/brkitr.patch
new file mode 100644
index 0000000..a78da75
--- /dev/null
+++ b/patches/brkitr.patch
@@ -0,0 +1,21 @@
+--- source/data/brkitr/line.txt 2009-06-24 14:06:38.000000000 -0700
++++ source/data/brkitr/line.txt 2009-08-07 17:27:26.991805000 -0700
+@@ -69,7 +69,8 @@
+ $HY = [:LineBreak = Hyphen:];
+ $H2 = [:LineBreak = H2:];
+ $H3 = [:LineBreak = H3:];
+-$ID = [:LineBreak = Ideographic:];
++$IDDelta = [\ufe45\ufe46\ufe51];
++$ID = [[:LineBreak = Ideographic:] - $IDDelta];
+ $IN = [:LineBreak = Inseperable:];
+ $IS = [:LineBreak = Infix_Numeric:];
+ $JL = [:LineBreak = JL:];
+@@ -77,7 +78,7 @@
+ $JT = [:LineBreak = JT:];
+ $LF = [:LineBreak = Line_Feed:];
+ $NL = [:LineBreak = Next_Line:];
+-$NS = [:LineBreak = Nonstarter:];
++$NS = [[:LineBreak = Nonstarter:] $IDDelta \u23b5];
+ $NU = [:LineBreak = Numeric:];
+ $OP = [:LineBreak = Open_Punctuation:];
+ $PO = [:LineBreak = Postfix_Numeric:];
diff --git a/source/data/brkitr/brklocal.mk b/source/data/brkitr/brklocal.mk
new file mode 100644
index 0000000..91754f1
--- /dev/null
+++ b/source/data/brkitr/brklocal.mk
@@ -0,0 +1,48 @@
+# * Copyright (C) 1998-2010, International Business Machines
+# * Corporation and others. All Rights Reserved.
+BRK_RES_CLDR_VERSION = 1.9
+# A list of txt's to build
+# Note:
+#
+# If you are thinking of modifying this file, READ THIS.
+#
+# Instead of changing this file [unless you want to check it back in],
+# you should consider creating a 'brklocal.mk' file in this same directory.
+# Then, you can have your local changes remain even if you upgrade or
+# reconfigure ICU.
+#
+# Example 'brklocal.mk' files:
+#
+# * To add an additional locale to the list:
+# _____________________________________________________
+# | BRK_RES_SOURCE_LOCAL = myLocale.txt ...
+#
+# * To REPLACE the default list and only build with a few
+# locales:
+# _____________________________________________________
+# | BRK_RES_SOURCE = ar.txt ar_AE.txt en.txt de.txt zh.txt
+#
+#
+# Generated by LDML2ICUConverter, from LDML source files.
+
+# Aliases without a corresponding xx.xml file (see icu-config.xml & build.xml)
+BRK_RES_SYNTHETIC_ALIAS =
+
+
+# All aliases (to not be included under 'installed'), but not including root.
+BRK_RES_ALIAS_SOURCE = $(BRK_RES_SYNTHETIC_ALIAS)
+
+
+# List of compact trie dictionary files (ctd).
+BRK_CTD_SOURCE = thaidict.txt cjdict.txt
+
+
+# List of break iterator files (brk).
+# Chrome change: remove word_ja.txt and line_he.txt
+BRK_SOURCE = sent_el.txt word_POSIX.txt line_fi.txt char.txt word.txt line.txt sent.txt title.txt char_th.txt
+
+
+# Ordinary resources
+# Chrome change: remove ja.txt and he.txt
+BRK_RES_SOURCE = el.txt en.txt en_US.txt en_US_POSIX.txt\
+ fi.txt th.txt
diff --git a/source/data/brkitr/line.txt b/source/data/brkitr/line.txt
index bd27b5e..ae6e493 100644
--- a/source/data/brkitr/line.txt
+++ b/source/data/brkitr/line.txt
@@ -11,6 +11,9 @@
# TODO: Rule LB 8 remains as it was in Unicode 5.2
# This is only because of a limitation of ICU break engine implementation,
# not because the older behavior is desirable.
+#
+# CHROME: Hebrew tailoring was incorporatd as well as some
+# other minor changes (CL, OP, ID).
#
# Character Classes defined by TR 14.
@@ -57,13 +60,15 @@
#
$AI = [:LineBreak = Ambiguous:];
-$AL = [:LineBreak = Alphabetic:];
-$BA = [:LineBreak = Break_After:];
+$AL = [[:LineBreak = Alphabetic:] - [[:Hebrew:] & [:Letter:]] - [\u23B4\u23B5]];
+$HL = [[:Hebrew:] & [:Letter:]];
+$BA = [[:LineBreak = Break_After:] - [\u2010]];
+$HH = [\u2010];
$BB = [:LineBreak = Break_Before:];
$BK = [:LineBreak = Mandatory_Break:];
$B2 = [:LineBreak = Break_Both:];
$CB = [:LineBreak = Contingent_Break:];
-$CL = [:LineBreak = Close_Punctuation:];
+$CL = [[:LineBreak = Close_Punctuation:] [\uFE51\uFE10\u23B5]];
$CM = [:LineBreak = Combining_Mark:];
$CP = [:LineBreak = Close_Parenthesis:];
$CR = [:LineBreak = Carriage_Return:];
@@ -72,9 +77,9 @@
$HY = [:LineBreak = Hyphen:];
$H2 = [:LineBreak = H2:];
$H3 = [:LineBreak = H3:];
-$ID = [:LineBreak = Ideographic:];
+$ID = [[:LineBreak = Ideographic:] - [\uFE51]];
$IN = [:LineBreak = Inseperable:];
-$IS = [:LineBreak = Infix_Numeric:];
+$IS = [[:LineBreak = Infix_Numeric:] - [\uFE10]];
$JL = [:LineBreak = JL:];
$JV = [:LineBreak = JV:];
$JT = [:LineBreak = JT:];
@@ -82,7 +87,7 @@
$NL = [:LineBreak = Next_Line:];
$NS = [:LineBreak = Nonstarter:];
$NU = [:LineBreak = Numeric:];
-$OP = [:LineBreak = Open_Punctuation:];
+$OP = [[:LineBreak = Open_Punctuation:] \u23B4];
$PO = [:LineBreak = Postfix_Numeric:];
$PR = [:LineBreak = Prefix_Numeric:];
$QU = [:LineBreak = Quotation:];
@@ -108,13 +113,15 @@
# XX (Unknown, unassigned)
# as $AL (Alphabetic)
#
-$ALPlus = [$AL $AI $SA $SG $XX];
+$ALPlus = [$AL $HL $AI $SA $SG $XX];
#
# Combining Marks. X $CM* behaves as if it were X. Rule LB6.
#
$ALcm = $ALPlus $CM*;
+$HLcm = $HL $CM*;
$BAcm = $BA $CM*;
+$HHcm = $HH $CM*;
$BBcm = $BB $CM*;
$B2cm = $B2 $CM*;
$CLcm = $CL $CM*;
@@ -148,6 +155,7 @@
#
$ALPlus $CM+;
$BA $CM+;
+$HH $CM+;
$BB $CM+;
$B2 $CM+;
$CL $CM+;
@@ -190,7 +198,7 @@
# so for this one case we need to manually list out longer sequences.
#
$AL_FOLLOW_NOCM = [$BK $CR $LF $NL $ZW $SP];
-$AL_FOLLOW_CM = [$CL $CP $EX $IS $SY $WJ $GL $OP $QU $BA $HY $NS $IN $NU $ALPlus];
+$AL_FOLLOW_CM = [$CL $CP $EX $IS $SY $WJ $GL $OP $QU $BA $HH $HY $NS $IN $NU $ALPlus];
$AL_FOLLOW = [$AL_FOLLOW_NOCM $AL_FOLLOW_CM];
@@ -252,7 +260,7 @@
# LB 12a Do not break before NBSP and related characters ...
# [^SP BA HY] x GL
#
-[[$LB8NonBreaks] - [$SP $BA $HY]] $CM* $GLcm;
+[[$LB8NonBreaks] - [$SP $BA $HH $HY]] $CM* $GLcm;
$CM+ GLcm;
@@ -325,7 +333,7 @@
# LB 21 x (BA | HY | NS)
# BB x
#
-$LB20NonBreaks $CM* ($BAcm | $HYcm | $NScm);
+$LB20NonBreaks $CM* ($BAcm | $HHcm | $HYcm | $NScm);
$BBcm [^$CB]; # $BB x
$BBcm $LB20NonBreaks $CM*;
@@ -381,6 +389,8 @@
$CM+ $OPcm; # The $CM+ is from rule 10, an unattached CM is treated as AL.
$CPcm ($ALcm | $NUcm);
+# (LB 31) Add new rule to prevent the break we do not want, this is the behavior change
+$HLcm ($HY | $HH) $ALcm;
#
# Reverse Rules.
@@ -391,6 +401,7 @@
$CM+ $ALPlus;
$CM+ $BA;
+$CM+ $HH;
$CM+ $BB;
$CM+ $B2;
$CM+ $CL;
@@ -479,7 +490,7 @@
# LB 12a
# [^SP BA HY] x GL
#
-$CM* $GL $CM* [$LB8NonBreaks-[$CM $SP $BA $HY]];
+$CM* $GL $CM* [$LB8NonBreaks-[$CM $SP $BA $HH $HY]];
# LB 12
# GL x
@@ -549,7 +560,7 @@
#
# LB 21
-$CM* ($BA | $HY | $NS) $CM* [$LB20NonBreaks-$CM]; # . x (BA | HY | NS)
+$CM* ($BA | $HH | $HY | $NS) $CM* [$LB20NonBreaks-$CM]; # . x (BA | HY | NS)
$CM* [$LB20NonBreaks-$CM] $CM* $BB; # BB x .
[^$CB] $CM* $BB; #
@@ -597,6 +608,8 @@
$CM* $OP $CM* ($ALPlus | $NU);
$CM* ($ALPlus | $NU) $CM* $CP;
+# (LB 31) Add new rule to prevent the break we do not want, this is the behavior change
+$CM* $ALPlus ($HY | $HH) $CM* $HL;
## -------------------------------------------------
diff --git a/source/data/brkitr/word.txt b/source/data/brkitr/word.txt
index 1efc65b..0b49377 100644
--- a/source/data/brkitr/word.txt
+++ b/source/data/brkitr/word.txt
@@ -29,29 +29,49 @@
$Newline = [\p{Word_Break = Newline}];
$Extend = [\p{Word_Break = Extend}];
$Format = [\p{Word_Break = Format}];
+$Hiragana = [:Hiragana:];
$Katakana = [\p{Word_Break = Katakana}];
+$Han = [:Han:];
$ALetter = [\p{Word_Break = ALetter}];
-$MidNumLet = [\p{Word_Break = MidNumLet}];
+# Remove two full stop characters from $MidNumLet and add them to $MidNum
+# to break a hostname into its components at the cost of breaking
+# 'e.g.' and 'i.e.' as well.
+# $MidNumLet is used in rules 6/7 (rules of our interest) and rules 11/12.
+# Because it's OR'd with $MidNum in rules 11/12, rules 11/12 are not affected
+# while rules 6/7 are reverted to the old behavior we want.
+$MidNumLet = [[\p{Word_Break = MidNumLet}] - [\u002E \uFF0E]];
$MidLetter = [\p{Word_Break = MidLetter}];
-$MidNum = [\p{Word_Break = MidNum}];
-$Numeric = [\p{Word_Break = Numeric}];
+$MidNum = [\p{Word_Break = MidNum}[\u002E \uFF0E]];
+$Numeric = [\p{Word_Break = Numeric}[\uff10-\uff19]]; #includes fullwidth digits
$ExtendNumLet = [\p{Word_Break = ExtendNumLet}];
+# Extra sets not to break 'HebrewLetter U+0022 HebrewLetter'.
+$HebrewLet = [\p{Word_Break = ALetter} & \p{Script = Hebrew} - [\u05F3]];
+# U+05F3 is ALetter and U+05F4 is MidLetter so that they're covered by
+# the current rule 6/7.
+$HebrewMidLet = [\u0022];
# Dictionary character set, for triggering language-based break engines. Currently
-# limited to LineBreak=Complex_Context. Note that this set only works in Unicode
-# 5.0 or later as the definition of Complex_Context was corrected to include all
+# limited to LineBreak=Complex_Context and CJK. Note that this set only works
+# in Unicode 5.0 or later as the definition of Complex_Context was corrected to include all
# characters requiring dictionary break.
-$dictionary = [:LineBreak = Complex_Context:];
$Control = [\p{Grapheme_Cluster_Break = Control}];
-$ALetterPlus = [$ALetter [$dictionary-$Extend-$Control]]; # Note: default ALetter does not
- # include the dictionary characters.
+$HangulSyllable = [\uac00-\ud7a3];
+$ComplexContext = [:LineBreak = Complex_Context:];
+$KanaKanji = [$Han $Hiragana $Katakana];
+$dictionaryCJK = [$KanaKanji $HangulSyllable];
+$dictionary = [$ComplexContext $dictionaryCJK];
+
+# leave CJK scripts out of ALetterPlus
+$ALetterPlus = [$ALetter-$dictionaryCJK [$ComplexContext-$Extend-$Control]];
+
#
# Rules 4 Ignore Format and Extend characters,
# except when they appear at the beginning of a region of text.
#
+# TODO: check if handling of katakana in dictionary makes rules incorrect/void.
$KatakanaEx = $Katakana ($Extend | $Format)*;
$ALetterEx = $ALetterPlus ($Extend | $Format)*;
$MidNumLetEx = $MidNumLet ($Extend | $Format)*;
@@ -59,8 +79,8 @@
$MidNumEx = $MidNum ($Extend | $Format)*;
$NumericEx = $Numeric ($Extend | $Format)*;
$ExtendNumLetEx = $ExtendNumLet ($Extend | $Format)*;
+$HebrewLetEx = $HebrewLet ($Extend | $Format)*;
-$Hiragana = [\p{script=Hiragana}];
$Ideographic = [\p{Ideographic}];
$HiraganaEx = $Hiragana ($Extend | $Format)*;
$IdeographicEx = $Ideographic ($Extend | $Format)*;
@@ -79,12 +99,14 @@
# begins with a group of Format chars, or with a "word" consisting of a single
# char that is not in any of the listed word break categories followed by
# format char(s).
-[^$CR $LF $Newline]? ($Extend | $Format)+;
+ # format char(s), or is not a CJK dictionary character.
+[^$CR $LF $Newline $dictionaryCJK]? ($Extend | $Format)+;
$NumericEx {100};
$ALetterEx {200};
-$KatakanaEx {300}; # note: these status values override those from rule 5
-$HiraganaEx {300}; # by virtual of being numerically larger.
+$HangulSyllable {200};
+$KatakanaEx {400}; #originally 300
+$HiraganaEx {400}; #originally 300
$IdeographicEx {400}; #
#
@@ -96,6 +118,9 @@
# rule 6 and 7
$ALetterEx ($MidLetterEx | $MidNumLetEx) $ALetterEx {200};
+# Chrome addition
+$HebrewLetEx $HebrewMidLet $HebrewLetEx {200};
+
# rule 8
$NumericEx $NumericEx {100};
@@ -114,19 +139,25 @@
# rule 13
-$KatakanaEx $KatakanaEx {300};
+# To be consistent with '$KanaKanji $KanaKanji', changed
+# from 300 to 400.
+# See also TestRuleStatus in intltest/rbbiapts.cpp
+$KatakanaEx $KatakanaEx {400};
# rule 13a/b
$ALetterEx $ExtendNumLetEx {200}; # (13a)
$NumericEx $ExtendNumLetEx {100}; # (13a)
-$KatakanaEx $ExtendNumLetEx {300}; # (13a)
+$KatakanaEx $ExtendNumLetEx {400}; # (13a)
$ExtendNumLetEx $ExtendNumLetEx {200}; # (13a)
$ExtendNumLetEx $ALetterEx {200}; # (13b)
$ExtendNumLetEx $NumericEx {100}; # (13b)
-$ExtendNumLetEx $KatakanaEx {300}; # (13b)
-
+$ExtendNumLetEx $KatakanaEx {400}; # (13b)
+
+# special handling for CJK characters: chain for later dictionary segmentation
+$HangulSyllable $HangulSyllable {200};
+$KanaKanji $KanaKanji {400}; #different rule status if both kanji and kana found
## -------------------------------------------------
@@ -139,13 +170,16 @@
$BackMidNumEx = ($Format | $Extend)* $MidNum;
$BackMidLetterEx = ($Format | $Extend)* $MidLetter;
$BackKatakanaEx = ($Format | $Extend)* $Katakana;
+$BackHiraganaEx = ($Extend | $Format)* $Hiragana;
$BackExtendNumLetEx= ($Format | $Extend)* $ExtendNumLet;
+$BackHebrewLetEx = ($Format | $Extend)* $HebrewLet;
+
# rule 3
$LF $CR;
# rule 4
-($Format | $Extend)* [^$CR $LF $Newline]?;
+($Format | $Extend)* [^$CR $LF $Newline $dictionaryCJK]?;
# rule 5
@@ -155,6 +189,8 @@
$BackALetterEx ($BackMidLetterEx | $BackMidNumLetEx) $BackALetterEx;
+# Chrome addition
+$BackHebrewLetEx $HebrewMidLet $BackHebrewLetEx;
# rule 8
@@ -181,6 +217,10 @@
$BackExtendNumLetEx ($BackALetterEx | $BackNumericEx | $BackKatakanaEx | $BackExtendNumLetEx);
($BackALetterEx | $BackNumericEx | $BackKatakanaEx) $BackExtendNumLetEx;
+# special handling for CJK characters: chain for later dictionary segmentation
+$HangulSyllable $HangulSyllable;
+$KanaKanji $KanaKanji; #different rule status if both kanji and kana found
+
## -------------------------------------------------
!!safe_reverse;
