INTEGRATION: CWS i18n08 (1.1.2); FILE ADDED

2003/08/08 23:33:24 khong 1.1.2.1: #i13494# #110146# fix word breakiterator rule to handle punctuations and signs correctly

i18npool/source/breakiterator/data/dict_word.txt

@@ -0,0 +1,135 @@
+#
+#   Copyright (C) 2002-2003, International Business Machines Corporation and others.
+#       All Rights Reserved.
+#
+#   file:  dict_word.txt   
+#
+#   ICU Word Break Rules
+#      See Unicode Standard Annex #29.
+#      These rules are based on Version 4.0.0, dated 2003-04-17
+#
+
+
+
+####################################################################################
+#
+#  Character class definitions from TR 29
+#
+####################################################################################
+$Katakana  = [[:Script = KATAKANA:] [:name = KATAKANA-HIRAGANA PROLONGED SOUND MARK:] 
+                                   [:name = HALFWIDTH KATAKANA-HIRAGANA PROLONGED SOUND MARK:]
+                                   [:name = HALFWIDTH KATAKANA VOICED SOUND MARK:]
+                                   [:name = HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK:]];
+
+
+$ALetter   = [\u0002 [:Alphabetic:] [:name= FULL STOP:] [:name= COMMERCIAL AT:] [:name= HEBREW PUNCTUATION GERESH:]
+                           - $Katakana
+                           - [:Script = Thai:]
+                           - [:Script = Lao:]
+                           - [:Script = Hiragana:]];
+                           
+$MidLetter = [[:name = APOSTROPHE:] [:name = GRAVE ACCENT:] \u0084 [:name = SOFT HYPHEN:] [:name = MIDDLE DOT:] [:name = GREEK TONOS:]
+              [:name = HEBREW PUNCTUATION GERSHAYIM:] [:name = DOUBLE VERTICAL LINE:] [:name = LEFT SINGLE QUOTATION MARK:]
+              [:name = RIGHT SINGLE QUOTATION MARK:] [:name = HYPHENATION POINT:] [:name = PRIME:] ];  
+              
+
+$MidNum    = [[:LineBreak = Infix_Numeric:] [:name= COMMERCIAL AT:] \u0084 [:name = GREEK TONOS:] [:name = ARABIC DECIMAL SEPARATOR:]
+             [:name = LEFT SINGLE QUOTATION MARK:] [:name = RIGHT SINGLE QUOTATION MARK:] [:name = SINGLE HIGH-REVERSED-9 QUOTATION MARK:]
+             [:name = PRIME:]];
+$Numeric   = [:LineBreak = Numeric:];
+
+
+#
+#  Character Class Definitions.
+#    The names are those from TR29.
+#
+$CR         = \u000d;
+$LF         = \u000a;
+$Control    = [[:Zl:] [:Zp:] [:Cc:] [:Cf:]];
+$Extend     = [[:Grapheme_Extend = TRUE:]]; 
+
+
+
+
+####################################################################################
+#
+#  Word Break Rules.    Definitions and Rules specific to word break begin Here. 
+#
+####################################################################################
+
+$Format    = [[:Cf:]];
+
+
+
+# Rule 3:  Treat a grapheme cluster as if it were a single character.
+#          Hangul Syllables are easier to deal with here than they are in Grapheme Clusters
+#          because we don't need to find the boundaries between adjacent syllables -
+#          they won't be word boundaries.
+#
+
+
+#
+#  "Extended"  definitions.  Grapheme Cluster + Format Chars, treated like the base char.
+#
+$ALetterEx    = $ALetter   $Extend*; 
+$NumericEx    = $Numeric   $Extend*;
+$MidNumEx     = $MidNum    $Extend*;
+$MidLetterEx  = $MidLetter $Extend*;
+$KatakanaEx   = $Katakana  $Extend*;
+$FormatEx     = $Format    $Extend*;
+
+
+#
+#  Numbers.  Rules 8, 11, 12 form the TR.
+#
+$NumberSequence = $NumericEx ($FormatEx* $MidNumEx? $FormatEx* $NumericEx)*;
+$NumberSequence {100};
+
+#
+#  Words.  Alpha-numerics.  Rule 5, 6, 7, 9, 10
+#     - must include at least one letter. 
+#     - may include both letters and numbers.
+#     - may include  MideLetter, MidNumber punctuation.
+#
+$LetterSequence = $ALetterEx ($FormatEx* $MidLetterEx? $FormatEx* $ALetterEx)*;     # rules #6, #7
+($NumberSequence $FormatEx*)? $LetterSequence ($FormatEx* ($NumberSequence | $LetterSequence))* {200};
+
+[[:P:][:S:]]*;
+
+#
+#  Do not break between Katakana.   Rule #13.
+#
+$KatakanaEx ($FormatEx* $KatakanaEx)* {300};
+[:Hiragana:] $Extend* {300};
+
+#
+#  Ideographic Characters.  Stand by themselves as words.
+#                           Separated from the "Everything Else" rule, below, only so that they
+#                           can be tagged with a return value.   TODO:  is this what we want?
+#
+# [:IDEOGRAPHIC:] $Extend* {400};
+
+#
+#  Everything Else, with no tag.
+#                   Non-Control chars combine with $Extend (combining) chars.
+#                   Controls are do not.
+#
+[^$Control [:Ideographic:]] $Extend*;
+$CR $LF;
+
+#
+#  Reverse Rules.   Back up over any of the chars that can group together.
+#                   (Reverse rules do not need to be exact; they can back up  too far,
+#                   but must back up at least enough, and must stop on a boundary.)
+#
+
+# NonStarters are the set of all characters that can appear at the 2nd - nth position of
+#    a word.   (They may also be the first.)   The reverse rule skips over these, until it
+#    reaches something that can only be the start (and probably only) char in a "word".
+#    A space or punctuation meets the test.
+#
+$NonStarters = [$Numeric $ALetter $Katakana [:P:] [:S:] $MidLetter $MidNum $Extend $Format];
+
+#!.*;
+! ($NonStarters* | \n \r) .;
+