| /* Copyright (C) 2007-2013 Google and others. All Rights Reserved. */ |
| /* Copyright (C) 2007-2013 IBM Corp. and others. All Rights Reserved. */ |
| |
| package org.unicode.cldr.test; |
| |
| import java.util.Arrays; |
| import java.util.HashMap; |
| import java.util.HashSet; |
| import java.util.List; |
| import java.util.Map; |
| import java.util.Set; |
| import java.util.TreeSet; |
| import java.util.regex.Matcher; |
| import java.util.regex.Pattern; |
| |
| import org.unicode.cldr.test.CheckExemplars.ExemplarType; |
| import org.unicode.cldr.util.AnnotationUtil; |
| import org.unicode.cldr.util.Builder; |
| import org.unicode.cldr.util.CLDRConfig; |
| import org.unicode.cldr.util.CLDRFile; |
| import org.unicode.cldr.util.CLDRLocale; |
| import org.unicode.cldr.util.CldrUtility; |
| import org.unicode.cldr.util.DateTimeCanonicalizer; |
| import org.unicode.cldr.util.DateTimeCanonicalizer.DateTimePatternType; |
| import org.unicode.cldr.util.Emoji; |
| import org.unicode.cldr.util.ICUServiceBuilder; |
| import org.unicode.cldr.util.PatternCache; |
| import org.unicode.cldr.util.SupplementalDataInfo; |
| import org.unicode.cldr.util.UnicodeSetPrettyPrinter; |
| import org.unicode.cldr.util.With; |
| import org.unicode.cldr.util.XPathParts; |
| |
| import com.google.common.base.Joiner; |
| import com.google.common.base.Splitter; |
| import com.google.myanmartools.ZawgyiDetector; |
| import com.ibm.icu.lang.UCharacter; |
| import com.ibm.icu.text.Collator; |
| import com.ibm.icu.text.DateIntervalInfo; |
| import com.ibm.icu.text.DateTimePatternGenerator; |
| import com.ibm.icu.text.DecimalFormat; |
| import com.ibm.icu.text.Normalizer; |
| import com.ibm.icu.text.RuleBasedCollator; |
| import com.ibm.icu.text.Transform; |
| import com.ibm.icu.text.Transliterator; |
| import com.ibm.icu.text.UnicodeSet; |
| import com.ibm.icu.text.UnicodeSetIterator; |
| import com.ibm.icu.util.ULocale; |
| |
| /** |
| * Class for processing the input and output of CLDR data for use in the |
| * Survey Tool and other tools. |
| */ |
| public class DisplayAndInputProcessor { |
| |
| private static final boolean FIX_YEARS = true; |
| |
| public static final boolean DEBUG_DAIP = CldrUtility.getProperty("DEBUG_DAIP", false); |
| |
| public static final UnicodeSet RTL = new UnicodeSet("[[:Bidi_Class=Arabic_Letter:][:Bidi_Class=Right_To_Left:]]") |
| .freeze(); |
| |
| public static final UnicodeSet TO_QUOTE = new UnicodeSet( |
| "[[:Cn:]" + |
| "[:Default_Ignorable_Code_Point:]" + |
| "[:patternwhitespace:]" + |
| "[:Me:][:Mn:]]" // add non-spacing marks |
| ).freeze(); |
| |
| public static final Pattern NUMBER_FORMAT_XPATH = Pattern |
| .compile("//ldml/numbers/.*Format\\[@type=\"standard\"]/pattern.*"); |
| |
| public static final Pattern NUMBER_SEPARATOR_PATTERN = Pattern |
| .compile("//ldml/numbers/symbols.*/(decimal|group)"); |
| |
| private static final Pattern APOSTROPHE_SKIP_PATHS = PatternCache.get("//ldml/(" |
| + "localeDisplayNames/languages/language\\[@type=\"mic\"].*|" |
| + "characters/.*|" |
| + "delimiters/.*|" |
| + "dates/.+/(pattern|intervalFormatItem|dateFormatItem).*|" |
| + "units/.+/unitPattern.*|" |
| + "units/.+/durationUnitPattern.*|" |
| + "numbers/symbols.*|" |
| + "numbers/miscPatterns.*|" |
| + "numbers/(decimal|currency|percent|scientific)Formats.+/(decimal|currency|percent|scientific)Format.*)"); |
| private static final Pattern INTERVAL_FORMAT_PATHS = PatternCache.get("//ldml/dates/.+/intervalFormat(Item.*|Fallback)"); |
| private static final Pattern NON_DECIMAL_PERIOD = PatternCache.get("(?<![0#'])\\.(?![0#'])"); |
| |
| // Pattern to match against paths that might have time formats with h or K (12-hour cycles) |
| private static final Pattern HOUR_FORMAT_XPATHS = PatternCache |
| .get("//ldml/dates/calendars/calendar\\[@type=\"[^\"]*\"]/(" |
| + "timeFormats/timeFormatLength\\[@type=\"[^\"]*\"]/timeFormat\\[@type=\"standard\"]/pattern\\[@type=\"standard\"].*|" |
| + "dateTimeFormats/availableFormats/dateFormatItem\\[@id=\"[A-GL-Ma-gl-m]*[hK][A-Za-z]*\"].*|" |
| + "dateTimeFormats/intervalFormats/intervalFormatItem\\[@id=\"[A-GL-Ma-gl-m]*[hK][A-Za-z]*\"].*)"); |
| |
| private static final Pattern AMPM_SPACE_BEFORE = PatternCache.get("([Khms])([ \\u00A0]+)(a+)"); // time, space, a+ |
| private static final Pattern AMPM_SPACE_AFTER = PatternCache.get("(a+)([ \\u00A0]+)([Kh])"); // a+, space, hour |
| |
| // Pattern to match against paths that might have date formats with y |
| private static final Pattern YEAR_FORMAT_XPATHS = PatternCache |
| .get("//ldml/dates/calendars/calendar\\[@type=\"[^\"]*\"]/(" |
| + "dateFormats/dateFormatLength\\[@type=\"[^\"]*\"]/dateFormat\\[@type=\"standard\"]/pattern\\[@type=\"standard\"].*|" |
| + "dateTimeFormats/availableFormats/dateFormatItem\\[@id=\"[A-XZa-xz]*y[A-Za-z]*\"].*|" |
| + "dateTimeFormats/intervalFormats/intervalFormatItem\\[@id=\"[A-XZa-xz]*y[A-Za-z]*\"].*)"); |
| |
| // Cyrillic year markers are or begin with (in various languages) \u0430 \u0433 \u0435 \u0436 \u043E \u0440 \u0441 |
| private static final Pattern YEAR_SPACE_YEARMARKER = PatternCache.get("y[ \\u00A0]+('?[агежорс])"); // y, space, Cyrillic year marker start |
| |
| public static final Pattern UNIT_NARROW_XPATHS = PatternCache |
| .get("//ldml/units/unitLength\\[@type=\"narrow\"]unit\\[@type=\"[^\"]*\"]/unitPattern.*"); |
| |
| public static final Pattern UNIT_SHORT_XPATHS = PatternCache |
| .get("//ldml/units/unitLength\\[@type=\"short\"]unit\\[@type=\"[^\"]*\"]/unitPattern.*"); |
| |
| private static final Pattern PLACEHOLDER_SPACE_AFTER = PatternCache.get("\\}[ \\u00A0\\u202F]+"); |
| private static final Pattern PLACEHOLDER_SPACE_BEFORE = PatternCache.get("[ \\u00A0\\u202F]+\\{"); |
| private static final Pattern INTERVAL_FALLBACK_RANGE = PatternCache.get("\\} [\\u2013-] \\{"); |
| |
| /** |
| * string of whitespace not including NBSP, i.e. [\t\n\r]+ |
| */ |
| private static final Pattern WHITESPACE_NO_NBSP_TO_NORMALIZE = PatternCache.get("\\s+"); // |
| |
| /** |
| * string of whitespace including NBSP, i.e. [\u00A0\t\n\r]+ |
| */ |
| private static final Pattern WHITESPACE_AND_NBSP_TO_NORMALIZE = PatternCache.get("[\\s\\u00A0]+"); |
| |
| /** |
| * one or more NBSP followed by one or more regular spaces |
| */ |
| private static final Pattern NBSP_PLUS_SPACE_TO_NORMALIZE = PatternCache.get("\\u00A0+\\u0020+"); |
| |
| /** |
| * one or more regular spaces followed by one or more NBSP |
| */ |
| private static final Pattern SPACE_PLUS_NBSP_TO_NORMALIZE = PatternCache.get("\\u0020+\\u00A0+"); |
| |
| private static final Pattern INITIAL_NBSP = PatternCache.get("^[\\u00A0\\u202F]+"); |
| private static final Pattern FINAL_NBSP = PatternCache.get("[\\u00A0\\u202F]+$"); |
| private static final Pattern MULTIPLE_NBSP = PatternCache.get("\\u00A0\\u00A0+"); |
| |
| // The following includes (among others) \u0009, \u0020, \u00A0, \u2007, \u2009, \u202F, \u3000 |
| private static final UnicodeSet UNICODE_WHITESPACE = new UnicodeSet("[:whitespace:]").freeze(); |
| |
| private static final CLDRLocale MALAYALAM = CLDRLocale.getInstance("ml"); |
| private static final CLDRLocale ROMANIAN = CLDRLocale.getInstance("ro"); |
| private static final CLDRLocale CATALAN = CLDRLocale.getInstance("ca"); |
| private static final CLDRLocale NGOMBA = CLDRLocale.getInstance("jgo"); |
| private static final CLDRLocale KWASIO = CLDRLocale.getInstance("nmg"); |
| private static final CLDRLocale HEBREW = CLDRLocale.getInstance("he"); |
| private static final CLDRLocale MYANMAR = CLDRLocale.getInstance("my"); |
| private static final CLDRLocale KYRGYZ = CLDRLocale.getInstance("ky"); |
| private static final CLDRLocale URDU = CLDRLocale.getInstance("ur"); |
| private static final CLDRLocale PASHTO = CLDRLocale.getInstance("ps"); |
| private static final CLDRLocale FARSI = CLDRLocale.getInstance("fa"); |
| private static final CLDRLocale GERMAN_SWITZERLAND = CLDRLocale.getInstance("de_CH"); |
| private static final CLDRLocale SWISS_GERMAN = CLDRLocale.getInstance("gsw"); |
| private static final CLDRLocale FF_ADLAM = CLDRLocale.getInstance("ff_Adlm"); |
| public static final Set<String> LANGUAGES_USING_MODIFIER_APOSTROPHE = new HashSet<>( |
| Arrays.asList("br", "bss", "cad", "cic", "cch", "gn", "ha", "ha_Latn", "lkt", "mgo", "moh", "mus", "nnh", "qu", "quc", "uk", "uz", "uz_Latn")); |
| |
| // Ş ş Ţ ţ => Ș ș Ț ț |
| private static final char[][] ROMANIAN_CONVERSIONS = { |
| { '\u015E', '\u0218' }, { '\u015F', '\u0219' }, { '\u0162', '\u021A' }, |
| { '\u0163', '\u021B' } }; |
| |
| private static final char[][] CATALAN_CONVERSIONS = { |
| { '\u013F', '\u004C', '\u00B7' }, // Ŀ -> L· |
| { '\u0140', '\u006C', '\u00B7' } }; // ŀ -> l· |
| |
| private static final char[][] NGOMBA_CONVERSIONS = { |
| { '\u0251', '\u0061' }, { '\u0261', '\u0067' }, // ɑ -> a , ɡ -> g , See ticket #5691 |
| { '\u2019', '\uA78C' }, { '\u02BC', '\uA78C' } }; // Saltillo, see ticket #6805 |
| |
| private static final char[][] KWASIO_CONVERSIONS = { |
| { '\u0306', '\u030C' }, // See ticket #6571, use caron instead of breve |
| { '\u0103', '\u01CE' }, { '\u0102', '\u01CD' }, // a-breve -> a-caron |
| { '\u0115', '\u011B' }, { '\u011A', '\u01CD' }, // e-breve -> e-caron |
| { '\u012D', '\u01D0' }, { '\u012C', '\u01CF' }, // i-breve -> i-caron |
| { '\u014F', '\u01D2' }, { '\u014E', '\u01D1' }, // o-breve -> o-caron |
| { '\u016D', '\u01D4' }, { '\u016C', '\u01D3' } // u-breve -> u-caron |
| }; |
| |
| private static final char[][] HEBREW_CONVERSIONS = { |
| { '\'', '\u05F3' }, { '"', '\u05F4' } }; // ' -> geresh " -> gershayim |
| |
| private static final char[][] KYRGYZ_CONVERSIONS = { |
| { 'ӊ', 'ң' }, { 'Ӊ', 'Ң' } }; // right modifier |
| |
| private static final char[][] URDU_PLUS_CONVERSIONS = { |
| { '\u0643', '\u06A9' }}; // wrong char |
| |
| private static final ZawgyiDetector detector = new ZawgyiDetector(); |
| private static final Transliterator zawgyiUnicodeTransliterator = |
| Transliterator.getInstance("Zawgyi-my"); |
| |
| private Collator col; |
| |
| private Collator spaceCol; |
| |
| private UnicodeSetPrettyPrinter pp = null; |
| |
| final private CLDRLocale locale; |
| private String scriptCode; // actual or default script code (not null after init) |
| private boolean isPosix; |
| |
| /** |
| * Constructor, taking cldrFile. |
| * |
| * @param cldrFileToCheck |
| */ |
| public DisplayAndInputProcessor(CLDRFile cldrFileToCheck, boolean needsCollator) { |
| init(this.locale = CLDRLocale.getInstance(cldrFileToCheck.getLocaleID()), needsCollator); |
| } |
| |
| public DisplayAndInputProcessor(CLDRFile cldrFileToCheck) { |
| init(this.locale = CLDRLocale.getInstance(cldrFileToCheck.getLocaleID()), true); |
| } |
| |
| void init(CLDRLocale locale, boolean needsCollator) { |
| isPosix = locale.toString().indexOf("POSIX") >= 0; |
| if (needsCollator) { |
| ICUServiceBuilder isb = null; |
| try { |
| isb = ICUServiceBuilder.forLocale(locale); |
| } catch (Exception e) { |
| } |
| |
| if (isb != null) { |
| try { |
| col = isb.getRuleBasedCollator(); |
| } catch (Exception e) { |
| col = Collator.getInstance(ULocale.ROOT); |
| } |
| } else { |
| col = Collator.getInstance(ULocale.ROOT); |
| } |
| |
| spaceCol = Collator.getInstance(locale.toULocale()); |
| if (spaceCol instanceof RuleBasedCollator) { |
| ((RuleBasedCollator) spaceCol).setAlternateHandlingShifted(false); |
| } |
| pp = new UnicodeSetPrettyPrinter().setOrdering(Collator.getInstance(ULocale.ROOT)) |
| .setSpaceComparator(Collator.getInstance(ULocale.ROOT).setStrength2(Collator.PRIMARY)) |
| .setCompressRanges(true) |
| .setToQuote(new UnicodeSet(TO_QUOTE)) |
| .setOrdering(col) |
| .setSpaceComparator(spaceCol); |
| } |
| String script = locale.getScript(); |
| if (script == null || script.length() < 4) { |
| SupplementalDataInfo sdi = CLDRConfig.getInstance().getSupplementalDataInfo(); |
| script = sdi.getDefaultScript(locale.getBaseName()); |
| if (script == null || script.length() < 4 || script.equals("Zzzz")) { |
| script = sdi.getDefaultScript(locale.getLanguage()); |
| } |
| if (script == null || script.length() < 4) { |
| script = "Zzzz"; |
| } |
| } |
| scriptCode = script; |
| } |
| |
| public UnicodeSetPrettyPrinter getPrettyPrinter() { |
| return pp; |
| } |
| |
| /** |
| * Constructor, taking ULocale and boolean. |
| * |
| * @param locale the ULocale |
| * @param needsCollator true or false |
| * |
| * Called by getProcessor, with locale = SurveyMain.TRANS_HINT_LOCALE |
| */ |
| public DisplayAndInputProcessor(ULocale locale, boolean needsCollator) { |
| init(this.locale = CLDRLocale.getInstance(locale), needsCollator); |
| } |
| |
| /** |
| * Constructor, taking ULocale. |
| * |
| * @param locale the ULocale |
| */ |
| public DisplayAndInputProcessor(ULocale locale) { |
| init(this.locale = CLDRLocale.getInstance(locale), true /* needsCollator */); |
| } |
| |
| /** |
| * Constructor, taking CLDRLocale and boolean. |
| * |
| * @param locale the CLDRLocale |
| * @param needsCollator true or false |
| */ |
| public DisplayAndInputProcessor(CLDRLocale locale, boolean needsCollator) { |
| init(this.locale = locale, needsCollator); |
| } |
| |
| /** |
| * Constructor, taking locale. |
| * |
| * @param locale |
| */ |
| public DisplayAndInputProcessor(CLDRLocale locale) { |
| init(this.locale = locale, true); |
| } |
| |
| /** |
| * Process the value for display. The result is a string for display in the |
| * Survey tool or similar program. |
| * |
| * @param path |
| * @param value |
| * @return |
| */ |
| public synchronized String processForDisplay(String path, String value) { |
| value = Normalizer.compose(value, false); // Always normalize all text to NFC. |
| if (hasUnicodeSetValue(path)) { |
| value = displayUnicodeSet(value); |
| } else if (path.contains("stopword")) { |
| return value.trim().isEmpty() ? "NONE" : value; |
| } else { |
| NumericType numericType = NumericType.getNumericType(path); |
| if (numericType != NumericType.NOT_NUMERIC) { |
| // Canonicalize existing values that aren't canonicalized yet. |
| // New values will be canonicalized on input using processInput(). |
| try { |
| value = getCanonicalPattern(value, numericType, isPosix); |
| } catch (IllegalArgumentException e) { |
| if (DEBUG_DAIP) System.err.println("Illegal pattern: " + value); |
| } |
| if (numericType != NumericType.CURRENCY && numericType != NumericType.CURRENCY_ABBREVIATED) { |
| value = value.replace("'", ""); |
| } |
| } |
| } |
| // Fix up any apostrophes in number symbols |
| if (NUMBER_SEPARATOR_PATTERN.matcher(path).matches()) { |
| value = value.replace('\'', '\u2019'); |
| } |
| // Fix up any apostrophes as appropriate (Don't do so for things like date patterns... |
| if (!APOSTROPHE_SKIP_PATHS.matcher(path).matches()) { |
| value = normalizeApostrophes(value); |
| } |
| // Fix up hyphens, replacing with N-dash as appropriate |
| if (INTERVAL_FORMAT_PATHS.matcher(path).matches()) { |
| value = normalizeIntervalHyphensAndSpaces(value); // This may also adjust spaces around en dash |
| } else { |
| value = normalizeHyphens(value); |
| } |
| return value; |
| } |
| |
| private boolean hasUnicodeSetValue(String path) { |
| return path.startsWith("//ldml/characters/exemplarCharacters") || path.startsWith("//ldml/characters/parseLenients"); |
| } |
| |
| static final UnicodeSet WHITESPACE = new UnicodeSet("[:whitespace:]").freeze(); |
| static final DateTimeCanonicalizer dtc = new DateTimeCanonicalizer(FIX_YEARS); |
| |
| private static final String BAR_VL = "\\|"; // U+007C VERTICAL LINE (pipe, bar) literal |
| private static final String BAR_EL = "\\s+l\\s+"; // U+006C LATIN SMALL LETTER L with space |
| private static final String BAR_DANDA = "।"; // U+0964 DEVANAGARI DANDA |
| private static final String BAR_REGEX = "(" + BAR_VL + "|" + BAR_EL + "|" + BAR_DANDA + ")"; |
| public static final Splitter SPLIT_BAR = Splitter.on(Pattern.compile(BAR_REGEX)).trimResults().omitEmptyStrings(); |
| static final Splitter SPLIT_SPACE = Splitter.on(' ').trimResults().omitEmptyStrings(); |
| static final Joiner JOIN_BAR = Joiner.on(" | "); |
| static final Joiner JOIN_SPACE = Joiner.on(' '); |
| |
| /** |
| * Process the value for input. The result is a cleaned-up value. For example, |
| * an exemplar set is modified to be in the normal format, and any missing [ ] |
| * are added (a common omission on entry). If there are any failures then the |
| * original value is returned, so that the proper error message can be given. |
| * |
| * @param path |
| * @param value |
| * @param internalException |
| * @return |
| */ |
| public synchronized String processInput(String path, String value, Exception[] internalException) { |
| String original = value; |
| value = stripProblematicControlCharacters(value); |
| value = Normalizer.compose(value, false); // Always normalize all input to NFC. |
| value = value.replace('\u00B5', '\u03BC'); // use the right Greek mu character |
| |
| if (internalException != null) { |
| internalException[0] = null; |
| } |
| // skip processing for inheritance marker |
| if (CldrUtility.INHERITANCE_MARKER.equals(value)) { |
| return value; // Reference: https://unicode.org/cldr/trac/ticket/11261 |
| } |
| // for root annotations |
| if (CLDRLocale.ROOT.equals(locale) && path.contains("/annotations")) { |
| return value; // Reference: https://unicode.org/cldr/trac/ticket/11261 |
| } |
| |
| try { |
| // Normalise Malayalam characters. |
| boolean isUnicodeSet = hasUnicodeSetValue(path); |
| if (locale.childOf(MALAYALAM)) { |
| String newvalue = normalizeMalayalam(value); |
| if (DEBUG_DAIP) System.out.println("DAIP: Normalized Malayalam '" + value + "' to '" + newvalue + "'"); |
| value = newvalue; |
| } else if (locale.childOf(ROMANIAN) && !isUnicodeSet) { |
| value = standardizeRomanian(value); |
| } else if (locale.childOf(CATALAN) && !isUnicodeSet) { |
| value = standardizeCatalan(value); |
| } else if (locale.childOf(NGOMBA) && !isUnicodeSet) { |
| value = standardizeNgomba(value); |
| } else if (locale.childOf(KWASIO) && !isUnicodeSet) { |
| value = standardizeKwasio(value); |
| } else if (locale.childOf(HEBREW) && !APOSTROPHE_SKIP_PATHS.matcher(path).matches()) { |
| value = replaceChars(path, value, HEBREW_CONVERSIONS, false); |
| } else if ((locale.childOf(SWISS_GERMAN) || locale.childOf(GERMAN_SWITZERLAND)) && !isUnicodeSet) { |
| value = standardizeSwissGerman(value); |
| } else if (locale.childOf(MYANMAR) && !isUnicodeSet) { |
| value = standardizeMyanmar(value); |
| } else if (locale.childOf(KYRGYZ)) { |
| value = replaceChars(path, value, KYRGYZ_CONVERSIONS, false); |
| } else if (locale.childOf(URDU) || locale.childOf(PASHTO) || locale.childOf(FARSI)) { |
| value = replaceChars(path, value, URDU_PLUS_CONVERSIONS, true); |
| } else if (locale.childOf(FF_ADLAM) && !isUnicodeSet) { |
| value = fixAdlamNasalization(value); |
| } |
| |
| if (UNICODE_WHITESPACE.containsSome(value)) { |
| value = normalizeWhitespace(path, value); |
| } |
| |
| // all of our values should not have leading or trailing spaces, except insertBetween, |
| // foreignSpaceReplacement, and anything with built-in attribute xml:space="preserve" |
| if (!path.contains("/insertBetween") && !path.contains("/foreignSpaceReplacement") && |
| !path.contains("[@xml:space=\"preserve\"]") && !isUnicodeSet) { |
| value = value.trim(); |
| } |
| |
| // fix grouping separator if space |
| if (path.startsWith("//ldml/numbers/symbols") && !path.contains("/alias")) { |
| if (value.isEmpty()) { |
| value = "\u00A0"; |
| } |
| value = value.replace(' ', '\u00A0'); |
| } |
| |
| // fix date patterns |
| DateTimePatternType datetimePatternType = DateTimePatternType.fromPath(path); |
| if (DateTimePatternType.STOCK_AVAILABLE_INTERVAL_PATTERNS.contains(datetimePatternType)) { |
| try { |
| value = dtc.getCanonicalDatePattern(path, value, datetimePatternType); |
| } catch (IllegalArgumentException ex) { |
| return value; |
| } |
| } |
| |
| if (path.startsWith("//ldml/numbers/currencies/currency") && path.contains("displayName")) { |
| value = normalizeCurrencyDisplayName(value); |
| } |
| NumericType numericType = NumericType.getNumericType(path); |
| if (numericType != NumericType.NOT_NUMERIC) { |
| if (numericType == NumericType.CURRENCY) { |
| value = value.replaceAll(" ", "\u00A0"); |
| if (numericType == NumericType.CURRENCY_ABBREVIATED) { |
| value = value.replaceAll("0\\.0+", "0"); |
| } |
| } else { |
| value = value.replaceAll("([%\u00A4]) ", "$1\u00A0") |
| .replaceAll(" ([%\u00A4])", "\u00A0$1"); |
| value = replace(NON_DECIMAL_PERIOD, value, "'.'"); |
| if (numericType == NumericType.DECIMAL_ABBREVIATED) { |
| value = value.replaceAll("0\\.0+", "0"); |
| } |
| } |
| value = getCanonicalPattern(value, numericType, isPosix); |
| } |
| |
| // fix [,] |
| if (path.startsWith("//ldml/localeDisplayNames/languages/language") |
| || path.startsWith("//ldml/localeDisplayNames/scripts/script") |
| || path.startsWith("//ldml/localeDisplayNames/territories/territory") |
| || path.startsWith("//ldml/localeDisplayNames/variants/variant") |
| || path.startsWith("//ldml/localeDisplayNames/keys/key") |
| || path.startsWith("//ldml/localeDisplayNames/types/type")) { |
| value = value.replace('[', '(').replace(']', ')').replace('[', '(').replace(']', ')'); |
| } |
| |
| // Normalize two single quotes for the inches symbol. |
| if (path.contains("/units")) { |
| value = value.replace("''", "″"); |
| } |
| |
| // check specific cases |
| if (isUnicodeSet) { |
| value = inputUnicodeSet(path, value); |
| } else if (path.contains("stopword")) { |
| if (value.equals("NONE")) { |
| value = ""; |
| } |
| } |
| |
| // Normalize ellipsis data. |
| if (path.startsWith("//ldml/characters/ellipsis")) { |
| value = value.replace("...", "…"); |
| } |
| |
| if (path.startsWith("//ldml/personNames/nameOrderLocales")) { |
| value = normalizeNameOrderLocales(value); |
| } |
| |
| // Replace Arabic presentation forms with their nominal counterparts |
| value = replaceArabicPresentationForms(value); |
| |
| // Fix up any apostrophes as appropriate (Don't do so for things like date patterns... |
| if (!APOSTROPHE_SKIP_PATHS.matcher(path).matches()) { |
| value = normalizeApostrophes(value); |
| } |
| // Fix up any apostrophes in number symbols |
| if (NUMBER_SEPARATOR_PATTERN.matcher(path).matches()) { |
| value = value.replace('\'', '\u2019'); |
| } |
| // Fix up hyphens, replacing with N-dash as appropriate |
| if (INTERVAL_FORMAT_PATHS.matcher(path).matches()) { |
| value = normalizeIntervalHyphensAndSpaces(value); // This may also adjust spaces around en dash |
| } else if (!isUnicodeSet) { |
| value = normalizeHyphens(value); |
| } |
| |
| if (AnnotationUtil.pathIsAnnotation(path)) { |
| if (path.contains(Emoji.TYPE_TTS)) { |
| // The row has something like "🦓 -name" in the first column. Cf. namePath, getNamePaths. |
| // Normally the value is like "zebra" or "unicorn face", without "|". |
| // If the user enters a value with "|", discard anything after "|"; e.g., change "a | b | c" to "a". |
| value = SPLIT_BAR.split(value).iterator().next(); |
| } else { |
| // The row has something like "🦓 –keywords" in the first column. Cf. keywordPath, getKeywordPaths. |
| // Normally the value is like "stripe | zebra", with "|". |
| value = annotationsForDisplay(value); |
| } |
| } |
| value = normalizeZeroWidthSpace(value); |
| return value; |
| } catch (RuntimeException e) { |
| if (internalException != null) { |
| internalException[0] = e; |
| } |
| return original; |
| } |
| } |
| |
| private String normalizeNameOrderLocales(String value) { |
| TreeSet<String> result = new TreeSet<>(SPLIT_SPACE.splitToList(value)); |
| result.remove("zxx"); |
| if (result.remove("und")) { // put und at the front |
| if (result.isEmpty()) { |
| return "und"; |
| } else { |
| return "und " + JOIN_SPACE.join(result); |
| } |
| } |
| return JOIN_SPACE.join(result); |
| } |
| |
| /** |
| * Strip out all code points less than U+0020 except for U+0009 tab, |
| * U+000A line feed, and U+000D carriage return. |
| * |
| * @param s the string |
| * @return the resulting string |
| */ |
| private String stripProblematicControlCharacters(String s) { |
| if (s == null || s.isEmpty()) { |
| return s; |
| } |
| return s.codePoints() |
| .filter(c -> (c >= 0x20 || c == 9 || c == 0xA || c == 0xD)) |
| .collect(StringBuilder::new, StringBuilder::appendCodePoint, StringBuilder::append) |
| .toString(); |
| } |
| |
| private static final boolean REMOVE_COVERED_KEYWORDS = true; |
| |
| /** |
| * Produce a modification of the given annotation by sorting its components and filtering covered keywords. |
| * |
| * Examples: Given "b | a", return "a | b". Given "bear | panda | panda bear", return "bear | panda". |
| * |
| * @param value the string |
| * @return the possibly modified string |
| */ |
| private static String annotationsForDisplay(String value) { |
| TreeSet<String> sorted = new TreeSet<>(Collator.getInstance(ULocale.ROOT)); |
| sorted.addAll(SPLIT_BAR.splitToList(value)); |
| if (REMOVE_COVERED_KEYWORDS) { |
| filterCoveredKeywords(sorted); |
| } |
| value = JOIN_BAR.join(sorted); |
| return value; |
| } |
| |
| /** |
| * Filter from the given set some keywords that include spaces, if they duplicate, |
| * or are "covered by", other keywords in the set. |
| * |
| * For example, if the set is {"bear", "panda", "panda bear"} (annotation was "bear | panda | panda bear"), |
| * then remove "panda bear", treating it as "covered" since the set already includes "panda" and "bear". |
| * |
| * @param sorted the set from which items may be removed |
| */ |
| public static void filterCoveredKeywords(TreeSet<String> sorted) { |
| // for now, just do single items |
| HashSet<String> toRemove = new HashSet<>(); |
| |
| for (String item : sorted) { |
| List<String> list = SPLIT_SPACE.splitToList(item); |
| if (list.size() < 2) { |
| continue; |
| } |
| if (sorted.containsAll(list)) { |
| toRemove.add(item); |
| } |
| } |
| sorted.removeAll(toRemove); |
| } |
| |
| private String displayUnicodeSet(String value) { |
| if (value.startsWith("[") && value.endsWith("]")) { |
| value = value.substring(1, value.length() - 1); |
| } |
| |
| value = replace(NEEDS_QUOTE1, value, "$1\\\\$2$3"); |
| value = replace(NEEDS_QUOTE2, value, "$1\\\\$2$3"); |
| |
| // if (RTL.containsSome(value) && value.startsWith("[") && value.endsWith("]")) { |
| // return "\u200E[\u200E" + value.substring(1,value.length()-2) + "\u200E]\u200E"; |
| // } |
| return value; |
| } |
| |
| private String inputUnicodeSet(String path, String value) { |
| // clean up the user's input. |
| // first, fix up the '[' |
| value = value.trim(); |
| |
| // remove brackets and trim again before regex |
| if (value.startsWith("[")) { |
| value = value.substring(1); |
| } |
| if (value.endsWith("]") && (!value.endsWith("\\]") || value.endsWith("\\\\]"))) { |
| value = value.substring(0, value.length() - 1); |
| } |
| value = value.trim(); |
| |
| value = replace(NEEDS_QUOTE1, value, "$1\\\\$2$3"); |
| value = replace(NEEDS_QUOTE2, value, "$1\\\\$2$3"); |
| |
| // re-add brackets. |
| value = "[" + value + "]"; |
| |
| UnicodeSet exemplar = new UnicodeSet(value); |
| XPathParts parts = XPathParts.getFrozenInstance(path); |
| if (parts.getElement(2).equals("parseLenients")) { |
| return exemplar.toPattern(false); |
| } |
| final String type = parts.getAttributeValue(-1, "type"); |
| ExemplarType exemplarType = type == null ? ExemplarType.main : ExemplarType.valueOf(type); |
| value = getCleanedUnicodeSet(exemplar, pp, exemplarType); |
| return value; |
| } |
| |
| private String normalizeCurrencyDisplayName(String value) { |
| StringBuilder result = new StringBuilder(); |
| boolean inParentheses = false; |
| for (int i = 0; i < value.length(); i++) { |
| char c = value.charAt(i); |
| if (c == '(') { |
| inParentheses = true; |
| } else if (c == ')') { |
| inParentheses = false; |
| } |
| if (inParentheses && c == '-' && Character.isDigit(value.charAt(i - 1))) { |
| c = 0x2013; /* Replace hyphen-minus with dash for date ranges */ |
| } |
| result.append(c); |
| } |
| return result.toString(); |
| } |
| |
| private String normalizeApostrophes(String value) { |
| // If our DAIP always had a CLDRFile to work with, then we could just check the exemplar set in it to see. |
| // But since we don't, we just maintain the list internally and use it. |
| if (LANGUAGES_USING_MODIFIER_APOSTROPHE.contains(locale.getLanguage())) { |
| return value.replace('\'', '\u02bc'); |
| } else { |
| char prev = 0; |
| StringBuilder builder = new StringBuilder(); |
| for (char c : value.toCharArray()) { |
| if (c == '\'') { |
| if (Character.isLetter(prev)) { |
| builder.append('\u2019'); |
| } else { |
| builder.append('\u2018'); |
| } |
| } else { |
| builder.append(c); |
| } |
| prev = c; |
| } |
| return builder.toString(); |
| } |
| } |
| |
| private String normalizeIntervalHyphensAndSpaces(String value) { |
| if (value.indexOf("{0}") >= 0) { |
| // intervalFormatFallback pattern, not handled by DateTimePatternGenerator.FormatParser |
| if (scriptCode.equals("Latn")) { |
| value = INTERVAL_FALLBACK_RANGE.matcher(value).replaceAll("}\u2009\u2013\u2009{"); |
| } |
| return value; |
| } |
| DateTimePatternGenerator.FormatParser fp = new DateTimePatternGenerator.FormatParser(); |
| fp.set(DateIntervalInfo.genPatternInfo(value, false).getFirstPart()); // first format & separator including spaces |
| List<Object> items = fp.getItems(); |
| Object last = items.get(items.size() - 1); |
| if (last instanceof String) { |
| String separator = last.toString(); // separator including spaces |
| String replacement = separator; |
| if (scriptCode.equals("Latn") && (separator.equals(" - ") || separator.equals(" \u2013 "))) { |
| replacement = "\u2009\u2013\u2009"; // Per CLDR-14032 |
| } else if (separator.contains("-")) { |
| replacement = separator.replace("-", "\u2013"); |
| } |
| if (!replacement.equals(separator)) { |
| StringBuilder sb = new StringBuilder(); |
| sb.append(DateIntervalInfo.genPatternInfo(value, false).getFirstPart()); |
| if (sb.lastIndexOf(separator) >= 0) { |
| sb.delete(sb.lastIndexOf(separator), sb.length()); |
| sb.append(replacement); |
| sb.append(DateIntervalInfo.genPatternInfo(value, false).getSecondPart()); // second format only |
| return sb.toString(); |
| } |
| } |
| } |
| return value; |
| } |
| |
| private String normalizeHyphens(String value) { |
| int hyphenLocation = value.indexOf("-"); |
| if (hyphenLocation > 0 && |
| Character.isDigit(value.charAt(hyphenLocation - 1)) && |
| hyphenLocation < value.length() - 1 && |
| Character.isDigit(value.charAt(hyphenLocation + 1))) { |
| StringBuilder sb = new StringBuilder(); |
| sb.append(value.substring(0, hyphenLocation)); |
| sb.append("\u2013"); |
| sb.append(value.substring(hyphenLocation + 1)); |
| return sb.toString(); |
| } |
| return value; |
| } |
| |
| private String standardizeRomanian(String value) { |
| StringBuilder builder = new StringBuilder(); |
| for (char c : value.toCharArray()) { |
| for (char[] pair : ROMANIAN_CONVERSIONS) { |
| if (c == pair[0]) { |
| c = pair[1]; |
| break; |
| } |
| } |
| builder.append(c); |
| } |
| return builder.toString(); |
| } |
| |
| private String standardizeKwasio(String value) { |
| StringBuilder builder = new StringBuilder(); |
| for (char c : value.toCharArray()) { |
| for (char[] pair : KWASIO_CONVERSIONS) { |
| if (c == pair[0]) { |
| c = pair[1]; |
| break; |
| } |
| } |
| builder.append(c); |
| } |
| return builder.toString(); |
| } |
| |
| // Use the myanmar-tools detector. |
| private String standardizeMyanmar(String value) { |
| if (detector.getZawgyiProbability(value) > 0.90) { |
| return zawgyiUnicodeTransliterator.transform(value); |
| } |
| return value; |
| } |
| |
| private String standardizeNgomba(String value) { |
| StringBuilder builder = new StringBuilder(); |
| char[] charArray = value.toCharArray(); |
| for (int i = 0; i < charArray.length; i++) { |
| char c = charArray[i]; |
| boolean convertedSaltillo = false; |
| for (char[] pair : NGOMBA_CONVERSIONS) { |
| if (c == pair[0]) { |
| c = pair[1]; |
| if (c == '\uA78C') { |
| convertedSaltillo = true; |
| } |
| break; |
| } |
| } |
| if (convertedSaltillo && |
| ((i > 0 && i < charArray.length - 1 && Character.isUpperCase(charArray[i - 1]) && Character.isUpperCase(charArray[i + 1])) || |
| (i > 1 && Character.isUpperCase(charArray[i - 1]) && Character.isUpperCase(charArray[i - 2])))) { |
| c = '\uA78B'; // UPPER CASE SALTILLO |
| } |
| builder.append(c); |
| } |
| return builder.toString(); |
| } |
| |
| private String replaceChars(String path, String value, char[][] charsToReplace, boolean skipAuxExemplars) { |
| if (skipAuxExemplars && path.contains("/exemplarCharacters[@type=\"auxiliary\"]")) { |
| return value; |
| } |
| StringBuilder builder = new StringBuilder(); |
| for (char c : value.toCharArray()) { |
| for (char[] pair : charsToReplace) { |
| if (c == pair[0]) { |
| c = pair[1]; |
| break; |
| } |
| } |
| builder.append(c); |
| } |
| return builder.toString(); |
| } |
| |
| private String standardizeSwissGerman(String value) { |
| return value.replaceAll("\u00DF", "ss"); |
| } |
| |
| private String standardizeCatalan(String value) { |
| StringBuilder builder = new StringBuilder(); |
| for (char c : value.toCharArray()) { |
| boolean didSubstitute = false; |
| for (char[] triple : CATALAN_CONVERSIONS) { |
| if (c == triple[0]) { |
| builder.append(triple[1]); |
| builder.append(triple[2]); |
| didSubstitute = true; |
| break; |
| } |
| } |
| if (!didSubstitute) { |
| builder.append(c); |
| } |
| } |
| return builder.toString(); |
| } |
| |
| private String replace(Pattern pattern, String value, String replacement) { |
| String value2 = pattern.matcher(value).replaceAll(replacement); |
| if (DEBUG_DAIP && !value.equals(value2)) { |
| System.out.println("\n" + value + " => " + value2); |
| } |
| return value2; |
| } |
| |
| private static Pattern UNNORMALIZED_MALAYALAM = PatternCache.get( |
| "(\u0D23|\u0D28|\u0D30|\u0D32|\u0D33|\u0D15)\u0D4D\u200D"); |
| |
| private static Map<Character, Character> NORMALIZING_MAP = Builder.with(new HashMap<Character, Character>()) |
| .put('\u0D23', '\u0D7A').put('\u0D28', '\u0D7B') |
| .put('\u0D30', '\u0D7C').put('\u0D32', '\u0D7D') |
| .put('\u0D33', '\u0D7E').put('\u0D15', '\u0D7F').get(); |
| |
| /** |
| * Normalizes the Malayalam characters in the specified input. |
| * |
| * @param value |
| * the input to be normalized |
| * @return |
| */ |
| private String normalizeMalayalam(String value) { |
| // Normalize Malayalam characters. |
| Matcher matcher = UNNORMALIZED_MALAYALAM.matcher(value); |
| if (matcher.find()) { |
| StringBuffer buffer = new StringBuffer(); |
| int start = 0; |
| do { |
| buffer.append(value.substring(start, matcher.start(0))); |
| char codePoint = matcher.group(1).charAt(0); |
| buffer.append(NORMALIZING_MAP.get(codePoint)); |
| start = matcher.end(0); |
| } while (matcher.find()); |
| buffer.append(value.substring(start)); |
| value = buffer.toString(); |
| } |
| return value; |
| } |
| |
| static final Transform<String, String> fixArabicPresentation = Transliterator.getInstance( |
| "[[:block=Arabic_Presentation_Forms_A:][:block=Arabic_Presentation_Forms_B:]] nfkc"); |
| |
| /** |
| * Normalizes the Arabic presentation forms characters in the specified input. |
| * |
| * @param value |
| * the input to be normalized |
| * @return |
| */ |
| private String replaceArabicPresentationForms(String value) { |
| value = fixArabicPresentation.transform(value); |
| return value; |
| } |
| |
| static Pattern ADLAM_MISNASALIZED = PatternCache.get("([𞤲𞤐])['’‘]([𞤁𞤔𞤘𞤄𞤣𞤦𞤶𞤺])"); |
| public static String ADLAM_NASALIZATION = "𞥋"; // U+1E94B (Unicode 12.0) |
| |
| public static String fixAdlamNasalization(String fromString) { |
| return ADLAM_MISNASALIZED.matcher(fromString) |
| .replaceAll("$1"+ADLAM_NASALIZATION+"$2"); // replace quote with 𞥋 |
| } |
| |
| static Pattern REMOVE_QUOTE1 = PatternCache.get("(\\s)(\\\\[-\\}\\]\\&])()"); |
| static Pattern REMOVE_QUOTE2 = PatternCache.get("(\\\\[\\-\\{\\[\\&])(\\s)"); // ([^\\])([\\-\\{\\[])(\\s) |
| |
| static Pattern NEEDS_QUOTE1 = PatternCache.get("(\\s|$)([-\\}\\]\\&])()"); |
| static Pattern NEEDS_QUOTE2 = PatternCache.get("([^\\\\])([\\-\\{\\[\\&])(\\s)"); // ([^\\])([\\-\\{\\[])(\\s) |
| |
| public static String getCleanedUnicodeSet(UnicodeSet exemplar, UnicodeSetPrettyPrinter prettyPrinter, |
| ExemplarType exemplarType) { |
| if (prettyPrinter == null) { |
| return exemplar.toPattern(false); |
| } |
| String value; |
| prettyPrinter.setCompressRanges(exemplar.size() > 300); |
| value = exemplar.toPattern(false); |
| UnicodeSet toAdd = new UnicodeSet(); |
| |
| for (UnicodeSetIterator usi = new UnicodeSetIterator(exemplar); usi.next();) { |
| String string = usi.getString(); |
| if (string.equals("ß") || string.equals("İ")) { |
| toAdd.add(string); |
| continue; |
| } |
| switch (string) { |
| case "\u2011": toAdd.add("-"); break; // nobreak hyphen |
| case "-": toAdd.add("\u2011"); break; // nobreak hyphen |
| |
| case " ": toAdd.add("\u00a0"); break; // nobreak space |
| case "\u00a0": toAdd.add(" "); break; // nobreak space |
| |
| case "\u202F": toAdd.add("\u2009"); break; // nobreak narrow space |
| case "\u2009": toAdd.add("\u202F"); break; // nobreak narrow space |
| } |
| if (exemplarType.convertUppercase) { |
| string = UCharacter.toLowerCase(ULocale.ENGLISH, string); |
| } |
| toAdd.add(string); |
| String composed = Normalizer.compose(string, false); |
| if (!string.equals(composed)) { |
| toAdd.add(composed); |
| } |
| } |
| |
| toAdd.removeAll(exemplarType.toRemove); |
| |
| if (DEBUG_DAIP && !toAdd.equals(exemplar)) { |
| UnicodeSet oldOnly = new UnicodeSet(exemplar).removeAll(toAdd); |
| UnicodeSet newOnly = new UnicodeSet(toAdd).removeAll(exemplar); |
| System.out.println("Exemplar:\t" + exemplarType + ",\tremoved\t" + oldOnly + ",\tadded\t" + newOnly); |
| } |
| |
| String fixedExemplar = prettyPrinter.format(toAdd); |
| UnicodeSet doubleCheck = new UnicodeSet(fixedExemplar); |
| if (!toAdd.equals(doubleCheck)) { |
| // something went wrong, leave as is |
| } else if (!value.equals(fixedExemplar)) { // put in this condition just for debugging |
| if (DEBUG_DAIP) { |
| System.out.println(TestMetadata.showDifference( |
| With.codePoints(value), |
| With.codePoints(fixedExemplar), |
| "\n")); |
| } |
| value = fixedExemplar; |
| } |
| return value; |
| } |
| |
| /** |
| * @return a canonical numeric pattern, based on the type, and the isPOSIX flag. The latter is set for en_US_POSIX. |
| */ |
| static final Splitter SEMI_SPLITTER = Splitter.on(';').trimResults(); |
| |
| public static String getCanonicalPattern(String inpattern, NumericType type, boolean isPOSIX) { |
| // TODO fix later to properly handle quoted ; |
| |
| DecimalFormat df = new DecimalFormat(inpattern); |
| if (type == NumericType.DECIMAL_ABBREVIATED || type == NumericType.CURRENCY_ABBREVIATED |
| || CldrUtility.INHERITANCE_MARKER.equals(inpattern)) { |
| return inpattern; // TODO fix when ICU bug is fixed |
| // df.setMaximumFractionDigits(df.getMinimumFractionDigits()); |
| // df.setMaximumIntegerDigits(Math.max(1, df.getMinimumIntegerDigits())); |
| } else { |
| // int decimals = type == CURRENCY_TYPE ? 2 : 1; |
| int[] digits = isPOSIX ? type.posixDigitCount : type.digitCount; |
| df.setMinimumIntegerDigits(digits[0]); |
| df.setMinimumFractionDigits(digits[1]); |
| df.setMaximumFractionDigits(digits[2]); |
| } |
| String pattern = df.toPattern(); |
| List<String> parts = SEMI_SPLITTER.splitToList(pattern); |
| String pattern2 = parts.get(0); |
| if (parts.size() > 1) { |
| pattern2 += ";" + parts.get(1); |
| } |
| if (!pattern2.equals(pattern)) { |
| pattern = pattern2; |
| } |
| // int pos = pattern.indexOf(';'); |
| // if (pos < 0) return pattern + ";-" + pattern; |
| return pattern; |
| } |
| |
| /* |
| * This tests what type a numeric pattern is. |
| */ |
| public enum NumericType { |
| CURRENCY(new int[] { 1, 2, 2 }, new int[] { 1, 2, 2 }), CURRENCY_ABBREVIATED(), DECIMAL(new int[] { 1, 0, 3 }, |
| new int[] { 1, 0, 6 }), DECIMAL_ABBREVIATED(), PERCENT(new int[] { 1, 0, 0 }, |
| new int[] { 1, 0, 0 }), SCIENTIFIC(new int[] { 0, 0, 0 }, new int[] { 1, 6, 6 }), NOT_NUMERIC; |
| |
| private static final Pattern NUMBER_PATH = Pattern |
| .compile("//ldml/numbers/((currency|decimal|percent|scientific)Formats|currencies/currency).*"); |
| private int[] digitCount; |
| private int[] posixDigitCount; |
| |
| private NumericType() { |
| } |
| |
| private NumericType(int[] digitCount, int[] posixDigitCount) { |
| this.digitCount = digitCount; |
| this.posixDigitCount = posixDigitCount; |
| } |
| |
| /** |
| * @return the numeric type of the xpath |
| */ |
| public static NumericType getNumericType(String xpath) { |
| Matcher matcher = NUMBER_PATH.matcher(xpath); |
| if (xpath.indexOf("/pattern") < 0) { |
| return NOT_NUMERIC; |
| } else if (matcher.matches()) { |
| if (matcher.group(1).equals("currencies/currency")) { |
| return CURRENCY; |
| } else { |
| NumericType type = NumericType.valueOf(matcher.group(2).toUpperCase()); |
| if (xpath.contains("=\"1000")) { |
| if (type == DECIMAL) { |
| type = DECIMAL_ABBREVIATED; |
| } else if (type == CURRENCY) { |
| type = CURRENCY_ABBREVIATED; |
| } else { |
| throw new IllegalArgumentException("Internal Error"); |
| } |
| } |
| return type; |
| } |
| } else { |
| return NOT_NUMERIC; |
| } |
| } |
| |
| public int[] getDigitCount() { |
| return digitCount; |
| } |
| |
| public int[] getPosixDigitCount() { |
| return posixDigitCount; |
| } |
| } |
| |
| /** |
| * Turn all whitespace sequences (including tab and newline, and NBSP for certain paths) |
| * into a single space or a single NBSP depending on path. |
| * Also trim initial/final NBSP, unless the value is only the one character, "\u00A0" |
| * |
| * @param path |
| * @param value |
| * @return the normalized value |
| */ |
| private String normalizeWhitespace(String path, String value) { |
| PathSpaceType pst = PathSpaceType.get(path); |
| if (pst == PathSpaceType.allowSp) { |
| value = WHITESPACE_AND_NBSP_TO_NORMALIZE.matcher(value).replaceAll(" "); // replace with regular space |
| } else if (pst == PathSpaceType.allowNbsp) { |
| value = WHITESPACE_AND_NBSP_TO_NORMALIZE.matcher(value).replaceAll("\u00A0"); // replace with NBSP |
| value = trimNBSP(value); |
| } else if (pst == PathSpaceType.allowNNbsp) { |
| value = WHITESPACE_AND_NBSP_TO_NORMALIZE.matcher(value).replaceAll("\u202F"); // replace with NNBSP |
| value = trimNBSP(value); |
| } else if (pst == PathSpaceType.allowSpOrNbsp) { |
| /* |
| * in this case don't normalize away NBSP |
| */ |
| value = WHITESPACE_NO_NBSP_TO_NORMALIZE.matcher(value).replaceAll(" "); // replace with regular space |
| /* |
| * if any NBSP and regular space are adjacent, replace with NBSP |
| */ |
| value = NBSP_PLUS_SPACE_TO_NORMALIZE.matcher(value).replaceAll("\u00A0"); |
| value = SPACE_PLUS_NBSP_TO_NORMALIZE.matcher(value).replaceAll("\u00A0"); |
| value = MULTIPLE_NBSP.matcher(value).replaceAll("\u00A0"); |
| value = trimNBSP(value); |
| } else { |
| throw new IllegalArgumentException("Unknown PathSpaceType " + pst); |
| } |
| |
| // Further whitespace adjustments per CLDR-14032 |
| if ((scriptCode.equals("Latn") || scriptCode.equals("Cyrl") || scriptCode.equals("Grek")) && |
| HOUR_FORMAT_XPATHS.matcher(path).matches()) { |
| String test = AMPM_SPACE_BEFORE.matcher(value).replaceAll("$1$2"); // value without a+ |
| if (value.length() - test.length() != 4) { // exclude patterns with aaaa |
| value = AMPM_SPACE_BEFORE.matcher(value).replaceAll("$1\u202F$3"); |
| } |
| test = AMPM_SPACE_AFTER.matcher(value).replaceAll("$2$3"); // value without a+ |
| if (value.length() - test.length() != 4) { // exclude patterns with aaaa |
| value = AMPM_SPACE_AFTER.matcher(value).replaceAll("$1\u202F$3"); |
| } |
| } |
| if (scriptCode.equals("Cyrl") && YEAR_FORMAT_XPATHS.matcher(path).matches()) { |
| value = YEAR_SPACE_YEARMARKER.matcher(value).replaceAll("y\u202F$1"); |
| } |
| if (UNIT_NARROW_XPATHS.matcher(path).matches()) { |
| value = PLACEHOLDER_SPACE_AFTER.matcher(value).replaceAll("}\u202F"); // Narrow NBSP |
| value = PLACEHOLDER_SPACE_BEFORE.matcher(value).replaceAll("\u202F{"); |
| } |
| if (UNIT_SHORT_XPATHS.matcher(path).matches()) { |
| value = PLACEHOLDER_SPACE_AFTER.matcher(value).replaceAll("}\u00A0"); // Regular NBSP |
| value = PLACEHOLDER_SPACE_BEFORE.matcher(value).replaceAll("\u00A0{"); |
| } |
| |
| return value; |
| } |
| |
| /** |
| * Delete any initial or final NBSP, unless the value is just NBSP |
| * |
| * @param value |
| * @return the trimmed value |
| */ |
| private String trimNBSP(String value) { |
| if (!value.equals("\u00A0") && !value.equals("\u202F")) { |
| value = INITIAL_NBSP.matcher(value).replaceAll(""); |
| value = FINAL_NBSP.matcher(value).replaceAll(""); |
| } |
| return value; |
| } |
| |
| /** |
| * Categorize xpaths according to whether they allow space, NBSP, or both |
| */ |
| public enum PathSpaceType { |
| allowSp, allowNbsp, allowNNbsp, allowSpOrNbsp; |
| |
| public static PathSpaceType get(String path) { |
| if (wantsRegularSpace(path)) { |
| return allowSp; |
| } else if (wantsNBSP(path)) { |
| return allowNbsp; |
| } else if (wantsNNBSP(path)) { |
| return allowNNbsp; |
| } else { |
| return allowSpOrNbsp; |
| } |
| } |
| |
| private static boolean wantsRegularSpace(String path) { |
| if ((path.contains("/dateFormatLength") && path.contains("/pattern")) || |
| path.contains("/availableFormats/dateFormatItem") || |
| (path.startsWith("//ldml/dates/timeZoneNames/metazone") && path.contains("/long")) || |
| path.startsWith("//ldml/dates/timeZoneNames/regionFormat") || |
| path.startsWith("//ldml/localeDisplayNames/codePatterns/codePattern") || |
| path.startsWith("//ldml/localeDisplayNames/languages/language") || |
| path.startsWith("//ldml/localeDisplayNames/territories/territory") || |
| path.startsWith("//ldml/localeDisplayNames/types/type") || |
| (path.startsWith("//ldml/numbers/currencies/currency") && path.contains("/displayName")) || |
| (path.contains("/decimalFormatLength[@type=\"long\"]") && path.contains("/pattern")) || |
| path.startsWith("//ldml/posix/messages") || |
| (path.startsWith("//ldml/units/uni") && path.contains("/unitPattern "))) { |
| return true; |
| } |
| return false; |
| } |
| |
| private static boolean wantsNBSP(String path) { |
| if ((path.contains("/currencies/currency") && (path.contains("/group") || path.contains("/pattern"))) || |
| (path.contains("/currencyFormatLength") && path.contains("/pattern")) || |
| (path.contains("/currencySpacing") && path.contains("/insertBetween")) || |
| (path.contains("/decimalFormatLength") && path.contains("/pattern")) || // i.e. the non-long ones |
| (path.contains("/percentFormatLength") && path.contains("/pattern")) || |
| (path.startsWith("//ldml/numbers/symbols") && (path.contains("/group") || path.contains("/nan")))) { |
| return true; |
| } |
| return false; |
| } |
| |
| private static boolean wantsNNBSP(String path) { |
| if ((path.contains("/dayPeriodWidth[@type=\"abbreviated\"]") || path.contains("/dayPeriodWidth[@type=\"narrow\"]")) && |
| (path.contains("/dayPeriod[@type=\"am\"]") || path.contains("/dayPeriod[@type=\"pm\"]")) ) { |
| return true; |
| } |
| return false; |
| } |
| } |
| |
| private static final Pattern ZERO_WIDTH_SPACES = PatternCache.get("\\u200B+"); |
| private static final Set<String> LOCALES_NOT_ALLOWING_ZWS = new HashSet<>(Arrays.asList("da", "fr")); |
| |
| /** |
| * Remove occurrences of U+200B ZERO_WIDTH_SPACE under certain conditions |
| * |
| * @param value the value to be normalized |
| * @return the normalized value |
| * |
| * TODO: extend this method to address more concerns, after clarifying the conditions |
| * - enlarge the set LOCALES_NOT_ALLOWING_ZWS? |
| * - strip initial and final ZWS in all locales? |
| * - reduce two or more adjacent ZWS to one ZWS? |
| * - allow or prohibit ZWS by itself as currency symbol, as currently in locales kea, pt_CV, pt_PT |
| * - allow or prohibit ZWS preceding URL as in "as per [U+200B]http://unicode.org/repos/cldr/trunk/specs/ldml/tr35-general.html#Annotations" |
| * Reference: https://unicode-org.atlassian.net/browse/CLDR-15976 |
| */ |
| private String normalizeZeroWidthSpace(String value) { |
| if (ZERO_WIDTH_SPACES.matcher(value).find()) { |
| final String localeId = locale.getBaseName(); |
| if (LOCALES_NOT_ALLOWING_ZWS.contains(localeId)) { |
| value = ZERO_WIDTH_SPACES.matcher(value).replaceAll(""); |
| } |
| } |
| return value; |
| } |
| } |