| package org.unicode.cldr.unittest; |
| |
| import java.io.IOException; |
| import java.util.ArrayList; |
| import java.util.Arrays; |
| import java.util.Collections; |
| import java.util.HashMap; |
| import java.util.HashSet; |
| import java.util.LinkedHashSet; |
| import java.util.List; |
| import java.util.Map; |
| import java.util.Map.Entry; |
| import java.util.Set; |
| import java.util.TreeMap; |
| import java.util.TreeSet; |
| import java.util.regex.Matcher; |
| |
| import org.unicode.cldr.draft.ScriptMetadata; |
| import org.unicode.cldr.draft.ScriptMetadata.Info; |
| import org.unicode.cldr.tool.GenerateMaximalLocales; |
| import org.unicode.cldr.tool.LikelySubtags; |
| import org.unicode.cldr.util.Builder; |
| import org.unicode.cldr.util.CLDRConfig; |
| import org.unicode.cldr.util.CLDRFile; |
| import org.unicode.cldr.util.CLDRLocale; |
| import org.unicode.cldr.util.ChainedMap; |
| import org.unicode.cldr.util.ChainedMap.M3; |
| import org.unicode.cldr.util.CldrUtility; |
| import org.unicode.cldr.util.LanguageTagParser; |
| import org.unicode.cldr.util.LocaleIDParser; |
| import org.unicode.cldr.util.PatternCache; |
| import org.unicode.cldr.util.StandardCodes; |
| import org.unicode.cldr.util.SupplementalDataInfo; |
| import org.unicode.cldr.util.SupplementalDataInfo.BasicLanguageData; |
| import org.unicode.cldr.util.SupplementalDataInfo.BasicLanguageData.Type; |
| import org.unicode.cldr.util.SupplementalDataInfo.OfficialStatus; |
| import org.unicode.cldr.util.SupplementalDataInfo.PopulationData; |
| import org.unicode.cldr.util.XPathParts; |
| |
| import com.ibm.icu.dev.test.TestFmwk; |
| import com.ibm.icu.dev.util.CollectionUtilities; |
| import com.ibm.icu.impl.Relation; |
| import com.ibm.icu.impl.Row.R2; |
| |
| public class TestInheritance extends TestFmwk { |
| |
| static CLDRConfig testInfo = CLDRConfig.getInstance(); |
| |
| private static boolean DEBUG = CldrUtility.getProperty("DEBUG", false); |
| |
| private static Matcher pathMatcher = PatternCache.get( |
| CldrUtility.getProperty("XPATH", ".*")).matcher(""); |
| |
| public static void main(String[] args) throws IOException { |
| new TestInheritance().run(args); |
| } |
| |
| private static final SupplementalDataInfo dataInfo = SupplementalDataInfo |
| .getInstance(); |
| private static final Set<String> defaultContents = dataInfo |
| .getDefaultContentLocales(); |
| |
| private static final boolean EXPECT_EQUALITY = false; |
| |
| private static Set<String> availableLocales = testInfo.getFullCldrFactory().getAvailable(); |
| |
| public void TestLocalesHaveOfficial() { |
| // If we have a language, we have all the region locales where the |
| // language is official |
| Set<String> SKIP_TERRITORIES = new HashSet<String>(Arrays.asList("001", |
| "150")); |
| for (Entry<String, R2<List<String>, String>> s : dataInfo |
| .getLocaleAliasInfo().get("territory").entrySet()) { |
| SKIP_TERRITORIES.add(s.getKey()); |
| } |
| |
| LanguageTagParser ltp = new LanguageTagParser(); |
| |
| Relation<String, String> languageLocalesSeen = Relation.of( |
| new TreeMap<String, Set<String>>(), TreeSet.class); |
| |
| Set<String> testOrg = testInfo.getStandardCodes() |
| .getLocaleCoverageLocales("google"); |
| ChainedMap.M4<String, OfficialStatus, String, Boolean> languageToOfficialChildren = ChainedMap |
| .of(new TreeMap<String, Object>(), |
| new TreeMap<OfficialStatus, Object>(), |
| new TreeMap<String, Object>(), Boolean.class); |
| |
| // gather the data |
| |
| for (String language : dataInfo |
| .getLanguagesForTerritoriesPopulationData()) { |
| for (String territory : dataInfo |
| .getTerritoriesForPopulationData(language)) { |
| if (SKIP_TERRITORIES.contains(territory)) { |
| continue; |
| } |
| PopulationData data = dataInfo |
| .getLanguageAndTerritoryPopulationData(language, |
| territory); |
| OfficialStatus status = data.getOfficialStatus(); |
| if (data.getOfficialStatus() != OfficialStatus.unknown) { |
| String locale = removeScript(language + "_" + territory); |
| String lang = removeScript(ltp.set(locale).getLanguage()); |
| languageToOfficialChildren.put(lang, status, locale, |
| Boolean.TRUE); |
| languageLocalesSeen.put(lang, locale); |
| } |
| } |
| } |
| |
| // flesh it out by adding 'clean' codes. |
| // also get the child locales in cldr. |
| |
| Relation<String, String> languageToChildren = Relation.of( |
| new TreeMap<String, Set<String>>(), TreeSet.class); |
| for (String locale : testInfo.getCldrFactory().getAvailable()) { |
| String lang = ltp.set(locale).getLanguage(); |
| if (SKIP_TERRITORIES.contains(ltp.getRegion())) { |
| continue; |
| } |
| lang = removeScript(lang); |
| locale = removeScript(locale); |
| |
| if (!lang.equals(locale)) { |
| languageToChildren.put(lang, locale); |
| Set<String> localesSeen = languageLocalesSeen.get(lang); |
| if (localesSeen == null || !localesSeen.contains(locale)) { |
| languageToOfficialChildren.put(lang, |
| OfficialStatus.unknown, locale, Boolean.TRUE); |
| } |
| } |
| } |
| |
| for (Entry<String, Set<String>> languageAndChildren : languageToChildren |
| .keyValuesSet()) { |
| String language = languageAndChildren.getKey(); |
| Set<String> children = languageAndChildren.getValue(); |
| M3<OfficialStatus, String, Boolean> officalStatusToChildren = languageToOfficialChildren |
| .get(language); |
| for (Entry<OfficialStatus, Map<String, Boolean>> entry : officalStatusToChildren) { |
| OfficialStatus status = entry.getKey(); |
| if (status != OfficialStatus.official |
| && status != OfficialStatus.de_facto_official) { |
| continue; |
| } |
| Set<String> officalChildren = entry.getValue().keySet(); |
| if (!children.containsAll(officalChildren)) { |
| Set<String> missing = new TreeSet<String>(officalChildren); |
| missing.removeAll(children); |
| String message = "Missing CLDR locales for " + status |
| + " languages: " + missing; |
| errln(message); |
| } else { |
| logln("CLDR locales " + children + " cover " + status |
| + " locales " + officalChildren); |
| } |
| |
| } |
| } |
| |
| if (DEBUG) { |
| Set<String> languages = new TreeSet<String>( |
| languageToChildren.keySet()); |
| languages.addAll(languageToOfficialChildren.keySet()); |
| System.out.print("\ncode\tlanguage"); |
| for (OfficialStatus status : OfficialStatus.values()) { |
| System.out.print("\tNo\t" + status); |
| } |
| System.out.println(); |
| for (String language : languages) { |
| if (!testOrg.contains(language)) { |
| continue; |
| } |
| System.out.print(language + "\t" |
| + testInfo.getEnglish().getName(language)); |
| |
| M3<OfficialStatus, String, Boolean> officialChildren = languageToOfficialChildren |
| .get(language); |
| for (OfficialStatus status : OfficialStatus.values()) { |
| Map<String, Boolean> children = officialChildren |
| .get(status); |
| if (children == null) { |
| System.out.print("\t" + 0 + "\t"); |
| } else { |
| System.out.print("\t" + children.size() + "\t" |
| + show(children.keySet(), false)); |
| } |
| } |
| System.out.println(); |
| } |
| } |
| } |
| |
| private String show(Set<String> joint, boolean showStatus) { |
| StringBuffer b = new StringBuffer(); |
| for (String s : joint) { |
| if (b.length() != 0) { |
| b.append(", "); |
| } |
| LanguageTagParser ltp = new LanguageTagParser().set(s); |
| String script = ltp.getScript(); |
| if (script.length() != 0) { |
| b.append(testInfo.getEnglish().getName(CLDRFile.SCRIPT_NAME, |
| script)); |
| } |
| String region = ltp.getRegion(); |
| if (region.length() != 0) { |
| if (script.length() != 0) { |
| b.append("-"); |
| } |
| b.append(testInfo.getEnglish().getName(CLDRFile.TERRITORY_NAME, |
| region)); |
| } |
| b.append(" [").append(s); |
| if (showStatus) { |
| PopulationData data = dataInfo |
| .getLanguageAndTerritoryPopulationData( |
| ltp.getLanguage(), region); |
| if (data == null) { |
| data = dataInfo.getLanguageAndTerritoryPopulationData( |
| ltp.getLanguageScript(), region); |
| } |
| b.append("; "); |
| b.append(data == null ? "?" : data.getOfficialStatus()); |
| } |
| b.append("]"); |
| |
| } |
| return b.toString(); |
| } |
| |
| private String removeScript(String lang) { |
| if (!lang.contains("_")) { |
| return lang; |
| } |
| LanguageTagParser ltp = new LanguageTagParser().set(lang); |
| // String ls = ltp.getLanguageScript(); |
| // if (defaultContents.contains(ls)) { |
| ltp.setScript(""); |
| // } |
| return ltp.toString(); |
| } |
| |
| public void TestLikelyAndDefaultConsistency() { |
| LikelySubtags likelySubtags = new LikelySubtags(); |
| LanguageTagParser ltp = new LanguageTagParser(); |
| // find multiscript locales |
| Relation<String, String> base2scripts = Relation.of( |
| new TreeMap<String, Set<String>>(), TreeSet.class); |
| Map<String, String> parent2default = new TreeMap<String, String>(); |
| Map<String, String> default2parent = new TreeMap<String, String>(); |
| Relation<String, String> base2locales = Relation.of( |
| new TreeMap<String, Set<String>>(), TreeSet.class); |
| |
| Set<String> knownMultiScriptLanguages = new HashSet<String>(Arrays.asList("bm", "ha")); |
| // get multiscript locales |
| for (String localeID : availableLocales) { |
| String script = ltp.set(localeID).getScript(); |
| final String base = ltp.getLanguage(); |
| if (!availableLocales.contains(base)) { |
| errln("Missing base locale for: " + localeID); |
| } |
| base2locales.put(base, localeID); |
| if (!script.isEmpty() && !base.equals("en")) { // HACK for en |
| base2scripts.put(base, script); |
| } |
| if (script.isEmpty() && knownMultiScriptLanguages.contains(base)) { |
| base2scripts.put(base, dataInfo.getDefaultScript(base)); |
| } |
| } |
| |
| // get default contents |
| for (String localeID : defaultContents) { |
| checkLocale(localeID, false); |
| String simpleParent = LocaleIDParser.getSimpleParent(localeID); |
| parent2default.put(simpleParent, localeID); |
| default2parent.put(localeID, simpleParent); |
| // if (!available.contains(simpleParent)) { |
| // // verify that base language has locale in CLDR (we don't want |
| // others) |
| // errln("Default contents contains locale not in CLDR:\t" + |
| // simpleParent); |
| // } |
| } |
| |
| // get likely |
| Map<String, String> likely2Maximized = likelySubtags.getToMaximized(); |
| for (Entry<String, String> likelyAndMaximized : likely2Maximized |
| .entrySet()) { |
| checkLocale(likelyAndMaximized.getKey(), true); |
| checkLocale(likelyAndMaximized.getValue(), true); |
| } |
| Map<String, String> exceptionDcLikely = new HashMap<String, String>(); |
| Map<String, String> exceptionLikelyDc = new HashMap<String, String>(); |
| for (String[] s : new String[][] { { "ar_001", "ar_Arab_EG" }, }) { |
| exceptionDcLikely.put(s[0], s[1]); |
| exceptionLikelyDc.put(s[1], s[0]); |
| } |
| |
| verifyDefaultContentsImplicationsForLikelySubtags(ltp, parent2default, |
| likely2Maximized, exceptionDcLikely); |
| |
| verifyLikelySubtagsImplicationsForDefaultContents(ltp, base2scripts, |
| parent2default, likely2Maximized, exceptionLikelyDc); |
| |
| verifyScriptsWithDefaultContents(ltp, base2scripts, parent2default, |
| base2locales); |
| } |
| |
| public void TestParentLocaleRelationships() { |
| // Testing invariant relationships between locales - See |
| // http://unicode.org/cldr/trac/ticket/5758 |
| Matcher langScript = PatternCache.get("^[a-z]{2,3}_[A-Z][a-z]{3}$") |
| .matcher(""); |
| for (String loc : availableLocales) { |
| if (langScript.reset(loc).matches()) { |
| String expectedParent = loc.split("_")[0]; |
| if (!defaultContents.contains(loc)) { |
| expectedParent = "root"; |
| } |
| String actualParent = dataInfo.getExplicitParentLocale(loc); |
| if (actualParent == null) { |
| actualParent = loc.split("_")[0]; |
| } |
| if (!actualParent.equals(expectedParent)) { |
| errln("Unexpected parent locale for locale " + loc |
| + ". Expected: " + expectedParent + " Got: " |
| + actualParent); |
| } |
| |
| if (dataInfo.getExplicitParentLocale(loc) != null |
| && defaultContents.contains(loc)) { |
| errln("Locale " |
| + loc |
| + " can't have an explicit parent AND be a default content locale"); |
| } |
| } |
| } |
| } |
| |
| public void TestParentLocaleInvariants() { |
| // Testing invariant relationships in parent locales - See |
| // http://unicode.org/cldr/trac/ticket/7887 |
| LocaleIDParser lp = new LocaleIDParser(); |
| for (String loc : availableLocales) { |
| String parentLocale = dataInfo.getExplicitParentLocale(loc); |
| if (parentLocale != null) { |
| if (!"root".equals(parentLocale) |
| && !lp.set(loc).getLanguage() |
| .equals(lp.set(parentLocale).getLanguage())) { |
| errln("Parent locale [" + parentLocale + "] for locale [" |
| + loc + "] cannot be a different language code."); |
| } |
| if (!"root".equals(parentLocale) |
| && !lp.set(loc).getScript() |
| .equals(lp.set(parentLocale).getScript())) { |
| errln("Parent locale [" + parentLocale + "] for locale [" |
| + loc + "] cannot be a different script code."); |
| } |
| lp.set(loc); |
| if (lp.getScript().length() == 0 && lp.getRegion().length() == 0) { |
| errln("Base language locale [" + loc + "] cannot have an explicit parent."); |
| } |
| |
| } |
| } |
| } |
| |
| public void TestParentLocalesForCycles() { |
| // Testing for cyclic relationships in parent locales - See |
| // http://unicode.org/cldr/trac/ticket/7887 |
| for (String loc : availableLocales) { |
| String currentLoc = loc; |
| boolean foundError = false; |
| List<String> inheritanceChain = new ArrayList<String>(Arrays.asList(loc)); |
| while (currentLoc != null && !foundError) { |
| currentLoc = LocaleIDParser.getParent(currentLoc); |
| if (inheritanceChain.contains(currentLoc)) { |
| foundError = true; |
| inheritanceChain.add(currentLoc); |
| errln("Inheritance chain for locale [" + loc + "] contains a cyclic relationship. " + inheritanceChain.toString()); |
| } |
| inheritanceChain.add(currentLoc); |
| } |
| } |
| } |
| |
| private void verifyScriptsWithDefaultContents(LanguageTagParser ltp, |
| Relation<String, String> base2scripts, |
| Map<String, String> parent2default, |
| Relation<String, String> base2locales) { |
| Set<String> skip = Builder.with(new HashSet<String>()) |
| .addAll("root", "und") |
| .freeze(); |
| Set<String> languagesWithOneOrLessLocaleScriptInCommon = new HashSet<String>(Arrays.asList("bm", "ha", "ms", "iu", "mn")); |
| // for each base we have to have, |
| // if multiscript, we have default contents for base+script, |
| // base+script+region; |
| // otherwise base+region. |
| for (String base : base2locales.keySet()) { |
| if (skip.contains(base)) { |
| continue; |
| } |
| String defaultContent = parent2default.get(base); |
| // Set<String> likely = base2likely.get(base); |
| // if (likely == null) { |
| // errln("Missing likely subtags for: " + base + " " + |
| // suggestLikelySubtagFor(base)); |
| // } |
| if (defaultContent == null) { |
| errln("Missing default content for: " + base + " " |
| + suggestLikelySubtagFor(base)); |
| continue; |
| } |
| Set<String> scripts = base2scripts.get(base); |
| ltp.set(defaultContent); |
| String script = ltp.getScript(); |
| String region = ltp.getRegion(); |
| if (scripts == null || languagesWithOneOrLessLocaleScriptInCommon.contains(base)) { |
| if (!script.isEmpty()) { |
| errln("Script should be empty in default content for: " |
| + base + "," + defaultContent); |
| } |
| if (region.isEmpty()) { |
| errln("Region must not be empty in default content for: " |
| + base + "," + defaultContent); |
| } |
| } else { |
| if (script.isEmpty()) { |
| errln("Script should not be empty in default content for: " |
| + base + "," + defaultContent); |
| } |
| if (!region.isEmpty()) { |
| errln("Region should be empty in default content for: " |
| + base + "," + defaultContent); |
| } |
| String defaultContent2 = parent2default.get(defaultContent); |
| if (defaultContent2 == null) { |
| errln("Missing default content for: " + defaultContent); |
| continue; |
| } |
| ltp.set(defaultContent2); |
| region = ltp.getRegion(); |
| if (region.isEmpty()) { |
| errln("Region must not be empty in default content for: " |
| + base + "," + defaultContent); |
| } |
| } |
| } |
| } |
| |
| private void verifyLikelySubtagsImplicationsForDefaultContents( |
| LanguageTagParser ltp, Relation<String, String> base2scripts, |
| Map<String, String> parent2default, |
| Map<String, String> likely2Maximized, |
| Map<String, String> exceptionLikelyDc) { |
| // Now check invariants for all LikelySubtags implications for Default |
| // Contents |
| // a) suppose likely max for la_Scrp => la_Scrp_RG |
| // Then default contents la_Scrp => la_Scrp_RG |
| // b) suppose likely max for la_RG => la_Scrp_RG |
| // Then we can draw no conclusions // was default contents la_Scrp => |
| // la_Scrp_RG |
| // c) suppose likely max for la => la_Scrp_RG |
| // Then default contents la => la_Scrp && la_Scrp => la_Scrp_RG |
| // or default contents la => la_RG && ! la_Scrp => la_Scrp_RG |
| |
| TreeSet<String> additionalDefaultContents = new TreeSet<String>(); |
| |
| for (Entry<String, String> entry : likely2Maximized.entrySet()) { |
| String source = entry.getKey(); |
| String likelyMax = entry.getValue(); |
| String sourceLang = ltp.set(source).getLanguage(); |
| if (sourceLang.equals("und") || source.equals("zh_Hani") |
| || source.equals("tl")) { |
| continue; |
| } |
| String sourceScript = ltp.getScript(); |
| String sourceRegion = ltp.getRegion(); |
| |
| String likelyMaxLang = ltp.set(likelyMax).getLanguage(); |
| String likelyMaxScript = ltp.getScript(); |
| String likelyMaxRegion = ltp.getRegion(); |
| |
| String dc = parent2default.get(source); |
| String possibleException = exceptionLikelyDc.get(likelyMax); |
| if (possibleException != null && possibleException.equals(dc)) { |
| continue; |
| } |
| String likelyLangScript = likelyMaxLang + "_" + likelyMaxScript; |
| String dcFromLangScript = parent2default.get(likelyLangScript); |
| |
| boolean consistent = true; |
| String caseNumber = null; |
| if (consistent) { |
| if (!sourceScript.isEmpty()) { |
| caseNumber = "a"; |
| if (dc == null) { |
| if (EXPECT_EQUALITY) { |
| String expected = likelyMax; |
| errln("Default contents null for " + source |
| + ", expected:\t" + expected); |
| additionalDefaultContents.add(expected); |
| } |
| continue; |
| } |
| consistent = likelyMax.equals(dc); |
| } else if (!sourceRegion.isEmpty()) { // a |
| caseNumber = "b"; |
| // consistent = likelyMax.equals(dcFromLangScript); |
| } else { // c |
| caseNumber = "c"; |
| if (dc == null) { |
| if (EXPECT_EQUALITY) { |
| String expected = base2scripts.get(source) == null ? likelyMaxLang |
| + "_" + likelyMaxRegion |
| : likelyMaxLang + "_" + likelyMaxScript; |
| errln("Default contents null for " + source |
| + ", expected:\t" + expected); |
| additionalDefaultContents.add(expected); |
| } |
| continue; |
| } |
| String dcScript = ltp.set(dc).getScript(); |
| consistent = likelyLangScript.equals(dc) |
| && likelyMax.equals(dcFromLangScript) |
| || dcScript.isEmpty() |
| && !likelyMax.equals(dcFromLangScript); |
| // || dcScript.isEmpty() && dcRegion.equals(likelyMaxRegion) |
| // && dcFromLangScript == null; |
| } |
| } |
| if (!consistent) { |
| errln("default contents inconsistent with likely subtag: (" |
| + caseNumber + ")" + "\n\t" + source + " => (ls) " |
| + likelyMax + "\n\t" + source + " => (dc) " + dc |
| + "\n\t" + likelyLangScript + " => (dc) " |
| + dcFromLangScript); |
| } |
| } |
| if (additionalDefaultContents.size() != 0) { |
| errln("Suggested additions to supplementalMetadata/../defaultContent:\n" |
| + CollectionUtilities.join(additionalDefaultContents, " ")); |
| } |
| } |
| |
| private void verifyDefaultContentsImplicationsForLikelySubtags( |
| LanguageTagParser ltp, Map<String, String> parent2default, |
| Map<String, String> likely2Maximized, |
| Map<String, String> exceptionDcLikely) { |
| // Now check invariants for all Default Contents implications for |
| // LikelySubtags |
| // a) suppose default contents la => la_Scrp. |
| // Then the likely contents for la => la_Scrp_* |
| // b) suppose default contents la => la_RG. |
| // Then the likely contents for la => la_*_RG |
| // c) suppose default contents la_Scrp => la_Scrp_RG. |
| // Then the likely contents of la_Scrp => la_Scrp_RG OR likely contents |
| // for la => la_*_* |
| for (Entry<String, String> parentAndDefault : parent2default.entrySet()) { |
| String source = parentAndDefault.getKey(); |
| String dc = parentAndDefault.getValue(); |
| String likelyMax = likely2Maximized.get(source); |
| |
| // skip special exceptions |
| String possibleException = exceptionDcLikely.get(dc); |
| if (possibleException != null |
| && possibleException.equals(likelyMax)) { |
| continue; |
| } |
| |
| String sourceLang = ltp.set(source).getLanguage(); |
| String sourceScript = ltp.getScript(); |
| // there cannot be a sourceRegion |
| |
| String dcScript = ltp.set(dc).getScript(); |
| String dcRegion = ltp.getRegion(); |
| |
| String likelyMaxLang = "", likelyMaxScript = "", likelyMaxRegion = ""; |
| if (likelyMax != null) { |
| likelyMaxLang = ltp.set(likelyMax).getLanguage(); |
| likelyMaxScript = ltp.getScript(); |
| likelyMaxRegion = ltp.getRegion(); |
| } |
| |
| String likelyMax2 = likely2Maximized.get(sourceLang); |
| |
| boolean consistent = true; |
| |
| if (sourceScript.isEmpty()) { // a or b |
| if (!dcScript.isEmpty()) { // a |
| consistent = likelyMaxLang.equals(source) |
| && likelyMaxScript.equals(dcScript); |
| } else { // b |
| consistent = likelyMaxLang.equals(source) |
| && likelyMaxRegion.equals(dcRegion); |
| } |
| } else { // c |
| consistent = dc.equals(likelyMax) || likelyMax2 != null; |
| } |
| if (!consistent) { |
| errln("likely subtag inconsistent with default contents: " |
| + "\n\t" |
| + source |
| + " =>( dc) " |
| + dc |
| + "\n\t" |
| + source |
| + " => (ls) " |
| + likelyMax |
| + (source.equals(sourceLang) ? "" : "\n\t" + sourceLang |
| + " => (ls) " + likelyMax2)); |
| } |
| } |
| } |
| |
| /** |
| * Suggest a likely subtag |
| * |
| * @param base |
| * @return |
| */ |
| static String suggestLikelySubtagFor(String base) { |
| SupplementalDataInfo sdi = SupplementalDataInfo.getInstance(); |
| |
| CLDRLocale loc = CLDRLocale.getInstance(base); |
| |
| if (!loc.getLanguage().equals(base)) { |
| return " (no suggestion- not a simple language locale)"; // no |
| // suggestion |
| // unless |
| // just |
| // a |
| // language |
| // locale. |
| } |
| Set<BasicLanguageData> basicData = sdi.getBasicLanguageData(base); |
| |
| for (BasicLanguageData bld : basicData) { |
| if (bld.getType() == org.unicode.cldr.util.SupplementalDataInfo.BasicLanguageData.Type.primary) { |
| Set<String> scripts = bld.getScripts(); |
| Set<String> territories = bld.getTerritories(); |
| |
| if (scripts.size() == 1) { |
| if (territories.size() == 1) { |
| return createSuggestion( |
| loc, |
| CLDRLocale.getInstance(base + "_" |
| + scripts.iterator().next() + "_" |
| + territories.iterator().next())); |
| } |
| } |
| return "(no suggestion - multiple scripts or territories)"; |
| } |
| } |
| return ("(no suggestion- no data)"); |
| } |
| |
| /** |
| * Format and return a suggested likelysubtag |
| */ |
| private static String createSuggestion(CLDRLocale loc, CLDRLocale toLoc) { |
| return " Suggest this to likelySubtags.xml: <likelySubtag from=\"" |
| + loc |
| + "\" to=\"" |
| + toLoc |
| + "\"/>\n" |
| + " <!--{ " |
| + loc.getDisplayName() |
| + "; ?; ? } => { " |
| + loc.getDisplayName() |
| + "; " |
| + toLoc.toULocale().getDisplayScript() |
| + "; " |
| + toLoc.toULocale().getDisplayCountry() + " }-->"; |
| |
| } |
| |
| public void TestDeprecatedTerritoryDataLocaleIds() { |
| HashSet<String> checked = new HashSet<String>(); |
| for (String language : dataInfo |
| .getLanguagesForTerritoriesPopulationData()) { |
| checkLocale(language, false); // checks la_Scrp and la |
| for (String region : dataInfo |
| .getTerritoriesForPopulationData(language)) { |
| if (!checked.contains(region)) { |
| checkValidCode(language + "_" + region, "territory", |
| region, false); |
| checked.add(region); |
| } |
| } |
| } |
| for (String language : dataInfo.getBasicLanguageDataLanguages()) { |
| checkLocale(language, false); // checks la_Scrp and la |
| Set<BasicLanguageData> data = dataInfo |
| .getBasicLanguageData(language); |
| for (BasicLanguageData datum : data) { |
| for (String script : datum.getScripts()) { |
| checkValidCode(language + "_" + script, "script", script, |
| false); |
| checked.add(script); |
| } |
| for (String region : datum.getTerritories()) { |
| checkValidCode(language + "_" + region, "territory", |
| region, false); |
| checked.add(region); |
| } |
| } |
| } |
| |
| } |
| |
| public void TestBasicLanguageDataAgainstScriptMetadata() { |
| // the invariants are: |
| // if there is primary data, the script must be there |
| // otherwise it must be in the secondary |
| main: for (String script : ScriptMetadata.getScripts()) { |
| Info info = ScriptMetadata.getInfo(script); |
| String language = info.likelyLanguage; |
| if (language.equals("und")) { |
| continue; |
| } |
| Map<Type, BasicLanguageData> data = dataInfo |
| .getBasicLanguageDataMap(language); |
| if (data == null) { |
| logln("Warning: ScriptMetadata has " + language + " for " |
| + script + "," + " but " + language |
| + " is missing in language_script.txt"); |
| continue; |
| } |
| for (BasicLanguageData entry : data.values()) { |
| if (entry.getScripts().contains(script)) { |
| continue main; |
| } |
| continue; |
| } |
| logln("Warning: ScriptMetadata has " + language + " for " + script |
| + "," + " but " + language + " doesn't have " + script |
| + " in language_script.txt"); |
| } |
| } |
| |
| public void TestCldrFileConsistency() { |
| boolean haveErrors = false; |
| for (String locale : testInfo.getCldrFactory().getAvailable()) { |
| CLDRFile cldrFileToCheck = testInfo.getCLDRFile(locale, |
| false); |
| int errors = 0; |
| for (String path : cldrFileToCheck) { |
| if (!pathMatcher.reset(path).find()) { |
| continue; |
| } |
| String fullPath = cldrFileToCheck.getFullXPath(path); |
| if (fullPath == null) { |
| // try again, for debugging |
| fullPath = cldrFileToCheck.getFullXPath(path); |
| String value = cldrFileToCheck.getStringValue(path); |
| if (DEBUG) { |
| errln("Invalid full path\t" + locale + ", " + path |
| + ", " + fullPath + ", " + value); |
| } |
| errors++; |
| haveErrors = true; |
| } |
| } |
| if (errors != 0) { |
| errln(locale |
| + (errors != 0 ? "\tinvalid getFullXPath() values:" |
| + errors : "")); |
| } else { |
| logln(locale); |
| } |
| } |
| if (haveErrors && !DEBUG) { |
| errln("Use -DDEBUG to see details"); |
| } |
| } |
| |
| static SupplementalDataInfo info = SupplementalDataInfo.getInstance(); |
| LanguageTagParser ltp = new LanguageTagParser(); |
| |
| // public void TestAliases() { |
| // Factory factory = Factory.make(CldrUtility.MAIN_DIRECTORY, fileMatcher); |
| // Set<String> allLocales = Factory.make(CldrUtility.MAIN_DIRECTORY, |
| // ".*").getAvailable(); |
| // |
| // LanguageTagCanonicalizer languageTagCanonicalizer = new |
| // LanguageTagCanonicalizer(); |
| // |
| // Set<String> defaultContents = info.getDefaultContentLocales(); |
| // |
| // Map<String, String> likelySubtags = info.getLikelySubtags(); |
| // |
| // XPathParts xpp = new XPathParts(); |
| // |
| // // get the top level aliases, and verify that they are consistent with |
| // // maximization |
| // Map<String, String> topLevelAliases = new TreeMap<String, String>(); |
| // Set<String> crossScriptSet = new TreeSet<String>(); |
| // Set<String> aliasPaths = new TreeSet<String>(); |
| // Set<String> locales = factory.getAvailable(); |
| // |
| // // get the languages that need scripts |
| // // TODO broaden to beyond CLDR |
| // Set<String> needScripts = new TreeSet<String>(); |
| // for (String locale : locales) { |
| // String script = ltp.set(locale).getScript(); |
| // if (script.length() != 0) { |
| // needScripts.add(ltp.getLanguage()); |
| // } |
| // } |
| // |
| // logln("Languages that have scripts:\t" + needScripts); |
| // |
| // for (String locale : locales) { |
| // |
| // // get alias locale |
| // String aliasLocale = locale; |
| // String explicitAlias = null; |
| // String aliasPathNew = null; |
| // CLDRFile cldrFileToCheck = factory.make(locale, false); |
| // aliasPaths.clear(); |
| // // examples: |
| // // in: <alias source="id" path="//ldml"/> |
| // // ar_IR: <alias source="az_Arab_IR" path="//ldml"/> |
| // |
| // cldrFileToCheck.getPaths("//ldml/alias", null, aliasPaths); |
| // if (aliasPaths.size() != 0) { |
| // String aliasPath = aliasPaths.iterator().next(); |
| // String fullPath = cldrFileToCheck.getFullXPath(aliasPath); |
| // explicitAlias = aliasLocale = xpp.set(fullPath).getAttributeValue(1, |
| // "source"); |
| // String aliasParent = LocaleIDParser.getParent(aliasLocale); |
| // if (!aliasParent.equals("root")) { |
| // topLevelAliases.put(locale, aliasParent); |
| // } |
| // aliasPathNew = xpp.set(fullPath).getAttributeValue(1, "path"); |
| // if ("//ldml/".equals(aliasPathNew)) { |
| // errln("Bad alias path:\t" + fullPath); |
| // } |
| // } |
| // |
| // checkAliasValues(cldrFileToCheck, allLocales); |
| // |
| // // get canonicalized |
| // String canonicalizedLocale = languageTagCanonicalizer.transform(locale); |
| // if (!locale.equals(canonicalizedLocale)) { |
| // logln("Locale\t" + locale + " => " + canonicalizedLocale); |
| // } |
| // |
| // String base = ltp.set(canonicalizedLocale).getLanguage(); |
| // String script = ltp.getScript(); |
| // if (canonicalizedLocale.equals(base)) { // eg, id, az |
| // continue; |
| // } |
| // |
| // // see if the locale's default script is the same as the base locale's |
| // |
| // String maximized = maximize(likelySubtags, canonicalizedLocale); |
| // if (maximized == null) { |
| // errln("Missing likely subtags for:\t" + locale + " " + |
| // suggestLikelySubtagFor(locale)); |
| // continue; |
| // } |
| // String maximizedScript = ltp.set(maximized).getScript(); |
| // |
| // String minimized = minimize(likelySubtags, canonicalizedLocale); |
| // |
| // String baseMaximized = maximize(likelySubtags, base); |
| // String baseScript = ltp.set(baseMaximized).getScript(); |
| // |
| // if (script.length() != 0 && !script.equals(baseScript)) { |
| // crossScriptSet.add(ltp.set(locale).getLanguageScript()); |
| // } |
| // |
| // // Finally, put together the expected alias for comparison. |
| // // It is the "best" alias, in that the default-content locales are |
| // skipped in favor of their parents |
| // |
| // String expectedAlias = |
| // !baseScript.equals(maximizedScript) ? minimized : |
| // !locale.equals(canonicalizedLocale) ? canonicalizedLocale : |
| // // needScripts.contains(base) ? ltp.getLanguageScript() : |
| // locale; |
| // |
| // if (!equals(aliasLocale, expectedAlias)) { |
| // String aliasMaximized = maximize(likelySubtags, aliasLocale); |
| // String expectedMaximized = maximize(likelySubtags, expectedAlias); |
| // if (!equals(aliasMaximized, expectedMaximized)) { |
| // errln("For locale:\t" + locale |
| // + ",\tbase-script:\t" + baseScript |
| // + ",\texpected alias Locale != actual alias Locale:\t" |
| // + expectedAlias + ", " + aliasLocale); |
| // } else if (explicitAlias == null) { |
| // // skip, we don't care in this case |
| // // but we emit warnings if the other conditions are true. The aliasing |
| // could be simpler. |
| // } else if (equals(expectedAlias, locale)) { |
| // logln("Warning; alias could be omitted. For locale:\t" + locale |
| // + ",\tbase-script:\t" + baseScript |
| // + ",\texpected alias Locale != actual alias Locale:\t" |
| // + expectedAlias + ", " + aliasLocale); |
| // } else { |
| // logln("Warning; alias could be minimized. For locale:\t" + locale |
| // + ",\tbase-script:\t" + baseScript |
| // + ",\texpected alias Locale != actual alias Locale:\t" |
| // + expectedAlias + ", " + aliasLocale); |
| // } |
| // } |
| // } |
| // |
| // // check the LocaleIDParser.TOP_LEVEL_ALIAS_LOCALES value and make sure |
| // it matches what is in the files in main/ |
| // |
| // if (!topLevelAliases.equals(LocaleIDParser.TOP_LEVEL_ALIAS_LOCALES) |
| // && locales.equals(allLocales)) { |
| // String diff = showDifferences(LocaleIDParser.TOP_LEVEL_ALIAS_LOCALES, |
| // topLevelAliases); |
| // if (!diff.isEmpty()) { |
| // errln("LocaleIDParser.TOP_LEVEL_ALIAS_LOCALES ≠ topLevelAliases: " + |
| // diff); |
| // } |
| // StringBuilder result = new StringBuilder( |
| // "Suggest changing LocaleIDParser.TOP_LEVEL_ALIAS_LOCALES to:\n"); |
| // for (Entry<String, String> entry : topLevelAliases.entrySet()) { |
| // result.append("\t.put(\"") |
| // .append(entry.getKey()) |
| // .append("\", \"") |
| // .append(entry.getValue()) |
| // .append("\")\n"); |
| // } |
| // errln(result.toString()); |
| // } else { |
| // logln("Top Level Aliases:\t" + topLevelAliases); |
| // } |
| // |
| // // verify that they are the same as what we would get if we were to |
| // maximize |
| // // all the locales and check against default_contents |
| // |
| // for (String locale : defaultContents) { |
| // CLDRFile cldrFileToCheck = null; |
| // try { |
| // cldrFileToCheck = factory.make(locale, false); |
| // } catch (Exception e) {} |
| // if (cldrFileToCheck == null) { |
| // logln("Present in default contents but has no XML file:\t" + locale); |
| // continue; |
| // } |
| // logln("Locale:\t" + locale); |
| // // verify empty, except for identity elements and alias |
| // for (String path : cldrFileToCheck) { |
| // if (path.contains("/identity/")) { |
| // continue; |
| // } |
| // errln("Default content locale not empty:\t" + locale + ", " + path); |
| // break; |
| // } |
| // } |
| // } |
| |
| Matcher aliasMatcher = PatternCache.get("//ldml.*/alias.*").matcher(""); |
| |
| private void checkAliasValues(CLDRFile cldrFileToCheck, Set<String> locales) { |
| Set<String> aliasPaths = new TreeSet<String>(); |
| Set<String> allAliasPaths = cldrFileToCheck.getPaths("//ldml/", |
| aliasMatcher, aliasPaths); |
| XPathParts xpp = new XPathParts(); |
| for (String aliasPath : allAliasPaths) { |
| if (aliasPath.startsWith("//ldml/alias")) { |
| continue; // we have different tests elsewhere |
| } |
| String fullPath = cldrFileToCheck.getFullXPath(aliasPath); |
| String aliasLocale = xpp.set(fullPath).getAttributeValue(-1, |
| "source"); |
| // just check to make sure that the alias is in the locales |
| if (aliasLocale != null && !aliasLocale.equals("locale")) { |
| if (!locales.contains(aliasLocale)) { |
| errln("Unknown Alias:\t" + aliasLocale + "\t in\t" |
| + fullPath); |
| } |
| } |
| String aliasPathNew = xpp.set(fullPath).getAttributeValue(-1, |
| "path"); |
| // just one check |
| if (".".equals(aliasPathNew)) { |
| errln("Illegal path, must not be .:\t" + aliasLocale |
| + "\t in\t" + fullPath); |
| } |
| |
| } |
| } |
| |
| private String minimize(Map<String, String> likelySubtags, String locale) { |
| String result = GenerateMaximalLocales.minimize(locale, likelySubtags, |
| false); |
| if (result == null) { |
| LanguageTagParser ltp3 = new LanguageTagParser().set(locale); |
| List<String> variants = ltp3.getVariants(); |
| Map<String, String> extensions = ltp3.getExtensions(); |
| Set<String> emptySet = Collections.emptySet(); |
| ltp3.setVariants(emptySet); |
| Map<String, String> emptyMap = Collections.emptyMap(); |
| ltp3.setExtensions(emptyMap); |
| String newLocale = ltp3.toString(); |
| result = GenerateMaximalLocales.minimize(newLocale, likelySubtags, |
| false); |
| if (result != null) { |
| ltp3.set(result); |
| ltp3.setVariants(variants); |
| ltp3.setExtensions(extensions); |
| result = ltp3.toString(); |
| } |
| } |
| return result; |
| } |
| |
| private String maximize(Map<String, String> likelySubtags, String locale) { |
| String result = GenerateMaximalLocales.maximize(locale, likelySubtags); |
| if (result == null) { |
| LanguageTagParser ltp3 = new LanguageTagParser().set(locale); |
| List<String> variants = ltp3.getVariants(); |
| Map<String, String> extensions = ltp3.getExtensions(); |
| Set<String> emptySet = Collections.emptySet(); |
| ltp3.setVariants(emptySet); |
| Map<String, String> emptyMap = Collections.emptyMap(); |
| ltp3.setExtensions(emptyMap); |
| String newLocale = ltp3.toString(); |
| result = GenerateMaximalLocales.maximize(newLocale, likelySubtags); |
| if (result != null) { |
| ltp3.set(result); |
| ltp3.setVariants(variants); |
| ltp3.setExtensions(extensions); |
| result = ltp3.toString(); |
| } |
| } |
| return result; |
| } |
| |
| // TODO move this into central utilities |
| public static boolean equals(CharSequence string, int codePoint) { |
| if (string == null) { |
| return false; |
| } |
| switch (string.length()) { |
| case 1: |
| return codePoint == string.charAt(0); |
| case 2: |
| return codePoint >= 0x10000 |
| && codePoint == Character.codePointAt(string, 0); |
| default: |
| return false; |
| } |
| } |
| |
| // TODO move this into central utilities |
| |
| private static final StandardCodes STANDARD_CODES = testInfo.getStandardCodes(); |
| private static final Map<String, Map<String, R2<List<String>, String>>> DEPRECATED_INFO = dataInfo |
| .getLocaleAliasInfo(); |
| |
| private void checkLocale(String localeID, boolean allowDeprecated) { |
| // verify that the localeID is valid |
| LanguageTagParser ltp = new LanguageTagParser().set(localeID); |
| String language = ltp.getLanguage(); |
| String script = ltp.getScript(); |
| String region = ltp.getRegion(); |
| // TODO check variants, extensions also. |
| checkValidCode(localeID, "language", language, allowDeprecated); |
| checkValidCode(localeID, "script", script, allowDeprecated); |
| checkValidCode(localeID, "territory", region, allowDeprecated); |
| } |
| |
| private void checkValidCode(String localeID, String subtagType, |
| String subtag, boolean allowDeprecated) { |
| if (subtagType.equals("language")) { |
| if (subtag.equals("und")) { |
| return; |
| } |
| } else { |
| if (subtag.isEmpty()) { |
| return; |
| } |
| } |
| if (!STANDARD_CODES.getAvailableCodes(subtagType).contains(subtag)) { |
| errln("Locale " + localeID + " contains illegal " |
| + showCode(subtagType, subtag)); |
| } else if (!allowDeprecated) { |
| // "language" -> "sh" -> <{"sr_Latn"}, reason> |
| R2<List<String>, String> deprecatedInfo = DEPRECATED_INFO.get( |
| subtagType).get(subtag); |
| if (deprecatedInfo != null) { |
| errln("Locale " + localeID + " contains deprecated " |
| + showCode(subtagType, subtag) + " " |
| + deprecatedInfo.get1() + "; suggest " |
| + showName(deprecatedInfo.get0(), subtagType)); |
| } |
| } |
| } |
| |
| private String showName(List<String> deprecatedInfo, String subtagType) { |
| StringBuilder result = new StringBuilder(); |
| for (String s : deprecatedInfo) { |
| result.append(showName(subtagType, s)).append(" "); |
| } |
| return result.toString(); |
| } |
| |
| private String showCode(String subtagType, String subtag) { |
| return subtagType + " code: " + showName(subtagType, subtag); |
| } |
| |
| private String showName(String subtagType, String subtag) { |
| return subtag + " (" + getName(subtagType, subtag) + ")"; |
| } |
| |
| private String getName(String subtagType, String subtag) { |
| Map<String, String> data = STANDARD_CODES.getLangData(subtagType, |
| subtag); |
| if (data == null) { |
| return "<no name>"; |
| } |
| return data.get("Description"); |
| } |
| |
| // TODO move this into central utilities |
| public static boolean equals(int codePoint, CharSequence string) { |
| return equals(string, codePoint); |
| } |
| |
| // TODO move this into central utilities |
| public static boolean equals(Object a, Object b) { |
| return a == b ? true : a == null || b == null ? false : a.equals(b); |
| } |
| |
| // TODO move this into central utilities |
| private <K, V> String showDifferences(Map<K, V> a, Map<K, V> b) { |
| StringBuilder result = new StringBuilder(); |
| Set<K> keys = new LinkedHashSet<K>(); |
| keys.addAll(a.keySet()); |
| keys.addAll(b.keySet()); |
| for (K key : keys) { |
| if (!a.containsKey(key)) { |
| result.append(key).append("→‹").append(a.get(key)) |
| .append("›,∅; "); |
| } else if (!b.containsKey(key)) { |
| result.append(key).append("→∅,‹").append(b.get(key)) |
| .append("›; "); |
| } else { |
| V aKey = a.get(key); |
| V bKey = b.get(key); |
| if (!equals(aKey, bKey)) { |
| result.append(key).append("→‹").append(a.get(key)) |
| .append("›,‹").append(b.get(key)).append("›; "); |
| } |
| } |
| } |
| return result.toString(); |
| } |
| |
| public void TestLanguageTagParser() { |
| LanguageTagParser ltp = new LanguageTagParser(); |
| ltp.set("en-Cyrl-US"); |
| assertEquals(null, "en", ltp.getLanguage()); |
| assertEquals(null, "en_Cyrl", ltp.getLanguageScript()); |
| assertEquals(null, "Cyrl", ltp.getScript()); |
| assertEquals(null, "US", ltp.getRegion()); |
| try { |
| ltp.set("$"); |
| assertFalse("expected exception", true); |
| } catch (Exception e) { |
| logln(e.getMessage()); |
| } |
| } |
| } |
